[Openmp-commits] [openmp] r289732 - Change source files from .c to .cpp
Jonathan Peyton via Openmp-commits
openmp-commits at lists.llvm.org
Wed Dec 14 14:39:12 PST 2016
Author: jlpeyton
Date: Wed Dec 14 16:39:11 2016
New Revision: 289732
URL: http://llvm.org/viewvc/llvm-project?rev=289732&view=rev
Log:
Change source files from .c to .cpp
Patch by Hansang Bae
Differential Revision: https://reviews.llvm.org/D26688
Added:
openmp/trunk/runtime/src/kmp_alloc.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_alloc.c
openmp/trunk/runtime/src/kmp_atomic.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_atomic.c
openmp/trunk/runtime/src/kmp_csupport.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_csupport.c
openmp/trunk/runtime/src/kmp_debug.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_debug.c
openmp/trunk/runtime/src/kmp_debugger.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_debugger.c
openmp/trunk/runtime/src/kmp_environment.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_environment.c
openmp/trunk/runtime/src/kmp_error.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_error.c
openmp/trunk/runtime/src/kmp_ftn_cdecl.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_ftn_cdecl.c
openmp/trunk/runtime/src/kmp_ftn_extra.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_ftn_extra.c
openmp/trunk/runtime/src/kmp_ftn_stdcall.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_ftn_stdcall.c
openmp/trunk/runtime/src/kmp_global.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_global.c
openmp/trunk/runtime/src/kmp_gsupport.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_gsupport.c
openmp/trunk/runtime/src/kmp_i18n.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_i18n.c
openmp/trunk/runtime/src/kmp_import.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_import.c
openmp/trunk/runtime/src/kmp_io.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_io.c
openmp/trunk/runtime/src/kmp_itt.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_itt.c
openmp/trunk/runtime/src/kmp_runtime.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_runtime.c
openmp/trunk/runtime/src/kmp_settings.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_settings.c
openmp/trunk/runtime/src/kmp_str.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_str.c
openmp/trunk/runtime/src/kmp_stub.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_stub.c
openmp/trunk/runtime/src/kmp_tasking.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_tasking.c
openmp/trunk/runtime/src/kmp_taskq.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_taskq.c
openmp/trunk/runtime/src/kmp_threadprivate.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_threadprivate.c
openmp/trunk/runtime/src/kmp_utility.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_utility.c
openmp/trunk/runtime/src/kmp_version.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/kmp_version.c
openmp/trunk/runtime/src/ompt-general.cpp
- copied, changed from r289725, openmp/trunk/runtime/src/ompt-general.c
openmp/trunk/runtime/src/ompt-specific.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/ompt-specific.c
openmp/trunk/runtime/src/tsan_annotations.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/tsan_annotations.c
openmp/trunk/runtime/src/z_Linux_util.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/z_Linux_util.c
openmp/trunk/runtime/src/z_Windows_NT-586_util.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/z_Windows_NT-586_util.c
openmp/trunk/runtime/src/z_Windows_NT_util.cpp
- copied unchanged from r289725, openmp/trunk/runtime/src/z_Windows_NT_util.c
Removed:
openmp/trunk/runtime/src/kmp_alloc.c
openmp/trunk/runtime/src/kmp_atomic.c
openmp/trunk/runtime/src/kmp_csupport.c
openmp/trunk/runtime/src/kmp_debug.c
openmp/trunk/runtime/src/kmp_debugger.c
openmp/trunk/runtime/src/kmp_environment.c
openmp/trunk/runtime/src/kmp_error.c
openmp/trunk/runtime/src/kmp_ftn_cdecl.c
openmp/trunk/runtime/src/kmp_ftn_extra.c
openmp/trunk/runtime/src/kmp_ftn_stdcall.c
openmp/trunk/runtime/src/kmp_global.c
openmp/trunk/runtime/src/kmp_gsupport.c
openmp/trunk/runtime/src/kmp_i18n.c
openmp/trunk/runtime/src/kmp_import.c
openmp/trunk/runtime/src/kmp_io.c
openmp/trunk/runtime/src/kmp_itt.c
openmp/trunk/runtime/src/kmp_runtime.c
openmp/trunk/runtime/src/kmp_settings.c
openmp/trunk/runtime/src/kmp_str.c
openmp/trunk/runtime/src/kmp_stub.c
openmp/trunk/runtime/src/kmp_tasking.c
openmp/trunk/runtime/src/kmp_taskq.c
openmp/trunk/runtime/src/kmp_threadprivate.c
openmp/trunk/runtime/src/kmp_utility.c
openmp/trunk/runtime/src/kmp_version.c
openmp/trunk/runtime/src/ompt-general.c
openmp/trunk/runtime/src/ompt-specific.c
openmp/trunk/runtime/src/tsan_annotations.c
openmp/trunk/runtime/src/z_Linux_util.c
openmp/trunk/runtime/src/z_Windows_NT-586_util.c
openmp/trunk/runtime/src/z_Windows_NT_util.c
Modified:
openmp/trunk/runtime/CMakeLists.txt
openmp/trunk/runtime/cmake/LibompUtils.cmake
openmp/trunk/runtime/src/CMakeLists.txt
Modified: openmp/trunk/runtime/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/CMakeLists.txt?rev=289732&r1=289731&r2=289732&view=diff
==============================================================================
--- openmp/trunk/runtime/CMakeLists.txt (original)
+++ openmp/trunk/runtime/CMakeLists.txt Wed Dec 14 16:39:11 2016
@@ -157,7 +157,7 @@ set(LIBOMP_USE_HWLOC FALSE CACHE BOOL
set(LIBOMP_HWLOC_INSTALL_DIR /usr/local CACHE PATH
"Install path for hwloc library")
-# Get the build number from kmp_version.c
+# Get the build number from kmp_version.cpp
libomp_get_build_number("${CMAKE_CURRENT_SOURCE_DIR}" LIBOMP_VERSION_BUILD)
math(EXPR LIBOMP_VERSION_BUILD_YEAR "${LIBOMP_VERSION_BUILD}/10000")
math(EXPR LIBOMP_VERSION_BUILD_MONTH_DAY "${LIBOMP_VERSION_BUILD}%10000")
Modified: openmp/trunk/runtime/cmake/LibompUtils.cmake
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/cmake/LibompUtils.cmake?rev=289732&r1=289731&r2=289732&view=diff
==============================================================================
--- openmp/trunk/runtime/cmake/LibompUtils.cmake (original)
+++ openmp/trunk/runtime/cmake/LibompUtils.cmake Wed Dec 14 16:39:11 2016
@@ -133,12 +133,12 @@ function(libomp_check_variable var)
endfunction()
# void libomp_get_build_number(string src_dir, string* return_build_number);
-# - grab the eight digit build number (or 00000000) from kmp_version.c
+# - grab the eight digit build number (or 00000000) from kmp_version.cpp
function(libomp_get_build_number src_dir return_build_number)
- # sets file_lines_list to a list of all lines in kmp_version.c
- file(STRINGS "${src_dir}/src/kmp_version.c" file_lines_list)
+ # sets file_lines_list to a list of all lines in kmp_version.cpp
+ file(STRINGS "${src_dir}/src/kmp_version.cpp" file_lines_list)
- # runs through each line in kmp_version.c
+ # runs through each line in kmp_version.cpp
foreach(line IN LISTS file_lines_list)
# if the line begins with "#define KMP_VERSION_BUILD" then we take not of the build number
string(REGEX MATCH "^[ \t]*#define[ \t]+KMP_VERSION_BUILD" valid "${line}")
Modified: openmp/trunk/runtime/src/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/CMakeLists.txt?rev=289732&r1=289731&r2=289732&view=diff
==============================================================================
--- openmp/trunk/runtime/src/CMakeLists.txt (original)
+++ openmp/trunk/runtime/src/CMakeLists.txt Wed Dec 14 16:39:11 2016
@@ -56,43 +56,27 @@ set(LIBOMP_CFILES)
set(LIBOMP_CXXFILES)
set(LIBOMP_ASMFILES)
if(${STUBS_LIBRARY})
- set(LIBOMP_CFILES kmp_stub.c)
+ set(LIBOMP_CFILES kmp_stub.cpp)
else()
- # Get C files
- set(LIBOMP_CFILES
- kmp_alloc.c
- kmp_atomic.c
- kmp_csupport.c
- kmp_debug.c
- kmp_itt.c
- kmp_environment.c
- kmp_error.c
- kmp_global.c
- kmp_i18n.c
- kmp_io.c
- kmp_runtime.c
- kmp_settings.c
- kmp_str.c
- kmp_tasking.c
- kmp_taskq.c
- kmp_threadprivate.c
- kmp_utility.c
- )
- if(WIN32)
- # Windows specific files
- libomp_append(LIBOMP_CFILES z_Windows_NT_util.c)
- libomp_append(LIBOMP_CFILES z_Windows_NT-586_util.c)
- libomp_append(LIBOMP_ASMFILES z_Windows_NT-586_asm.asm) # Windows assembly file
- else()
- # Unix specific files
- libomp_append(LIBOMP_CFILES z_Linux_util.c)
- libomp_append(LIBOMP_CFILES kmp_gsupport.c)
- libomp_append(LIBOMP_ASMFILES z_Linux_asm.s) # Unix assembly file
- endif()
- libomp_append(LIBOMP_CFILES thirdparty/ittnotify/ittnotify_static.c LIBOMP_USE_ITT_NOTIFY)
- libomp_append(LIBOMP_CFILES kmp_debugger.c LIBOMP_USE_DEBUGGER)
# Get C++ files
set(LIBOMP_CXXFILES
+ kmp_alloc.cpp
+ kmp_atomic.cpp
+ kmp_csupport.cpp
+ kmp_debug.cpp
+ kmp_itt.cpp
+ kmp_environment.cpp
+ kmp_error.cpp
+ kmp_global.cpp
+ kmp_i18n.cpp
+ kmp_io.cpp
+ kmp_runtime.cpp
+ kmp_settings.cpp
+ kmp_str.cpp
+ kmp_tasking.cpp
+ kmp_taskq.cpp
+ kmp_threadprivate.cpp
+ kmp_utility.cpp
kmp_barrier.cpp
kmp_wait_release.cpp
kmp_affinity.cpp
@@ -100,6 +84,19 @@ else()
kmp_lock.cpp
kmp_sched.cpp
)
+ if(WIN32)
+ # Windows specific files
+ libomp_append(LIBOMP_CXXFILES z_Windows_NT_util.cpp)
+ libomp_append(LIBOMP_CXXFILES z_Windows_NT-586_util.cpp)
+ libomp_append(LIBOMP_ASMFILES z_Windows_NT-586_asm.asm) # Windows assembly file
+ else()
+ # Unix specific files
+ libomp_append(LIBOMP_CXXFILES z_Linux_util.cpp)
+ libomp_append(LIBOMP_CXXFILES kmp_gsupport.cpp)
+ libomp_append(LIBOMP_ASMFILES z_Linux_asm.s) # Unix assembly file
+ endif()
+ libomp_append(LIBOMP_CFILES thirdparty/ittnotify/ittnotify_static.c LIBOMP_USE_ITT_NOTIFY)
+ libomp_append(LIBOMP_CXXFILES kmp_debugger.cpp LIBOMP_USE_DEBUGGER)
libomp_append(LIBOMP_CXXFILES kmp_stats.cpp LIBOMP_STATS)
libomp_append(LIBOMP_CXXFILES kmp_stats_timing.cpp LIBOMP_STATS)
if(${LIBOMP_OMP_VERSION} GREATER 40 OR ${LIBOMP_OMP_VERSION} EQUAL 40)
@@ -108,12 +105,11 @@ else()
endif()
endif()
# Files common to stubs and normal library
-libomp_append(LIBOMP_CFILES kmp_ftn_cdecl.c)
-libomp_append(LIBOMP_CFILES kmp_ftn_extra.c)
-libomp_append(LIBOMP_CFILES kmp_version.c)
-libomp_append(LIBOMP_CFILES ompt-general.c IF_TRUE LIBOMP_OMPT_SUPPORT)
-
-libomp_append(LIBOMP_CFILES tsan_annotations.c IF_TRUE LIBOMP_TSAN_SUPPORT)
+libomp_append(LIBOMP_CXXFILES kmp_ftn_cdecl.cpp)
+libomp_append(LIBOMP_CXXFILES kmp_ftn_extra.cpp)
+libomp_append(LIBOMP_CXXFILES kmp_version.cpp)
+libomp_append(LIBOMP_CXXFILES ompt-general.cpp IF_TRUE LIBOMP_OMPT_SUPPORT)
+libomp_append(LIBOMP_CXXFILES tsan_annotations.cpp IF_TRUE LIBOMP_TSAN_SUPPORT)
set(LIBOMP_SOURCE_FILES ${LIBOMP_CFILES} ${LIBOMP_CXXFILES} ${LIBOMP_ASMFILES})
# For Windows, there is a resource file (.rc -> .res) that is also compiled
@@ -202,9 +198,9 @@ if(WIN32)
set_source_files_properties(thirdparty/ittnotify/ittnotify_static.c PROPERTIES COMPILE_DEFINITIONS "UNICODE")
# Create Windows import library
- # the import library is "re-linked" to include kmp_import.c which prevents
+ # the import library is "re-linked" to include kmp_import.cpp which prevents
# linking of both Visual Studio OpenMP and newly built OpenMP
- set_source_files_properties(kmp_import.c PROPERTIES COMPILE_FLAGS "${LIBOMP_CONFIGURED_CFLAGS}")
+ set_source_files_properties(kmp_import.cpp PROPERTIES COMPILE_FLAGS "${LIBOMP_CONFIGURED_CFLAGS}")
set(LIBOMP_IMP_LIB_FILE ${LIBOMP_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(LIBOMP_GENERATED_IMP_LIB_FILENAME ${LIBOMP_LIB_FILE}${CMAKE_STATIC_LIBRARY_SUFFIX})
set_target_properties(omp PROPERTIES
@@ -220,8 +216,8 @@ if(WIN32)
set(LIBOMP_GENERATED_IMP_LIB ${CMAKE_CURRENT_BINARY_DIR}/${LIBOMP_GENERATED_IMP_LIB_FILENAME})
endif()
set_source_files_properties(${LIBOMP_GENERATED_IMP_LIB} PROPERTIES GENERATED TRUE EXTERNAL_OBJECT TRUE)
- # Create new import library that is just the previously created one + kmp_import.c
- add_library(ompimp STATIC ${LIBOMP_GENERATED_IMP_LIB} kmp_import.c)
+ # Create new import library that is just the previously created one + kmp_import.cpp
+ add_library(ompimp STATIC ${LIBOMP_GENERATED_IMP_LIB} kmp_import.cpp)
set_target_properties(ompimp PROPERTIES
PREFIX "" SUFFIX "" OUTPUT_NAME "${LIBOMP_IMP_LIB_FILE}"
LINKER_LANGUAGE C
Removed: openmp/trunk/runtime/src/kmp_alloc.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_alloc.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_alloc.c (original)
+++ openmp/trunk/runtime/src/kmp_alloc.c (removed)
@@ -1,2078 +0,0 @@
-/*
- * kmp_alloc.c -- private/shared dynamic memory allocation and management
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_wrapper_malloc.h"
-#include "kmp_io.h"
-
-// Disable bget when it is not used
-#if KMP_USE_BGET
-
-/* Thread private buffer management code */
-
-typedef int (*bget_compact_t)(size_t, int);
-typedef void *(*bget_acquire_t)(size_t);
-typedef void (*bget_release_t)(void *);
-
-/* NOTE: bufsize must be a signed datatype */
-
-#if KMP_OS_WINDOWS
-# if KMP_ARCH_X86 || KMP_ARCH_ARM
- typedef kmp_int32 bufsize;
-# else
- typedef kmp_int64 bufsize;
-# endif
-#else
- typedef ssize_t bufsize;
-#endif
-
-/* The three modes of operation are, fifo search, lifo search, and best-fit */
-
-typedef enum bget_mode {
- bget_mode_fifo = 0,
- bget_mode_lifo = 1,
- bget_mode_best = 2
-} bget_mode_t;
-
-
-static void bpool( kmp_info_t *th, void *buffer, bufsize len);
-static void *bget( kmp_info_t *th, bufsize size);
-static void *bgetz( kmp_info_t *th, bufsize size);
-static void *bgetr( kmp_info_t *th, void *buffer, bufsize newsize);
-static void brel( kmp_info_t *th, void *buf);
-static void bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr );
-
-#ifdef KMP_DEBUG
-static void bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree, long *nget, long *nrel);
-static void bstatse( kmp_info_t *th, bufsize *pool_incr, long *npool, long *npget, long *nprel, long *ndget, long *ndrel);
-static void bufdump( kmp_info_t *th, void *buf);
-static void bpoold( kmp_info_t *th, void *pool, int dumpalloc, int dumpfree);
-static int bpoolv( kmp_info_t *th, void *pool);
-#endif
-
-/* BGET CONFIGURATION */
- /* Buffer allocation size quantum:
- all buffers allocated are a
- multiple of this size. This
- MUST be a power of two. */
-
- /* On IA-32 architecture with Linux* OS,
- malloc() does not
- ensure 16 byte alignmnent */
-
-#if KMP_ARCH_X86 || !KMP_HAVE_QUAD
-
-#define SizeQuant 8
-#define AlignType double
-
-#else
-
-#define SizeQuant 16
-#define AlignType _Quad
-
-#endif
-
-#define BufStats 1 /* Define this symbol to enable the
- bstats() function which calculates
- the total free space in the buffer
- pool, the largest available
- buffer, and the total space
- currently allocated. */
-
-#ifdef KMP_DEBUG
-
-#define BufDump 1 /* Define this symbol to enable the
- bpoold() function which dumps the
- buffers in a buffer pool. */
-
-#define BufValid 1 /* Define this symbol to enable the
- bpoolv() function for validating
- a buffer pool. */
-
-#define DumpData 1 /* Define this symbol to enable the
- bufdump() function which allows
- dumping the contents of an allocated
- or free buffer. */
-#ifdef NOT_USED_NOW
-
-#define FreeWipe 1 /* Wipe free buffers to a guaranteed
- pattern of garbage to trip up
- miscreants who attempt to use
- pointers into released buffers. */
-
-#define BestFit 1 /* Use a best fit algorithm when
- searching for space for an
- allocation request. This uses
- memory more efficiently, but
- allocation will be much slower. */
-#endif /* NOT_USED_NOW */
-#endif /* KMP_DEBUG */
-
-
-static bufsize bget_bin_size[ ] = {
- 0,
-// 1 << 6, /* .5 Cache line */
- 1 << 7, /* 1 Cache line, new */
- 1 << 8, /* 2 Cache lines */
- 1 << 9, /* 4 Cache lines, new */
- 1 << 10, /* 8 Cache lines */
- 1 << 11, /* 16 Cache lines, new */
- 1 << 12,
- 1 << 13, /* new */
- 1 << 14,
- 1 << 15, /* new */
- 1 << 16,
- 1 << 17,
- 1 << 18,
- 1 << 19,
- 1 << 20, /* 1MB */
- 1 << 21, /* 2MB */
- 1 << 22, /* 4MB */
- 1 << 23, /* 8MB */
- 1 << 24, /* 16MB */
- 1 << 25, /* 32MB */
-};
-
-#define MAX_BGET_BINS (int)(sizeof(bget_bin_size) / sizeof(bufsize))
-
-struct bfhead;
-
-/* Declare the interface, including the requested buffer size type,
- bufsize. */
-
-/* Queue links */
-
-typedef struct qlinks {
- struct bfhead *flink; /* Forward link */
- struct bfhead *blink; /* Backward link */
-} qlinks_t;
-
-/* Header in allocated and free buffers */
-
-typedef struct bhead2 {
- kmp_info_t *bthr; /* The thread which owns the buffer pool */
- bufsize prevfree; /* Relative link back to previous
- free buffer in memory or 0 if
- previous buffer is allocated. */
- bufsize bsize; /* Buffer size: positive if free,
- negative if allocated. */
-} bhead2_t;
-
-/* Make sure the bhead structure is a multiple of SizeQuant in size. */
-
-typedef union bhead {
- KMP_ALIGN( SizeQuant )
- AlignType b_align;
- char b_pad[ sizeof(bhead2_t) + (SizeQuant - (sizeof(bhead2_t) % SizeQuant)) ];
- bhead2_t bb;
-} bhead_t;
-#define BH(p) ((bhead_t *) (p))
-
-/* Header in directly allocated buffers (by acqfcn) */
-
-typedef struct bdhead
-{
- bufsize tsize; /* Total size, including overhead */
- bhead_t bh; /* Common header */
-} bdhead_t;
-#define BDH(p) ((bdhead_t *) (p))
-
-/* Header in free buffers */
-
-typedef struct bfhead {
- bhead_t bh; /* Common allocated/free header */
- qlinks_t ql; /* Links on free list */
-} bfhead_t;
-#define BFH(p) ((bfhead_t *) (p))
-
-typedef struct thr_data {
- bfhead_t freelist[ MAX_BGET_BINS ];
-#if BufStats
- size_t totalloc; /* Total space currently allocated */
- long numget, numrel; /* Number of bget() and brel() calls */
- long numpblk; /* Number of pool blocks */
- long numpget, numprel; /* Number of block gets and rels */
- long numdget, numdrel; /* Number of direct gets and rels */
-#endif /* BufStats */
-
- /* Automatic expansion block management functions */
- bget_compact_t compfcn;
- bget_acquire_t acqfcn;
- bget_release_t relfcn;
-
- bget_mode_t mode; /* what allocation mode to use? */
-
- bufsize exp_incr; /* Expansion block size */
- bufsize pool_len; /* 0: no bpool calls have been made
- -1: not all pool blocks are
- the same size
- >0: (common) block size for all
- bpool calls made so far
- */
- bfhead_t * last_pool; /* Last pool owned by this thread (delay dealocation) */
-} thr_data_t;
-
-/* Minimum allocation quantum: */
-
-#define QLSize (sizeof(qlinks_t))
-#define SizeQ ((SizeQuant > QLSize) ? SizeQuant : QLSize)
-#define MaxSize (bufsize)( ~ ( ( (bufsize)( 1 ) << ( sizeof( bufsize ) * CHAR_BIT - 1 ) ) | ( SizeQuant - 1 ) ) )
- // Maximun for the requested size.
-
-/* End sentinel: value placed in bsize field of dummy block delimiting
- end of pool block. The most negative number which will fit in a
- bufsize, defined in a way that the compiler will accept. */
-
-#define ESent ((bufsize) (-(((((bufsize)1)<<((int)sizeof(bufsize)*8-2))-1)*2)-2))
-
-/* ------------------------------------------------------------------------ */
-
-/* Thread Data management routines */
-
-static int
-bget_get_bin( bufsize size )
-{
- // binary chop bins
- int lo = 0, hi = MAX_BGET_BINS - 1;
-
- KMP_DEBUG_ASSERT( size > 0 );
-
- while ( (hi - lo) > 1 ) {
- int mid = (lo + hi) >> 1;
- if (size < bget_bin_size[ mid ])
- hi = mid - 1;
- else
- lo = mid;
- }
-
- KMP_DEBUG_ASSERT( (lo >= 0) && (lo < MAX_BGET_BINS) );
-
- return lo;
-}
-
-static void
-set_thr_data( kmp_info_t *th )
-{
- int i;
- thr_data_t *data;
-
- data =
- (thr_data_t *)(
- ( ! th->th.th_local.bget_data ) ? __kmp_allocate( sizeof( *data ) ) : th->th.th_local.bget_data
- );
-
- memset( data, '\0', sizeof( *data ) );
-
- for (i = 0; i < MAX_BGET_BINS; ++i) {
- data->freelist[ i ].ql.flink = & data->freelist[ i ];
- data->freelist[ i ].ql.blink = & data->freelist[ i ];
- }
-
- th->th.th_local.bget_data = data;
- th->th.th_local.bget_list = 0;
-#if ! USE_CMP_XCHG_FOR_BGET
-#ifdef USE_QUEUING_LOCK_FOR_BGET
- __kmp_init_lock( & th->th.th_local.bget_lock );
-#else
- __kmp_init_bootstrap_lock( & th->th.th_local.bget_lock );
-#endif /* USE_LOCK_FOR_BGET */
-#endif /* ! USE_CMP_XCHG_FOR_BGET */
-}
-
-static thr_data_t *
-get_thr_data( kmp_info_t *th )
-{
- thr_data_t *data;
-
- data = (thr_data_t *) th->th.th_local.bget_data;
-
- KMP_DEBUG_ASSERT( data != 0 );
-
- return data;
-}
-
-
-#ifdef KMP_DEBUG
-
-static void
-__kmp_bget_validate_queue( kmp_info_t *th )
-{
- /* NOTE: assume that the global_lock is held */
-
- void *p = (void *) th->th.th_local.bget_list;
-
- while (p != 0) {
- bfhead_t *b = BFH(((char *) p) - sizeof(bhead_t));
-
- KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
- p = (void *) b->ql.flink;
- }
-}
-
-#endif
-
-/* Walk the free list and release the enqueued buffers */
-
-static void
-__kmp_bget_dequeue( kmp_info_t *th )
-{
- void *p = TCR_SYNC_PTR(th->th.th_local.bget_list);
-
- if (p != 0) {
- #if USE_CMP_XCHG_FOR_BGET
- {
- volatile void *old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
- while ( ! KMP_COMPARE_AND_STORE_PTR(
- & th->th.th_local.bget_list, old_value, NULL ) )
- {
- KMP_CPU_PAUSE();
- old_value = TCR_SYNC_PTR(th->th.th_local.bget_list);
- }
- p = (void *) old_value;
- }
- #else /* ! USE_CMP_XCHG_FOR_BGET */
- #ifdef USE_QUEUING_LOCK_FOR_BGET
- __kmp_acquire_lock( & th->th.th_local.bget_lock,
- __kmp_gtid_from_thread(th) );
- #else
- __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
- #endif /* USE_QUEUING_LOCK_FOR_BGET */
-
- p = (void *) th->th.th_local.bget_list;
- th->th.th_local.bget_list = 0;
-
- #ifdef USE_QUEUING_LOCK_FOR_BGET
- __kmp_release_lock( & th->th.th_local.bget_lock,
- __kmp_gtid_from_thread(th) );
- #else
- __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
- #endif
- #endif /* USE_CMP_XCHG_FOR_BGET */
-
- /* Check again to make sure the list is not empty */
-
- while (p != 0) {
- void *buf = p;
- bfhead_t *b = BFH(((char *) p) - sizeof(bhead_t));
-
- KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
- KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
- (kmp_uintptr_t)th ); // clear possible mark
- KMP_DEBUG_ASSERT( b->ql.blink == 0 );
-
- p = (void *) b->ql.flink;
-
- brel( th, buf );
- }
- }
-}
-
-/* Chain together the free buffers by using the thread owner field */
-
-static void
-__kmp_bget_enqueue( kmp_info_t *th, void *buf
-#ifdef USE_QUEUING_LOCK_FOR_BGET
- , kmp_int32 rel_gtid
-#endif
- )
-{
- bfhead_t *b = BFH(((char *) buf) - sizeof(bhead_t));
-
- KMP_DEBUG_ASSERT( b->bh.bb.bsize != 0 );
- KMP_DEBUG_ASSERT( ( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ) ==
- (kmp_uintptr_t)th ); // clear possible mark
-
- b->ql.blink = 0;
-
- KC_TRACE( 10, ( "__kmp_bget_enqueue: moving buffer to T#%d list\n",
- __kmp_gtid_from_thread( th ) ) );
-
-#if USE_CMP_XCHG_FOR_BGET
- {
- volatile void *old_value = TCR_PTR(th->th.th_local.bget_list);
- /* the next pointer must be set before setting bget_list to buf to avoid
- exposing a broken list to other threads, even for an instant. */
- b->ql.flink = BFH( old_value );
-
- while ( ! KMP_COMPARE_AND_STORE_PTR(
- & th->th.th_local.bget_list, old_value, buf ) )
- {
- KMP_CPU_PAUSE();
- old_value = TCR_PTR(th->th.th_local.bget_list);
- /* the next pointer must be set before setting bget_list to buf to avoid
- exposing a broken list to other threads, even for an instant. */
- b->ql.flink = BFH( old_value );
- }
- }
-#else /* ! USE_CMP_XCHG_FOR_BGET */
-# ifdef USE_QUEUING_LOCK_FOR_BGET
- __kmp_acquire_lock( & th->th.th_local.bget_lock, rel_gtid );
-# else
- __kmp_acquire_bootstrap_lock( & th->th.th_local.bget_lock );
- # endif
-
- b->ql.flink = BFH( th->th.th_local.bget_list );
- th->th.th_local.bget_list = (void *) buf;
-
-# ifdef USE_QUEUING_LOCK_FOR_BGET
- __kmp_release_lock( & th->th.th_local.bget_lock, rel_gtid );
-# else
- __kmp_release_bootstrap_lock( & th->th.th_local.bget_lock );
-# endif
-#endif /* USE_CMP_XCHG_FOR_BGET */
-}
-
-/* insert buffer back onto a new freelist */
-
-static void
-__kmp_bget_insert_into_freelist( thr_data_t *thr, bfhead_t *b )
-{
- int bin;
-
- KMP_DEBUG_ASSERT( ((size_t)b ) % SizeQuant == 0 );
- KMP_DEBUG_ASSERT( b->bh.bb.bsize % SizeQuant == 0 );
-
- bin = bget_get_bin( b->bh.bb.bsize );
-
- KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.blink->ql.flink == &thr->freelist[ bin ]);
- KMP_DEBUG_ASSERT(thr->freelist[ bin ].ql.flink->ql.blink == &thr->freelist[ bin ]);
-
- b->ql.flink = &thr->freelist[ bin ];
- b->ql.blink = thr->freelist[ bin ].ql.blink;
-
- thr->freelist[ bin ].ql.blink = b;
- b->ql.blink->ql.flink = b;
-}
-
-/* unlink the buffer from the old freelist */
-
-static void
-__kmp_bget_remove_from_freelist( bfhead_t *b )
-{
- KMP_DEBUG_ASSERT(b->ql.blink->ql.flink == b);
- KMP_DEBUG_ASSERT(b->ql.flink->ql.blink == b);
-
- b->ql.blink->ql.flink = b->ql.flink;
- b->ql.flink->ql.blink = b->ql.blink;
-}
-
-/* ------------------------------------------------------------------------ */
-
-/* GET STATS -- check info on free list */
-
-static void
-bcheck( kmp_info_t *th, bufsize *max_free, bufsize *total_free )
-{
- thr_data_t *thr = get_thr_data( th );
- int bin;
-
- *total_free = *max_free = 0;
-
- for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
- bfhead_t *b, *best;
-
- best = &thr->freelist[ bin ];
- b = best->ql.flink;
-
- while (b != &thr->freelist[ bin ]) {
- *total_free += (b->bh.bb.bsize - sizeof( bhead_t ));
- if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize))
- best = b;
-
- /* Link to next buffer */
- b = b->ql.flink;
- }
-
- if (*max_free < best->bh.bb.bsize)
- *max_free = best->bh.bb.bsize;
- }
-
- if (*max_free > (bufsize)sizeof( bhead_t ))
- *max_free -= sizeof( bhead_t );
-}
-
-/* ------------------------------------------------------------------------ */
-
-/* BGET -- Allocate a buffer. */
-
-static void *
-bget( kmp_info_t *th, bufsize requested_size )
-{
- thr_data_t *thr = get_thr_data( th );
- bufsize size = requested_size;
- bfhead_t *b;
- void *buf;
- int compactseq = 0;
- int use_blink = 0;
-/* For BestFit */
- bfhead_t *best;
-
- if ( size < 0 || size + sizeof( bhead_t ) > MaxSize ) {
- return NULL;
- }; // if
-
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
-
- if (size < (bufsize)SizeQ) { /* Need at least room for the */
- size = SizeQ; /* queue links. */
- }
- #if defined( SizeQuant ) && ( SizeQuant > 1 )
- size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
- #endif
-
- size += sizeof(bhead_t); /* Add overhead in allocated buffer
- to size required. */
- KMP_DEBUG_ASSERT( size >= 0 );
- KMP_DEBUG_ASSERT( size % SizeQuant == 0 );
-
- use_blink = ( thr->mode == bget_mode_lifo );
-
- /* If a compact function was provided in the call to bectl(), wrap
- a loop around the allocation process to allow compaction to
- intervene in case we don't find a suitable buffer in the chain. */
-
- for (;;) {
- int bin;
-
- for (bin = bget_get_bin( size ); bin < MAX_BGET_BINS; ++bin) {
- /* Link to next buffer */
- b = ( use_blink ? thr->freelist[ bin ].ql.blink : thr->freelist[ bin ].ql.flink );
-
- if (thr->mode == bget_mode_best) {
- best = &thr->freelist[ bin ];
-
- /* Scan the free list searching for the first buffer big enough
- to hold the requested size buffer. */
-
- while (b != &thr->freelist[ bin ]) {
- if (b->bh.bb.bsize >= (bufsize) size) {
- if ((best == &thr->freelist[ bin ]) || (b->bh.bb.bsize < best->bh.bb.bsize)) {
- best = b;
- }
- }
-
- /* Link to next buffer */
- b = ( use_blink ? b->ql.blink : b->ql.flink );
- }
- b = best;
- }
-
- while (b != &thr->freelist[ bin ]) {
- if ((bufsize) b->bh.bb.bsize >= (bufsize) size) {
-
- /* Buffer is big enough to satisfy the request. Allocate it
- to the caller. We must decide whether the buffer is large
- enough to split into the part given to the caller and a
- free buffer that remains on the free list, or whether the
- entire buffer should be removed from the free list and
- given to the caller in its entirety. We only split the
- buffer if enough room remains for a header plus the minimum
- quantum of allocation. */
-
- if ((b->bh.bb.bsize - (bufsize) size) > (bufsize)(SizeQ + (sizeof(bhead_t)))) {
- bhead_t *ba, *bn;
-
- ba = BH(((char *) b) + (b->bh.bb.bsize - (bufsize) size));
- bn = BH(((char *) ba) + size);
-
- KMP_DEBUG_ASSERT(bn->bb.prevfree == b->bh.bb.bsize);
-
- /* Subtract size from length of free block. */
- b->bh.bb.bsize -= (bufsize) size;
-
- /* Link allocated buffer to the previous free buffer. */
- ba->bb.prevfree = b->bh.bb.bsize;
-
- /* Plug negative size into user buffer. */
- ba->bb.bsize = -size;
-
- /* Mark this buffer as owned by this thread. */
- TCW_PTR(ba->bb.bthr, th); // not an allocated address (do not mark it)
- /* Mark buffer after this one not preceded by free block. */
- bn->bb.prevfree = 0;
-
- /* unlink the buffer from the old freelist, and reinsert it into the new freelist */
- __kmp_bget_remove_from_freelist( b );
- __kmp_bget_insert_into_freelist( thr, b );
-#if BufStats
- thr->totalloc += (size_t) size;
- thr->numget++; /* Increment number of bget() calls */
-#endif
- buf = (void *) ((((char *) ba) + sizeof(bhead_t)));
- KMP_DEBUG_ASSERT( ((size_t)buf) % SizeQuant == 0 );
- return buf;
- } else {
- bhead_t *ba;
-
- ba = BH(((char *) b) + b->bh.bb.bsize);
-
- KMP_DEBUG_ASSERT(ba->bb.prevfree == b->bh.bb.bsize);
-
- /* The buffer isn't big enough to split. Give the whole
- shebang to the caller and remove it from the free list. */
-
- __kmp_bget_remove_from_freelist( b );
-#if BufStats
- thr->totalloc += (size_t) b->bh.bb.bsize;
- thr->numget++; /* Increment number of bget() calls */
-#endif
- /* Negate size to mark buffer allocated. */
- b->bh.bb.bsize = -(b->bh.bb.bsize);
-
- /* Mark this buffer as owned by this thread. */
- TCW_PTR(ba->bb.bthr, th); // not an allocated address (do not mark it)
- /* Zero the back pointer in the next buffer in memory
- to indicate that this buffer is allocated. */
- ba->bb.prevfree = 0;
-
- /* Give user buffer starting at queue links. */
- buf = (void *) &(b->ql);
- KMP_DEBUG_ASSERT( ((size_t)buf) % SizeQuant == 0 );
- return buf;
- }
- }
-
- /* Link to next buffer */
- b = ( use_blink ? b->ql.blink : b->ql.flink );
- }
- }
-
- /* We failed to find a buffer. If there's a compact function
- defined, notify it of the size requested. If it returns
- TRUE, try the allocation again. */
-
- if ((thr->compfcn == 0) || (!(*thr->compfcn)(size, ++compactseq))) {
- break;
- }
- }
-
- /* No buffer available with requested size free. */
-
- /* Don't give up yet -- look in the reserve supply. */
-
- if (thr->acqfcn != 0) {
- if (size > (bufsize) (thr->exp_incr - sizeof(bhead_t))) {
-
- /* Request is too large to fit in a single expansion
- block. Try to satisy it by a direct buffer acquisition. */
-
- bdhead_t *bdh;
-
- size += sizeof(bdhead_t) - sizeof(bhead_t);
-
- KE_TRACE( 10, ("%%%%%% MALLOC( %d )\n", (int) size ) );
-
- /* richryan */
- bdh = BDH((*thr->acqfcn)((bufsize) size));
- if (bdh != NULL) {
-
- /* Mark the buffer special by setting the size field
- of its header to zero. */
- bdh->bh.bb.bsize = 0;
-
- /* Mark this buffer as owned by this thread. */
- TCW_PTR(bdh->bh.bb.bthr, th); // don't mark buffer as allocated,
- // because direct buffer never goes to free list
- bdh->bh.bb.prevfree = 0;
- bdh->tsize = size;
-#if BufStats
- thr->totalloc += (size_t) size;
- thr->numget++; /* Increment number of bget() calls */
- thr->numdget++; /* Direct bget() call count */
-#endif
- buf = (void *) (bdh + 1);
- KMP_DEBUG_ASSERT( ((size_t)buf) % SizeQuant == 0 );
- return buf;
- }
-
- } else {
-
- /* Try to obtain a new expansion block */
-
- void *newpool;
-
- KE_TRACE( 10, ("%%%%%% MALLOCB( %d )\n", (int) thr->exp_incr ) );
-
- /* richryan */
- newpool = (*thr->acqfcn)((bufsize) thr->exp_incr);
- KMP_DEBUG_ASSERT( ((size_t)newpool) % SizeQuant == 0 );
- if (newpool != NULL) {
- bpool( th, newpool, thr->exp_incr);
- buf = bget( th, requested_size); /* This can't, I say, can't get into a loop. */
- return buf;
- }
- }
- }
-
- /* Still no buffer available */
-
- return NULL;
-}
-
-/* BGETZ -- Allocate a buffer and clear its contents to zero. We clear
- the entire contents of the buffer to zero, not just the
- region requested by the caller. */
-
-static void *
-bgetz( kmp_info_t *th, bufsize size )
-{
- char *buf = (char *) bget( th, size);
-
- if (buf != NULL) {
- bhead_t *b;
- bufsize rsize;
-
- b = BH(buf - sizeof(bhead_t));
- rsize = -(b->bb.bsize);
- if (rsize == 0) {
- bdhead_t *bd;
-
- bd = BDH(buf - sizeof(bdhead_t));
- rsize = bd->tsize - (bufsize) sizeof(bdhead_t);
- } else {
- rsize -= sizeof(bhead_t);
- }
-
- KMP_DEBUG_ASSERT(rsize >= size);
-
- (void) memset(buf, 0, (bufsize) rsize);
- }
- return ((void *) buf);
-}
-
-/* BGETR -- Reallocate a buffer. This is a minimal implementation,
- simply in terms of brel() and bget(). It could be
- enhanced to allow the buffer to grow into adjacent free
- blocks and to avoid moving data unnecessarily. */
-
-static void *
-bgetr( kmp_info_t *th, void *buf, bufsize size)
-{
- void *nbuf;
- bufsize osize; /* Old size of buffer */
- bhead_t *b;
-
- nbuf = bget( th, size );
- if ( nbuf == NULL ) { /* Acquire new buffer */
- return NULL;
- }
- if ( buf == NULL ) {
- return nbuf;
- }
- b = BH(((char *) buf) - sizeof(bhead_t));
- osize = -b->bb.bsize;
- if (osize == 0) {
- /* Buffer acquired directly through acqfcn. */
- bdhead_t *bd;
-
- bd = BDH(((char *) buf) - sizeof(bdhead_t));
- osize = bd->tsize - (bufsize) sizeof(bdhead_t);
- } else {
- osize -= sizeof(bhead_t);
- };
-
- KMP_DEBUG_ASSERT(osize > 0);
-
- (void) KMP_MEMCPY((char *) nbuf, (char *) buf, /* Copy the data */
- (size_t) ((size < osize) ? size : osize));
- brel( th, buf );
-
- return nbuf;
-}
-
-/* BREL -- Release a buffer. */
-
-static void
-brel( kmp_info_t *th, void *buf )
-{
- thr_data_t *thr = get_thr_data( th );
- bfhead_t *b, *bn;
- kmp_info_t *bth;
-
- KMP_DEBUG_ASSERT(buf != NULL);
- KMP_DEBUG_ASSERT( ((size_t)buf) % SizeQuant == 0 );
-
- b = BFH(((char *) buf) - sizeof(bhead_t));
-
- if (b->bh.bb.bsize == 0) { /* Directly-acquired buffer? */
- bdhead_t *bdh;
-
- bdh = BDH(((char *) buf) - sizeof(bdhead_t));
- KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
-#if BufStats
- thr->totalloc -= (size_t) bdh->tsize;
- thr->numdrel++; /* Number of direct releases */
- thr->numrel++; /* Increment number of brel() calls */
-#endif /* BufStats */
-#ifdef FreeWipe
- (void) memset((char *) buf, 0x55,
- (size_t) (bdh->tsize - sizeof(bdhead_t)));
-#endif /* FreeWipe */
-
- KE_TRACE( 10, ("%%%%%% FREE( %p )\n", (void *) bdh ) );
-
- KMP_DEBUG_ASSERT( thr->relfcn != 0 );
- (*thr->relfcn)((void *) bdh); /* Release it directly. */
- return;
- }
-
- bth = (kmp_info_t *)( (kmp_uintptr_t)TCR_PTR(b->bh.bb.bthr) & ~1 ); // clear possible mark before comparison
- if ( bth != th ) {
- /* Add this buffer to be released by the owning thread later */
- __kmp_bget_enqueue( bth, buf
-#ifdef USE_QUEUING_LOCK_FOR_BGET
- , __kmp_gtid_from_thread( th )
-#endif
- );
- return;
- }
-
- /* Buffer size must be negative, indicating that the buffer is
- allocated. */
-
- if (b->bh.bb.bsize >= 0) {
- bn = NULL;
- }
- KMP_DEBUG_ASSERT(b->bh.bb.bsize < 0);
-
- /* Back pointer in next buffer must be zero, indicating the
- same thing: */
-
- KMP_DEBUG_ASSERT(BH((char *) b - b->bh.bb.bsize)->bb.prevfree == 0);
-
-#if BufStats
- thr->numrel++; /* Increment number of brel() calls */
- thr->totalloc += (size_t) b->bh.bb.bsize;
-#endif
-
- /* If the back link is nonzero, the previous buffer is free. */
-
- if (b->bh.bb.prevfree != 0) {
- /* The previous buffer is free. Consolidate this buffer with it
- by adding the length of this buffer to the previous free
- buffer. Note that we subtract the size in the buffer being
- released, since it's negative to indicate that the buffer is
- allocated. */
-
- register bufsize size = b->bh.bb.bsize;
-
- /* Make the previous buffer the one we're working on. */
- KMP_DEBUG_ASSERT(BH((char *) b - b->bh.bb.prevfree)->bb.bsize == b->bh.bb.prevfree);
- b = BFH(((char *) b) - b->bh.bb.prevfree);
- b->bh.bb.bsize -= size;
-
- /* unlink the buffer from the old freelist */
- __kmp_bget_remove_from_freelist( b );
- }
- else {
- /* The previous buffer isn't allocated. Mark this buffer
- size as positive (i.e. free) and fall through to place
- the buffer on the free list as an isolated free block. */
-
- b->bh.bb.bsize = -b->bh.bb.bsize;
- }
-
- /* insert buffer back onto a new freelist */
- __kmp_bget_insert_into_freelist( thr, b );
-
-
- /* Now we look at the next buffer in memory, located by advancing from
- the start of this buffer by its size, to see if that buffer is
- free. If it is, we combine this buffer with the next one in
- memory, dechaining the second buffer from the free list. */
-
- bn = BFH(((char *) b) + b->bh.bb.bsize);
- if (bn->bh.bb.bsize > 0) {
-
- /* The buffer is free. Remove it from the free list and add
- its size to that of our buffer. */
-
- KMP_DEBUG_ASSERT(BH((char *) bn + bn->bh.bb.bsize)->bb.prevfree == bn->bh.bb.bsize);
-
- __kmp_bget_remove_from_freelist( bn );
-
- b->bh.bb.bsize += bn->bh.bb.bsize;
-
- /* unlink the buffer from the old freelist, and reinsert it into the new freelist */
-
- __kmp_bget_remove_from_freelist( b );
- __kmp_bget_insert_into_freelist( thr, b );
-
- /* Finally, advance to the buffer that follows the newly
- consolidated free block. We must set its backpointer to the
- head of the consolidated free block. We know the next block
- must be an allocated block because the process of recombination
- guarantees that two free blocks will never be contiguous in
- memory. */
-
- bn = BFH(((char *) b) + b->bh.bb.bsize);
- }
-#ifdef FreeWipe
- (void) memset(((char *) b) + sizeof(bfhead_t), 0x55,
- (size_t) (b->bh.bb.bsize - sizeof(bfhead_t)));
-#endif
- KMP_DEBUG_ASSERT(bn->bh.bb.bsize < 0);
-
- /* The next buffer is allocated. Set the backpointer in it to point
- to this buffer; the previous free buffer in memory. */
-
- bn->bh.bb.prevfree = b->bh.bb.bsize;
-
- /* If a block-release function is defined, and this free buffer
- constitutes the entire block, release it. Note that pool_len
- is defined in such a way that the test will fail unless all
- pool blocks are the same size. */
-
- if (thr->relfcn != 0 &&
- b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t)))
- {
-#if BufStats
- if (thr->numpblk != 1) { /* Do not release the last buffer until finalization time */
-#endif
-
- KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
- KMP_DEBUG_ASSERT(BH((char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
- KMP_DEBUG_ASSERT(BH((char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
-
- /* Unlink the buffer from the free list */
- __kmp_bget_remove_from_freelist( b );
-
- KE_TRACE( 10, ("%%%%%% FREE( %p )\n", (void *) b ) );
-
- (*thr->relfcn)(b);
-#if BufStats
- thr->numprel++; /* Nr of expansion block releases */
- thr->numpblk--; /* Total number of blocks */
- KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
-
- /* avoid leaving stale last_pool pointer around if it is being dealloced */
- if (thr->last_pool == b) thr->last_pool = 0;
- }
- else {
- thr->last_pool = b;
- }
-#endif /* BufStats */
- }
-}
-
-/* BECTL -- Establish automatic pool expansion control */
-
-static void
-bectl( kmp_info_t *th, bget_compact_t compact, bget_acquire_t acquire, bget_release_t release, bufsize pool_incr)
-{
- thr_data_t *thr = get_thr_data( th );
-
- thr->compfcn = compact;
- thr->acqfcn = acquire;
- thr->relfcn = release;
- thr->exp_incr = pool_incr;
-}
-
-/* BPOOL -- Add a region of memory to the buffer pool. */
-
-static void
-bpool( kmp_info_t *th, void *buf, bufsize len)
-{
-/* int bin = 0; */
- thr_data_t *thr = get_thr_data( th );
- bfhead_t *b = BFH(buf);
- bhead_t *bn;
-
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
-
-#ifdef SizeQuant
- len &= ~(SizeQuant - 1);
-#endif
- if (thr->pool_len == 0) {
- thr->pool_len = len;
- } else if (len != thr->pool_len) {
- thr->pool_len = -1;
- }
-#if BufStats
- thr->numpget++; /* Number of block acquisitions */
- thr->numpblk++; /* Number of blocks total */
- KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
-#endif /* BufStats */
-
- /* Since the block is initially occupied by a single free buffer,
- it had better not be (much) larger than the largest buffer
- whose size we can store in bhead.bb.bsize. */
-
- KMP_DEBUG_ASSERT(len - sizeof(bhead_t) <= -((bufsize) ESent + 1));
-
- /* Clear the backpointer at the start of the block to indicate that
- there is no free block prior to this one. That blocks
- recombination when the first block in memory is released. */
-
- b->bh.bb.prevfree = 0;
-
- /* Create a dummy allocated buffer at the end of the pool. This dummy
- buffer is seen when a buffer at the end of the pool is released and
- blocks recombination of the last buffer with the dummy buffer at
- the end. The length in the dummy buffer is set to the largest
- negative number to denote the end of the pool for diagnostic
- routines (this specific value is not counted on by the actual
- allocation and release functions). */
-
- len -= sizeof(bhead_t);
- b->bh.bb.bsize = (bufsize) len;
- /* Set the owner of this buffer */
- TCW_PTR( b->bh.bb.bthr, (kmp_info_t*)((kmp_uintptr_t)th | 1) ); // mark the buffer as allocated address
-
- /* Chain the new block to the free list. */
- __kmp_bget_insert_into_freelist( thr, b );
-
-#ifdef FreeWipe
- (void) memset(((char *) b) + sizeof(bfhead_t), 0x55,
- (size_t) (len - sizeof(bfhead_t)));
-#endif
- bn = BH(((char *) b) + len);
- bn->bb.prevfree = (bufsize) len;
- /* Definition of ESent assumes two's complement! */
- KMP_DEBUG_ASSERT( (~0) == -1 && (bn != 0) );
-
- bn->bb.bsize = ESent;
-}
-
-/* ------------------------------------------------------------------------ */
-
-/* BFREED -- Dump the free lists for this thread. */
-
-static void
-bfreed( kmp_info_t *th )
-{
- int bin = 0, count = 0;
- int gtid = __kmp_gtid_from_thread( th );
- thr_data_t *thr = get_thr_data( th );
-
-#if BufStats
- __kmp_printf_no_lock("__kmp_printpool: T#%d total=%" KMP_UINT64_SPEC " get=%" KMP_INT64_SPEC " rel=%" \
- KMP_INT64_SPEC " pblk=%" KMP_INT64_SPEC " pget=%" KMP_INT64_SPEC " prel=%" KMP_INT64_SPEC \
- " dget=%" KMP_INT64_SPEC " drel=%" KMP_INT64_SPEC "\n",
- gtid, (kmp_uint64) thr->totalloc,
- (kmp_int64) thr->numget, (kmp_int64) thr->numrel,
- (kmp_int64) thr->numpblk,
- (kmp_int64) thr->numpget, (kmp_int64) thr->numprel,
- (kmp_int64) thr->numdget, (kmp_int64) thr->numdrel );
-#endif
-
- for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
- bfhead_t *b;
-
- for (b = thr->freelist[ bin ].ql.flink; b != &thr->freelist[ bin ]; b = b->ql.flink) {
- bufsize bs = b->bh.bb.bsize;
-
- KMP_DEBUG_ASSERT( b->ql.blink->ql.flink == b );
- KMP_DEBUG_ASSERT( b->ql.flink->ql.blink == b );
- KMP_DEBUG_ASSERT( bs > 0 );
-
- count += 1;
-
- __kmp_printf_no_lock("__kmp_printpool: T#%d Free block: 0x%p size %6ld bytes.\n", gtid, b, (long) bs );
-#ifdef FreeWipe
- {
- char *lerr = ((char *) b) + sizeof(bfhead_t);
- if ((bs > sizeof(bfhead_t)) && ((*lerr != 0x55) || (memcmp(lerr, lerr + 1, (size_t) (bs - (sizeof(bfhead_t) + 1))) != 0))) {
- __kmp_printf_no_lock( "__kmp_printpool: T#%d (Contents of above free block have been overstored.)\n", gtid );
- }
- }
-#endif
- }
- }
-
- if (count == 0)
- __kmp_printf_no_lock("__kmp_printpool: T#%d No free blocks\n", gtid );
-}
-
-/* ------------------------------------------------------------------------ */
-
-#ifdef KMP_DEBUG
-
-#if BufStats
-
-/* BSTATS -- Return buffer allocation free space statistics. */
-
-static void
-bstats( kmp_info_t *th, bufsize *curalloc, bufsize *totfree, bufsize *maxfree, long *nget, long *nrel)
-{
- int bin = 0;
- thr_data_t *thr = get_thr_data( th );
-
- *nget = thr->numget;
- *nrel = thr->numrel;
- *curalloc = (bufsize) thr->totalloc;
- *totfree = 0;
- *maxfree = -1;
-
- for (bin = 0; bin < MAX_BGET_BINS; ++bin) {
- bfhead_t *b = thr->freelist[ bin ].ql.flink;
-
- while (b != &thr->freelist[ bin ]) {
- KMP_DEBUG_ASSERT(b->bh.bb.bsize > 0);
- *totfree += b->bh.bb.bsize;
- if (b->bh.bb.bsize > *maxfree) {
- *maxfree = b->bh.bb.bsize;
- }
- b = b->ql.flink; /* Link to next buffer */
- }
- }
-}
-
-/* BSTATSE -- Return extended statistics */
-
-static void
-bstatse( kmp_info_t *th, bufsize *pool_incr, long *npool, long *npget, long *nprel, long *ndget, long *ndrel)
-{
- thr_data_t *thr = get_thr_data( th );
-
- *pool_incr = (thr->pool_len < 0) ? -thr->exp_incr : thr->exp_incr;
- *npool = thr->numpblk;
- *npget = thr->numpget;
- *nprel = thr->numprel;
- *ndget = thr->numdget;
- *ndrel = thr->numdrel;
-}
-
-#endif /* BufStats */
-
-/* BUFDUMP -- Dump the data in a buffer. This is called with the user
- data pointer, and backs up to the buffer header. It will
- dump either a free block or an allocated one. */
-
-static void
-bufdump( kmp_info_t *th, void *buf )
-{
- bfhead_t *b;
- unsigned char *bdump;
- bufsize bdlen;
-
- b = BFH(((char *) buf) - sizeof(bhead_t));
- KMP_DEBUG_ASSERT(b->bh.bb.bsize != 0);
- if (b->bh.bb.bsize < 0) {
- bdump = (unsigned char *) buf;
- bdlen = (-b->bh.bb.bsize) - (bufsize) sizeof(bhead_t);
- } else {
- bdump = (unsigned char *) (((char *) b) + sizeof(bfhead_t));
- bdlen = b->bh.bb.bsize - (bufsize) sizeof(bfhead_t);
- }
-
- while (bdlen > 0) {
- int i, dupes = 0;
- bufsize l = bdlen;
- char bhex[50], bascii[20];
-
- if (l > 16) {
- l = 16;
- }
-
- for (i = 0; i < l; i++) {
- (void) KMP_SNPRINTF(bhex + i * 3, sizeof(bhex) - i * 3, "%02X ", bdump[i]);
- if (bdump[i] > 0x20 && bdump[i] < 0x7F)
- bascii[ i ] = bdump[ i ];
- else
- bascii[ i ] = ' ';
- }
- bascii[i] = 0;
- (void) __kmp_printf_no_lock("%-48s %s\n", bhex, bascii);
- bdump += l;
- bdlen -= l;
- while ((bdlen > 16) && (memcmp((char *) (bdump - 16),
- (char *) bdump, 16) == 0)) {
- dupes++;
- bdump += 16;
- bdlen -= 16;
- }
- if (dupes > 1) {
- (void) __kmp_printf_no_lock(
- " (%d lines [%d bytes] identical to above line skipped)\n",
- dupes, dupes * 16);
- } else if (dupes == 1) {
- bdump -= 16;
- bdlen += 16;
- }
- }
-}
-
-/* BPOOLD -- Dump a buffer pool. The buffer headers are always listed.
- If DUMPALLOC is nonzero, the contents of allocated buffers
- are dumped. If DUMPFREE is nonzero, free blocks are
- dumped as well. If FreeWipe checking is enabled, free
- blocks which have been clobbered will always be dumped. */
-
-static void
-bpoold( kmp_info_t *th, void *buf, int dumpalloc, int dumpfree)
-{
- bfhead_t *b = BFH( (char*)buf - sizeof(bhead_t));
-
- while (b->bh.bb.bsize != ESent) {
- bufsize bs = b->bh.bb.bsize;
-
- if (bs < 0) {
- bs = -bs;
- (void) __kmp_printf_no_lock("Allocated buffer: size %6ld bytes.\n", (long) bs);
- if (dumpalloc) {
- bufdump( th, (void *) (((char *) b) + sizeof(bhead_t)));
- }
- } else {
- const char *lerr = "";
-
- KMP_DEBUG_ASSERT(bs > 0);
- if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
- lerr = " (Bad free list links)";
- }
- (void) __kmp_printf_no_lock("Free block: size %6ld bytes.%s\n",
- (long) bs, lerr);
-#ifdef FreeWipe
- lerr = ((char *) b) + sizeof(bfhead_t);
- if ((bs > sizeof(bfhead_t)) && ((*lerr != 0x55) ||
- (memcmp(lerr, lerr + 1,
- (size_t) (bs - (sizeof(bfhead_t) + 1))) != 0))) {
- (void) __kmp_printf_no_lock(
- "(Contents of above free block have been overstored.)\n");
- bufdump( th, (void *) (((char *) b) + sizeof(bhead_t)));
- } else
-#endif
- if (dumpfree) {
- bufdump( th, (void *) (((char *) b) + sizeof(bhead_t)));
- }
- }
- b = BFH(((char *) b) + bs);
- }
-}
-
-/* BPOOLV -- Validate a buffer pool. */
-
-static int
-bpoolv( kmp_info_t *th, void *buf )
-{
- bfhead_t *b = BFH(buf);
-
- while (b->bh.bb.bsize != ESent) {
- bufsize bs = b->bh.bb.bsize;
-
- if (bs < 0) {
- bs = -bs;
- } else {
-#ifdef FreeWipe
- char *lerr = "";
-#endif
-
- KMP_DEBUG_ASSERT(bs > 0);
- if (bs <= 0) {
- return 0;
- }
- if ((b->ql.blink->ql.flink != b) || (b->ql.flink->ql.blink != b)) {
- (void) __kmp_printf_no_lock("Free block: size %6ld bytes. (Bad free list links)\n",
- (long) bs);
- KMP_DEBUG_ASSERT(0);
- return 0;
- }
-#ifdef FreeWipe
- lerr = ((char *) b) + sizeof(bfhead_t);
- if ((bs > sizeof(bfhead_t)) && ((*lerr != 0x55) ||
- (memcmp(lerr, lerr + 1,
- (size_t) (bs - (sizeof(bfhead_t) + 1))) != 0))) {
- (void) __kmp_printf_no_lock(
- "(Contents of above free block have been overstored.)\n");
- bufdump( th, (void *) (((char *) b) + sizeof(bhead_t)));
- KMP_DEBUG_ASSERT(0);
- return 0;
- }
-#endif /* FreeWipe */
- }
- b = BFH(((char *) b) + bs);
- }
- return 1;
-}
-
-#endif /* KMP_DEBUG */
-
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_initialize_bget( kmp_info_t *th )
-{
- KMP_DEBUG_ASSERT( SizeQuant >= sizeof( void * ) && (th != 0) );
-
- set_thr_data( th );
-
- bectl( th, (bget_compact_t) 0, (bget_acquire_t) malloc, (bget_release_t) free,
- (bufsize) __kmp_malloc_pool_incr );
-}
-
-void
-__kmp_finalize_bget( kmp_info_t *th )
-{
- thr_data_t *thr;
- bfhead_t *b;
-
- KMP_DEBUG_ASSERT( th != 0 );
-
-#if BufStats
- thr = (thr_data_t *) th->th.th_local.bget_data;
- KMP_DEBUG_ASSERT( thr != NULL );
- b = thr->last_pool;
-
- /* If a block-release function is defined, and this free buffer
- constitutes the entire block, release it. Note that pool_len
- is defined in such a way that the test will fail unless all
- pool blocks are the same size. */
-
- /* Deallocate the last pool if one exists because we no longer do it in brel() */
- if (thr->relfcn != 0 && b != 0 && thr->numpblk != 0 &&
- b->bh.bb.bsize == (bufsize)(thr->pool_len - sizeof(bhead_t)))
- {
- KMP_DEBUG_ASSERT(b->bh.bb.prevfree == 0);
- KMP_DEBUG_ASSERT(BH((char *) b + b->bh.bb.bsize)->bb.bsize == ESent);
- KMP_DEBUG_ASSERT(BH((char *) b + b->bh.bb.bsize)->bb.prevfree == b->bh.bb.bsize);
-
- /* Unlink the buffer from the free list */
- __kmp_bget_remove_from_freelist( b );
-
- KE_TRACE( 10, ("%%%%%% FREE( %p )\n", (void *) b ) );
-
- (*thr->relfcn)(b);
- thr->numprel++; /* Nr of expansion block releases */
- thr->numpblk--; /* Total number of blocks */
- KMP_DEBUG_ASSERT(thr->numpblk == thr->numpget - thr->numprel);
- }
-#endif /* BufStats */
-
- /* Deallocate bget_data */
- if ( th->th.th_local.bget_data != NULL ) {
- __kmp_free( th->th.th_local.bget_data );
- th->th.th_local.bget_data = NULL;
- }; // if
-}
-
-void
-kmpc_set_poolsize( size_t size )
-{
- bectl( __kmp_get_thread(), (bget_compact_t) 0, (bget_acquire_t) malloc,
- (bget_release_t) free, (bufsize) size );
-}
-
-size_t
-kmpc_get_poolsize( void )
-{
- thr_data_t *p;
-
- p = get_thr_data( __kmp_get_thread() );
-
- return p->exp_incr;
-}
-
-void
-kmpc_set_poolmode( int mode )
-{
- thr_data_t *p;
-
- if (mode == bget_mode_fifo || mode == bget_mode_lifo || mode == bget_mode_best) {
- p = get_thr_data( __kmp_get_thread() );
- p->mode = (bget_mode_t) mode;
- }
-}
-
-int
-kmpc_get_poolmode( void )
-{
- thr_data_t *p;
-
- p = get_thr_data( __kmp_get_thread() );
-
- return p->mode;
-}
-
-void
-kmpc_get_poolstat( size_t *maxmem, size_t *allmem )
-{
- kmp_info_t *th = __kmp_get_thread();
- bufsize a, b;
-
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
-
- bcheck( th, &a, &b );
-
- *maxmem = a;
- *allmem = b;
-}
-
-void
-kmpc_poolprint( void )
-{
- kmp_info_t *th = __kmp_get_thread();
-
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
-
- bfreed( th );
-}
-
-#endif // #if KMP_USE_BGET
-
-/* ------------------------------------------------------------------------ */
-
-void *
-kmpc_malloc( size_t size )
-{
- void * ptr;
- ptr = bget( __kmp_entry_thread(), (bufsize)(size + sizeof(ptr)) );
- if( ptr != NULL ) {
- // save allocated pointer just before one returned to user
- *(void**)ptr = ptr;
- ptr = (void**)ptr + 1;
- }
- return ptr;
-}
-
-#define IS_POWER_OF_TWO(n) (((n)&((n)-1))==0)
-
-void *
-kmpc_aligned_malloc( size_t size, size_t alignment )
-{
- void * ptr;
- void * ptr_allocated;
- KMP_DEBUG_ASSERT( alignment < 32 * 1024 ); // Alignment should not be too big
- if( !IS_POWER_OF_TWO(alignment) ) {
- // AC: do we need to issue a warning here?
- errno = EINVAL;
- return NULL;
- }
- size = size + sizeof( void* ) + alignment;
- ptr_allocated = bget( __kmp_entry_thread(), (bufsize)size );
- if( ptr_allocated != NULL ) {
- // save allocated pointer just before one returned to user
- ptr = (void*)(((kmp_uintptr_t)ptr_allocated + sizeof( void* ) + alignment) & ~(alignment - 1));
- *((void**)ptr - 1) = ptr_allocated;
- } else {
- ptr = NULL;
- }
- return ptr;
-}
-
-void *
-kmpc_calloc( size_t nelem, size_t elsize )
-{
- void * ptr;
- ptr = bgetz( __kmp_entry_thread(), (bufsize) (nelem * elsize + sizeof(ptr)) );
- if( ptr != NULL ) {
- // save allocated pointer just before one returned to user
- *(void**)ptr = ptr;
- ptr = (void**)ptr + 1;
- }
- return ptr;
-}
-
-void *
-kmpc_realloc( void * ptr, size_t size )
-{
- void * result = NULL;
- if ( ptr == NULL ) {
- // If pointer is NULL, realloc behaves like malloc.
- result = bget( __kmp_entry_thread(), (bufsize)(size + sizeof(ptr)) );
- // save allocated pointer just before one returned to user
- if( result != NULL ) {
- *(void**)result = result;
- result = (void**)result + 1;
- }
- } else if ( size == 0 ) {
- // If size is 0, realloc behaves like free.
- // The thread must be registered by the call to kmpc_malloc() or kmpc_calloc() before.
- // So it should be safe to call __kmp_get_thread(), not __kmp_entry_thread().
- KMP_ASSERT(*((void**)ptr - 1));
- brel( __kmp_get_thread(), *((void**)ptr - 1) );
- } else {
- result = bgetr( __kmp_entry_thread(), *((void**)ptr - 1), (bufsize)(size + sizeof(ptr)) );
- if( result != NULL ) {
- *(void**)result = result;
- result = (void**)result + 1;
- }
- }; // if
- return result;
-}
-
-/* NOTE: the library must have already been initialized by a previous allocate */
-
-void
-kmpc_free( void * ptr )
-{
- if ( ! __kmp_init_serial ) {
- return;
- }; // if
- if ( ptr != NULL ) {
- kmp_info_t *th = __kmp_get_thread();
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
- // extract allocated pointer and free it
- KMP_ASSERT(*((void**)ptr - 1));
- brel( th, *((void**)ptr - 1) );
- };
-}
-
-
-/* ------------------------------------------------------------------------ */
-
-void *
-___kmp_thread_malloc( kmp_info_t *th, size_t size KMP_SRC_LOC_DECL )
-{
- void * ptr;
- KE_TRACE( 30, (
- "-> __kmp_thread_malloc( %p, %d ) called from %s:%d\n",
- th,
- (int) size
- KMP_SRC_LOC_PARM
- ) );
- ptr = bget( th, (bufsize) size );
- KE_TRACE( 30, ( "<- __kmp_thread_malloc() returns %p\n", ptr ) );
- return ptr;
-}
-
-void *
-___kmp_thread_calloc( kmp_info_t *th, size_t nelem, size_t elsize KMP_SRC_LOC_DECL )
-{
- void * ptr;
- KE_TRACE( 30, (
- "-> __kmp_thread_calloc( %p, %d, %d ) called from %s:%d\n",
- th,
- (int) nelem,
- (int) elsize
- KMP_SRC_LOC_PARM
- ) );
- ptr = bgetz( th, (bufsize) (nelem * elsize) );
- KE_TRACE( 30, ( "<- __kmp_thread_calloc() returns %p\n", ptr ) );
- return ptr;
-}
-
-void *
-___kmp_thread_realloc( kmp_info_t *th, void *ptr, size_t size KMP_SRC_LOC_DECL )
-{
- KE_TRACE( 30, (
- "-> __kmp_thread_realloc( %p, %p, %d ) called from %s:%d\n",
- th,
- ptr,
- (int) size
- KMP_SRC_LOC_PARM
- ) );
- ptr = bgetr( th, ptr, (bufsize) size );
- KE_TRACE( 30, ( "<- __kmp_thread_realloc() returns %p\n", ptr ) );
- return ptr;
-}
-
-void
-___kmp_thread_free( kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL )
-{
- KE_TRACE( 30, (
- "-> __kmp_thread_free( %p, %p ) called from %s:%d\n",
- th,
- ptr
- KMP_SRC_LOC_PARM
- ) );
- if ( ptr != NULL ) {
- __kmp_bget_dequeue( th ); /* Release any queued buffers */
- brel( th, ptr );
- }
- KE_TRACE( 30, ( "<- __kmp_thread_free()\n" ) );
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-/*
- If LEAK_MEMORY is defined, __kmp_free() will *not* free memory. It causes memory leaks, but it
- may be useful for debugging memory corruptions, used freed pointers, etc.
-*/
-/* #define LEAK_MEMORY */
-
-struct kmp_mem_descr { // Memory block descriptor.
- void * ptr_allocated; // Pointer returned by malloc(), subject for free().
- size_t size_allocated; // Size of allocated memory block.
- void * ptr_aligned; // Pointer to aligned memory, to be used by client code.
- size_t size_aligned; // Size of aligned memory block.
-};
-typedef struct kmp_mem_descr kmp_mem_descr_t;
-
-/*
- Allocate memory on requested boundary, fill allocated memory with 0x00.
- NULL is NEVER returned, __kmp_abort() is called in case of memory allocation error.
- Must use __kmp_free when freeing memory allocated by this routine!
- */
-static
-void *
-___kmp_allocate_align( size_t size, size_t alignment KMP_SRC_LOC_DECL )
-{
- /*
- __kmp_allocate() allocates (by call to malloc()) bigger memory block than requested to
- return properly aligned pointer. Original pointer returned by malloc() and size of allocated
- block is saved in descriptor just before the aligned pointer. This information used by
- __kmp_free() -- it has to pass to free() original pointer, not aligned one.
-
- +---------+------------+-----------------------------------+---------+
- | padding | descriptor | aligned block | padding |
- +---------+------------+-----------------------------------+---------+
- ^ ^
- | |
- | +- Aligned pointer returned to caller
- +- Pointer returned by malloc()
-
- Aligned block is filled with zeros, paddings are filled with 0xEF.
- */
-
- kmp_mem_descr_t descr;
- kmp_uintptr_t addr_allocated; // Address returned by malloc().
- kmp_uintptr_t addr_aligned; // Aligned address to return to caller.
- kmp_uintptr_t addr_descr; // Address of memory block descriptor.
-
- KE_TRACE( 25, (
- "-> ___kmp_allocate_align( %d, %d ) called from %s:%d\n",
- (int) size,
- (int) alignment
- KMP_SRC_LOC_PARM
- ) );
-
- KMP_DEBUG_ASSERT( alignment < 32 * 1024 ); // Alignment should not be too
- KMP_DEBUG_ASSERT( sizeof( void * ) <= sizeof( kmp_uintptr_t ) );
- // Make sure kmp_uintptr_t is enough to store addresses.
-
- descr.size_aligned = size;
- descr.size_allocated = descr.size_aligned + sizeof( kmp_mem_descr_t ) + alignment;
-
-#if KMP_DEBUG
- descr.ptr_allocated = _malloc_src_loc( descr.size_allocated, _file_, _line_ );
-#else
- descr.ptr_allocated = malloc_src_loc( descr.size_allocated KMP_SRC_LOC_PARM );
-#endif
- KE_TRACE( 10, (
- " malloc( %d ) returned %p\n",
- (int) descr.size_allocated,
- descr.ptr_allocated
- ) );
- if ( descr.ptr_allocated == NULL ) {
- KMP_FATAL( OutOfHeapMemory );
- };
-
- addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
- addr_aligned =
- ( addr_allocated + sizeof( kmp_mem_descr_t ) + alignment )
- & ~ ( alignment - 1 );
- addr_descr = addr_aligned - sizeof( kmp_mem_descr_t );
-
- descr.ptr_aligned = (void *) addr_aligned;
-
- KE_TRACE( 26, (
- " ___kmp_allocate_align: "
- "ptr_allocated=%p, size_allocated=%d, "
- "ptr_aligned=%p, size_aligned=%d\n",
- descr.ptr_allocated,
- (int) descr.size_allocated,
- descr.ptr_aligned,
- (int) descr.size_aligned
- ) );
-
- KMP_DEBUG_ASSERT( addr_allocated <= addr_descr );
- KMP_DEBUG_ASSERT( addr_descr + sizeof( kmp_mem_descr_t ) == addr_aligned );
- KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
- KMP_DEBUG_ASSERT( addr_aligned % alignment == 0 );
-#ifdef KMP_DEBUG
- memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
- // Fill allocated memory block with 0xEF.
-#endif
- memset( descr.ptr_aligned, 0x00, descr.size_aligned );
- // Fill the aligned memory block (which is intended for using by caller) with 0x00. Do not
- // put this filling under KMP_DEBUG condition! Many callers expect zeroed memory. (Padding
- // bytes remain filled with 0xEF in debugging library.)
- * ( (kmp_mem_descr_t *) addr_descr ) = descr;
-
- KMP_MB();
-
- KE_TRACE( 25, ( "<- ___kmp_allocate_align() returns %p\n", descr.ptr_aligned ) );
- return descr.ptr_aligned;
-} // func ___kmp_allocate_align
-
-
-/*
- Allocate memory on cache line boundary, fill allocated memory with 0x00.
- Do not call this func directly! Use __kmp_allocate macro instead.
- NULL is NEVER returned, __kmp_abort() is called in case of memory allocation error.
- Must use __kmp_free when freeing memory allocated by this routine!
- */
-void *
-___kmp_allocate( size_t size KMP_SRC_LOC_DECL )
-{
- void * ptr;
- KE_TRACE( 25, ( "-> __kmp_allocate( %d ) called from %s:%d\n", (int) size KMP_SRC_LOC_PARM ) );
- ptr = ___kmp_allocate_align( size, __kmp_align_alloc KMP_SRC_LOC_PARM );
- KE_TRACE( 25, ( "<- __kmp_allocate() returns %p\n", ptr ) );
- return ptr;
-} // func ___kmp_allocate
-
-#if (BUILD_MEMORY==FIRST_TOUCH)
-void *
-__kmp_ft_page_allocate(size_t size)
-{
- void *adr, *aadr;
-
- const int page_size = KMP_GET_PAGE_SIZE();
-
- adr = (void *) __kmp_thread_malloc( __kmp_get_thread(),
- size + page_size + KMP_PTR_SKIP);
- if ( adr == 0 )
- KMP_FATAL( OutOfHeapMemory );
-
- /* check to see if adr is on a page boundary. */
- if ( ( (kmp_uintptr_t) adr & (page_size - 1)) == 0)
- /* nothing to do if adr is already on a page boundary. */
- aadr = adr;
- else
- /* else set aadr to the first page boundary in the allocated memory. */
- aadr = (void *) ( ( (kmp_uintptr_t) adr + page_size) & ~(page_size - 1) );
-
- /* the first touch by the owner thread. */
- *((void**)aadr) = adr;
-
- /* skip the memory space used for storing adr above. */
- return (void*)((char*)aadr + KMP_PTR_SKIP);
-}
-#endif
-
-/*
- Allocate memory on page boundary, fill allocated memory with 0x00.
- Does not call this func directly! Use __kmp_page_allocate macro instead.
- NULL is NEVER returned, __kmp_abort() is called in case of memory allocation error.
- Must use __kmp_free when freeing memory allocated by this routine!
- */
-void *
-___kmp_page_allocate( size_t size KMP_SRC_LOC_DECL )
-{
- int page_size = 8 * 1024;
- void * ptr;
-
- KE_TRACE( 25, (
- "-> __kmp_page_allocate( %d ) called from %s:%d\n",
- (int) size
- KMP_SRC_LOC_PARM
- ) );
- ptr = ___kmp_allocate_align( size, page_size KMP_SRC_LOC_PARM );
- KE_TRACE( 25, ( "<- __kmp_page_allocate( %d ) returns %p\n", (int) size, ptr ) );
- return ptr;
-} // ___kmp_page_allocate
-
-/*
- Free memory allocated by __kmp_allocate() and __kmp_page_allocate().
- In debug mode, fill the memory block with 0xEF before call to free().
-*/
-void
-___kmp_free( void * ptr KMP_SRC_LOC_DECL )
-{
- kmp_mem_descr_t descr;
- kmp_uintptr_t addr_allocated; // Address returned by malloc().
- kmp_uintptr_t addr_aligned; // Aligned address passed by caller.
-
- KE_TRACE( 25, ( "-> __kmp_free( %p ) called from %s:%d\n", ptr KMP_SRC_LOC_PARM ) );
- KMP_ASSERT( ptr != NULL );
-
- descr = * ( kmp_mem_descr_t *) ( (kmp_uintptr_t) ptr - sizeof( kmp_mem_descr_t ) );
-
- KE_TRACE( 26, ( " __kmp_free: "
- "ptr_allocated=%p, size_allocated=%d, "
- "ptr_aligned=%p, size_aligned=%d\n",
- descr.ptr_allocated, (int) descr.size_allocated,
- descr.ptr_aligned, (int) descr.size_aligned ));
-
- addr_allocated = (kmp_uintptr_t) descr.ptr_allocated;
- addr_aligned = (kmp_uintptr_t) descr.ptr_aligned;
-
- KMP_DEBUG_ASSERT( addr_aligned % CACHE_LINE == 0 );
- KMP_DEBUG_ASSERT( descr.ptr_aligned == ptr );
- KMP_DEBUG_ASSERT( addr_allocated + sizeof( kmp_mem_descr_t ) <= addr_aligned );
- KMP_DEBUG_ASSERT( descr.size_aligned < descr.size_allocated );
- KMP_DEBUG_ASSERT( addr_aligned + descr.size_aligned <= addr_allocated + descr.size_allocated );
-
- #ifdef KMP_DEBUG
- memset( descr.ptr_allocated, 0xEF, descr.size_allocated );
- // Fill memory block with 0xEF, it helps catch using freed memory.
- #endif
-
- #ifndef LEAK_MEMORY
- KE_TRACE( 10, ( " free( %p )\n", descr.ptr_allocated ) );
- # ifdef KMP_DEBUG
- _free_src_loc( descr.ptr_allocated, _file_, _line_ );
- # else
- free_src_loc( descr.ptr_allocated KMP_SRC_LOC_PARM );
- # endif
- #endif
- KMP_MB();
- KE_TRACE( 25, ( "<- __kmp_free() returns\n" ) );
-} // func ___kmp_free
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-#if USE_FAST_MEMORY == 3
-// Allocate fast memory by first scanning the thread's free lists
-// If a chunk the right size exists, grab it off the free list.
-// Otherwise allocate normally using kmp_thread_malloc.
-
-// AC: How to choose the limit? Just get 16 for now...
-#define KMP_FREE_LIST_LIMIT 16
-
-// Always use 128 bytes for determining buckets for caching memory blocks
-#define DCACHE_LINE 128
-
-void *
-___kmp_fast_allocate( kmp_info_t *this_thr, size_t size KMP_SRC_LOC_DECL )
-{
- void * ptr;
- int num_lines;
- int idx;
- int index;
- void * alloc_ptr;
- size_t alloc_size;
- kmp_mem_descr_t * descr;
-
- KE_TRACE( 25, ( "-> __kmp_fast_allocate( T#%d, %d ) called from %s:%d\n",
- __kmp_gtid_from_thread(this_thr), (int) size KMP_SRC_LOC_PARM ) );
-
- num_lines = ( size + DCACHE_LINE - 1 ) / DCACHE_LINE;
- idx = num_lines - 1;
- KMP_DEBUG_ASSERT( idx >= 0 );
- if ( idx < 2 ) {
- index = 0; // idx is [ 0, 1 ], use first free list
- num_lines = 2; // 1, 2 cache lines or less than cache line
- } else if ( ( idx >>= 2 ) == 0 ) {
- index = 1; // idx is [ 2, 3 ], use second free list
- num_lines = 4; // 3, 4 cache lines
- } else if ( ( idx >>= 2 ) == 0 ) {
- index = 2; // idx is [ 4, 15 ], use third free list
- num_lines = 16; // 5, 6, ..., 16 cache lines
- } else if ( ( idx >>= 2 ) == 0 ) {
- index = 3; // idx is [ 16, 63 ], use fourth free list
- num_lines = 64; // 17, 18, ..., 64 cache lines
- } else {
- goto alloc_call; // 65 or more cache lines ( > 8KB ), don't use free lists
- }
-
- ptr = this_thr->th.th_free_lists[index].th_free_list_self;
- if ( ptr != NULL ) {
- // pop the head of no-sync free list
- this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr);
- KMP_DEBUG_ASSERT( this_thr ==
- ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t) ))->ptr_aligned );
- goto end;
- };
- ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
- if ( ptr != NULL ) {
- // no-sync free list is empty, use sync free list (filled in by other threads only)
- // pop the head of the sync free list, push NULL instead
- while ( ! KMP_COMPARE_AND_STORE_PTR(
- &this_thr->th.th_free_lists[index].th_free_list_sync, ptr, NULL ) )
- {
- KMP_CPU_PAUSE();
- ptr = TCR_SYNC_PTR( this_thr->th.th_free_lists[index].th_free_list_sync );
- }
- // push the rest of chain into no-sync free list (can be NULL if there was the only block)
- this_thr->th.th_free_lists[index].th_free_list_self = *((void **)ptr);
- KMP_DEBUG_ASSERT( this_thr ==
- ((kmp_mem_descr_t *)( (kmp_uintptr_t)ptr - sizeof(kmp_mem_descr_t) ))->ptr_aligned );
- goto end;
- }
-
- alloc_call:
- // haven't found block in the free lists, thus allocate it
- size = num_lines * DCACHE_LINE;
-
- alloc_size = size + sizeof( kmp_mem_descr_t ) + DCACHE_LINE;
- KE_TRACE( 25, ( "__kmp_fast_allocate: T#%d Calling __kmp_thread_malloc with alloc_size %d\n",
- __kmp_gtid_from_thread( this_thr ), alloc_size ) );
- alloc_ptr = bget( this_thr, (bufsize) alloc_size );
-
- // align ptr to DCACHE_LINE
- ptr = (void *)(( ((kmp_uintptr_t)alloc_ptr) + sizeof(kmp_mem_descr_t) + DCACHE_LINE ) & ~( DCACHE_LINE - 1 ));
- descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t) );
-
- descr->ptr_allocated = alloc_ptr; // remember allocated pointer
- // we don't need size_allocated
- descr->ptr_aligned = (void *)this_thr; // remember allocating thread
- // (it is already saved in bget buffer,
- // but we may want to use another allocator in future)
- descr->size_aligned = size;
-
- end:
- KE_TRACE( 25, ( "<- __kmp_fast_allocate( T#%d ) returns %p\n",
- __kmp_gtid_from_thread( this_thr ), ptr ) );
- return ptr;
-} // func __kmp_fast_allocate
-
-// Free fast memory and place it on the thread's free list if it is of
-// the correct size.
-void
-___kmp_fast_free( kmp_info_t *this_thr, void * ptr KMP_SRC_LOC_DECL )
-{
- kmp_mem_descr_t * descr;
- kmp_info_t * alloc_thr;
- size_t size;
- size_t idx;
- int index;
-
- KE_TRACE( 25, ( "-> __kmp_fast_free( T#%d, %p ) called from %s:%d\n",
- __kmp_gtid_from_thread(this_thr), ptr KMP_SRC_LOC_PARM ) );
- KMP_ASSERT( ptr != NULL );
-
- descr = (kmp_mem_descr_t *)( ((kmp_uintptr_t)ptr) - sizeof(kmp_mem_descr_t) );
-
- KE_TRACE(26, (" __kmp_fast_free: size_aligned=%d\n",
- (int) descr->size_aligned ) );
-
- size = descr->size_aligned; // 2, 4, 16, 64, 65, 66, ... cache lines
-
- idx = DCACHE_LINE * 2; // 2 cache lines is minimal size of block
- if ( idx == size ) {
- index = 0; // 2 cache lines
- } else if ( ( idx <<= 1 ) == size ) {
- index = 1; // 4 cache lines
- } else if ( ( idx <<= 2 ) == size ) {
- index = 2; // 16 cache lines
- } else if ( ( idx <<= 2 ) == size ) {
- index = 3; // 64 cache lines
- } else {
- KMP_DEBUG_ASSERT( size > DCACHE_LINE * 64 );
- goto free_call; // 65 or more cache lines ( > 8KB )
- }
-
- alloc_thr = (kmp_info_t *)descr->ptr_aligned; // get thread owning the block
- if ( alloc_thr == this_thr ) {
- // push block to self no-sync free list, linking previous head (LIFO)
- *((void **)ptr) = this_thr->th.th_free_lists[index].th_free_list_self;
- this_thr->th.th_free_lists[index].th_free_list_self = ptr;
- } else {
- void * head = this_thr->th.th_free_lists[index].th_free_list_other;
- if ( head == NULL ) {
- // Create new free list
- this_thr->th.th_free_lists[index].th_free_list_other = ptr;
- *((void **)ptr) = NULL; // mark the tail of the list
- descr->size_allocated = (size_t)1; // head of the list keeps its length
- } else {
- // need to check existed "other" list's owner thread and size of queue
- kmp_mem_descr_t * dsc = (kmp_mem_descr_t *)( (char*)head - sizeof(kmp_mem_descr_t) );
- kmp_info_t * q_th = (kmp_info_t *)(dsc->ptr_aligned); // allocating thread, same for all queue nodes
- size_t q_sz = dsc->size_allocated + 1; // new size in case we add current task
- if ( q_th == alloc_thr && q_sz <= KMP_FREE_LIST_LIMIT ) {
- // we can add current task to "other" list, no sync needed
- *((void **)ptr) = head;
- descr->size_allocated = q_sz;
- this_thr->th.th_free_lists[index].th_free_list_other = ptr;
- } else {
- // either queue blocks owner is changing or size limit exceeded
- // return old queue to allocating thread (q_th) synchroneously,
- // and start new list for alloc_thr's tasks
- void * old_ptr;
- void * tail = head;
- void * next = *((void **)head);
- while ( next != NULL ) {
- KMP_DEBUG_ASSERT(
- // queue size should decrease by 1 each step through the list
- ((kmp_mem_descr_t*)((char*)next - sizeof(kmp_mem_descr_t)))->size_allocated + 1 ==
- ((kmp_mem_descr_t*)((char*)tail - sizeof(kmp_mem_descr_t)))->size_allocated );
- tail = next; // remember tail node
- next = *((void **)next);
- }
- KMP_DEBUG_ASSERT( q_th != NULL );
- // push block to owner's sync free list
- old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
- /* the next pointer must be set before setting free_list to ptr to avoid
- exposing a broken list to other threads, even for an instant. */
- *((void **)tail) = old_ptr;
-
- while ( ! KMP_COMPARE_AND_STORE_PTR(
- &q_th->th.th_free_lists[index].th_free_list_sync,
- old_ptr,
- head ) )
- {
- KMP_CPU_PAUSE();
- old_ptr = TCR_PTR( q_th->th.th_free_lists[index].th_free_list_sync );
- *((void **)tail) = old_ptr;
- }
-
- // start new list of not-selt tasks
- this_thr->th.th_free_lists[index].th_free_list_other = ptr;
- *((void **)ptr) = NULL;
- descr->size_allocated = (size_t)1; // head of queue keeps its length
- }
- }
- }
- goto end;
-
- free_call:
- KE_TRACE(25, ( "__kmp_fast_free: T#%d Calling __kmp_thread_free for size %d\n",
- __kmp_gtid_from_thread( this_thr), size ) );
- __kmp_bget_dequeue( this_thr ); /* Release any queued buffers */
- brel( this_thr, descr->ptr_allocated );
-
- end:
- KE_TRACE( 25, ( "<- __kmp_fast_free() returns\n" ) );
-
-} // func __kmp_fast_free
-
-
-// Initialize the thread free lists related to fast memory
-// Only do this when a thread is initially created.
-void
-__kmp_initialize_fast_memory( kmp_info_t *this_thr )
-{
- KE_TRACE(10, ( "__kmp_initialize_fast_memory: Called from th %p\n", this_thr ) );
-
- memset ( this_thr->th.th_free_lists, 0, NUM_LISTS * sizeof( kmp_free_list_t ) );
-}
-
-// Free the memory in the thread free lists related to fast memory
-// Only do this when a thread is being reaped (destroyed).
-void
-__kmp_free_fast_memory( kmp_info_t *th )
-{
- // Suppose we use BGET underlying allocator, walk through its structures...
- int bin;
- thr_data_t * thr = get_thr_data( th );
- void ** lst = NULL;
-
- KE_TRACE(5, ( "__kmp_free_fast_memory: Called T#%d\n",
- __kmp_gtid_from_thread( th ) ) );
-
- __kmp_bget_dequeue( th ); // Release any queued buffers
-
- // Dig through free lists and extract all allocated blocks
- for ( bin = 0; bin < MAX_BGET_BINS; ++bin ) {
- bfhead_t * b = thr->freelist[ bin ].ql.flink;
- while ( b != &thr->freelist[ bin ] ) {
- if ( (kmp_uintptr_t)b->bh.bb.bthr & 1 ) { // if the buffer is an allocated address?
- *((void**)b) = lst; // link the list (override bthr, but keep flink yet)
- lst = (void**)b; // push b into lst
- }
- b = b->ql.flink; // get next buffer
- }
- }
- while ( lst != NULL ) {
- void * next = *lst;
- KE_TRACE(10, ( "__kmp_free_fast_memory: freeing %p, next=%p th %p (%d)\n",
- lst, next, th, __kmp_gtid_from_thread( th ) ) );
- (*thr->relfcn)(lst);
- #if BufStats
- // count blocks to prevent problems in __kmp_finalize_bget()
- thr->numprel++; /* Nr of expansion block releases */
- thr->numpblk--; /* Total number of blocks */
- #endif
- lst = (void**)next;
- }
-
- KE_TRACE(5, ( "__kmp_free_fast_memory: Freed T#%d\n",
- __kmp_gtid_from_thread( th ) ) );
-}
-
-#endif // USE_FAST_MEMORY
Removed: openmp/trunk/runtime/src/kmp_atomic.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_atomic.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_atomic.c (original)
+++ openmp/trunk/runtime/src/kmp_atomic.c (removed)
@@ -1,3120 +0,0 @@
-/*
- * kmp_atomic.c -- ATOMIC implementation routines
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp_atomic.h"
-#include "kmp.h" // TRUE, asm routines prototypes
-
-typedef unsigned char uchar;
-typedef unsigned short ushort;
-
-/*!
- at defgroup ATOMIC_OPS Atomic Operations
-These functions are used for implementing the many different varieties of atomic operations.
-
-The compiler is at liberty to inline atomic operations that are naturally supported
-by the target architecture. For instance on IA-32 architecture an atomic like this can be inlined
- at code
-static int s = 0;
-#pragma omp atomic
- s++;
- at endcode
-using the single instruction: `lock; incl s`
-
-However the runtime does provide entrypoints for these operations to support compilers that choose
-not to inline them. (For instance, `__kmpc_atomic_fixed4_add` could be used to perform the
-increment above.)
-
-The names of the functions are encoded by using the data type name and the operation name, as in these tables.
-
-Data Type | Data type encoding
------------|---------------
-int8_t | `fixed1`
-uint8_t | `fixed1u`
-int16_t | `fixed2`
-uint16_t | `fixed2u`
-int32_t | `fixed4`
-uint32_t | `fixed4u`
-int32_t | `fixed8`
-uint32_t | `fixed8u`
-float | `float4`
-double | `float8`
-float 10 (8087 eighty bit float) | `float10`
-complex<float> | `cmplx4`
-complex<double> | `cmplx8`
-complex<float10> | `cmplx10`
-<br>
-
-Operation | Operation encoding
-----------|-------------------
-+ | add
-- | sub
-\* | mul
-/ | div
-& | andb
-<< | shl
-\>\> | shr
-\| | orb
-^ | xor
-&& | andl
-\|\| | orl
-maximum | max
-minimum | min
-.eqv. | eqv
-.neqv. | neqv
-
-<br>
-For non-commutative operations, `_rev` can also be added for the reversed operation.
-For the functions that capture the result, the suffix `_cpt` is added.
-
-Update Functions
-================
-The general form of an atomic function that just performs an update (without a `capture`)
- at code
-void __kmpc_atomic_<datatype>_<operation>( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs );
- at endcode
- at param ident_t a pointer to source location
- at param gtid the global thread id
- at param lhs a pointer to the left operand
- at param rhs the right operand
-
-`capture` functions
-===================
-The capture functions perform an atomic update and return a result, which is either the value
-before the capture, or that after. They take an additional argument to determine which result is returned.
-Their general form is therefore
- at code
-TYPE __kmpc_atomic_<datatype>_<operation>_cpt( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag );
- at endcode
- at param ident_t a pointer to source location
- at param gtid the global thread id
- at param lhs a pointer to the left operand
- at param rhs the right operand
- at param flag one if the result is to be captured *after* the operation, zero if captured *before*.
-
-The one set of exceptions to this is the `complex<float>` type where the value is not returned,
-rather an extra argument pointer is passed.
-
-They look like
- at code
-void __kmpc_atomic_cmplx4_<op>_cpt( ident_t *id_ref, int gtid, kmp_cmplx32 * lhs, kmp_cmplx32 rhs, kmp_cmplx32 * out, int flag );
- at endcode
-
-Read and Write Operations
-=========================
-The OpenMP<sup>*</sup> standard now supports atomic operations that simply ensure that the
-value is read or written atomically, with no modification
-performed. In many cases on IA-32 architecture these operations can be inlined since
-the architecture guarantees that no tearing occurs on aligned objects
-accessed with a single memory operation of up to 64 bits in size.
-
-The general form of the read operations is
- at code
-TYPE __kmpc_atomic_<type>_rd ( ident_t *id_ref, int gtid, TYPE * loc );
- at endcode
-
-For the write operations the form is
- at code
-void __kmpc_atomic_<type>_wr ( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs );
- at endcode
-
-Full list of functions
-======================
-This leads to the generation of 376 atomic functions, as follows.
-
-Functons for integers
----------------------
-There are versions here for integers of size 1,2,4 and 8 bytes both signed and unsigned (where that matters).
- at code
- __kmpc_atomic_fixed1_add
- __kmpc_atomic_fixed1_add_cpt
- __kmpc_atomic_fixed1_add_fp
- __kmpc_atomic_fixed1_andb
- __kmpc_atomic_fixed1_andb_cpt
- __kmpc_atomic_fixed1_andl
- __kmpc_atomic_fixed1_andl_cpt
- __kmpc_atomic_fixed1_div
- __kmpc_atomic_fixed1_div_cpt
- __kmpc_atomic_fixed1_div_cpt_rev
- __kmpc_atomic_fixed1_div_float8
- __kmpc_atomic_fixed1_div_fp
- __kmpc_atomic_fixed1_div_rev
- __kmpc_atomic_fixed1_eqv
- __kmpc_atomic_fixed1_eqv_cpt
- __kmpc_atomic_fixed1_max
- __kmpc_atomic_fixed1_max_cpt
- __kmpc_atomic_fixed1_min
- __kmpc_atomic_fixed1_min_cpt
- __kmpc_atomic_fixed1_mul
- __kmpc_atomic_fixed1_mul_cpt
- __kmpc_atomic_fixed1_mul_float8
- __kmpc_atomic_fixed1_mul_fp
- __kmpc_atomic_fixed1_neqv
- __kmpc_atomic_fixed1_neqv_cpt
- __kmpc_atomic_fixed1_orb
- __kmpc_atomic_fixed1_orb_cpt
- __kmpc_atomic_fixed1_orl
- __kmpc_atomic_fixed1_orl_cpt
- __kmpc_atomic_fixed1_rd
- __kmpc_atomic_fixed1_shl
- __kmpc_atomic_fixed1_shl_cpt
- __kmpc_atomic_fixed1_shl_cpt_rev
- __kmpc_atomic_fixed1_shl_rev
- __kmpc_atomic_fixed1_shr
- __kmpc_atomic_fixed1_shr_cpt
- __kmpc_atomic_fixed1_shr_cpt_rev
- __kmpc_atomic_fixed1_shr_rev
- __kmpc_atomic_fixed1_sub
- __kmpc_atomic_fixed1_sub_cpt
- __kmpc_atomic_fixed1_sub_cpt_rev
- __kmpc_atomic_fixed1_sub_fp
- __kmpc_atomic_fixed1_sub_rev
- __kmpc_atomic_fixed1_swp
- __kmpc_atomic_fixed1_wr
- __kmpc_atomic_fixed1_xor
- __kmpc_atomic_fixed1_xor_cpt
- __kmpc_atomic_fixed1u_add_fp
- __kmpc_atomic_fixed1u_sub_fp
- __kmpc_atomic_fixed1u_mul_fp
- __kmpc_atomic_fixed1u_div
- __kmpc_atomic_fixed1u_div_cpt
- __kmpc_atomic_fixed1u_div_cpt_rev
- __kmpc_atomic_fixed1u_div_fp
- __kmpc_atomic_fixed1u_div_rev
- __kmpc_atomic_fixed1u_shr
- __kmpc_atomic_fixed1u_shr_cpt
- __kmpc_atomic_fixed1u_shr_cpt_rev
- __kmpc_atomic_fixed1u_shr_rev
- __kmpc_atomic_fixed2_add
- __kmpc_atomic_fixed2_add_cpt
- __kmpc_atomic_fixed2_add_fp
- __kmpc_atomic_fixed2_andb
- __kmpc_atomic_fixed2_andb_cpt
- __kmpc_atomic_fixed2_andl
- __kmpc_atomic_fixed2_andl_cpt
- __kmpc_atomic_fixed2_div
- __kmpc_atomic_fixed2_div_cpt
- __kmpc_atomic_fixed2_div_cpt_rev
- __kmpc_atomic_fixed2_div_float8
- __kmpc_atomic_fixed2_div_fp
- __kmpc_atomic_fixed2_div_rev
- __kmpc_atomic_fixed2_eqv
- __kmpc_atomic_fixed2_eqv_cpt
- __kmpc_atomic_fixed2_max
- __kmpc_atomic_fixed2_max_cpt
- __kmpc_atomic_fixed2_min
- __kmpc_atomic_fixed2_min_cpt
- __kmpc_atomic_fixed2_mul
- __kmpc_atomic_fixed2_mul_cpt
- __kmpc_atomic_fixed2_mul_float8
- __kmpc_atomic_fixed2_mul_fp
- __kmpc_atomic_fixed2_neqv
- __kmpc_atomic_fixed2_neqv_cpt
- __kmpc_atomic_fixed2_orb
- __kmpc_atomic_fixed2_orb_cpt
- __kmpc_atomic_fixed2_orl
- __kmpc_atomic_fixed2_orl_cpt
- __kmpc_atomic_fixed2_rd
- __kmpc_atomic_fixed2_shl
- __kmpc_atomic_fixed2_shl_cpt
- __kmpc_atomic_fixed2_shl_cpt_rev
- __kmpc_atomic_fixed2_shl_rev
- __kmpc_atomic_fixed2_shr
- __kmpc_atomic_fixed2_shr_cpt
- __kmpc_atomic_fixed2_shr_cpt_rev
- __kmpc_atomic_fixed2_shr_rev
- __kmpc_atomic_fixed2_sub
- __kmpc_atomic_fixed2_sub_cpt
- __kmpc_atomic_fixed2_sub_cpt_rev
- __kmpc_atomic_fixed2_sub_fp
- __kmpc_atomic_fixed2_sub_rev
- __kmpc_atomic_fixed2_swp
- __kmpc_atomic_fixed2_wr
- __kmpc_atomic_fixed2_xor
- __kmpc_atomic_fixed2_xor_cpt
- __kmpc_atomic_fixed2u_add_fp
- __kmpc_atomic_fixed2u_sub_fp
- __kmpc_atomic_fixed2u_mul_fp
- __kmpc_atomic_fixed2u_div
- __kmpc_atomic_fixed2u_div_cpt
- __kmpc_atomic_fixed2u_div_cpt_rev
- __kmpc_atomic_fixed2u_div_fp
- __kmpc_atomic_fixed2u_div_rev
- __kmpc_atomic_fixed2u_shr
- __kmpc_atomic_fixed2u_shr_cpt
- __kmpc_atomic_fixed2u_shr_cpt_rev
- __kmpc_atomic_fixed2u_shr_rev
- __kmpc_atomic_fixed4_add
- __kmpc_atomic_fixed4_add_cpt
- __kmpc_atomic_fixed4_add_fp
- __kmpc_atomic_fixed4_andb
- __kmpc_atomic_fixed4_andb_cpt
- __kmpc_atomic_fixed4_andl
- __kmpc_atomic_fixed4_andl_cpt
- __kmpc_atomic_fixed4_div
- __kmpc_atomic_fixed4_div_cpt
- __kmpc_atomic_fixed4_div_cpt_rev
- __kmpc_atomic_fixed4_div_float8
- __kmpc_atomic_fixed4_div_fp
- __kmpc_atomic_fixed4_div_rev
- __kmpc_atomic_fixed4_eqv
- __kmpc_atomic_fixed4_eqv_cpt
- __kmpc_atomic_fixed4_max
- __kmpc_atomic_fixed4_max_cpt
- __kmpc_atomic_fixed4_min
- __kmpc_atomic_fixed4_min_cpt
- __kmpc_atomic_fixed4_mul
- __kmpc_atomic_fixed4_mul_cpt
- __kmpc_atomic_fixed4_mul_float8
- __kmpc_atomic_fixed4_mul_fp
- __kmpc_atomic_fixed4_neqv
- __kmpc_atomic_fixed4_neqv_cpt
- __kmpc_atomic_fixed4_orb
- __kmpc_atomic_fixed4_orb_cpt
- __kmpc_atomic_fixed4_orl
- __kmpc_atomic_fixed4_orl_cpt
- __kmpc_atomic_fixed4_rd
- __kmpc_atomic_fixed4_shl
- __kmpc_atomic_fixed4_shl_cpt
- __kmpc_atomic_fixed4_shl_cpt_rev
- __kmpc_atomic_fixed4_shl_rev
- __kmpc_atomic_fixed4_shr
- __kmpc_atomic_fixed4_shr_cpt
- __kmpc_atomic_fixed4_shr_cpt_rev
- __kmpc_atomic_fixed4_shr_rev
- __kmpc_atomic_fixed4_sub
- __kmpc_atomic_fixed4_sub_cpt
- __kmpc_atomic_fixed4_sub_cpt_rev
- __kmpc_atomic_fixed4_sub_fp
- __kmpc_atomic_fixed4_sub_rev
- __kmpc_atomic_fixed4_swp
- __kmpc_atomic_fixed4_wr
- __kmpc_atomic_fixed4_xor
- __kmpc_atomic_fixed4_xor_cpt
- __kmpc_atomic_fixed4u_add_fp
- __kmpc_atomic_fixed4u_sub_fp
- __kmpc_atomic_fixed4u_mul_fp
- __kmpc_atomic_fixed4u_div
- __kmpc_atomic_fixed4u_div_cpt
- __kmpc_atomic_fixed4u_div_cpt_rev
- __kmpc_atomic_fixed4u_div_fp
- __kmpc_atomic_fixed4u_div_rev
- __kmpc_atomic_fixed4u_shr
- __kmpc_atomic_fixed4u_shr_cpt
- __kmpc_atomic_fixed4u_shr_cpt_rev
- __kmpc_atomic_fixed4u_shr_rev
- __kmpc_atomic_fixed8_add
- __kmpc_atomic_fixed8_add_cpt
- __kmpc_atomic_fixed8_add_fp
- __kmpc_atomic_fixed8_andb
- __kmpc_atomic_fixed8_andb_cpt
- __kmpc_atomic_fixed8_andl
- __kmpc_atomic_fixed8_andl_cpt
- __kmpc_atomic_fixed8_div
- __kmpc_atomic_fixed8_div_cpt
- __kmpc_atomic_fixed8_div_cpt_rev
- __kmpc_atomic_fixed8_div_float8
- __kmpc_atomic_fixed8_div_fp
- __kmpc_atomic_fixed8_div_rev
- __kmpc_atomic_fixed8_eqv
- __kmpc_atomic_fixed8_eqv_cpt
- __kmpc_atomic_fixed8_max
- __kmpc_atomic_fixed8_max_cpt
- __kmpc_atomic_fixed8_min
- __kmpc_atomic_fixed8_min_cpt
- __kmpc_atomic_fixed8_mul
- __kmpc_atomic_fixed8_mul_cpt
- __kmpc_atomic_fixed8_mul_float8
- __kmpc_atomic_fixed8_mul_fp
- __kmpc_atomic_fixed8_neqv
- __kmpc_atomic_fixed8_neqv_cpt
- __kmpc_atomic_fixed8_orb
- __kmpc_atomic_fixed8_orb_cpt
- __kmpc_atomic_fixed8_orl
- __kmpc_atomic_fixed8_orl_cpt
- __kmpc_atomic_fixed8_rd
- __kmpc_atomic_fixed8_shl
- __kmpc_atomic_fixed8_shl_cpt
- __kmpc_atomic_fixed8_shl_cpt_rev
- __kmpc_atomic_fixed8_shl_rev
- __kmpc_atomic_fixed8_shr
- __kmpc_atomic_fixed8_shr_cpt
- __kmpc_atomic_fixed8_shr_cpt_rev
- __kmpc_atomic_fixed8_shr_rev
- __kmpc_atomic_fixed8_sub
- __kmpc_atomic_fixed8_sub_cpt
- __kmpc_atomic_fixed8_sub_cpt_rev
- __kmpc_atomic_fixed8_sub_fp
- __kmpc_atomic_fixed8_sub_rev
- __kmpc_atomic_fixed8_swp
- __kmpc_atomic_fixed8_wr
- __kmpc_atomic_fixed8_xor
- __kmpc_atomic_fixed8_xor_cpt
- __kmpc_atomic_fixed8u_add_fp
- __kmpc_atomic_fixed8u_sub_fp
- __kmpc_atomic_fixed8u_mul_fp
- __kmpc_atomic_fixed8u_div
- __kmpc_atomic_fixed8u_div_cpt
- __kmpc_atomic_fixed8u_div_cpt_rev
- __kmpc_atomic_fixed8u_div_fp
- __kmpc_atomic_fixed8u_div_rev
- __kmpc_atomic_fixed8u_shr
- __kmpc_atomic_fixed8u_shr_cpt
- __kmpc_atomic_fixed8u_shr_cpt_rev
- __kmpc_atomic_fixed8u_shr_rev
- at endcode
-
-Functions for floating point
-----------------------------
-There are versions here for floating point numbers of size 4, 8, 10 and 16 bytes.
-(Ten byte floats are used by X87, but are now rare).
- at code
- __kmpc_atomic_float4_add
- __kmpc_atomic_float4_add_cpt
- __kmpc_atomic_float4_add_float8
- __kmpc_atomic_float4_add_fp
- __kmpc_atomic_float4_div
- __kmpc_atomic_float4_div_cpt
- __kmpc_atomic_float4_div_cpt_rev
- __kmpc_atomic_float4_div_float8
- __kmpc_atomic_float4_div_fp
- __kmpc_atomic_float4_div_rev
- __kmpc_atomic_float4_max
- __kmpc_atomic_float4_max_cpt
- __kmpc_atomic_float4_min
- __kmpc_atomic_float4_min_cpt
- __kmpc_atomic_float4_mul
- __kmpc_atomic_float4_mul_cpt
- __kmpc_atomic_float4_mul_float8
- __kmpc_atomic_float4_mul_fp
- __kmpc_atomic_float4_rd
- __kmpc_atomic_float4_sub
- __kmpc_atomic_float4_sub_cpt
- __kmpc_atomic_float4_sub_cpt_rev
- __kmpc_atomic_float4_sub_float8
- __kmpc_atomic_float4_sub_fp
- __kmpc_atomic_float4_sub_rev
- __kmpc_atomic_float4_swp
- __kmpc_atomic_float4_wr
- __kmpc_atomic_float8_add
- __kmpc_atomic_float8_add_cpt
- __kmpc_atomic_float8_add_fp
- __kmpc_atomic_float8_div
- __kmpc_atomic_float8_div_cpt
- __kmpc_atomic_float8_div_cpt_rev
- __kmpc_atomic_float8_div_fp
- __kmpc_atomic_float8_div_rev
- __kmpc_atomic_float8_max
- __kmpc_atomic_float8_max_cpt
- __kmpc_atomic_float8_min
- __kmpc_atomic_float8_min_cpt
- __kmpc_atomic_float8_mul
- __kmpc_atomic_float8_mul_cpt
- __kmpc_atomic_float8_mul_fp
- __kmpc_atomic_float8_rd
- __kmpc_atomic_float8_sub
- __kmpc_atomic_float8_sub_cpt
- __kmpc_atomic_float8_sub_cpt_rev
- __kmpc_atomic_float8_sub_fp
- __kmpc_atomic_float8_sub_rev
- __kmpc_atomic_float8_swp
- __kmpc_atomic_float8_wr
- __kmpc_atomic_float10_add
- __kmpc_atomic_float10_add_cpt
- __kmpc_atomic_float10_add_fp
- __kmpc_atomic_float10_div
- __kmpc_atomic_float10_div_cpt
- __kmpc_atomic_float10_div_cpt_rev
- __kmpc_atomic_float10_div_fp
- __kmpc_atomic_float10_div_rev
- __kmpc_atomic_float10_mul
- __kmpc_atomic_float10_mul_cpt
- __kmpc_atomic_float10_mul_fp
- __kmpc_atomic_float10_rd
- __kmpc_atomic_float10_sub
- __kmpc_atomic_float10_sub_cpt
- __kmpc_atomic_float10_sub_cpt_rev
- __kmpc_atomic_float10_sub_fp
- __kmpc_atomic_float10_sub_rev
- __kmpc_atomic_float10_swp
- __kmpc_atomic_float10_wr
- __kmpc_atomic_float16_add
- __kmpc_atomic_float16_add_cpt
- __kmpc_atomic_float16_div
- __kmpc_atomic_float16_div_cpt
- __kmpc_atomic_float16_div_cpt_rev
- __kmpc_atomic_float16_div_rev
- __kmpc_atomic_float16_max
- __kmpc_atomic_float16_max_cpt
- __kmpc_atomic_float16_min
- __kmpc_atomic_float16_min_cpt
- __kmpc_atomic_float16_mul
- __kmpc_atomic_float16_mul_cpt
- __kmpc_atomic_float16_rd
- __kmpc_atomic_float16_sub
- __kmpc_atomic_float16_sub_cpt
- __kmpc_atomic_float16_sub_cpt_rev
- __kmpc_atomic_float16_sub_rev
- __kmpc_atomic_float16_swp
- __kmpc_atomic_float16_wr
- at endcode
-
-Functions for Complex types
----------------------------
-Functions for complex types whose component floating point variables are of size 4,8,10 or 16 bytes.
-The names here are based on the size of the component float, *not* the size of the complex type. So
-`__kmpc_atomc_cmplx8_add` is an operation on a `complex<double>` or `complex(kind=8)`, *not* `complex<float>`.
-
- at code
- __kmpc_atomic_cmplx4_add
- __kmpc_atomic_cmplx4_add_cmplx8
- __kmpc_atomic_cmplx4_add_cpt
- __kmpc_atomic_cmplx4_div
- __kmpc_atomic_cmplx4_div_cmplx8
- __kmpc_atomic_cmplx4_div_cpt
- __kmpc_atomic_cmplx4_div_cpt_rev
- __kmpc_atomic_cmplx4_div_rev
- __kmpc_atomic_cmplx4_mul
- __kmpc_atomic_cmplx4_mul_cmplx8
- __kmpc_atomic_cmplx4_mul_cpt
- __kmpc_atomic_cmplx4_rd
- __kmpc_atomic_cmplx4_sub
- __kmpc_atomic_cmplx4_sub_cmplx8
- __kmpc_atomic_cmplx4_sub_cpt
- __kmpc_atomic_cmplx4_sub_cpt_rev
- __kmpc_atomic_cmplx4_sub_rev
- __kmpc_atomic_cmplx4_swp
- __kmpc_atomic_cmplx4_wr
- __kmpc_atomic_cmplx8_add
- __kmpc_atomic_cmplx8_add_cpt
- __kmpc_atomic_cmplx8_div
- __kmpc_atomic_cmplx8_div_cpt
- __kmpc_atomic_cmplx8_div_cpt_rev
- __kmpc_atomic_cmplx8_div_rev
- __kmpc_atomic_cmplx8_mul
- __kmpc_atomic_cmplx8_mul_cpt
- __kmpc_atomic_cmplx8_rd
- __kmpc_atomic_cmplx8_sub
- __kmpc_atomic_cmplx8_sub_cpt
- __kmpc_atomic_cmplx8_sub_cpt_rev
- __kmpc_atomic_cmplx8_sub_rev
- __kmpc_atomic_cmplx8_swp
- __kmpc_atomic_cmplx8_wr
- __kmpc_atomic_cmplx10_add
- __kmpc_atomic_cmplx10_add_cpt
- __kmpc_atomic_cmplx10_div
- __kmpc_atomic_cmplx10_div_cpt
- __kmpc_atomic_cmplx10_div_cpt_rev
- __kmpc_atomic_cmplx10_div_rev
- __kmpc_atomic_cmplx10_mul
- __kmpc_atomic_cmplx10_mul_cpt
- __kmpc_atomic_cmplx10_rd
- __kmpc_atomic_cmplx10_sub
- __kmpc_atomic_cmplx10_sub_cpt
- __kmpc_atomic_cmplx10_sub_cpt_rev
- __kmpc_atomic_cmplx10_sub_rev
- __kmpc_atomic_cmplx10_swp
- __kmpc_atomic_cmplx10_wr
- __kmpc_atomic_cmplx16_add
- __kmpc_atomic_cmplx16_add_cpt
- __kmpc_atomic_cmplx16_div
- __kmpc_atomic_cmplx16_div_cpt
- __kmpc_atomic_cmplx16_div_cpt_rev
- __kmpc_atomic_cmplx16_div_rev
- __kmpc_atomic_cmplx16_mul
- __kmpc_atomic_cmplx16_mul_cpt
- __kmpc_atomic_cmplx16_rd
- __kmpc_atomic_cmplx16_sub
- __kmpc_atomic_cmplx16_sub_cpt
- __kmpc_atomic_cmplx16_sub_cpt_rev
- __kmpc_atomic_cmplx16_swp
- __kmpc_atomic_cmplx16_wr
- at endcode
-*/
-
-/*!
- at ingroup ATOMIC_OPS
-@{
-*/
-
-/*
- * Global vars
- */
-
-#ifndef KMP_GOMP_COMPAT
-int __kmp_atomic_mode = 1; // Intel perf
-#else
-int __kmp_atomic_mode = 2; // GOMP compatibility
-#endif /* KMP_GOMP_COMPAT */
-
-KMP_ALIGN(128)
-
-kmp_atomic_lock_t __kmp_atomic_lock; /* Control access to all user coded atomics in Gnu compat mode */
-kmp_atomic_lock_t __kmp_atomic_lock_1i; /* Control access to all user coded atomics for 1-byte fixed data types */
-kmp_atomic_lock_t __kmp_atomic_lock_2i; /* Control access to all user coded atomics for 2-byte fixed data types */
-kmp_atomic_lock_t __kmp_atomic_lock_4i; /* Control access to all user coded atomics for 4-byte fixed data types */
-kmp_atomic_lock_t __kmp_atomic_lock_4r; /* Control access to all user coded atomics for kmp_real32 data type */
-kmp_atomic_lock_t __kmp_atomic_lock_8i; /* Control access to all user coded atomics for 8-byte fixed data types */
-kmp_atomic_lock_t __kmp_atomic_lock_8r; /* Control access to all user coded atomics for kmp_real64 data type */
-kmp_atomic_lock_t __kmp_atomic_lock_8c; /* Control access to all user coded atomics for complex byte data type */
-kmp_atomic_lock_t __kmp_atomic_lock_10r; /* Control access to all user coded atomics for long double data type */
-kmp_atomic_lock_t __kmp_atomic_lock_16r; /* Control access to all user coded atomics for _Quad data type */
-kmp_atomic_lock_t __kmp_atomic_lock_16c; /* Control access to all user coded atomics for double complex data type*/
-kmp_atomic_lock_t __kmp_atomic_lock_20c; /* Control access to all user coded atomics for long double complex type*/
-kmp_atomic_lock_t __kmp_atomic_lock_32c; /* Control access to all user coded atomics for _Quad complex data type */
-
-
-/*
- 2007-03-02:
- Without "volatile" specifier in OP_CMPXCHG and MIN_MAX_CMPXCHG we have a
- bug on *_32 and *_32e. This is just a temporary workaround for the problem.
- It seems the right solution is writing OP_CMPXCHG and MIN_MAX_CMPXCHG
- routines in assembler language.
-*/
-#define KMP_ATOMIC_VOLATILE volatile
-
-#if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD
-
- static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
- static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
- static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
- static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
- static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) { return lhs.q < rhs.q; }
- static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) { return lhs.q > rhs.q; }
-
- static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
- static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
- static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
- static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
- static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) { return lhs.q < rhs.q; }
- static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) { return lhs.q > rhs.q; }
-
- static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
- static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
- static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
- static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
-
- static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
- static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
- static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
- static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
-
-#endif
-
-/* ------------------------------------------------------------------------ */
-/* ATOMIC implementation routines */
-/* one routine for each operation and operand type */
-/* ------------------------------------------------------------------------ */
-
-// All routines declarations looks like
-// void __kmpc_atomic_RTYPE_OP( ident_t*, int, TYPE *lhs, TYPE rhs );
-// ------------------------------------------------------------------------
-
-#define KMP_CHECK_GTID \
- if ( gtid == KMP_GTID_UNKNOWN ) { \
- gtid = __kmp_entry_gtid(); \
- } // check and get gtid when needed
-
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-#define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
-RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
-
-// ------------------------------------------------------------------------
-// Lock variables used for critical sections for various size operands
-#define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
-#define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
-#define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
-#define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
-#define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
-#define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
-#define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
-#define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
-#define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
-#define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
-#define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
-#define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
-#define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- (*lhs) OP (rhs); \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-
-// ------------------------------------------------------------------------
-// For GNU compatibility, we may need to use a critical section,
-// even though it is not required by the ISA.
-//
-// On IA-32 architecture, all atomic operations except for fixed 4 byte add,
-// sub, and bitwise logical ops, and 1 & 2 byte logical ops use a common
-// critical section. On Intel(R) 64, all atomic operations are done with fetch
-// and add or compare and exchange. Therefore, the FLAG parameter to this
-// macro is either KMP_ARCH_X86 or 0 (or 1, for Intel-specific extension which
-// require a critical section, where we predict that they will be implemented
-// in the Gnu codegen by calling GOMP_atomic_start() / GOMP_atomic_end()).
-//
-// When the OP_GOMP_CRITICAL macro is used in a *CRITICAL* macro construct,
-// the FLAG parameter should always be 1. If we know that we will be using
-// a critical section, then we want to make certain that we use the generic
-// lock __kmp_atomic_lock to protect the atomic update, and not of of the
-// locks that are specialized based upon the size or type of the data.
-//
-// If FLAG is 0, then we are relying on dead code elimination by the build
-// compiler to get rid of the useless block of code, and save a needless
-// branch at runtime.
-//
-
-#ifdef KMP_GOMP_COMPAT
-# define OP_GOMP_CRITICAL(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL( OP, 0 ); \
- return; \
- }
-# else
-# define OP_GOMP_CRITICAL(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-#if KMP_MIC
-# define KMP_DO_PAUSE _mm_delay_32( 1 )
-#else
-# define KMP_DO_PAUSE KMP_CPU_PAUSE()
-#endif /* KMP_MIC */
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-#define OP_CMPXCHG(TYPE,BITS,OP) \
- { \
- TYPE old_value, new_value; \
- old_value = *(TYPE volatile *)lhs; \
- new_value = old_value OP rhs; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_DO_PAUSE; \
- \
- old_value = *(TYPE volatile *)lhs; \
- new_value = old_value OP rhs; \
- } \
- }
-
-#if USE_CMPXCHG_FIX
-// 2007-06-25:
-// workaround for C78287 (complex(kind=4) data type)
-// lin_32, lin_32e, win_32 and win_32e are affected (I verified the asm)
-// Compiler ignores the volatile qualifier of the temp_val in the OP_CMPXCHG macro.
-// This is a problem of the compiler.
-// Related tracker is C76005, targeted to 11.0.
-// I verified the asm of the workaround.
-#define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
- { \
- struct _sss { \
- TYPE cmp; \
- kmp_int##BITS *vvv; \
- }; \
- struct _sss old_value, new_value; \
- old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \
- new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \
- *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
- new_value.cmp = old_value.cmp OP rhs; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
- *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \
- { \
- KMP_DO_PAUSE; \
- \
- *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \
- new_value.cmp = old_value.cmp OP rhs; \
- } \
- }
-// end of the first part of the workaround for C78287
-#endif // USE_CMPXCHG_FIX
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-// ------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ====================================
-#define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
- KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
-}
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- OP_CMPXCHG(TYPE,BITS,OP) \
-}
-#if USE_CMPXCHG_FIX
-// -------------------------------------------------------------------------
-// workaround for C78287 (complex(kind=4) data type)
-#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
-}
-// end of the second part of the workaround for C78287
-#endif
-
-#else
-// -------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
- KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(OP##=,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(OP##=,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-#if USE_CMPXCHG_FIX
-// -------------------------------------------------------------------------
-// workaround for C78287 (complex(kind=4) data type)
-#define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(OP##=,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-// end of the second part of the workaround for C78287
-#endif // USE_CMPXCHG_FIX
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-// Routines for ATOMIC 4-byte operands addition and subtraction
-ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 ) // __kmpc_atomic_fixed4_add
-ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 ) // __kmpc_atomic_fixed4_sub
-
-ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add
-ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub
-
-// Routines for ATOMIC 8-byte operands addition and subtraction
-ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add
-ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub
-
-ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_add
-ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub
-
-// ------------------------------------------------------------------------
-// Entries definition for integer operands
-// TYPE_ID - operands type and size (fixed4, float4)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operand type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator (used in critical section)
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-// MASK - used for alignment check
-
-// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID,MASK,GOMP_FLAG
-// ------------------------------------------------------------------------
-// Routines for ATOMIC integer operands, other operators
-// ------------------------------------------------------------------------
-// TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
-ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add
-ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 ) // __kmpc_atomic_fixed1_andb
-ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div
-ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div
-ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul
-ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 ) // __kmpc_atomic_fixed1_orb
-ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shl
-ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shr
-ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_shr
-ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub
-ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 ) // __kmpc_atomic_fixed1_xor
-ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add
-ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 ) // __kmpc_atomic_fixed2_andb
-ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div
-ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div
-ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul
-ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 ) // __kmpc_atomic_fixed2_orb
-ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shl
-ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shr
-ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_shr
-ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub
-ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 ) // __kmpc_atomic_fixed2_xor
-ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 ) // __kmpc_atomic_fixed4_andb
-ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_div
-ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_div
-ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_mul
-ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 ) // __kmpc_atomic_fixed4_orb
-ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shl
-ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shr
-ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_shr
-ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 ) // __kmpc_atomic_fixed4_xor
-ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_andb
-ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div
-ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div
-ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul
-ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_orb
-ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shl
-ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shr
-ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_shr
-ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_xor
-ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div
-ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul
-ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div
-ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_mul
-// TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
-
-
-/* ------------------------------------------------------------------------ */
-/* Routines for C/C++ Reduction operators && and || */
-/* ------------------------------------------------------------------------ */
-
-// ------------------------------------------------------------------------
-// Need separate macros for &&, || because there is no combined assignment
-// TODO: eliminate ATOMIC_CRIT_{L,EQV} macros as not used
-#define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
- OP_CRITICAL( = *lhs OP, LCK_ID ) \
-}
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-// ------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ===================================
-#define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \
- OP_CMPXCHG(TYPE,BITS,OP) \
-}
-
-#else
-// ------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(= *lhs OP,LCK_ID) /* unaligned - use critical */ \
- } \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-ATOMIC_CMPX_L( fixed1, andl, char, 8, &&, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_andl
-ATOMIC_CMPX_L( fixed1, orl, char, 8, ||, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_orl
-ATOMIC_CMPX_L( fixed2, andl, short, 16, &&, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_andl
-ATOMIC_CMPX_L( fixed2, orl, short, 16, ||, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_orl
-ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 ) // __kmpc_atomic_fixed4_andl
-ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 ) // __kmpc_atomic_fixed4_orl
-ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_andl
-ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_orl
-
-
-/* ------------------------------------------------------------------------- */
-/* Routines for Fortran operators that matched no one in C: */
-/* MAX, MIN, .EQV., .NEQV. */
-/* Operators .AND., .OR. are covered by __kmpc_atomic_*_{andl,orl} */
-/* Intrinsics IAND, IOR, IEOR are covered by __kmpc_atomic_*_{andb,orb,xor} */
-/* ------------------------------------------------------------------------- */
-
-// -------------------------------------------------------------------------
-// MIN and MAX need separate macros
-// OP - operator to check if we need any actions?
-#define MIN_MAX_CRITSECT(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if ( *lhs OP rhs ) { /* still need actions? */ \
- *lhs = rhs; \
- } \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-
-// -------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \
- if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
- KMP_CHECK_GTID; \
- MIN_MAX_CRITSECT( OP, 0 ); \
- return; \
- }
-#else
-#define GOMP_MIN_MAX_CRITSECT(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// -------------------------------------------------------------------------
-#define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- while ( old_value OP rhs && /* still need actions? */ \
- ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
- { \
- KMP_CPU_PAUSE(); \
- temp_val = *lhs; \
- old_value = temp_val; \
- } \
- }
-
-// -------------------------------------------------------------------------
-// 1-byte, 2-byte operands - use critical section
-#define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- if ( *lhs OP rhs ) { /* need actions? */ \
- GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
- MIN_MAX_CRITSECT(OP,LCK_ID) \
- } \
-}
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-// -------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ====================================
-#define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- if ( *lhs OP rhs ) { \
- GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
- MIN_MAX_CMPXCHG(TYPE,BITS,OP) \
- } \
-}
-
-#else
-// -------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- if ( *lhs OP rhs ) { \
- GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- MIN_MAX_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- MIN_MAX_CRITSECT(OP,LCK_ID) /* unaligned address */ \
- } \
- } \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-MIN_MAX_COMPXCHG( fixed1, max, char, 8, <, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_max
-MIN_MAX_COMPXCHG( fixed1, min, char, 8, >, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_min
-MIN_MAX_COMPXCHG( fixed2, max, short, 16, <, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_max
-MIN_MAX_COMPXCHG( fixed2, min, short, 16, >, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_min
-MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 ) // __kmpc_atomic_fixed4_max
-MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 ) // __kmpc_atomic_fixed4_min
-MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_max
-MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_min
-MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_max
-MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_min
-MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_max
-MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_min
-#if KMP_HAVE_QUAD
-MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 ) // __kmpc_atomic_float16_max
-MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 ) // __kmpc_atomic_float16_min
-#if ( KMP_ARCH_X86 )
- MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 ) // __kmpc_atomic_float16_max_a16
- MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 ) // __kmpc_atomic_float16_min_a16
-#endif
-#endif
-// ------------------------------------------------------------------------
-// Need separate macros for .EQV. because of the need of complement (~)
-// OP ignored for critical sections, ^=~ used instead
-#define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(^=~,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL(^=~,LCK_ID) /* send assignment and complement */ \
-}
-
-// ------------------------------------------------------------------------
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-// ------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ===================================
-#define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(^=~,GOMP_FLAG) /* send assignment */ \
- OP_CMPXCHG(TYPE,BITS,OP) \
-}
-// ------------------------------------------------------------------------
-#else
-// ------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(^=~,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_neqv
-ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_neqv
-ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_neqv
-ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_neqv
-ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_eqv
-ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_eqv
-ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_eqv
-ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_eqv
-
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL(OP##=,LCK_ID) /* send assignment */ \
-}
-
-/* ------------------------------------------------------------------------- */
-// routines for long double type
-ATOMIC_CRITICAL( float10, add, long double, +, 10r, 1 ) // __kmpc_atomic_float10_add
-ATOMIC_CRITICAL( float10, sub, long double, -, 10r, 1 ) // __kmpc_atomic_float10_sub
-ATOMIC_CRITICAL( float10, mul, long double, *, 10r, 1 ) // __kmpc_atomic_float10_mul
-ATOMIC_CRITICAL( float10, div, long double, /, 10r, 1 ) // __kmpc_atomic_float10_div
-#if KMP_HAVE_QUAD
-// routines for _Quad type
-ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 ) // __kmpc_atomic_float16_add
-ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 ) // __kmpc_atomic_float16_sub
-ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 ) // __kmpc_atomic_float16_mul
-ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 ) // __kmpc_atomic_float16_div
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 ) // __kmpc_atomic_float16_add_a16
- ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 ) // __kmpc_atomic_float16_sub_a16
- ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 ) // __kmpc_atomic_float16_mul_a16
- ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 ) // __kmpc_atomic_float16_div_a16
-#endif
-#endif
-// routines for complex types
-
-#if USE_CMPXCHG_FIX
-// workaround for C78287 (complex(kind=4) data type)
-ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 ) // __kmpc_atomic_cmplx4_add
-ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 ) // __kmpc_atomic_cmplx4_sub
-ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 ) // __kmpc_atomic_cmplx4_mul
-ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 ) // __kmpc_atomic_cmplx4_div
-// end of the workaround for C78287
-#else
-ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 ) // __kmpc_atomic_cmplx4_add
-ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 ) // __kmpc_atomic_cmplx4_sub
-ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 ) // __kmpc_atomic_cmplx4_mul
-ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 ) // __kmpc_atomic_cmplx4_div
-#endif // USE_CMPXCHG_FIX
-
-ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 ) // __kmpc_atomic_cmplx8_add
-ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 ) // __kmpc_atomic_cmplx8_sub
-ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 ) // __kmpc_atomic_cmplx8_mul
-ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 ) // __kmpc_atomic_cmplx8_div
-ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 ) // __kmpc_atomic_cmplx10_add
-ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 ) // __kmpc_atomic_cmplx10_sub
-ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 ) // __kmpc_atomic_cmplx10_mul
-ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 ) // __kmpc_atomic_cmplx10_div
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 ) // __kmpc_atomic_cmplx16_add
-ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub
-ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 ) // __kmpc_atomic_cmplx16_mul
-ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 ) // __kmpc_atomic_cmplx16_add_a16
- ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_a16
- ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 ) // __kmpc_atomic_cmplx16_mul_a16
- ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_a16
-#endif
-#endif
-
-#if OMP_40_ENABLED
-
-// OpenMP 4.0: x = expr binop x for non-commutative operations.
-// Supported only on IA-32 architecture and Intel(R) 64
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL_REV(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- (*lhs) = (rhs) OP (*lhs); \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_REV(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_REV( OP, 0 ); \
- return; \
- }
-#else
-#define OP_GOMP_CRITICAL_REV(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-#define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
-RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid ));
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-// Note: temp_val introduced in order to force the compiler to read
-// *lhs only once (w/o it the compiler reads *lhs twice)
-#define OP_CMPXCHG_REV(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value, new_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs OP old_value; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_DO_PAUSE; \
- \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs OP old_value; \
- } \
- }
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
- OP_CMPXCHG_REV(TYPE,BITS,OP) \
-}
-
-// ------------------------------------------------------------------------
-// Entries definition for integer operands
-// TYPE_ID - operands type and size (fixed4, float4)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operand type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator (used in critical section)
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-
-// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID,GOMP_FLAG
-// ------------------------------------------------------------------------
-// Routines for ATOMIC integer operands, other operators
-// ------------------------------------------------------------------------
-// TYPE_ID,OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG
-ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_rev
-ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_rev
-ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shl_rev
-ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shr_rev
-ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_shr_rev
-ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_rev
-
-ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_rev
-ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_rev
-ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shl_rev
-ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shr_rev
-ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_shr_rev
-ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_rev
-
-ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_div_rev
-ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_div_rev
-ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shl_rev
-ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shr_rev
-ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_shr_rev
-ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_sub_rev
-
-ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_rev
-ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_rev
-ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shl_rev
-ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shr_rev
-ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_shr_rev
-ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_rev
-
-ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_rev
-ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_rev
-
-ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_rev
-ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_rev
-// TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID, GOMP_FLAG
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
- OP_CRITICAL_REV(OP,LCK_ID) \
-}
-
-/* ------------------------------------------------------------------------- */
-// routines for long double type
-ATOMIC_CRITICAL_REV( float10, sub, long double, -, 10r, 1 ) // __kmpc_atomic_float10_sub_rev
-ATOMIC_CRITICAL_REV( float10, div, long double, /, 10r, 1 ) // __kmpc_atomic_float10_div_rev
-#if KMP_HAVE_QUAD
-// routines for _Quad type
-ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 ) // __kmpc_atomic_float16_sub_rev
-ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 ) // __kmpc_atomic_float16_div_rev
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 ) // __kmpc_atomic_float16_sub_a16_rev
- ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 ) // __kmpc_atomic_float16_div_a16_rev
-#endif
-#endif
-
-// routines for complex types
-ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 ) // __kmpc_atomic_cmplx4_sub_rev
-ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 ) // __kmpc_atomic_cmplx4_div_rev
-ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 ) // __kmpc_atomic_cmplx8_sub_rev
-ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 ) // __kmpc_atomic_cmplx8_div_rev
-ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 ) // __kmpc_atomic_cmplx10_sub_rev
-ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 ) // __kmpc_atomic_cmplx10_div_rev
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_rev
-ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_rev
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_a16_rev
- ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_a16_rev
-#endif
-#endif
-
-
-#endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
-// End of OpenMP 4.0: x = expr binop x for non-commutative operations.
-
-#endif //OMP_40_ENABLED
-
-
-/* ------------------------------------------------------------------------ */
-/* Routines for mixed types of LHS and RHS, when RHS is "larger" */
-/* Note: in order to reduce the total number of types combinations */
-/* it is supposed that compiler converts RHS to longest floating type,*/
-/* that is _Quad, before call to any of these routines */
-/* Conversion to _Quad will be done by the compiler during calculation, */
-/* conversion back to TYPE - before the assignment, like: */
-/* *lhs = (TYPE)( (_Quad)(*lhs) OP rhs ) */
-/* Performance penalty expected because of SW emulation use */
-/* ------------------------------------------------------------------------ */
-
-#define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
-void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL(OP##=,LCK_ID) /* send assignment */ \
-}
-
-// -------------------------------------------------------------------------
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-// -------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ====================================
-#define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- OP_CMPXCHG(TYPE,BITS,OP) \
-}
-// -------------------------------------------------------------------------
-#else
-// ------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(OP##=,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-// -------------------------------------------------------------------------
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
- OP_CMPXCHG_REV(TYPE,BITS,OP) \
-}
-#define ATOMIC_CRITICAL_REV_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \
- OP_CRITICAL_REV(OP,LCK_ID) \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-// RHS=float8
-ATOMIC_CMPXCHG_MIX( fixed1, char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_float8
-ATOMIC_CMPXCHG_MIX( fixed1, char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_float8
-ATOMIC_CMPXCHG_MIX( fixed2, short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_float8
-ATOMIC_CMPXCHG_MIX( fixed2, short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_float8
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 ) // __kmpc_atomic_fixed4_mul_float8
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 ) // __kmpc_atomic_fixed4_div_float8
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_float8
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_float8
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add_float8
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_float8
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul_float8
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_float8
-
-// RHS=float16 (deprecated, to be removed when we are sure the compiler does not use them)
-#if KMP_HAVE_QUAD
-ATOMIC_CMPXCHG_MIX( fixed1, char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add_fp
-ATOMIC_CMPXCHG_MIX( fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_add_fp
-ATOMIC_CMPXCHG_MIX( fixed1, char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed1, char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed1, char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_fp
-ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_fp
-
-ATOMIC_CMPXCHG_MIX( fixed2, short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add_fp
-ATOMIC_CMPXCHG_MIX( fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_add_fp
-ATOMIC_CMPXCHG_MIX( fixed2, short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed2, short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed2, short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_fp
-ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_fp
-
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_add_fp
-ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_add_fp
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_div_fp
-ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_div_fp
-
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add_fp
-ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_add_fp
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_fp
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_mul_fp
-ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_fp
-ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_fp
-
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add_fp
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_fp
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul_fp
-ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_fp
-
-ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_add_fp
-ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_fp
-ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_mul_fp
-ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_fp
-
-ATOMIC_CRITICAL_FP( float10, long double, add, +, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_add_fp
-ATOMIC_CRITICAL_FP( float10, long double, sub, -, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_sub_fp
-ATOMIC_CRITICAL_FP( float10, long double, mul, *, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_mul_fp
-ATOMIC_CRITICAL_FP( float10, long double, div, /, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_div_fp
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-// Reverse operations
-ATOMIC_CMPXCHG_REV_MIX( fixed1, char, sub_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed1, char, div_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_rev_fp
-
-ATOMIC_CMPXCHG_REV_MIX( fixed2, short, sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed2, short, div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_rev_fp
-
-ATOMIC_CMPXCHG_REV_MIX( fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_div_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_div_rev_fp
-
-ATOMIC_CMPXCHG_REV_MIX( fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_rev_fp
-
-ATOMIC_CMPXCHG_REV_MIX( float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_rev_fp
-
-ATOMIC_CMPXCHG_REV_MIX( float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_rev_fp
-ATOMIC_CMPXCHG_REV_MIX( float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_rev_fp
-
-ATOMIC_CRITICAL_REV_FP( float10, long double, sub_rev, -, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_sub_rev_fp
-ATOMIC_CRITICAL_REV_FP( float10, long double, div_rev, /, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_div_rev_fp
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-#endif
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-// ------------------------------------------------------------------------
-// X86 or X86_64: no alignment problems ====================================
-#if USE_CMPXCHG_FIX
-// workaround for C78287 (complex(kind=4) data type)
-#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \
-}
-// end of the second part of the workaround for C78287
-#else
-#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- OP_CMPXCHG(TYPE,BITS,OP) \
-}
-#endif // USE_CMPXCHG_FIX
-#else
-// ------------------------------------------------------------------------
-// Code for other architectures that don't handle unaligned accesses.
-#define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \
- OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \
- if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \
- OP_CMPXCHG(TYPE,BITS,OP) /* aligned address */ \
- } else { \
- KMP_CHECK_GTID; \
- OP_CRITICAL(OP##=,LCK_ID) /* unaligned address - use critical */ \
- } \
-}
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_add_cmplx8
-ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_sub_cmplx8
-ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_mul_cmplx8
-ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 ) // __kmpc_atomic_cmplx4_div_cmplx8
-
-// READ, WRITE, CAPTURE are supported only on IA-32 architecture and Intel(R) 64
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////
-// ------------------------------------------------------------------------
-// Atomic READ routines
-// ------------------------------------------------------------------------
-
-// ------------------------------------------------------------------------
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-#define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \
-RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store_ret" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-// Note: temp_val introduced in order to force the compiler to read
-// *lhs only once (w/o it the compiler reads *lhs twice)
-// TODO: check if it is still necessary
-// Return old value regardless of the result of "compare & swap# operation
-
-#define OP_CMPXCHG_READ(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- union f_i_union { \
- TYPE f_val; \
- kmp_int##BITS i_val; \
- }; \
- union f_i_union old_value; \
- temp_val = *loc; \
- old_value.f_val = temp_val; \
- old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \
- new_value = old_value.f_val; \
- return new_value; \
- }
-
-// -------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL_READ(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- new_value = (*loc); \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-
-// -------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_READ(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_READ( OP, 0 ); \
- return new_value; \
- }
-#else
-#define OP_GOMP_CRITICAL_READ(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// -------------------------------------------------------------------------
-#define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
- new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \
- return new_value; \
-}
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \
- OP_CMPXCHG_READ(TYPE,BITS,OP) \
-}
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL_READ(OP,LCK_ID) /* send assignment */ \
- return new_value; \
-}
-
-// ------------------------------------------------------------------------
-// Fix for cmplx4 read (CQ220361) on Windows* OS. Regular routine with return value doesn't work.
-// Let's return the read value through the additional parameter.
-
-#if ( KMP_OS_WINDOWS )
-
-#define OP_CRITICAL_READ_WRK(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- (*out) = (*loc); \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_READ_WRK( OP, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-// ------------------------------------------------------------------------
-#define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
-void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
-
-// ------------------------------------------------------------------------
-#define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \
- OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL_READ_WRK(OP,LCK_ID) /* send assignment */ \
-}
-
-#endif // KMP_OS_WINDOWS
-
-// ------------------------------------------------------------------------
-// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
-ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 ) // __kmpc_atomic_fixed4_rd
-ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_rd
-ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 ) // __kmpc_atomic_float4_rd
-ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 ) // __kmpc_atomic_float8_rd
-
-// !!! TODO: Remove lock operations for "char" since it can't be non-atomic
-ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_rd
-ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_rd
-
-ATOMIC_CRITICAL_READ( float10, rd, long double, +, 10r, 1 ) // __kmpc_atomic_float10_rd
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 ) // __kmpc_atomic_float16_rd
-#endif // KMP_HAVE_QUAD
-
-// Fix for CQ220361 on Windows* OS
-#if ( KMP_OS_WINDOWS )
- ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 ) // __kmpc_atomic_cmplx4_rd
-#else
- ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 ) // __kmpc_atomic_cmplx4_rd
-#endif
-ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 ) // __kmpc_atomic_cmplx8_rd
-ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 ) // __kmpc_atomic_cmplx10_rd
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 ) // __kmpc_atomic_cmplx16_rd
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 ) // __kmpc_atomic_float16_a16_rd
- ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 ) // __kmpc_atomic_cmplx16_a16_rd
-#endif
-#endif
-
-
-// ------------------------------------------------------------------------
-// Atomic WRITE routines
-// ------------------------------------------------------------------------
-
-#define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
- KMP_XCHG_FIXED##BITS( lhs, rhs ); \
-}
-// ------------------------------------------------------------------------
-#define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
- KMP_XCHG_REAL##BITS( lhs, rhs ); \
-}
-
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-// Note: temp_val introduced in order to force the compiler to read
-// *lhs only once (w/o it the compiler reads *lhs twice)
-#define OP_CMPXCHG_WR(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value, new_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_CPU_PAUSE(); \
- \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs; \
- } \
- }
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP,GOMP_FLAG) \
- OP_CMPXCHG_WR(TYPE,BITS,OP) \
-}
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \
- OP_GOMP_CRITICAL(OP,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL(OP,LCK_ID) /* send assignment */ \
-}
-// -------------------------------------------------------------------------
-
-ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_wr
-ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_wr
-ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_wr
-#if ( KMP_ARCH_X86 )
- ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_wr
-#else
- ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_wr
-#endif
-
-ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 ) // __kmpc_atomic_float4_wr
-#if ( KMP_ARCH_X86 )
- ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 ) // __kmpc_atomic_float8_wr
-#else
- ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 ) // __kmpc_atomic_float8_wr
-#endif
-
-ATOMIC_CRITICAL_WR( float10, wr, long double, =, 10r, 1 ) // __kmpc_atomic_float10_wr
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 ) // __kmpc_atomic_float16_wr
-#endif
-ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 ) // __kmpc_atomic_cmplx4_wr
-ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 ) // __kmpc_atomic_cmplx8_wr
-ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 ) // __kmpc_atomic_cmplx10_wr
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 ) // __kmpc_atomic_cmplx16_wr
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 ) // __kmpc_atomic_float16_a16_wr
- ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 ) // __kmpc_atomic_cmplx16_a16_wr
-#endif
-#endif
-
-
-// ------------------------------------------------------------------------
-// Atomic CAPTURE routines
-// ------------------------------------------------------------------------
-
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-#define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \
-RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
-
-// -------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL_CPT(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if( flag ) { \
- (*lhs) OP rhs; \
- new_value = (*lhs); \
- } else { \
- new_value = (*lhs); \
- (*lhs) OP rhs; \
- } \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return new_value;
-
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_CPT(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_CPT( OP##=, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_CPT(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-// Note: temp_val introduced in order to force the compiler to read
-// *lhs only once (w/o it the compiler reads *lhs twice)
-#define OP_CMPXCHG_CPT(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value, new_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = old_value OP rhs; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_CPU_PAUSE(); \
- \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = old_value OP rhs; \
- } \
- if( flag ) { \
- return new_value; \
- } else \
- return old_value; \
- }
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
- OP_CMPXCHG_CPT(TYPE,BITS,OP) \
-}
-
-// -------------------------------------------------------------------------
-#define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE old_value, new_value; \
- OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
- /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
- old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \
- if( flag ) { \
- return old_value OP rhs; \
- } else \
- return old_value; \
-}
-// -------------------------------------------------------------------------
-
-ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 ) // __kmpc_atomic_fixed4_add_cpt
-ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 ) // __kmpc_atomic_fixed4_sub_cpt
-ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add_cpt
-ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt
-
-ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add_cpt
-ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt
-ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 ) // __kmpc_atomic_float8_add_cpt
-ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt
-
-// ------------------------------------------------------------------------
-// Entries definition for integer operands
-// TYPE_ID - operands type and size (fixed4, float4)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operand type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator (used in critical section)
-// TYPE_ID,OP_ID, TYPE, BITS,OP,GOMP_FLAG
-// ------------------------------------------------------------------------
-// Routines for ATOMIC integer operands, other operators
-// ------------------------------------------------------------------------
-// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
-ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 ) // __kmpc_atomic_fixed1_andb_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 ) // __kmpc_atomic_fixed1_orb_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shl_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt
-ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 ) // __kmpc_atomic_fixed1_xor_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 ) // __kmpc_atomic_fixed2_andb_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 ) // __kmpc_atomic_fixed2_orb_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shl_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 ) // __kmpc_atomic_fixed2_xor_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 ) // __kmpc_atomic_fixed4_andb_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_mul_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 ) // __kmpc_atomic_fixed4_orb_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shl_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 ) // __kmpc_atomic_fixed4_xor_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_andb_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_orb_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shl_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_shr_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_xor_cpt
-ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt
-ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul_cpt
-ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt
-ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 ) // __kmpc_atomic_float8_mul_cpt
-// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
-
-//////////////////////////////////
-
-// CAPTURE routines for mixed types RHS=float16
-#if KMP_HAVE_QUAD
-
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-#define ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \
-TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs, int flag ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid ));
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \
- OP_CMPXCHG_CPT(TYPE,BITS,OP) \
-}
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL_CPT(OP##=,LCK_ID) /* send assignment */ \
-}
-
-ATOMIC_CMPXCHG_CPT_MIX( fixed1, char, add_cpt, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1, char, sub_cpt, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1, char, mul_cpt, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1, char, div_cpt, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt_fp
-
-ATOMIC_CMPXCHG_CPT_MIX( fixed2, short, add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2, short, sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2, short, mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2, short, div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt_fp
-
-ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_div_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_div_cpt_fp
-
-ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt_fp
-
-ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt_fp
-
-ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_add_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_mul_cpt_fp
-ATOMIC_CMPXCHG_CPT_MIX( float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt_fp
-
-ATOMIC_CRITICAL_CPT_MIX( float10, long double, add_cpt, +, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_add_cpt_fp
-ATOMIC_CRITICAL_CPT_MIX( float10, long double, sub_cpt, -, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_sub_cpt_fp
-ATOMIC_CRITICAL_CPT_MIX( float10, long double, mul_cpt, *, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_mul_cpt_fp
-ATOMIC_CRITICAL_CPT_MIX( float10, long double, div_cpt, /, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_div_cpt_fp
-
-#endif //KMP_HAVE_QUAD
-
-///////////////////////////////////
-
-// ------------------------------------------------------------------------
-// Routines for C/C++ Reduction operators && and ||
-// ------------------------------------------------------------------------
-
-// -------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL_L_CPT(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if( flag ) { \
- new_value OP rhs; \
- } else \
- new_value = (*lhs); \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid );
-
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_L_CPT( OP, 0 ); \
- return new_value; \
- }
-#else
-#define OP_GOMP_CRITICAL_L_CPT(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// ------------------------------------------------------------------------
-// Need separate macros for &&, || because there is no combined assignment
-#define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \
- OP_CMPXCHG_CPT(TYPE,BITS,OP) \
-}
-
-ATOMIC_CMPX_L_CPT( fixed1, andl_cpt, char, 8, &&, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_andl_cpt
-ATOMIC_CMPX_L_CPT( fixed1, orl_cpt, char, 8, ||, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_orl_cpt
-ATOMIC_CMPX_L_CPT( fixed2, andl_cpt, short, 16, &&, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_andl_cpt
-ATOMIC_CMPX_L_CPT( fixed2, orl_cpt, short, 16, ||, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_orl_cpt
-ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 ) // __kmpc_atomic_fixed4_andl_cpt
-ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 ) // __kmpc_atomic_fixed4_orl_cpt
-ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_andl_cpt
-ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_orl_cpt
-
-
-// -------------------------------------------------------------------------
-// Routines for Fortran operators that matched no one in C:
-// MAX, MIN, .EQV., .NEQV.
-// Operators .AND., .OR. are covered by __kmpc_atomic_*_{andl,orl}_cpt
-// Intrinsics IAND, IOR, IEOR are covered by __kmpc_atomic_*_{andb,orb,xor}_cpt
-// -------------------------------------------------------------------------
-
-// -------------------------------------------------------------------------
-// MIN and MAX need separate macros
-// OP - operator to check if we need any actions?
-#define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if ( *lhs OP rhs ) { /* still need actions? */ \
- old_value = *lhs; \
- *lhs = rhs; \
- if ( flag ) \
- new_value = rhs; \
- else \
- new_value = old_value; \
- } \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return new_value; \
-
-// -------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \
- if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \
- KMP_CHECK_GTID; \
- MIN_MAX_CRITSECT_CPT( OP, 0 ); \
- }
-#else
-#define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// -------------------------------------------------------------------------
-#define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- /*TYPE old_value; */ \
- temp_val = *lhs; \
- old_value = temp_val; \
- while ( old_value OP rhs && /* still need actions? */ \
- ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \
- { \
- KMP_CPU_PAUSE(); \
- temp_val = *lhs; \
- old_value = temp_val; \
- } \
- if( flag ) \
- return rhs; \
- else \
- return old_value; \
- }
-
-// -------------------------------------------------------------------------
-// 1-byte, 2-byte operands - use critical section
-#define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value, old_value; \
- if ( *lhs OP rhs ) { /* need actions? */ \
- GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
- MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \
- } \
- return *lhs; \
-}
-
-#define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value, old_value; \
- if ( *lhs OP rhs ) { \
- GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \
- MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \
- } \
- return *lhs; \
-}
-
-
-MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt, char, 8, <, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_max_cpt
-MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt, char, 8, >, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_min_cpt
-MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt, short, 16, <, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_max_cpt
-MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt, short, 16, >, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_min_cpt
-MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 ) // __kmpc_atomic_fixed4_max_cpt
-MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 ) // __kmpc_atomic_fixed4_min_cpt
-MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_max_cpt
-MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_min_cpt
-MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 ) // __kmpc_atomic_float4_max_cpt
-MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 ) // __kmpc_atomic_float4_min_cpt
-MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 ) // __kmpc_atomic_float8_max_cpt
-MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 ) // __kmpc_atomic_float8_min_cpt
-#if KMP_HAVE_QUAD
-MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 ) // __kmpc_atomic_float16_max_cpt
-MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 ) // __kmpc_atomic_float16_min_cpt
-#if ( KMP_ARCH_X86 )
- MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 ) // __kmpc_atomic_float16_max_a16_cpt
- MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 ) // __kmpc_atomic_float16_mix_a16_cpt
-#endif
-#endif
-
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_CPT( OP, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-// ------------------------------------------------------------------------
-#define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) /* send assignment */ \
- OP_CMPXCHG_CPT(TYPE,BITS,OP) \
-}
-
-// ------------------------------------------------------------------------
-
-ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_neqv_cpt
-ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_neqv_cpt
-ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_neqv_cpt
-ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_neqv_cpt
-ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_eqv_cpt
-ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_eqv_cpt
-ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_eqv_cpt
-ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_eqv_cpt
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL_CPT(OP##=,LCK_ID) /* send assignment */ \
-}
-
-// ------------------------------------------------------------------------
-
-// Workaround for cmplx4. Regular routines with return value don't work
-// on Win_32e. Let's return captured values through the additional parameter.
-#define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if( flag ) { \
- (*lhs) OP rhs; \
- (*out) = (*lhs); \
- } else { \
- (*out) = (*lhs); \
- (*lhs) OP rhs; \
- } \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return;
-// ------------------------------------------------------------------------
-
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_CPT_WRK( OP##=, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-// ------------------------------------------------------------------------
-
-#define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
-void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid ));
-// ------------------------------------------------------------------------
-
-#define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
- OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \
- OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \
-}
-// The end of workaround for cmplx4
-
-/* ------------------------------------------------------------------------- */
-// routines for long double type
-ATOMIC_CRITICAL_CPT( float10, add_cpt, long double, +, 10r, 1 ) // __kmpc_atomic_float10_add_cpt
-ATOMIC_CRITICAL_CPT( float10, sub_cpt, long double, -, 10r, 1 ) // __kmpc_atomic_float10_sub_cpt
-ATOMIC_CRITICAL_CPT( float10, mul_cpt, long double, *, 10r, 1 ) // __kmpc_atomic_float10_mul_cpt
-ATOMIC_CRITICAL_CPT( float10, div_cpt, long double, /, 10r, 1 ) // __kmpc_atomic_float10_div_cpt
-#if KMP_HAVE_QUAD
-// routines for _Quad type
-ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 ) // __kmpc_atomic_float16_add_cpt
-ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 ) // __kmpc_atomic_float16_sub_cpt
-ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 ) // __kmpc_atomic_float16_mul_cpt
-ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 ) // __kmpc_atomic_float16_div_cpt
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 ) // __kmpc_atomic_float16_add_a16_cpt
- ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 ) // __kmpc_atomic_float16_sub_a16_cpt
- ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 ) // __kmpc_atomic_float16_mul_a16_cpt
- ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 ) // __kmpc_atomic_float16_div_a16_cpt
-#endif
-#endif
-
-// routines for complex types
-
-// cmplx4 routines to return void
-ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 ) // __kmpc_atomic_cmplx4_add_cpt
-ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 ) // __kmpc_atomic_cmplx4_sub_cpt
-ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 ) // __kmpc_atomic_cmplx4_mul_cpt
-ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 ) // __kmpc_atomic_cmplx4_div_cpt
-
-ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 ) // __kmpc_atomic_cmplx8_add_cpt
-ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 ) // __kmpc_atomic_cmplx8_sub_cpt
-ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 ) // __kmpc_atomic_cmplx8_mul_cpt
-ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 ) // __kmpc_atomic_cmplx8_div_cpt
-ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 ) // __kmpc_atomic_cmplx10_add_cpt
-ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 ) // __kmpc_atomic_cmplx10_sub_cpt
-ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 ) // __kmpc_atomic_cmplx10_mul_cpt
-ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 ) // __kmpc_atomic_cmplx10_div_cpt
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 ) // __kmpc_atomic_cmplx16_add_cpt
-ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_cpt
-ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 ) // __kmpc_atomic_cmplx16_mul_cpt
-ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_cpt
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 ) // __kmpc_atomic_cmplx16_add_a16_cpt
- ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_a16_cpt
- ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 ) // __kmpc_atomic_cmplx16_mul_a16_cpt
- ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_a16_cpt
-#endif
-#endif
-
-#if OMP_40_ENABLED
-
-// OpenMP 4.0: v = x = expr binop x; { v = x; x = expr binop x; } { x = expr binop x; v = x; } for non-commutative operations.
-// Supported only on IA-32 architecture and Intel(R) 64
-
-// -------------------------------------------------------------------------
-// Operation on *lhs, rhs bound by critical section
-// OP - operator (it's supposed to contain an assignment)
-// LCK_ID - lock identifier
-// Note: don't check gtid as it should always be valid
-// 1, 2-byte - expect valid parameter, other - check before this macro
-#define OP_CRITICAL_CPT_REV(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if( flag ) { \
- /*temp_val = (*lhs);*/\
- (*lhs) = (rhs) OP (*lhs); \
- new_value = (*lhs); \
- } else { \
- new_value = (*lhs);\
- (*lhs) = (rhs) OP (*lhs); \
- } \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return new_value;
-
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_CPT_REV( OP, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-// ------------------------------------------------------------------------
-// Operation on *lhs, rhs using "compare_and_store" routine
-// TYPE - operands' type
-// BITS - size in bits, used to distinguish low level calls
-// OP - operator
-// Note: temp_val introduced in order to force the compiler to read
-// *lhs only once (w/o it the compiler reads *lhs twice)
-#define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value, new_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs OP old_value; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_CPU_PAUSE(); \
- \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs OP old_value; \
- } \
- if( flag ) { \
- return new_value; \
- } else \
- return old_value; \
- }
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
- OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
-}
-
-
-ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shl_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shl_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shl_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4u_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_sub_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shl_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_shr_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt_rev
-ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt_rev
-// TYPE_ID,OP_ID, TYPE, OP, GOMP_FLAG
-
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-// TYPE_ID, OP_ID, TYPE - detailed above
-// OP - operator
-// LCK_ID - lock identifier, used to possibly distinguish lock variable
-#define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \
- TYPE new_value; \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- /*printf("__kmp_atomic_mode = %d\n", __kmp_atomic_mode);*/\
- OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
- OP_CRITICAL_CPT_REV(OP,LCK_ID) \
-}
-
-
-/* ------------------------------------------------------------------------- */
-// routines for long double type
-ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev, long double, -, 10r, 1 ) // __kmpc_atomic_float10_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev, long double, /, 10r, 1 ) // __kmpc_atomic_float10_div_cpt_rev
-#if KMP_HAVE_QUAD
-// routines for _Quad type
-ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 ) // __kmpc_atomic_float16_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 ) // __kmpc_atomic_float16_div_cpt_rev
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 ) // __kmpc_atomic_float16_sub_a16_cpt_rev
- ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 ) // __kmpc_atomic_float16_div_a16_cpt_rev
-#endif
-#endif
-
-// routines for complex types
-
-// ------------------------------------------------------------------------
-
-// Workaround for cmplx4. Regular routines with return value don't work
-// on Win_32e. Let's return captured values through the additional parameter.
-#define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- if( flag ) { \
- (*lhs) = (rhs) OP (*lhs); \
- (*out) = (*lhs); \
- } else { \
- (*out) = (*lhs); \
- (*lhs) = (rhs) OP (*lhs); \
- } \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return;
-// ------------------------------------------------------------------------
-
-#ifdef KMP_GOMP_COMPAT
-#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \
- }
-#else
-#define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG)
-#endif /* KMP_GOMP_COMPAT */
-// ------------------------------------------------------------------------
-
-#define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \
- OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \
- OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \
-}
-// The end of workaround for cmplx4
-
-
-// !!! TODO: check if we need to return void for cmplx4 routines
-// cmplx4 routines to return void
-ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 ) // __kmpc_atomic_cmplx4_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 ) // __kmpc_atomic_cmplx4_div_cpt_rev
-
-ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 ) // __kmpc_atomic_cmplx8_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 ) // __kmpc_atomic_cmplx8_div_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 ) // __kmpc_atomic_cmplx10_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 ) // __kmpc_atomic_cmplx10_div_cpt_rev
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_cpt_rev
-ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_cpt_rev
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 ) // __kmpc_atomic_cmplx16_sub_a16_cpt_rev
- ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 ) // __kmpc_atomic_cmplx16_div_a16_cpt_rev
-#endif
-#endif
-
-// Capture reverse for mixed type: RHS=float16
-#if KMP_HAVE_QUAD
-
-// Beginning of a definition (provides name, parameters, gebug trace)
-// TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned fixed)
-// OP_ID - operation identifier (add, sub, mul, ...)
-// TYPE - operands' type
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \
- OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \
-}
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_CPT_MIX(TYPE_ID,OP_ID,TYPE,RTYPE_ID,RTYPE) \
- TYPE new_value; \
- OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) /* send assignment */ \
- OP_CRITICAL_CPT_REV(OP,LCK_ID) /* send assignment */ \
-}
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1, char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1, char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_div_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1u_div_cpt_rev_fp
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2, short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2, short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_div_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2u_div_cpt_rev_fp
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4_div_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad, 4i, 3, 0 ) // __kmpc_atomic_fixed4u_div_cpt_rev_fp
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_div_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8u_div_cpt_rev_fp
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 ) // __kmpc_atomic_float4_div_cpt_rev_fp
-
-ATOMIC_CMPXCHG_CPT_REV_MIX( float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_sub_cpt_rev_fp
-ATOMIC_CMPXCHG_CPT_REV_MIX( float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 ) // __kmpc_atomic_float8_div_cpt_rev_fp
-
-ATOMIC_CRITICAL_CPT_REV_MIX( float10, long double, sub_cpt_rev, -, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_sub_cpt_rev_fp
-ATOMIC_CRITICAL_CPT_REV_MIX( float10, long double, div_cpt_rev, /, fp, _Quad, 10r, 1 ) // __kmpc_atomic_float10_div_cpt_rev_fp
-
-#endif //KMP_HAVE_QUAD
-
-
-// OpenMP 4.0 Capture-write (swap): {v = x; x = expr;}
-
-#define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
-TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
-
-#define CRITICAL_SWP(LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- old_value = (*lhs); \
- (*lhs) = rhs; \
- \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return old_value;
-
-// ------------------------------------------------------------------------
-#ifdef KMP_GOMP_COMPAT
-#define GOMP_CRITICAL_SWP(FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- CRITICAL_SWP( 0 ); \
- }
-#else
-#define GOMP_CRITICAL_SWP(FLAG)
-#endif /* KMP_GOMP_COMPAT */
-
-
-#define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
-ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
- TYPE old_value; \
- GOMP_CRITICAL_SWP(GOMP_FLAG) \
- old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \
- return old_value; \
-}
-// ------------------------------------------------------------------------
-#define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
-ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
- TYPE old_value; \
- GOMP_CRITICAL_SWP(GOMP_FLAG) \
- old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \
- return old_value; \
-}
-
-// ------------------------------------------------------------------------
-#define CMPXCHG_SWP(TYPE,BITS) \
- { \
- TYPE KMP_ATOMIC_VOLATILE temp_val; \
- TYPE old_value, new_value; \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs; \
- while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \
- *VOLATILE_CAST(kmp_int##BITS *) &old_value, \
- *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \
- { \
- KMP_CPU_PAUSE(); \
- \
- temp_val = *lhs; \
- old_value = temp_val; \
- new_value = rhs; \
- } \
- return old_value; \
- }
-
-// -------------------------------------------------------------------------
-#define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \
-ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
- TYPE old_value; \
- GOMP_CRITICAL_SWP(GOMP_FLAG) \
- CMPXCHG_SWP(TYPE,BITS) \
-}
-
-ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 ) // __kmpc_atomic_fixed1_swp
-ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 ) // __kmpc_atomic_fixed2_swp
-ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 ) // __kmpc_atomic_fixed4_swp
-
-ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 ) // __kmpc_atomic_float4_swp
-
-#if ( KMP_ARCH_X86 )
- ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_swp
- ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 ) // __kmpc_atomic_float8_swp
-#else
- ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 ) // __kmpc_atomic_fixed8_swp
- ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 ) // __kmpc_atomic_float8_swp
-#endif
-
-// ------------------------------------------------------------------------
-// Routines for Extended types: long double, _Quad, complex flavours (use critical section)
-#define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \
- TYPE old_value; \
- GOMP_CRITICAL_SWP(GOMP_FLAG) \
- CRITICAL_SWP(LCK_ID) \
-}
-
-// ------------------------------------------------------------------------
-
-// !!! TODO: check if we need to return void for cmplx4 routines
-// Workaround for cmplx4. Regular routines with return value don't work
-// on Win_32e. Let's return captured values through the additional parameter.
-
-#define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
-void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \
-{ \
- KMP_DEBUG_ASSERT( __kmp_init_serial ); \
- KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid ));
-
-
-#define CRITICAL_SWP_WRK(LCK_ID) \
- __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- \
- tmp = (*lhs); \
- (*lhs) = (rhs); \
- (*out) = tmp; \
- __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \
- return;
-
-// ------------------------------------------------------------------------
-
-#ifdef KMP_GOMP_COMPAT
-#define GOMP_CRITICAL_SWP_WRK(FLAG) \
- if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \
- KMP_CHECK_GTID; \
- CRITICAL_SWP_WRK( 0 ); \
- }
-#else
-#define GOMP_CRITICAL_SWP_WRK(FLAG)
-#endif /* KMP_GOMP_COMPAT */
-// ------------------------------------------------------------------------
-
-#define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \
-ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \
- TYPE tmp; \
- GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
- CRITICAL_SWP_WRK(LCK_ID) \
-}
-// The end of workaround for cmplx4
-
-
-ATOMIC_CRITICAL_SWP( float10, long double, 10r, 1 ) // __kmpc_atomic_float10_swp
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 ) // __kmpc_atomic_float16_swp
-#endif
-// cmplx4 routine to return void
-ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 ) // __kmpc_atomic_cmplx4_swp
-
-//ATOMIC_CRITICAL_SWP( cmplx4, kmp_cmplx32, 8c, 1 ) // __kmpc_atomic_cmplx4_swp
-
-
-ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 ) // __kmpc_atomic_cmplx8_swp
-ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 ) // __kmpc_atomic_cmplx10_swp
-#if KMP_HAVE_QUAD
-ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 ) // __kmpc_atomic_cmplx16_swp
-#if ( KMP_ARCH_X86 )
- ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 ) // __kmpc_atomic_float16_a16_swp
- ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 ) // __kmpc_atomic_cmplx16_a16_swp
-#endif
-#endif
-
-
-// End of OpenMP 4.0 Capture
-
-#endif //OMP_40_ENABLED
-
-#endif //KMP_ARCH_X86 || KMP_ARCH_X86_64
-
-
-#undef OP_CRITICAL
-
-/* ------------------------------------------------------------------------ */
-/* Generic atomic routines */
-/* ------------------------------------------------------------------------ */
-
-void
-__kmpc_atomic_1( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- if (
-#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
- FALSE /* must use lock */
-#else
- TRUE
-#endif
- )
- {
- kmp_int8 old_value, new_value;
-
- old_value = *(kmp_int8 *) lhs;
- (*f)( &new_value, &old_value, rhs );
-
- /* TODO: Should this be acquire or release? */
- while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
- *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
- {
- KMP_CPU_PAUSE();
-
- old_value = *(kmp_int8 *) lhs;
- (*f)( &new_value, &old_value, rhs );
- }
-
- return;
- }
- else {
- //
- // All 1-byte data is of integer data type.
- //
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
- }
-}
-
-void
-__kmpc_atomic_2( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- if (
-#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
- FALSE /* must use lock */
-#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
- TRUE /* no alignment problems */
-#else
- ! ( (kmp_uintptr_t) lhs & 0x1) /* make sure address is 2-byte aligned */
-#endif
- )
- {
- kmp_int16 old_value, new_value;
-
- old_value = *(kmp_int16 *) lhs;
- (*f)( &new_value, &old_value, rhs );
-
- /* TODO: Should this be acquire or release? */
- while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
- *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
- {
- KMP_CPU_PAUSE();
-
- old_value = *(kmp_int16 *) lhs;
- (*f)( &new_value, &old_value, rhs );
- }
-
- return;
- }
- else {
- //
- // All 2-byte data is of integer data type.
- //
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
- }
-}
-
-void
-__kmpc_atomic_4( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- if (
- //
- // FIXME: On IA-32 architecture, gcc uses cmpxchg only for 4-byte ints.
- // Gomp compatibility is broken if this routine is called for floats.
- //
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
- TRUE /* no alignment problems */
-#else
- ! ( (kmp_uintptr_t) lhs & 0x3) /* make sure address is 4-byte aligned */
-#endif
- )
- {
- kmp_int32 old_value, new_value;
-
- old_value = *(kmp_int32 *) lhs;
- (*f)( &new_value, &old_value, rhs );
-
- /* TODO: Should this be acquire or release? */
- while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
- *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
- {
- KMP_CPU_PAUSE();
-
- old_value = *(kmp_int32 *) lhs;
- (*f)( &new_value, &old_value, rhs );
- }
-
- return;
- }
- else {
- //
- // Use __kmp_atomic_lock_4i for all 4-byte data,
- // even if it isn't of integer data type.
- //
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
- }
-}
-
-void
-__kmpc_atomic_8( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
- if (
-
-#if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
- FALSE /* must use lock */
-#elif KMP_ARCH_X86 || KMP_ARCH_X86_64
- TRUE /* no alignment problems */
-#else
- ! ( (kmp_uintptr_t) lhs & 0x7) /* make sure address is 8-byte aligned */
-#endif
- )
- {
- kmp_int64 old_value, new_value;
-
- old_value = *(kmp_int64 *) lhs;
- (*f)( &new_value, &old_value, rhs );
- /* TODO: Should this be acquire or release? */
- while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
- *(kmp_int64 *) &old_value,
- *(kmp_int64 *) &new_value ) )
- {
- KMP_CPU_PAUSE();
-
- old_value = *(kmp_int64 *) lhs;
- (*f)( &new_value, &old_value, rhs );
- }
-
- return;
- } else {
- //
- // Use __kmp_atomic_lock_8i for all 8-byte data,
- // even if it isn't of integer data type.
- //
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
- }
-}
-
-void
-__kmpc_atomic_10( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
-}
-
-void
-__kmpc_atomic_16( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
-}
-
-void
-__kmpc_atomic_20( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
-}
-
-void
-__kmpc_atomic_32( ident_t *id_ref, int gtid, void* lhs, void* rhs, void (*f)( void *, void *, void * ) )
-{
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
-
- (*f)( lhs, lhs, rhs );
-
-#ifdef KMP_GOMP_COMPAT
- if ( __kmp_atomic_mode == 2 ) {
- __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
- }
- else
-#endif /* KMP_GOMP_COMPAT */
- __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
-}
-
-// AC: same two routines as GOMP_atomic_start/end, but will be called by our compiler
-// duplicated in order to not use 3-party names in pure Intel code
-// TODO: consider adding GTID parameter after consultation with Ernesto/Xinmin.
-void
-__kmpc_atomic_start(void)
-{
- int gtid = __kmp_entry_gtid();
- KA_TRACE(20, ("__kmpc_atomic_start: T#%d\n", gtid));
- __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
-}
-
-
-void
-__kmpc_atomic_end(void)
-{
- int gtid = __kmp_get_gtid();
- KA_TRACE(20, ("__kmpc_atomic_end: T#%d\n", gtid));
- __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-/*!
-@}
-*/
-
-// end of file
Removed: openmp/trunk/runtime/src/kmp_csupport.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_csupport.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_csupport.c (original)
+++ openmp/trunk/runtime/src/kmp_csupport.c (removed)
@@ -1,3337 +0,0 @@
-/*
- * kmp_csupport.c -- kfront linkage support for OpenMP.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "omp.h" /* extern "C" declarations of user-visible routines */
-#include "kmp.h"
-#include "kmp_i18n.h"
-#include "kmp_itt.h"
-#include "kmp_lock.h"
-#include "kmp_error.h"
-#include "kmp_stats.h"
-
-#if OMPT_SUPPORT
-#include "ompt-internal.h"
-#include "ompt-specific.h"
-#endif
-
-#define MAX_MESSAGE 512
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* flags will be used in future, e.g., to implement */
-/* openmp_strict library restrictions */
-
-/*!
- * @ingroup STARTUP_SHUTDOWN
- * @param loc in source location information
- * @param flags in for future use (currently ignored)
- *
- * Initialize the runtime library. This call is optional; if it is not made then
- * it will be implicitly called by attempts to use other library functions.
- *
- */
-void
-__kmpc_begin(ident_t *loc, kmp_int32 flags)
-{
- // By default __kmp_ignore_mppbeg() returns TRUE.
- if (__kmp_ignore_mppbeg() == FALSE) {
- __kmp_internal_begin();
-
- KC_TRACE( 10, ("__kmpc_begin: called\n" ) );
- }
-}
-
-/*!
- * @ingroup STARTUP_SHUTDOWN
- * @param loc source location information
- *
- * Shutdown the runtime library. This is also optional, and even if called will not
- * do anything unless the `KMP_IGNORE_MPPEND` environment variable is set to zero.
- */
-void
-__kmpc_end(ident_t *loc)
-{
- // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end() call no-op.
- // However, this can be overridden with KMP_IGNORE_MPPEND environment variable.
- // If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend() returns FALSE and __kmpc_end()
- // will unregister this root (it can cause library shut down).
- if (__kmp_ignore_mppend() == FALSE) {
- KC_TRACE( 10, ("__kmpc_end: called\n" ) );
- KA_TRACE( 30, ("__kmpc_end\n" ));
-
- __kmp_internal_end_thread( -1 );
- }
-}
-
-/*!
- at ingroup THREAD_STATES
- at param loc Source location information.
- at return The global thread index of the active thread.
-
-This function can be called in any context.
-
-If the runtime has ony been entered at the outermost level from a
-single (necessarily non-OpenMP<sup>*</sup>) thread, then the thread number is that
-which would be returned by omp_get_thread_num() in the outermost
-active parallel construct. (Or zero if there is no active parallel
-construct, since the master thread is necessarily thread zero).
-
-If multiple non-OpenMP threads all enter an OpenMP construct then this
-will be a unique thread identifier among all the threads created by
-the OpenMP runtime (but the value cannote be defined in terms of
-OpenMP thread ids returned by omp_get_thread_num()).
-
-*/
-kmp_int32
-__kmpc_global_thread_num(ident_t *loc)
-{
- kmp_int32 gtid = __kmp_entry_gtid();
-
- KC_TRACE( 10, ("__kmpc_global_thread_num: T#%d\n", gtid ) );
-
- return gtid;
-}
-
-/*!
- at ingroup THREAD_STATES
- at param loc Source location information.
- at return The number of threads under control of the OpenMP<sup>*</sup> runtime
-
-This function can be called in any context.
-It returns the total number of threads under the control of the OpenMP runtime. That is
-not a number that can be determined by any OpenMP standard calls, since the library may be
-called from more than one non-OpenMP thread, and this reflects the total over all such calls.
-Similarly the runtime maintains underlying threads even when they are not active (since the cost
-of creating and destroying OS threads is high), this call counts all such threads even if they are not
-waiting for work.
-*/
-kmp_int32
-__kmpc_global_num_threads(ident_t *loc)
-{
- KC_TRACE( 10, ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_nth ) );
-
- return TCR_4(__kmp_nth);
-}
-
-/*!
- at ingroup THREAD_STATES
- at param loc Source location information.
- at return The thread number of the calling thread in the innermost active parallel construct.
-
-*/
-kmp_int32
-__kmpc_bound_thread_num(ident_t *loc)
-{
- KC_TRACE( 10, ("__kmpc_bound_thread_num: called\n" ) );
- return __kmp_tid_from_gtid( __kmp_entry_gtid() );
-}
-
-/*!
- at ingroup THREAD_STATES
- at param loc Source location information.
- at return The number of threads in the innermost active parallel construct.
-*/
-kmp_int32
-__kmpc_bound_num_threads(ident_t *loc)
-{
- KC_TRACE( 10, ("__kmpc_bound_num_threads: called\n" ) );
-
- return __kmp_entry_thread() -> th.th_team -> t.t_nproc;
-}
-
-/*!
- * @ingroup DEPRECATED
- * @param loc location description
- *
- * This function need not be called. It always returns TRUE.
- */
-kmp_int32
-__kmpc_ok_to_fork(ident_t *loc)
-{
-#ifndef KMP_DEBUG
-
- return TRUE;
-
-#else
-
- const char *semi2;
- const char *semi3;
- int line_no;
-
- if (__kmp_par_range == 0) {
- return TRUE;
- }
- semi2 = loc->psource;
- if (semi2 == NULL) {
- return TRUE;
- }
- semi2 = strchr(semi2, ';');
- if (semi2 == NULL) {
- return TRUE;
- }
- semi2 = strchr(semi2 + 1, ';');
- if (semi2 == NULL) {
- return TRUE;
- }
- if (__kmp_par_range_filename[0]) {
- const char *name = semi2 - 1;
- while ((name > loc->psource) && (*name != '/') && (*name != ';')) {
- name--;
- }
- if ((*name == '/') || (*name == ';')) {
- name++;
- }
- if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
- return __kmp_par_range < 0;
- }
- }
- semi3 = strchr(semi2 + 1, ';');
- if (__kmp_par_range_routine[0]) {
- if ((semi3 != NULL) && (semi3 > semi2)
- && (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
- return __kmp_par_range < 0;
- }
- }
- if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) {
- if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
- return __kmp_par_range > 0;
- }
- return __kmp_par_range < 0;
- }
- return TRUE;
-
-#endif /* KMP_DEBUG */
-
-}
-
-/*!
- at ingroup THREAD_STATES
- at param loc Source location information.
- at return 1 if this thread is executing inside an active parallel region, zero if not.
-*/
-kmp_int32
-__kmpc_in_parallel( ident_t *loc )
-{
- return __kmp_entry_thread() -> th.th_root -> r.r_active;
-}
-
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param global_tid global thread number
- at param num_threads number of threads requested for this parallel construct
-
-Set the number of threads to be used by the next fork spawned by this thread.
-This call is only required if the parallel construct has a `num_threads` clause.
-*/
-void
-__kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads )
-{
- KA_TRACE( 20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
- global_tid, num_threads ) );
-
- __kmp_push_num_threads( loc, global_tid, num_threads );
-}
-
-void
-__kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid )
-{
- KA_TRACE( 20, ("__kmpc_pop_num_threads: enter\n" ) );
-
- /* the num_threads are automatically popped */
-}
-
-
-#if OMP_40_ENABLED
-
-void
-__kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, kmp_int32 proc_bind )
-{
- KA_TRACE( 20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n",
- global_tid, proc_bind ) );
-
- __kmp_push_proc_bind( loc, global_tid, (kmp_proc_bind_t)proc_bind );
-}
-
-#endif /* OMP_40_ENABLED */
-
-
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param argc total number of arguments in the ellipsis
- at param microtask pointer to callback routine consisting of outlined parallel construct
- at param ... pointers to shared variables that aren't global
-
-Do the actual fork and call the microtask in the relevant number of threads.
-*/
-void
-__kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
-{
- int gtid = __kmp_entry_gtid();
-
-#if (KMP_STATS_ENABLED)
- int inParallel = __kmpc_in_parallel(loc);
- if (inParallel)
- {
- KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL);
- }
- else
- {
- KMP_COUNT_BLOCK(OMP_PARALLEL);
- }
-#endif
-
- // maybe to save thr_state is enough here
- {
- va_list ap;
- va_start( ap, microtask );
-
-#if OMPT_SUPPORT
- ompt_frame_t* ompt_frame;
- if (ompt_enabled) {
- kmp_info_t *master_th = __kmp_threads[ gtid ];
- kmp_team_t *parent_team = master_th->th.th_team;
- ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
- if (lwt)
- ompt_frame = &(lwt->ompt_task_info.frame);
- else
- {
- int tid = __kmp_tid_from_gtid( gtid );
- ompt_frame = &(parent_team->t.t_implicit_task_taskdata[tid].
- ompt_task_info.frame);
- }
- ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
-
-#if INCLUDE_SSC_MARKS
- SSC_MARK_FORKING();
-#endif
- __kmp_fork_call( loc, gtid, fork_context_intel,
- argc,
-#if OMPT_SUPPORT
- VOLATILE_CAST(void *) microtask, // "unwrapped" task
-#endif
- VOLATILE_CAST(microtask_t) microtask, // "wrapped" task
- VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
-/* TODO: revert workaround for Intel(R) 64 tracker #96 */
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- &ap
-#else
- ap
-#endif
- );
-#if INCLUDE_SSC_MARKS
- SSC_MARK_JOINING();
-#endif
- __kmp_join_call( loc, gtid
-#if OMPT_SUPPORT
- , fork_context_intel
-#endif
- );
-
- va_end( ap );
-
- }
-}
-
-#if OMP_40_ENABLED
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param global_tid global thread number
- at param num_teams number of teams requested for the teams construct
- at param num_threads number of threads per team requested for the teams construct
-
-Set the number of teams to be used by the teams construct.
-This call is only required if the teams construct has a `num_teams` clause
-or a `thread_limit` clause (or both).
-*/
-void
-__kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads )
-{
- KA_TRACE( 20, ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
- global_tid, num_teams, num_threads ) );
-
- __kmp_push_num_teams( loc, global_tid, num_teams, num_threads );
-}
-
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param argc total number of arguments in the ellipsis
- at param microtask pointer to callback routine consisting of outlined teams construct
- at param ... pointers to shared variables that aren't global
-
-Do the actual fork and call the microtask in the relevant number of threads.
-*/
-void
-__kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...)
-{
- int gtid = __kmp_entry_gtid();
- kmp_info_t *this_thr = __kmp_threads[ gtid ];
- va_list ap;
- va_start( ap, microtask );
-
- KMP_COUNT_BLOCK(OMP_TEAMS);
-
- // remember teams entry point and nesting level
- this_thr->th.th_teams_microtask = microtask;
- this_thr->th.th_teams_level = this_thr->th.th_team->t.t_level; // AC: can be >0 on host
-
-#if OMPT_SUPPORT
- kmp_team_t *parent_team = this_thr->th.th_team;
- int tid = __kmp_tid_from_gtid( gtid );
- if (ompt_enabled) {
- parent_team->t.t_implicit_task_taskdata[tid].
- ompt_task_info.frame.reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
-
- // check if __kmpc_push_num_teams called, set default number of teams otherwise
- if ( this_thr->th.th_teams_size.nteams == 0 ) {
- __kmp_push_num_teams( loc, gtid, 0, 0 );
- }
- KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
- KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
- KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
-
- __kmp_fork_call( loc, gtid, fork_context_intel,
- argc,
-#if OMPT_SUPPORT
- VOLATILE_CAST(void *) microtask, // "unwrapped" task
-#endif
- VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task
- VOLATILE_CAST(launch_t) __kmp_invoke_teams_master,
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- &ap
-#else
- ap
-#endif
- );
- __kmp_join_call( loc, gtid
-#if OMPT_SUPPORT
- , fork_context_intel
-#endif
- );
-
- this_thr->th.th_teams_microtask = NULL;
- this_thr->th.th_teams_level = 0;
- *(kmp_int64*)(&this_thr->th.th_teams_size) = 0L;
- va_end( ap );
-}
-#endif /* OMP_40_ENABLED */
-
-
-//
-// I don't think this function should ever have been exported.
-// The __kmpc_ prefix was misapplied. I'm fairly certain that no generated
-// openmp code ever called it, but it's been exported from the RTL for so
-// long that I'm afraid to remove the definition.
-//
-int
-__kmpc_invoke_task_func( int gtid )
-{
- return __kmp_invoke_task_func( gtid );
-}
-
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param global_tid global thread number
-
-Enter a serialized parallel construct. This interface is used to handle a
-conditional parallel region, like this,
- at code
-#pragma omp parallel if (condition)
- at endcode
-when the condition is false.
-*/
-void
-__kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
-{
- __kmp_serialized_parallel(loc, global_tid); /* The implementation is now in kmp_runtime.c so that it can share static functions with
- * kmp_fork_call since the tasks to be done are similar in each case.
- */
-}
-
-/*!
- at ingroup PARALLEL
- at param loc source location information
- at param global_tid global thread number
-
-Leave a serialized parallel construct.
-*/
-void
-__kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
-{
- kmp_internal_control_t *top;
- kmp_info_t *this_thr;
- kmp_team_t *serial_team;
-
- KC_TRACE( 10, ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid ) );
-
- /* skip all this code for autopar serialized loops since it results in
- unacceptable overhead */
- if( loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR ) )
- return;
-
- // Not autopar code
- if( ! TCR_4( __kmp_init_parallel ) )
- __kmp_parallel_initialize();
-
- this_thr = __kmp_threads[ global_tid ];
- serial_team = this_thr->th.th_serial_team;
-
- #if OMP_45_ENABLED
- kmp_task_team_t * task_team = this_thr->th.th_task_team;
-
- // we need to wait for the proxy tasks before finishing the thread
- if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks )
- __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL) ); // is an ITT object needed here?
- #endif
-
- KMP_MB();
- KMP_DEBUG_ASSERT( serial_team );
- KMP_ASSERT( serial_team -> t.t_serialized );
- KMP_DEBUG_ASSERT( this_thr -> th.th_team == serial_team );
- KMP_DEBUG_ASSERT( serial_team != this_thr->th.th_root->r.r_root_team );
- KMP_DEBUG_ASSERT( serial_team -> t.t_threads );
- KMP_DEBUG_ASSERT( serial_team -> t.t_threads[0] == this_thr );
-
- /* If necessary, pop the internal control stack values and replace the team values */
- top = serial_team -> t.t_control_stack_top;
- if ( top && top -> serial_nesting_level == serial_team -> t.t_serialized ) {
- copy_icvs( &serial_team -> t.t_threads[0] -> th.th_current_task -> td_icvs, top );
- serial_team -> t.t_control_stack_top = top -> next;
- __kmp_free(top);
- }
-
- //if( serial_team -> t.t_serialized > 1 )
- serial_team -> t.t_level--;
-
- /* pop dispatch buffers stack */
- KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
- {
- dispatch_private_info_t * disp_buffer = serial_team->t.t_dispatch->th_disp_buffer;
- serial_team->t.t_dispatch->th_disp_buffer =
- serial_team->t.t_dispatch->th_disp_buffer->next;
- __kmp_free( disp_buffer );
- }
-
- -- serial_team -> t.t_serialized;
- if ( serial_team -> t.t_serialized == 0 ) {
-
- /* return to the parallel section */
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
- if ( __kmp_inherit_fp_control && serial_team->t.t_fp_control_saved ) {
- __kmp_clear_x87_fpu_status_word();
- __kmp_load_x87_fpu_control_word( &serial_team->t.t_x87_fpu_control_word );
- __kmp_load_mxcsr( &serial_team->t.t_mxcsr );
- }
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
- this_thr -> th.th_team = serial_team -> t.t_parent;
- this_thr -> th.th_info.ds.ds_tid = serial_team -> t.t_master_tid;
-
- /* restore values cached in the thread */
- this_thr -> th.th_team_nproc = serial_team -> t.t_parent -> t.t_nproc; /* JPH */
- this_thr -> th.th_team_master = serial_team -> t.t_parent -> t.t_threads[0]; /* JPH */
- this_thr -> th.th_team_serialized = this_thr -> th.th_team -> t.t_serialized;
-
- /* TODO the below shouldn't need to be adjusted for serialized teams */
- this_thr -> th.th_dispatch = & this_thr -> th.th_team ->
- t.t_dispatch[ serial_team -> t.t_master_tid ];
-
- __kmp_pop_current_task_from_thread( this_thr );
-
- KMP_ASSERT( this_thr -> th.th_current_task -> td_flags.executing == 0 );
- this_thr -> th.th_current_task -> td_flags.executing = 1;
-
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- // Copy the task team from the new child / old parent team to the thread.
- this_thr->th.th_task_team = this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
- KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d restoring task_team %p / team %p\n",
- global_tid, this_thr -> th.th_task_team, this_thr -> th.th_team ) );
- }
- } else {
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KA_TRACE( 20, ( "__kmpc_end_serialized_parallel: T#%d decreasing nesting depth of serial team %p to %d\n",
- global_tid, serial_team, serial_team -> t.t_serialized ) );
- }
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_parallel( global_tid, NULL );
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information.
-
-Execute <tt>flush</tt>. This is implemented as a full memory fence. (Though
-depending on the memory ordering convention obeyed by the compiler
-even that may not be necessary).
-*/
-void
-__kmpc_flush(ident_t *loc)
-{
- KC_TRACE( 10, ("__kmpc_flush: called\n" ) );
-
- /* need explicit __mf() here since use volatile instead in library */
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 )
- #if KMP_MIC
- // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used.
- // We shouldn't need it, though, since the ABI rules require that
- // * If the compiler generates NGO stores it also generates the fence
- // * If users hand-code NGO stores they should insert the fence
- // therefore no incomplete unordered stores should be visible.
- #else
- // C74404
- // This is to address non-temporal store instructions (sfence needed).
- // The clflush instruction is addressed either (mfence needed).
- // Probably the non-temporal load monvtdqa instruction should also be addressed.
- // mfence is a SSE2 instruction. Do not execute it if CPU is not SSE2.
- if ( ! __kmp_cpuinfo.initialized ) {
- __kmp_query_cpuid( & __kmp_cpuinfo );
- }; // if
- if ( ! __kmp_cpuinfo.sse2 ) {
- // CPU cannot execute SSE2 instructions.
- } else {
- #if KMP_COMPILER_ICC
- _mm_mfence();
- #elif KMP_COMPILER_MSVC
- MemoryBarrier();
- #else
- __sync_synchronize();
- #endif // KMP_COMPILER_ICC
- }; // if
- #endif // KMP_MIC
- #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64)
- // Nothing to see here move along
- #elif KMP_ARCH_PPC64
- // Nothing needed here (we have a real MB above).
- #if KMP_OS_CNK
- // The flushing thread needs to yield here; this prevents a
- // busy-waiting thread from saturating the pipeline. flush is
- // often used in loops like this:
- // while (!flag) {
- // #pragma omp flush(flag)
- // }
- // and adding the yield here is good for at least a 10x speedup
- // when running >2 threads per core (on the NAS LU benchmark).
- __kmp_yield(TRUE);
- #endif
- #else
- #error Unknown or unsupported architecture
- #endif
-
-}
-
-/* -------------------------------------------------------------------------- */
-
-/* -------------------------------------------------------------------------- */
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid thread id.
-
-Execute a barrier.
-*/
-void
-__kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
-{
- KMP_COUNT_BLOCK(OMP_BARRIER);
- KC_TRACE( 10, ("__kmpc_barrier: called T#%d\n", global_tid ) );
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
- if ( __kmp_env_consistency_check ) {
- if ( loc == 0 ) {
- KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user?
- }; // if
-
- __kmp_check_barrier( global_tid, ct_barrier, loc );
- }
-
-#if OMPT_SUPPORT && OMPT_TRACE
- ompt_frame_t * ompt_frame;
- if (ompt_enabled ) {
- ompt_frame = __ompt_get_task_frame_internal(0);
- if ( ompt_frame->reenter_runtime_frame == NULL )
- ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
- __kmp_threads[ global_tid ]->th.th_ident = loc;
- // TODO: explicit barrier_wait_id:
- // this function is called when 'barrier' directive is present or
- // implicit barrier at the end of a worksharing construct.
- // 1) better to add a per-thread barrier counter to a thread data structure
- // 2) set to 0 when a new team is created
- // 4) no sync is required
-
- __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled ) {
- ompt_frame->reenter_runtime_frame = NULL;
- }
-#endif
-}
-
-/* The BARRIER for a MASTER section is always explicit */
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param global_tid global thread number .
- at return 1 if this thread should execute the <tt>master</tt> block, 0 otherwise.
-*/
-kmp_int32
-__kmpc_master(ident_t *loc, kmp_int32 global_tid)
-{
- int status = 0;
-
- KC_TRACE( 10, ("__kmpc_master: called T#%d\n", global_tid ) );
-
- if( ! TCR_4( __kmp_init_parallel ) )
- __kmp_parallel_initialize();
-
- if( KMP_MASTER_GTID( global_tid )) {
- KMP_COUNT_BLOCK(OMP_MASTER);
- KMP_PUSH_PARTITIONED_TIMER(OMP_master);
- status = 1;
- }
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (status) {
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_master_begin)) {
- kmp_info_t *this_thr = __kmp_threads[ global_tid ];
- kmp_team_t *team = this_thr -> th.th_team;
-
- int tid = __kmp_tid_from_gtid( global_tid );
- ompt_callbacks.ompt_callback(ompt_event_master_begin)(
- team->t.ompt_team_info.parallel_id,
- team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
- }
- }
-#endif
-
- if ( __kmp_env_consistency_check ) {
-#if KMP_USE_DYNAMIC_LOCK
- if (status)
- __kmp_push_sync( global_tid, ct_master, loc, NULL, 0 );
- else
- __kmp_check_sync( global_tid, ct_master, loc, NULL, 0 );
-#else
- if (status)
- __kmp_push_sync( global_tid, ct_master, loc, NULL );
- else
- __kmp_check_sync( global_tid, ct_master, loc, NULL );
-#endif
- }
-
- return status;
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param global_tid global thread number .
-
-Mark the end of a <tt>master</tt> region. This should only be called by the thread
-that executes the <tt>master</tt> region.
-*/
-void
-__kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
-{
- KC_TRACE( 10, ("__kmpc_end_master: called T#%d\n", global_tid ) );
-
- KMP_DEBUG_ASSERT( KMP_MASTER_GTID( global_tid ));
- KMP_POP_PARTITIONED_TIMER();
-
-#if OMPT_SUPPORT && OMPT_TRACE
- kmp_info_t *this_thr = __kmp_threads[ global_tid ];
- kmp_team_t *team = this_thr -> th.th_team;
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_master_end)) {
- int tid = __kmp_tid_from_gtid( global_tid );
- ompt_callbacks.ompt_callback(ompt_event_master_end)(
- team->t.ompt_team_info.parallel_id,
- team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
- }
-#endif
-
- if ( __kmp_env_consistency_check ) {
- if( global_tid < 0 )
- KMP_WARNING( ThreadIdentInvalid );
-
- if( KMP_MASTER_GTID( global_tid ))
- __kmp_pop_sync( global_tid, ct_master, loc );
- }
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param gtid global thread number.
-
-Start execution of an <tt>ordered</tt> construct.
-*/
-void
-__kmpc_ordered( ident_t * loc, kmp_int32 gtid )
-{
- int cid = 0;
- kmp_info_t *th;
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- KC_TRACE( 10, ("__kmpc_ordered: called T#%d\n", gtid ));
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
-#if USE_ITT_BUILD
- __kmp_itt_ordered_prep( gtid );
- // TODO: ordered_wait_id
-#endif /* USE_ITT_BUILD */
-
- th = __kmp_threads[ gtid ];
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled) {
- /* OMPT state update */
- th->th.ompt_thread_info.wait_id = (uint64_t) loc;
- th->th.ompt_thread_info.state = ompt_state_wait_ordered;
-
- /* OMPT event callback */
- if (ompt_callbacks.ompt_callback(ompt_event_wait_ordered)) {
- ompt_callbacks.ompt_callback(ompt_event_wait_ordered)(
- th->th.ompt_thread_info.wait_id);
- }
- }
-#endif
-
- if ( th -> th.th_dispatch -> th_deo_fcn != 0 )
- (*th->th.th_dispatch->th_deo_fcn)( & gtid, & cid, loc );
- else
- __kmp_parallel_deo( & gtid, & cid, loc );
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled) {
- /* OMPT state update */
- th->th.ompt_thread_info.state = ompt_state_work_parallel;
- th->th.ompt_thread_info.wait_id = 0;
-
- /* OMPT event callback */
- if (ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)) {
- ompt_callbacks.ompt_callback(ompt_event_acquired_ordered)(
- th->th.ompt_thread_info.wait_id);
- }
- }
-#endif
-
-#if USE_ITT_BUILD
- __kmp_itt_ordered_start( gtid );
-#endif /* USE_ITT_BUILD */
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param gtid global thread number.
-
-End execution of an <tt>ordered</tt> construct.
-*/
-void
-__kmpc_end_ordered( ident_t * loc, kmp_int32 gtid )
-{
- int cid = 0;
- kmp_info_t *th;
-
- KC_TRACE( 10, ("__kmpc_end_ordered: called T#%d\n", gtid ) );
-
-#if USE_ITT_BUILD
- __kmp_itt_ordered_end( gtid );
- // TODO: ordered_wait_id
-#endif /* USE_ITT_BUILD */
-
- th = __kmp_threads[ gtid ];
-
- if ( th -> th.th_dispatch -> th_dxo_fcn != 0 )
- (*th->th.th_dispatch->th_dxo_fcn)( & gtid, & cid, loc );
- else
- __kmp_parallel_dxo( & gtid, & cid, loc );
-
-#if OMPT_SUPPORT && OMPT_BLAME
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
- ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
- th->th.ompt_thread_info.wait_id);
- }
-#endif
-}
-
-#if KMP_USE_DYNAMIC_LOCK
-
-static __forceinline void
-__kmp_init_indirect_csptr(kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid, kmp_indirect_locktag_t tag)
-{
- // Pointer to the allocated indirect lock is written to crit, while indexing is ignored.
- void *idx;
- kmp_indirect_lock_t **lck;
- lck = (kmp_indirect_lock_t **)crit;
- kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
- KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
- KMP_SET_I_LOCK_LOCATION(ilk, loc);
- KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
- KA_TRACE(20, ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
-#if USE_ITT_BUILD
- __kmp_itt_critical_creating(ilk->lock, loc);
-#endif
- int status = KMP_COMPARE_AND_STORE_PTR(lck, 0, ilk);
- if (status == 0) {
-#if USE_ITT_BUILD
- __kmp_itt_critical_destroyed(ilk->lock);
-#endif
- // We don't really need to destroy the unclaimed lock here since it will be cleaned up at program exit.
- //KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx);
- }
- KMP_DEBUG_ASSERT(*lck != NULL);
-}
-
-// Fast-path acquire tas lock
-#define KMP_ACQUIRE_TAS_LOCK(lock, gtid) { \
- kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
- if (l->lk.poll != KMP_LOCK_FREE(tas) || \
- ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \
- kmp_uint32 spins; \
- KMP_FSYNC_PREPARE(l); \
- KMP_INIT_YIELD(spins); \
- if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
- KMP_YIELD(TRUE); \
- } else { \
- KMP_YIELD_SPIN(spins); \
- } \
- kmp_backoff_t backoff = __kmp_spin_backoff_params; \
- while (l->lk.poll != KMP_LOCK_FREE(tas) || \
- ! KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas))) { \
- __kmp_spin_backoff(&backoff); \
- if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
- KMP_YIELD(TRUE); \
- } else { \
- KMP_YIELD_SPIN(spins); \
- } \
- } \
- } \
- KMP_FSYNC_ACQUIRED(l); \
-}
-
-// Fast-path test tas lock
-#define KMP_TEST_TAS_LOCK(lock, gtid, rc) { \
- kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
- rc = l->lk.poll == KMP_LOCK_FREE(tas) && \
- KMP_COMPARE_AND_STORE_ACQ32(&(l->lk.poll), KMP_LOCK_FREE(tas), KMP_LOCK_BUSY(gtid+1, tas)); \
-}
-
-// Fast-path release tas lock
-#define KMP_RELEASE_TAS_LOCK(lock, gtid) { \
- TCW_4(((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); \
- KMP_MB(); \
-}
-
-#if KMP_USE_FUTEX
-
-# include <unistd.h>
-# include <sys/syscall.h>
-# ifndef FUTEX_WAIT
-# define FUTEX_WAIT 0
-# endif
-# ifndef FUTEX_WAKE
-# define FUTEX_WAKE 1
-# endif
-
-// Fast-path acquire futex lock
-#define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) { \
- kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
- kmp_int32 gtid_code = (gtid+1) << 1; \
- KMP_MB(); \
- KMP_FSYNC_PREPARE(ftx); \
- kmp_int32 poll_val; \
- while ((poll_val = KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
- KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
- kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
- if (!cond) { \
- if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, poll_val | KMP_LOCK_BUSY(1, futex))) { \
- continue; \
- } \
- poll_val |= KMP_LOCK_BUSY(1, futex); \
- } \
- kmp_int32 rc; \
- if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, NULL, NULL, 0)) != 0) { \
- continue; \
- } \
- gtid_code |= 1; \
- } \
- KMP_FSYNC_ACQUIRED(ftx); \
-}
-
-// Fast-path test futex lock
-#define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) { \
- kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
- if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), KMP_LOCK_BUSY(gtid+1 << 1, futex))) { \
- KMP_FSYNC_ACQUIRED(ftx); \
- rc = TRUE; \
- } else { \
- rc = FALSE; \
- } \
-}
-
-// Fast-path release futex lock
-#define KMP_RELEASE_FUTEX_LOCK(lock, gtid) { \
- kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
- KMP_MB(); \
- KMP_FSYNC_RELEASING(ftx); \
- kmp_int32 poll_val = KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
- if (KMP_LOCK_STRIP(poll_val) & 1) { \
- syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
- } \
- KMP_MB(); \
- KMP_YIELD(TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)); \
-}
-
-#endif // KMP_USE_FUTEX
-
-#else // KMP_USE_DYNAMIC_LOCK
-
-static kmp_user_lock_p
-__kmp_get_critical_section_ptr( kmp_critical_name * crit, ident_t const * loc, kmp_int32 gtid )
-{
- kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
-
- //
- // Because of the double-check, the following load
- // doesn't need to be volatile.
- //
- kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
-
- if ( lck == NULL ) {
- void * idx;
-
- // Allocate & initialize the lock.
- // Remember allocated locks in table in order to free them in __kmp_cleanup()
- lck = __kmp_user_lock_allocate( &idx, gtid, kmp_lf_critical_section );
- __kmp_init_user_lock_with_checks( lck );
- __kmp_set_user_lock_location( lck, loc );
-#if USE_ITT_BUILD
- __kmp_itt_critical_creating( lck );
- // __kmp_itt_critical_creating() should be called *before* the first usage of underlying
- // lock. It is the only place where we can guarantee it. There are chances the lock will
- // destroyed with no usage, but it is not a problem, because this is not real event seen
- // by user but rather setting name for object (lock). See more details in kmp_itt.h.
-#endif /* USE_ITT_BUILD */
-
- //
- // Use a cmpxchg instruction to slam the start of the critical
- // section with the lock pointer. If another thread beat us
- // to it, deallocate the lock, and use the lock that the other
- // thread allocated.
- //
- int status = KMP_COMPARE_AND_STORE_PTR( lck_pp, 0, lck );
-
- if ( status == 0 ) {
- // Deallocate the lock and reload the value.
-#if USE_ITT_BUILD
- __kmp_itt_critical_destroyed( lck );
- // Let ITT know the lock is destroyed and the same memory location may be reused for
- // another purpose.
-#endif /* USE_ITT_BUILD */
- __kmp_destroy_user_lock_with_checks( lck );
- __kmp_user_lock_free( &idx, gtid, lck );
- lck = (kmp_user_lock_p)TCR_PTR( *lck_pp );
- KMP_DEBUG_ASSERT( lck != NULL );
- }
- }
- return lck;
-}
-
-#endif // KMP_USE_DYNAMIC_LOCK
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param global_tid global thread number .
- at param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or
-some other suitably unique value.
-
-Enter code protected by a `critical` construct.
-This function blocks until the executing thread can enter the critical section.
-*/
-void
-__kmpc_critical( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit )
-{
-#if KMP_USE_DYNAMIC_LOCK
- __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
-#else
- KMP_COUNT_BLOCK(OMP_CRITICAL);
- KMP_TIME_PARTITIONED_BLOCK(OMP_critical_wait); /* Time spent waiting to enter the critical section */
- kmp_user_lock_p lck;
-
- KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) );
-
- //TODO: add THR_OVHD_STATE
-
- KMP_CHECK_USER_LOCK_INIT();
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
- lck = (kmp_user_lock_p)crit;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
- lck = (kmp_user_lock_p)crit;
- }
-#endif
- else { // ticket, queuing or drdpa
- lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_critical, loc, lck );
-
- /* since the critical directive binds to all threads, not just
- * the current team we have to check this even if we are in a
- * serialized team */
- /* also, even if we are the uber thread, we still have to conduct the lock,
- * as we have to contend with sibling threads */
-
-#if USE_ITT_BUILD
- __kmp_itt_critical_acquiring( lck );
-#endif /* USE_ITT_BUILD */
- // Value of 'crit' should be good for using as a critical_id of the critical section directive.
- __kmp_acquire_user_lock_with_checks( lck, global_tid );
-
-#if USE_ITT_BUILD
- __kmp_itt_critical_acquired( lck );
-#endif /* USE_ITT_BUILD */
-
- KMP_START_EXPLICIT_TIMER(OMP_critical);
- KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-#if KMP_USE_DYNAMIC_LOCK
-
-// Converts the given hint to an internal lock implementation
-static __forceinline kmp_dyna_lockseq_t
-__kmp_map_hint_to_lock(uintptr_t hint)
-{
-#if KMP_USE_TSX
-# define KMP_TSX_LOCK(seq) lockseq_##seq
-#else
-# define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
-#endif
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-# define KMP_CPUINFO_RTM (__kmp_cpuinfo.rtm)
-#else
-# define KMP_CPUINFO_RTM 0
-#endif
-
- // Hints that do not require further logic
- if (hint & kmp_lock_hint_hle)
- return KMP_TSX_LOCK(hle);
- if (hint & kmp_lock_hint_rtm)
- return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm): __kmp_user_lock_seq;
- if (hint & kmp_lock_hint_adaptive)
- return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive): __kmp_user_lock_seq;
-
- // Rule out conflicting hints first by returning the default lock
- if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
- return __kmp_user_lock_seq;
- if ((hint & omp_lock_hint_speculative) && (hint & omp_lock_hint_nonspeculative))
- return __kmp_user_lock_seq;
-
- // Do not even consider speculation when it appears to be contended
- if (hint & omp_lock_hint_contended)
- return lockseq_queuing;
-
- // Uncontended lock without speculation
- if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
- return lockseq_tas;
-
- // HLE lock for speculation
- if (hint & omp_lock_hint_speculative)
- return KMP_TSX_LOCK(hle);
-
- return __kmp_user_lock_seq;
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param global_tid global thread number.
- at param crit identity of the critical section. This could be a pointer to a lock associated with the critical section,
-or some other suitably unique value.
- at param hint the lock hint.
-
-Enter code protected by a `critical` construct with a hint. The hint value is used to suggest a lock implementation.
-This function blocks until the executing thread can enter the critical section unless the hint suggests use of
-speculative execution and the hardware supports it.
-*/
-void
-__kmpc_critical_with_hint( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit, uintptr_t hint )
-{
- KMP_COUNT_BLOCK(OMP_CRITICAL);
- kmp_user_lock_p lck;
-
- KC_TRACE( 10, ("__kmpc_critical: called T#%d\n", global_tid ) );
-
- kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
- // Check if it is initialized.
- if (*lk == 0) {
- kmp_dyna_lockseq_t lckseq = __kmp_map_hint_to_lock(hint);
- if (KMP_IS_D_LOCK(lckseq)) {
- KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(lckseq));
- } else {
- __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lckseq));
- }
- }
- // Branch for accessing the actual lock object and set operation. This branching is inevitable since
- // this lock initialization does not follow the normal dispatch path (lock table is not used).
- if (KMP_EXTRACT_D_TAG(lk) != 0) {
- lck = (kmp_user_lock_p)lk;
- if (__kmp_env_consistency_check) {
- __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
- }
-# if USE_ITT_BUILD
- __kmp_itt_critical_acquiring(lck);
-# endif
-# if KMP_USE_INLINED_TAS
- if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
- KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
- } else
-# elif KMP_USE_INLINED_FUTEX
- if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
- KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
- } else
-# endif
- {
- KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
- }
- } else {
- kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
- lck = ilk->lock;
- if (__kmp_env_consistency_check) {
- __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_map_hint_to_lock(hint));
- }
-# if USE_ITT_BUILD
- __kmp_itt_critical_acquiring(lck);
-# endif
- KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_critical_acquired( lck );
-#endif /* USE_ITT_BUILD */
-
- KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
- KA_TRACE( 15, ("__kmpc_critical: done T#%d\n", global_tid ));
-} // __kmpc_critical_with_hint
-
-#endif // KMP_USE_DYNAMIC_LOCK
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param global_tid global thread number .
- at param crit identity of the critical section. This could be a pointer to a lock associated with the critical section, or
-some other suitably unique value.
-
-Leave a critical section, releasing any lock that was held during its execution.
-*/
-void
-__kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
-{
- kmp_user_lock_p lck;
-
- KC_TRACE( 10, ("__kmpc_end_critical: called T#%d\n", global_tid ));
-
-#if KMP_USE_DYNAMIC_LOCK
- if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
- lck = (kmp_user_lock_p)crit;
- KMP_ASSERT(lck != NULL);
- if (__kmp_env_consistency_check) {
- __kmp_pop_sync(global_tid, ct_critical, loc);
- }
-# if USE_ITT_BUILD
- __kmp_itt_critical_releasing( lck );
-# endif
-# if KMP_USE_INLINED_TAS
- if (__kmp_user_lock_seq == lockseq_tas && !__kmp_env_consistency_check) {
- KMP_RELEASE_TAS_LOCK(lck, global_tid);
- } else
-# elif KMP_USE_INLINED_FUTEX
- if (__kmp_user_lock_seq == lockseq_futex && !__kmp_env_consistency_check) {
- KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
- } else
-# endif
- {
- KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
- }
- } else {
- kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
- KMP_ASSERT(ilk != NULL);
- lck = ilk->lock;
- if (__kmp_env_consistency_check) {
- __kmp_pop_sync(global_tid, ct_critical, loc);
- }
-# if USE_ITT_BUILD
- __kmp_itt_critical_releasing( lck );
-# endif
- KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
- }
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
- lck = (kmp_user_lock_p)crit;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_CRITICAL_SIZE ) ) {
- lck = (kmp_user_lock_p)crit;
- }
-#endif
- else { // ticket, queuing or drdpa
- lck = (kmp_user_lock_p) TCR_PTR(*((kmp_user_lock_p *)crit));
- }
-
- KMP_ASSERT(lck != NULL);
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_sync( global_tid, ct_critical, loc );
-
-#if USE_ITT_BUILD
- __kmp_itt_critical_releasing( lck );
-#endif /* USE_ITT_BUILD */
- // Value of 'crit' should be good for using as a critical_id of the critical section directive.
- __kmp_release_user_lock_with_checks( lck, global_tid );
-
-#if OMPT_SUPPORT && OMPT_BLAME
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_release_critical)) {
- ompt_callbacks.ompt_callback(ompt_event_release_critical)(
- (uint64_t) lck);
- }
-#endif
-
-#endif // KMP_USE_DYNAMIC_LOCK
- KMP_POP_PARTITIONED_TIMER();
- KA_TRACE( 15, ("__kmpc_end_critical: done T#%d\n", global_tid ));
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid thread id.
- at return one if the thread should execute the master block, zero otherwise
-
-Start execution of a combined barrier and master. The barrier is executed inside this function.
-*/
-kmp_int32
-__kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
-{
- int status;
-
- KC_TRACE( 10, ("__kmpc_barrier_master: called T#%d\n", global_tid ) );
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
- if ( __kmp_env_consistency_check )
- __kmp_check_barrier( global_tid, ct_barrier, loc );
-
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- status = __kmp_barrier( bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL );
-
- return (status != 0) ? 0 : 1;
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid thread id.
-
-Complete the execution of a combined barrier and master. This function should
-only be called at the completion of the <tt>master</tt> code. Other threads will
-still be waiting at the barrier and this call releases them.
-*/
-void
-__kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
-{
- KC_TRACE( 10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid ));
-
- __kmp_end_split_barrier ( bs_plain_barrier, global_tid );
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid thread id.
- at return one if the thread should execute the master block, zero otherwise
-
-Start execution of a combined barrier and master(nowait) construct.
-The barrier is executed inside this function.
-There is no equivalent "end" function, since the
-*/
-kmp_int32
-__kmpc_barrier_master_nowait( ident_t * loc, kmp_int32 global_tid )
-{
- kmp_int32 ret;
-
- KC_TRACE( 10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid ));
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
- if ( __kmp_env_consistency_check ) {
- if ( loc == 0 ) {
- KMP_WARNING( ConstructIdentInvalid ); // ??? What does it mean for the user?
- }
- __kmp_check_barrier( global_tid, ct_barrier, loc );
- }
-
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
-
- ret = __kmpc_master (loc, global_tid);
-
- if ( __kmp_env_consistency_check ) {
- /* there's no __kmpc_end_master called; so the (stats) */
- /* actions of __kmpc_end_master are done here */
-
- if ( global_tid < 0 ) {
- KMP_WARNING( ThreadIdentInvalid );
- }
- if (ret) {
- /* only one thread should do the pop since only */
- /* one did the push (see __kmpc_master()) */
-
- __kmp_pop_sync( global_tid, ct_master, loc );
- }
- }
-
- return (ret);
-}
-
-/* The BARRIER for a SINGLE process section is always explicit */
-/*!
- at ingroup WORK_SHARING
- at param loc source location information
- at param global_tid global thread number
- at return One if this thread should execute the single construct, zero otherwise.
-
-Test whether to execute a <tt>single</tt> construct.
-There are no implicit barriers in the two "single" calls, rather the compiler should
-introduce an explicit barrier if it is required.
-*/
-
-kmp_int32
-__kmpc_single(ident_t *loc, kmp_int32 global_tid)
-{
- kmp_int32 rc = __kmp_enter_single( global_tid, loc, TRUE );
-
- if (rc) {
- // We are going to execute the single statement, so we should count it.
- KMP_COUNT_BLOCK(OMP_SINGLE);
- KMP_PUSH_PARTITIONED_TIMER(OMP_single);
- }
-
-#if OMPT_SUPPORT && OMPT_TRACE
- kmp_info_t *this_thr = __kmp_threads[ global_tid ];
- kmp_team_t *team = this_thr -> th.th_team;
- int tid = __kmp_tid_from_gtid( global_tid );
-
- if (ompt_enabled) {
- if (rc) {
- if (ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_single_in_block_begin)(
- team->t.ompt_team_info.parallel_id,
- team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id,
- team->t.ompt_team_info.microtask);
- }
- } else {
- if (ompt_callbacks.ompt_callback(ompt_event_single_others_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_single_others_begin)(
- team->t.ompt_team_info.parallel_id,
- team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
- }
- this_thr->th.ompt_thread_info.state = ompt_state_wait_single;
- }
- }
-#endif
-
- return rc;
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc source location information
- at param global_tid global thread number
-
-Mark the end of a <tt>single</tt> construct. This function should
-only be called by the thread that executed the block of code protected
-by the `single` construct.
-*/
-void
-__kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
-{
- __kmp_exit_single( global_tid );
- KMP_POP_PARTITIONED_TIMER();
-
-#if OMPT_SUPPORT && OMPT_TRACE
- kmp_info_t *this_thr = __kmp_threads[ global_tid ];
- kmp_team_t *team = this_thr -> th.th_team;
- int tid = __kmp_tid_from_gtid( global_tid );
-
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)) {
- ompt_callbacks.ompt_callback(ompt_event_single_in_block_end)(
- team->t.ompt_team_info.parallel_id,
- team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id);
- }
-#endif
-}
-
-/*!
- at ingroup WORK_SHARING
- at param loc Source location
- at param global_tid Global thread id
-
-Mark the end of a statically scheduled loop.
-*/
-void
-__kmpc_for_static_fini( ident_t *loc, kmp_int32 global_tid )
-{
- KE_TRACE( 10, ("__kmpc_for_static_fini called T#%d\n", global_tid));
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_loop_end)) {
- ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
- ompt_callbacks.ompt_callback(ompt_event_loop_end)(
- team_info->parallel_id, task_info->task_id);
- }
-#endif
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_workshare( global_tid, ct_pdo, loc );
-}
-
-/*
- * User routines which take C-style arguments (call by value)
- * different from the Fortran equivalent routines
- */
-
-void
-ompc_set_num_threads( int arg )
-{
-// !!!!! TODO: check the per-task binding
- __kmp_set_num_threads( arg, __kmp_entry_gtid() );
-}
-
-void
-ompc_set_dynamic( int flag )
-{
- kmp_info_t *thread;
-
- /* For the thread-private implementation of the internal controls */
- thread = __kmp_entry_thread();
-
- __kmp_save_internal_controls( thread );
-
- set__dynamic( thread, flag ? TRUE : FALSE );
-}
-
-void
-ompc_set_nested( int flag )
-{
- kmp_info_t *thread;
-
- /* For the thread-private internal controls implementation */
- thread = __kmp_entry_thread();
-
- __kmp_save_internal_controls( thread );
-
- set__nested( thread, flag ? TRUE : FALSE );
-}
-
-void
-ompc_set_max_active_levels( int max_active_levels )
-{
- /* TO DO */
- /* we want per-task implementation of this internal control */
-
- /* For the per-thread internal controls implementation */
- __kmp_set_max_active_levels( __kmp_entry_gtid(), max_active_levels );
-}
-
-void
-ompc_set_schedule( omp_sched_t kind, int modifier )
-{
-// !!!!! TODO: check the per-task binding
- __kmp_set_schedule( __kmp_entry_gtid(), ( kmp_sched_t ) kind, modifier );
-}
-
-int
-ompc_get_ancestor_thread_num( int level )
-{
- return __kmp_get_ancestor_thread_num( __kmp_entry_gtid(), level );
-}
-
-int
-ompc_get_team_size( int level )
-{
- return __kmp_get_team_size( __kmp_entry_gtid(), level );
-}
-
-void
-kmpc_set_stacksize( int arg )
-{
- // __kmp_aux_set_stacksize initializes the library if needed
- __kmp_aux_set_stacksize( arg );
-}
-
-void
-kmpc_set_stacksize_s( size_t arg )
-{
- // __kmp_aux_set_stacksize initializes the library if needed
- __kmp_aux_set_stacksize( arg );
-}
-
-void
-kmpc_set_blocktime( int arg )
-{
- int gtid, tid;
- kmp_info_t *thread;
-
- gtid = __kmp_entry_gtid();
- tid = __kmp_tid_from_gtid(gtid);
- thread = __kmp_thread_from_gtid(gtid);
-
- __kmp_aux_set_blocktime( arg, thread, tid );
-}
-
-void
-kmpc_set_library( int arg )
-{
- // __kmp_user_set_library initializes the library if needed
- __kmp_user_set_library( (enum library_type)arg );
-}
-
-void
-kmpc_set_defaults( char const * str )
-{
- // __kmp_aux_set_defaults initializes the library if needed
- __kmp_aux_set_defaults( str, KMP_STRLEN( str ) );
-}
-
-void
-kmpc_set_disp_num_buffers( int arg )
-{
- // ignore after initialization because some teams have already
- // allocated dispatch buffers
- if( __kmp_init_serial == 0 && arg > 0 )
- __kmp_dispatch_num_buffers = arg;
-}
-
-int
-kmpc_set_affinity_mask_proc( int proc, void **mask )
-{
-#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
- return -1;
-#else
- if ( ! TCR_4(__kmp_init_middle) ) {
- __kmp_middle_initialize();
- }
- return __kmp_aux_set_affinity_mask_proc( proc, mask );
-#endif
-}
-
-int
-kmpc_unset_affinity_mask_proc( int proc, void **mask )
-{
-#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
- return -1;
-#else
- if ( ! TCR_4(__kmp_init_middle) ) {
- __kmp_middle_initialize();
- }
- return __kmp_aux_unset_affinity_mask_proc( proc, mask );
-#endif
-}
-
-int
-kmpc_get_affinity_mask_proc( int proc, void **mask )
-{
-#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
- return -1;
-#else
- if ( ! TCR_4(__kmp_init_middle) ) {
- __kmp_middle_initialize();
- }
- return __kmp_aux_get_affinity_mask_proc( proc, mask );
-#endif
-}
-
-
-/* -------------------------------------------------------------------------- */
-/*!
- at ingroup THREADPRIVATE
- at param loc source location information
- at param gtid global thread number
- at param cpy_size size of the cpy_data buffer
- at param cpy_data pointer to data to be copied
- at param cpy_func helper function to call for copying data
- at param didit flag variable: 1=single thread; 0=not single thread
-
-__kmpc_copyprivate implements the interface for the private data broadcast needed for
-the copyprivate clause associated with a single region in an OpenMP<sup>*</sup> program (both C and Fortran).
-All threads participating in the parallel region call this routine.
-One of the threads (called the single thread) should have the <tt>didit</tt> variable set to 1
-and all other threads should have that variable set to 0.
-All threads pass a pointer to a data buffer (cpy_data) that they have built.
-
-The OpenMP specification forbids the use of nowait on the single region when a copyprivate
-clause is present. However, @ref __kmpc_copyprivate implements a barrier internally to avoid
-race conditions, so the code generation for the single region should avoid generating a barrier
-after the call to @ref __kmpc_copyprivate.
-
-The <tt>gtid</tt> parameter is the global thread id for the current thread.
-The <tt>loc</tt> parameter is a pointer to source location information.
-
-Internal implementation: The single thread will first copy its descriptor address (cpy_data)
-to a team-private location, then the other threads will each call the function pointed to by
-the parameter cpy_func, which carries out the copy by copying the data using the cpy_data buffer.
-
-The cpy_func routine used for the copy and the contents of the data area defined by cpy_data
-and cpy_size may be built in any fashion that will allow the copy to be done. For instance,
-the cpy_data buffer can hold the actual data to be copied or it may hold a list of pointers
-to the data. The cpy_func routine must interpret the cpy_data buffer appropriately.
-
-The interface to cpy_func is as follows:
- at code
-void cpy_func( void *destination, void *source )
- at endcode
-where void *destination is the cpy_data pointer for the thread being copied to
-and void *source is the cpy_data pointer for the thread being copied from.
-*/
-void
-__kmpc_copyprivate( ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void*,void*), kmp_int32 didit )
-{
- void **data_ptr;
-
- KC_TRACE( 10, ("__kmpc_copyprivate: called T#%d\n", gtid ));
-
- KMP_MB();
-
- data_ptr = & __kmp_team_from_gtid( gtid )->t.t_copypriv_data;
-
- if ( __kmp_env_consistency_check ) {
- if ( loc == 0 ) {
- KMP_WARNING( ConstructIdentInvalid );
- }
- }
-
- /* ToDo: Optimize the following two barriers into some kind of split barrier */
-
- if (didit) *data_ptr = cpy_data;
-
- /* This barrier is not a barrier region boundary */
-#if USE_ITT_NOTIFY
- __kmp_threads[gtid]->th.th_ident = loc;
-#endif
- __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
-
- if (! didit) (*cpy_func)( cpy_data, *data_ptr );
-
- /* Consider next barrier the user-visible barrier for barrier region boundaries */
- /* Nesting checks are already handled by the single construct checks */
-
-#if USE_ITT_NOTIFY
- __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g. tasks can overwrite the location)
-#endif
- __kmp_barrier( bs_plain_barrier, gtid, FALSE , 0, NULL, NULL );
-}
-
-/* -------------------------------------------------------------------------- */
-
-#define INIT_LOCK __kmp_init_user_lock_with_checks
-#define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
-#define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
-#define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
-#define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
-#define ACQUIRE_NESTED_LOCK_TIMED __kmp_acquire_nested_user_lock_with_checks_timed
-#define RELEASE_LOCK __kmp_release_user_lock_with_checks
-#define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
-#define TEST_LOCK __kmp_test_user_lock_with_checks
-#define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
-#define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
-#define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
-
-
-/*
- * TODO: Make check abort messages use location info & pass it
- * into with_checks routines
- */
-
-#if KMP_USE_DYNAMIC_LOCK
-
-// internal lock initializer
-static __forceinline void
-__kmp_init_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq)
-{
- if (KMP_IS_D_LOCK(seq)) {
- KMP_INIT_D_LOCK(lock, seq);
-#if USE_ITT_BUILD
- __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
-#endif
- } else {
- KMP_INIT_I_LOCK(lock, seq);
-#if USE_ITT_BUILD
- kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
- __kmp_itt_lock_creating(ilk->lock, loc);
-#endif
- }
-}
-
-// internal nest lock initializer
-static __forceinline void
-__kmp_init_nest_lock_with_hint(ident_t *loc, void **lock, kmp_dyna_lockseq_t seq)
-{
-#if KMP_USE_TSX
- // Don't have nested lock implementation for speculative locks
- if (seq == lockseq_hle || seq == lockseq_rtm || seq == lockseq_adaptive)
- seq = __kmp_user_lock_seq;
-#endif
- switch (seq) {
- case lockseq_tas:
- seq = lockseq_nested_tas;
- break;
-#if KMP_USE_FUTEX
- case lockseq_futex:
- seq = lockseq_nested_futex;
- break;
-#endif
- case lockseq_ticket:
- seq = lockseq_nested_ticket;
- break;
- case lockseq_queuing:
- seq = lockseq_nested_queuing;
- break;
- case lockseq_drdpa:
- seq = lockseq_nested_drdpa;
- break;
- default:
- seq = lockseq_nested_queuing;
- }
- KMP_INIT_I_LOCK(lock, seq);
-#if USE_ITT_BUILD
- kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
- __kmp_itt_lock_creating(ilk->lock, loc);
-#endif
-}
-
-/* initialize the lock with a hint */
-void
-__kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
-{
- KMP_DEBUG_ASSERT(__kmp_init_serial);
- if (__kmp_env_consistency_check && user_lock == NULL) {
- KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint");
- }
-
- __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
-}
-
-/* initialize the lock with a hint */
-void
-__kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock, uintptr_t hint)
-{
- KMP_DEBUG_ASSERT(__kmp_init_serial);
- if (__kmp_env_consistency_check && user_lock == NULL) {
- KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint");
- }
-
- __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
-}
-
-#endif // KMP_USE_DYNAMIC_LOCK
-
-/* initialize the lock */
-void
-__kmpc_init_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
-#if KMP_USE_DYNAMIC_LOCK
- KMP_DEBUG_ASSERT(__kmp_init_serial);
- if (__kmp_env_consistency_check && user_lock == NULL) {
- KMP_FATAL(LockIsUninitialized, "omp_init_lock");
- }
- __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- static char const * const func = "omp_init_lock";
- kmp_user_lock_p lck;
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- if ( __kmp_env_consistency_check ) {
- if ( user_lock == NULL ) {
- KMP_FATAL( LockIsUninitialized, func );
- }
- }
-
- KMP_CHECK_USER_LOCK_INIT();
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
- }
- INIT_LOCK( lck );
- __kmp_set_user_lock_location( lck, loc );
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_init_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_init_lock)((uint64_t) lck);
- }
-#endif
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_creating( lck );
-#endif /* USE_ITT_BUILD */
-
-#endif // KMP_USE_DYNAMIC_LOCK
-} // __kmpc_init_lock
-
-/* initialize the lock */
-void
-__kmpc_init_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
-#if KMP_USE_DYNAMIC_LOCK
-
- KMP_DEBUG_ASSERT(__kmp_init_serial);
- if (__kmp_env_consistency_check && user_lock == NULL) {
- KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock");
- }
- __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- static char const * const func = "omp_init_nest_lock";
- kmp_user_lock_p lck;
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- if ( __kmp_env_consistency_check ) {
- if ( user_lock == NULL ) {
- KMP_FATAL( LockIsUninitialized, func );
- }
- }
-
- KMP_CHECK_USER_LOCK_INIT();
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_user_lock_allocate( user_lock, gtid, 0 );
- }
-
- INIT_NESTED_LOCK( lck );
- __kmp_set_user_lock_location( lck, loc );
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_init_nest_lock)((uint64_t) lck);
- }
-#endif
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_creating( lck );
-#endif /* USE_ITT_BUILD */
-
-#endif // KMP_USE_DYNAMIC_LOCK
-} // __kmpc_init_nest_lock
-
-void
-__kmpc_destroy_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
-#if KMP_USE_DYNAMIC_LOCK
-
-# if USE_ITT_BUILD
- kmp_user_lock_p lck;
- if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
- lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
- } else {
- lck = (kmp_user_lock_p)user_lock;
- }
- __kmp_itt_lock_destroyed(lck);
-# endif
- KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
-#else
- kmp_user_lock_p lck;
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_lock" );
- }
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_destroy_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_destroy_lock)((uint64_t) lck);
- }
-#endif
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_destroyed( lck );
-#endif /* USE_ITT_BUILD */
- DESTROY_LOCK( lck );
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- ;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- ;
- }
-#endif
- else {
- __kmp_user_lock_free( user_lock, gtid, lck );
- }
-#endif // KMP_USE_DYNAMIC_LOCK
-} // __kmpc_destroy_lock
-
-/* destroy the lock */
-void
-__kmpc_destroy_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
-#if KMP_USE_DYNAMIC_LOCK
-
-# if USE_ITT_BUILD
- kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
- __kmp_itt_lock_destroyed(ilk->lock);
-# endif
- KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_destroy_nest_lock" );
- }
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_destroy_nest_lock)((uint64_t) lck);
- }
-#endif
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_destroyed( lck );
-#endif /* USE_ITT_BUILD */
-
- DESTROY_NESTED_LOCK( lck );
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
- ;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- ;
- }
-#endif
- else {
- __kmp_user_lock_free( user_lock, gtid, lck );
- }
-#endif // KMP_USE_DYNAMIC_LOCK
-} // __kmpc_destroy_nest_lock
-
-void
-__kmpc_set_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
- KMP_COUNT_BLOCK(OMP_set_lock);
-#if KMP_USE_DYNAMIC_LOCK
- int tag = KMP_EXTRACT_D_TAG(user_lock);
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock); // itt function will get to the right lock object.
-# endif
-# if KMP_USE_INLINED_TAS
- if (tag == locktag_tas && !__kmp_env_consistency_check) {
- KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
- } else
-# elif KMP_USE_INLINED_FUTEX
- if (tag == locktag_futex && !__kmp_env_consistency_check) {
- KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
- } else
-# endif
- {
- __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
- }
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
-# endif
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_set_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquiring( lck );
-#endif /* USE_ITT_BUILD */
-
- ACQUIRE_LOCK( lck, gtid );
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquired( lck );
-#endif /* USE_ITT_BUILD */
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_acquired_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_acquired_lock)((uint64_t) lck);
- }
-#endif
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-void
-__kmpc_set_nest_lock( ident_t * loc, kmp_int32 gtid, void ** user_lock ) {
-#if KMP_USE_DYNAMIC_LOCK
-
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
-# endif
- KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
-#endif
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled) {
- // missing support here: need to know whether acquired first or not
- }
-#endif
-
-#else // KMP_USE_DYNAMIC_LOCK
- int acquire_status;
- kmp_user_lock_p lck;
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_set_nest_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquiring( lck );
-#endif /* USE_ITT_BUILD */
-
- ACQUIRE_NESTED_LOCK( lck, gtid, &acquire_status );
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquired( lck );
-#endif /* USE_ITT_BUILD */
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled) {
- if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
- if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first))
- ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_first)((uint64_t) lck);
- } else {
- if(ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next))
- ompt_callbacks.ompt_callback(ompt_event_acquired_nest_lock_next)((uint64_t) lck);
- }
- }
-#endif
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-void
-__kmpc_unset_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
-{
-#if KMP_USE_DYNAMIC_LOCK
-
- int tag = KMP_EXTRACT_D_TAG(user_lock);
-# if USE_ITT_BUILD
- __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
-# endif
-# if KMP_USE_INLINED_TAS
- if (tag == locktag_tas && !__kmp_env_consistency_check) {
- KMP_RELEASE_TAS_LOCK(user_lock, gtid);
- } else
-# elif KMP_USE_INLINED_FUTEX
- if (tag == locktag_futex && !__kmp_env_consistency_check) {
- KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
- } else
-# endif
- {
- __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
- }
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
-
- /* Can't use serial interval since not block structured */
- /* release the lock */
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
- // "fast" path implemented to fix customer performance issue
-#if USE_ITT_BUILD
- __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
-#endif /* USE_ITT_BUILD */
- TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
- KMP_MB();
- return;
-#else
- lck = (kmp_user_lock_p)user_lock;
-#endif
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_unset_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_releasing( lck );
-#endif /* USE_ITT_BUILD */
-
- RELEASE_LOCK( lck, gtid );
-
-#if OMPT_SUPPORT && OMPT_BLAME
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_release_lock)) {
- ompt_callbacks.ompt_callback(ompt_event_release_lock)((uint64_t) lck);
- }
-#endif
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-/* release the lock */
-void
-__kmpc_unset_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
-{
-#if KMP_USE_DYNAMIC_LOCK
-
-# if USE_ITT_BUILD
- __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
-# endif
- KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
-
- /* Can't use serial interval since not block structured */
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
-#if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
- // "fast" path implemented to fix customer performance issue
- kmp_tas_lock_t *tl = (kmp_tas_lock_t*)user_lock;
-#if USE_ITT_BUILD
- __kmp_itt_lock_releasing( (kmp_user_lock_p)user_lock );
-#endif /* USE_ITT_BUILD */
- if ( --(tl->lk.depth_locked) == 0 ) {
- TCW_4(tl->lk.poll, 0);
- }
- KMP_MB();
- return;
-#else
- lck = (kmp_user_lock_p)user_lock;
-#endif
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_unset_nest_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_releasing( lck );
-#endif /* USE_ITT_BUILD */
-
- int release_status;
- release_status = RELEASE_NESTED_LOCK( lck, gtid );
-#if OMPT_SUPPORT && OMPT_BLAME
- if (ompt_enabled) {
- if (release_status == KMP_LOCK_RELEASED) {
- if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)) {
- ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_last)(
- (uint64_t) lck);
- }
- } else if (ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)) {
- ompt_callbacks.ompt_callback(ompt_event_release_nest_lock_prev)(
- (uint64_t) lck);
- }
- }
-#endif
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-/* try to acquire the lock */
-int
-__kmpc_test_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
-{
- KMP_COUNT_BLOCK(OMP_test_lock);
-
-#if KMP_USE_DYNAMIC_LOCK
- int rc;
- int tag = KMP_EXTRACT_D_TAG(user_lock);
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
-# endif
-# if KMP_USE_INLINED_TAS
- if (tag == locktag_tas && !__kmp_env_consistency_check) {
- KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
- } else
-# elif KMP_USE_INLINED_FUTEX
- if (tag == locktag_futex && !__kmp_env_consistency_check) {
- KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
- } else
-# endif
- {
- rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
- }
- if (rc) {
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
-# endif
- return FTN_TRUE;
- } else {
-# if USE_ITT_BUILD
- __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
-# endif
- return FTN_FALSE;
- }
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
- int rc;
-
- if ( ( __kmp_user_lock_kind == lk_tas )
- && ( sizeof( lck->tas.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) <= OMP_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_test_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquiring( lck );
-#endif /* USE_ITT_BUILD */
-
- rc = TEST_LOCK( lck, gtid );
-#if USE_ITT_BUILD
- if ( rc ) {
- __kmp_itt_lock_acquired( lck );
- } else {
- __kmp_itt_lock_cancelled( lck );
- }
-#endif /* USE_ITT_BUILD */
- return ( rc ? FTN_TRUE : FTN_FALSE );
-
- /* Can't use serial interval since not block structured */
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-/* try to acquire the lock */
-int
-__kmpc_test_nest_lock( ident_t *loc, kmp_int32 gtid, void **user_lock )
-{
-#if KMP_USE_DYNAMIC_LOCK
- int rc;
-# if USE_ITT_BUILD
- __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
-# endif
- rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
-# if USE_ITT_BUILD
- if (rc) {
- __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
- } else {
- __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
- }
-# endif
- return rc;
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- kmp_user_lock_p lck;
- int rc;
-
- if ( ( __kmp_user_lock_kind == lk_tas ) && ( sizeof( lck->tas.lk.poll )
- + sizeof( lck->tas.lk.depth_locked ) <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#if KMP_USE_FUTEX
- else if ( ( __kmp_user_lock_kind == lk_futex )
- && ( sizeof( lck->futex.lk.poll ) + sizeof( lck->futex.lk.depth_locked )
- <= OMP_NEST_LOCK_T_SIZE ) ) {
- lck = (kmp_user_lock_p)user_lock;
- }
-#endif
- else {
- lck = __kmp_lookup_user_lock( user_lock, "omp_test_nest_lock" );
- }
-
-#if USE_ITT_BUILD
- __kmp_itt_lock_acquiring( lck );
-#endif /* USE_ITT_BUILD */
-
- rc = TEST_NESTED_LOCK( lck, gtid );
-#if USE_ITT_BUILD
- if ( rc ) {
- __kmp_itt_lock_acquired( lck );
- } else {
- __kmp_itt_lock_cancelled( lck );
- }
-#endif /* USE_ITT_BUILD */
- return rc;
-
- /* Can't use serial interval since not block structured */
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-
-/*--------------------------------------------------------------------------------------------------------------------*/
-
-/*
- * Interface to fast scalable reduce methods routines
- */
-
-// keep the selected method in a thread local structure for cross-function usage: will be used in __kmpc_end_reduce* functions;
-// another solution: to re-determine the method one more time in __kmpc_end_reduce* functions (new prototype required then)
-// AT: which solution is better?
-#define __KMP_SET_REDUCTION_METHOD(gtid,rmethod) \
- ( ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method ) = ( rmethod ) )
-
-#define __KMP_GET_REDUCTION_METHOD(gtid) \
- ( __kmp_threads[ ( gtid ) ] -> th.th_local.packed_reduction_method )
-
-// description of the packed_reduction_method variable: look at the macros in kmp.h
-
-
-// used in a critical section reduce block
-static __forceinline void
-__kmp_enter_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
-
- // this lock was visible to a customer and to the threading profile tool as a serial overhead span
- // (although it's used for an internal purpose only)
- // why was it visible in previous implementation?
- // should we keep it visible in new reduce block?
- kmp_user_lock_p lck;
-
-#if KMP_USE_DYNAMIC_LOCK
-
- kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
- // Check if it is initialized.
- if (*lk == 0) {
- if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
- KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0, KMP_GET_D_TAG(__kmp_user_lock_seq));
- } else {
- __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(__kmp_user_lock_seq));
- }
- }
- // Branch for accessing the actual lock object and set operation. This branching is inevitable since
- // this lock initialization does not follow the normal dispatch path (lock table is not used).
- if (KMP_EXTRACT_D_TAG(lk) != 0) {
- lck = (kmp_user_lock_p)lk;
- KMP_DEBUG_ASSERT(lck != NULL);
- if (__kmp_env_consistency_check) {
- __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
- }
- KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
- } else {
- kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
- lck = ilk->lock;
- KMP_DEBUG_ASSERT(lck != NULL);
- if (__kmp_env_consistency_check) {
- __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
- }
- KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
- }
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- // We know that the fast reduction code is only emitted by Intel compilers
- // with 32 byte critical sections. If there isn't enough space, then we
- // have to use a pointer.
- if ( __kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE ) {
- lck = (kmp_user_lock_p)crit;
- }
- else {
- lck = __kmp_get_critical_section_ptr( crit, loc, global_tid );
- }
- KMP_DEBUG_ASSERT( lck != NULL );
-
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_critical, loc, lck );
-
- __kmp_acquire_user_lock_with_checks( lck, global_tid );
-
-#endif // KMP_USE_DYNAMIC_LOCK
-}
-
-// used in a critical section reduce block
-static __forceinline void
-__kmp_end_critical_section_reduce_block( ident_t * loc, kmp_int32 global_tid, kmp_critical_name * crit ) {
-
- kmp_user_lock_p lck;
-
-#if KMP_USE_DYNAMIC_LOCK
-
- if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
- lck = (kmp_user_lock_p)crit;
- if (__kmp_env_consistency_check)
- __kmp_pop_sync(global_tid, ct_critical, loc);
- KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
- } else {
- kmp_indirect_lock_t *ilk = (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
- if (__kmp_env_consistency_check)
- __kmp_pop_sync(global_tid, ct_critical, loc);
- KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
- }
-
-#else // KMP_USE_DYNAMIC_LOCK
-
- // We know that the fast reduction code is only emitted by Intel compilers with 32 byte critical
- // sections. If there isn't enough space, then we have to use a pointer.
- if ( __kmp_base_user_lock_size > 32 ) {
- lck = *( (kmp_user_lock_p *) crit );
- KMP_ASSERT( lck != NULL );
- } else {
- lck = (kmp_user_lock_p) crit;
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_sync( global_tid, ct_critical, loc );
-
- __kmp_release_user_lock_with_checks( lck, global_tid );
-
-#endif // KMP_USE_DYNAMIC_LOCK
-} // __kmp_end_critical_section_reduce_block
-
-
-/* 2.a.i. Reduce Block without a terminating barrier */
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid global thread number
- at param num_vars number of items (variables) to be reduced
- at param reduce_size size of data in bytes to be reduced
- at param reduce_data pointer to data to be reduced
- at param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data
- at param lck pointer to the unique lock data structure
- at result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed
-
-The nowait version is used for a reduce clause with the nowait argument.
-*/
-kmp_int32
-__kmpc_reduce_nowait(
- ident_t *loc, kmp_int32 global_tid,
- kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
- kmp_critical_name *lck ) {
-
- KMP_COUNT_BLOCK(REDUCE_nowait);
- int retval = 0;
- PACKED_REDUCTION_METHOD_T packed_reduction_method;
-#if OMP_40_ENABLED
- kmp_team_t *team;
- kmp_info_t *th;
- int teams_swapped = 0, task_state;
-#endif
- KA_TRACE( 10, ( "__kmpc_reduce_nowait() enter: called T#%d\n", global_tid ) );
-
- // why do we need this initialization here at all?
- // Reduction clause can not be used as a stand-alone directive.
-
- // do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed
- // possible detection of false-positive race by the threadchecker ???
- if( ! TCR_4( __kmp_init_parallel ) )
- __kmp_parallel_initialize();
-
- // check correctness of reduce block nesting
-#if KMP_USE_DYNAMIC_LOCK
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
-#else
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
-#endif
-
-#if OMP_40_ENABLED
- th = __kmp_thread_from_gtid(global_tid);
- if( th->th.th_teams_microtask ) { // AC: check if we are inside the teams construct?
- team = th->th.th_team;
- if( team->t.t_level == th->th.th_teams_level ) {
- // this is reduction at teams construct
- KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0
- // Let's swap teams temporarily for the reduction barrier
- teams_swapped = 1;
- th->th.th_info.ds.ds_tid = team->t.t_master_tid;
- th->th.th_team = team->t.t_parent;
- th->th.th_team_nproc = th->th.th_team->t.t_nproc;
- th->th.th_task_team = th->th.th_team->t.t_task_team[0];
- task_state = th->th.th_task_state;
- th->th.th_task_state = 0;
- }
- }
-#endif // OMP_40_ENABLED
-
- // packed_reduction_method value will be reused by __kmp_end_reduce* function, the value should be kept in a variable
- // the variable should be either a construct-specific or thread-specific property, not a team specific property
- // (a thread can reach the next reduce block on the next construct, reduce method may differ on the next construct)
- // an ident_t "loc" parameter could be used as a construct-specific property (what if loc == 0?)
- // (if both construct-specific and team-specific variables were shared, then unness extra syncs should be needed)
- // a thread-specific variable is better regarding two issues above (next construct and extra syncs)
- // a thread-specific "th_local.reduction_method" variable is used currently
- // each thread executes 'determine' and 'set' lines (no need to execute by one thread, to avoid unness extra syncs)
-
- packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
- __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
-
- if( packed_reduction_method == critical_reduce_block ) {
-
- __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
- retval = 1;
-
- } else if( packed_reduction_method == empty_reduce_block ) {
-
- // usage: if team size == 1, no synchronization is required ( Intel platforms only )
- retval = 1;
-
- } else if( packed_reduction_method == atomic_reduce_block ) {
-
- retval = 2;
-
- // all threads should do this pop here (because __kmpc_end_reduce_nowait() won't be called by the code gen)
- // (it's not quite good, because the checking block has been closed by this 'pop',
- // but atomic operation has not been executed yet, will be executed slightly later, literally on next instruction)
- if ( __kmp_env_consistency_check )
- __kmp_pop_sync( global_tid, ct_reduce, loc );
-
- } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
-
- //AT: performance issue: a real barrier here
- //AT: (if master goes slow, other threads are blocked here waiting for the master to come and release them)
- //AT: (it's not what a customer might expect specifying NOWAIT clause)
- //AT: (specifying NOWAIT won't result in improvement of performance, it'll be confusing to a customer)
- //AT: another implementation of *barrier_gather*nowait() (or some other design) might go faster
- // and be more in line with sense of NOWAIT
- //AT: TO DO: do epcc test and compare times
-
- // this barrier should be invisible to a customer and to the threading profile tool
- // (it's neither a terminating barrier nor customer's code, it's used for an internal purpose)
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, FALSE, reduce_size, reduce_data, reduce_func );
- retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
-
- // all other workers except master should do this pop here
- // ( none of other workers will get to __kmpc_end_reduce_nowait() )
- if ( __kmp_env_consistency_check ) {
- if( retval == 0 ) {
- __kmp_pop_sync( global_tid, ct_reduce, loc );
- }
- }
-
- } else {
-
- // should never reach this block
- KMP_ASSERT( 0 ); // "unexpected method"
-
- }
-#if OMP_40_ENABLED
- if( teams_swapped ) {
- // Restore thread structure
- th->th.th_info.ds.ds_tid = 0;
- th->th.th_team = team;
- th->th.th_team_nproc = team->t.t_nproc;
- th->th.th_task_team = team->t.t_task_team[task_state];
- th->th.th_task_state = task_state;
- }
-#endif
- KA_TRACE( 10, ( "__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
-
- return retval;
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid global thread id.
- at param lck pointer to the unique lock data structure
-
-Finish the execution of a reduce nowait.
-*/
-void
-__kmpc_end_reduce_nowait( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) {
-
- PACKED_REDUCTION_METHOD_T packed_reduction_method;
-
- KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid ) );
-
- packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
-
- if( packed_reduction_method == critical_reduce_block ) {
-
- __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
-
- } else if( packed_reduction_method == empty_reduce_block ) {
-
- // usage: if team size == 1, no synchronization is required ( on Intel platforms only )
-
- } else if( packed_reduction_method == atomic_reduce_block ) {
-
- // neither master nor other workers should get here
- // (code gen does not generate this call in case 2: atomic reduce block)
- // actually it's better to remove this elseif at all;
- // after removal this value will checked by the 'else' and will assert
-
- } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
-
- // only master gets here
-
- } else {
-
- // should never reach this block
- KMP_ASSERT( 0 ); // "unexpected method"
-
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_sync( global_tid, ct_reduce, loc );
-
- KA_TRACE( 10, ( "__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
-
- return;
-}
-
-/* 2.a.ii. Reduce Block with a terminating barrier */
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid global thread number
- at param num_vars number of items (variables) to be reduced
- at param reduce_size size of data in bytes to be reduced
- at param reduce_data pointer to data to be reduced
- at param reduce_func callback function providing reduction operation on two operands and returning result of reduction in lhs_data
- at param lck pointer to the unique lock data structure
- at result 1 for the master thread, 0 for all other team threads, 2 for all team threads if atomic reduction needed
-
-A blocking reduce that includes an implicit barrier.
-*/
-kmp_int32
-__kmpc_reduce(
- ident_t *loc, kmp_int32 global_tid,
- kmp_int32 num_vars, size_t reduce_size, void *reduce_data,
- void (*reduce_func)(void *lhs_data, void *rhs_data),
- kmp_critical_name *lck )
-{
- KMP_COUNT_BLOCK(REDUCE_wait);
- int retval = 0;
- PACKED_REDUCTION_METHOD_T packed_reduction_method;
-
- KA_TRACE( 10, ( "__kmpc_reduce() enter: called T#%d\n", global_tid ) );
-
- // why do we need this initialization here at all?
- // Reduction clause can not be a stand-alone directive.
-
- // do not call __kmp_serial_initialize(), it will be called by __kmp_parallel_initialize() if needed
- // possible detection of false-positive race by the threadchecker ???
- if( ! TCR_4( __kmp_init_parallel ) )
- __kmp_parallel_initialize();
-
- // check correctness of reduce block nesting
-#if KMP_USE_DYNAMIC_LOCK
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_reduce, loc, NULL, 0 );
-#else
- if ( __kmp_env_consistency_check )
- __kmp_push_sync( global_tid, ct_reduce, loc, NULL );
-#endif
-
- packed_reduction_method = __kmp_determine_reduction_method( loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck );
- __KMP_SET_REDUCTION_METHOD( global_tid, packed_reduction_method );
-
- if( packed_reduction_method == critical_reduce_block ) {
-
- __kmp_enter_critical_section_reduce_block( loc, global_tid, lck );
- retval = 1;
-
- } else if( packed_reduction_method == empty_reduce_block ) {
-
- // usage: if team size == 1, no synchronization is required ( Intel platforms only )
- retval = 1;
-
- } else if( packed_reduction_method == atomic_reduce_block ) {
-
- retval = 2;
-
- } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
-
- //case tree_reduce_block:
- // this barrier should be visible to a customer and to the threading profile tool
- // (it's a terminating barrier on constructs if NOWAIT not specified)
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc; // needed for correct notification of frames
-#endif
- retval = __kmp_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid, TRUE, reduce_size, reduce_data, reduce_func );
- retval = ( retval != 0 ) ? ( 0 ) : ( 1 );
-
- // all other workers except master should do this pop here
- // ( none of other workers except master will enter __kmpc_end_reduce() )
- if ( __kmp_env_consistency_check ) {
- if( retval == 0 ) { // 0: all other workers; 1: master
- __kmp_pop_sync( global_tid, ct_reduce, loc );
- }
- }
-
- } else {
-
- // should never reach this block
- KMP_ASSERT( 0 ); // "unexpected method"
-
- }
-
- KA_TRACE( 10, ( "__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n", global_tid, packed_reduction_method, retval ) );
-
- return retval;
-}
-
-/*!
- at ingroup SYNCHRONIZATION
- at param loc source location information
- at param global_tid global thread id.
- at param lck pointer to the unique lock data structure
-
-Finish the execution of a blocking reduce.
-The <tt>lck</tt> pointer must be the same as that used in the corresponding start function.
-*/
-void
-__kmpc_end_reduce( ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck ) {
-
- PACKED_REDUCTION_METHOD_T packed_reduction_method;
-
- KA_TRACE( 10, ( "__kmpc_end_reduce() enter: called T#%d\n", global_tid ) );
-
- packed_reduction_method = __KMP_GET_REDUCTION_METHOD( global_tid );
-
- // this barrier should be visible to a customer and to the threading profile tool
- // (it's a terminating barrier on constructs if NOWAIT not specified)
-
- if( packed_reduction_method == critical_reduce_block ) {
-
- __kmp_end_critical_section_reduce_block( loc, global_tid, lck );
-
- // TODO: implicit barrier: should be exposed
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
-
- } else if( packed_reduction_method == empty_reduce_block ) {
-
- // usage: if team size == 1, no synchronization is required ( Intel platforms only )
-
- // TODO: implicit barrier: should be exposed
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
-
- } else if( packed_reduction_method == atomic_reduce_block ) {
-
- // TODO: implicit barrier: should be exposed
-#if USE_ITT_NOTIFY
- __kmp_threads[global_tid]->th.th_ident = loc;
-#endif
- __kmp_barrier( bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL );
-
- } else if( TEST_REDUCTION_METHOD( packed_reduction_method, tree_reduce_block ) ) {
-
- // only master executes here (master releases all other workers)
- __kmp_end_split_barrier( UNPACK_REDUCTION_BARRIER( packed_reduction_method ), global_tid );
-
- } else {
-
- // should never reach this block
- KMP_ASSERT( 0 ); // "unexpected method"
-
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_pop_sync( global_tid, ct_reduce, loc );
-
- KA_TRACE( 10, ( "__kmpc_end_reduce() exit: called T#%d: method %08x\n", global_tid, packed_reduction_method ) );
-
- return;
-}
-
-#undef __KMP_GET_REDUCTION_METHOD
-#undef __KMP_SET_REDUCTION_METHOD
-
-/*-- end of interface to fast scalable reduce routines ---------------------------------------------------------------*/
-
-kmp_uint64
-__kmpc_get_taskid() {
-
- kmp_int32 gtid;
- kmp_info_t * thread;
-
- gtid = __kmp_get_gtid();
- if ( gtid < 0 ) {
- return 0;
- }; // if
- thread = __kmp_thread_from_gtid( gtid );
- return thread->th.th_current_task->td_task_id;
-
-} // __kmpc_get_taskid
-
-
-kmp_uint64
-__kmpc_get_parent_taskid() {
-
- kmp_int32 gtid;
- kmp_info_t * thread;
- kmp_taskdata_t * parent_task;
-
- gtid = __kmp_get_gtid();
- if ( gtid < 0 ) {
- return 0;
- }; // if
- thread = __kmp_thread_from_gtid( gtid );
- parent_task = thread->th.th_current_task->td_parent;
- return ( parent_task == NULL ? 0 : parent_task->td_task_id );
-
-} // __kmpc_get_parent_taskid
-
-void __kmpc_place_threads(int nS, int sO, int nC, int cO, int nT)
-{
- if ( ! __kmp_init_serial ) {
- __kmp_serial_initialize();
- }
- __kmp_place_num_sockets = nS;
- __kmp_place_socket_offset = sO;
- __kmp_place_num_cores = nC;
- __kmp_place_core_offset = cO;
- __kmp_place_num_threads_per_core = nT;
-}
-
-#if OMP_45_ENABLED
-/*!
- at ingroup WORK_SHARING
- at param loc source location information.
- at param gtid global thread number.
- at param num_dims number of associated doacross loops.
- at param dims info on loops bounds.
-
-Initialize doacross loop information.
-Expect compiler send us inclusive bounds,
-e.g. for(i=2;i<9;i+=2) lo=2, up=8, st=2.
-*/
-void
-__kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, struct kmp_dim * dims)
-{
- int j, idx;
- kmp_int64 last, trace_count;
- kmp_info_t *th = __kmp_threads[gtid];
- kmp_team_t *team = th->th.th_team;
- kmp_uint32 *flags;
- kmp_disp_t *pr_buf = th->th.th_dispatch;
- dispatch_shared_info_t *sh_buf;
-
- KA_TRACE(20,("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
- gtid, num_dims, !team->t.t_serialized));
- KMP_DEBUG_ASSERT(dims != NULL);
- KMP_DEBUG_ASSERT(num_dims > 0);
-
- if( team->t.t_serialized ) {
- KA_TRACE(20,("__kmpc_doacross_init() exit: serialized team\n"));
- return; // no dependencies if team is serialized
- }
- KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
- idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for the next loop
- sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
-
- // Save bounds info into allocated private buffer
- KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
- pr_buf->th_doacross_info =
- (kmp_int64*)__kmp_thread_malloc(th, sizeof(kmp_int64)*(4 * num_dims + 1));
- KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
- pr_buf->th_doacross_info[0] = (kmp_int64)num_dims; // first element is number of dimensions
- // Save also address of num_done in order to access it later without knowing the buffer index
- pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
- pr_buf->th_doacross_info[2] = dims[0].lo;
- pr_buf->th_doacross_info[3] = dims[0].up;
- pr_buf->th_doacross_info[4] = dims[0].st;
- last = 5;
- for( j = 1; j < num_dims; ++j ) {
- kmp_int64 range_length; // To keep ranges of all dimensions but the first dims[0]
- if( dims[j].st == 1 ) { // most common case
- // AC: should we care of ranges bigger than LLONG_MAX? (not for now)
- range_length = dims[j].up - dims[j].lo + 1;
- } else {
- if( dims[j].st > 0 ) {
- KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
- range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
- } else { // negative increment
- KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
- range_length = (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
- }
- }
- pr_buf->th_doacross_info[last++] = range_length;
- pr_buf->th_doacross_info[last++] = dims[j].lo;
- pr_buf->th_doacross_info[last++] = dims[j].up;
- pr_buf->th_doacross_info[last++] = dims[j].st;
- }
-
- // Compute total trip count.
- // Start with range of dims[0] which we don't need to keep in the buffer.
- if( dims[0].st == 1 ) { // most common case
- trace_count = dims[0].up - dims[0].lo + 1;
- } else if( dims[0].st > 0 ) {
- KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
- trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
- } else { // negative increment
- KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
- trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
- }
- for( j = 1; j < num_dims; ++j ) {
- trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
- }
- KMP_DEBUG_ASSERT(trace_count > 0);
-
- // Check if shared buffer is not occupied by other loop (idx - __kmp_dispatch_num_buffers)
- if( idx != sh_buf->doacross_buf_idx ) {
- // Shared buffer is occupied, wait for it to be free
- __kmp_wait_yield_4( (kmp_uint32*)&sh_buf->doacross_buf_idx, idx, __kmp_eq_4, NULL );
- }
- // Check if we are the first thread. After the CAS the first thread gets 0,
- // others get 1 if initialization is in progress, allocated pointer otherwise.
- flags = (kmp_uint32*)KMP_COMPARE_AND_STORE_RET64(
- (kmp_int64*)&sh_buf->doacross_flags,NULL,(kmp_int64)1);
- if( flags == NULL ) {
- // we are the first thread, allocate the array of flags
- kmp_int64 size = trace_count / 8 + 8; // in bytes, use single bit per iteration
- sh_buf->doacross_flags = (kmp_uint32*)__kmp_thread_calloc(th, size, 1);
- } else if( (kmp_int64)flags == 1 ) {
- // initialization is still in progress, need to wait
- while( (volatile kmp_int64)sh_buf->doacross_flags == 1 ) {
- KMP_YIELD(TRUE);
- }
- }
- KMP_DEBUG_ASSERT((kmp_int64)sh_buf->doacross_flags > 1); // check value of pointer
- pr_buf->th_doacross_flags = sh_buf->doacross_flags; // save private copy in order to not
- // touch shared buffer on each iteration
- KA_TRACE(20,("__kmpc_doacross_init() exit: T#%d\n", gtid));
-}
-
-void
-__kmpc_doacross_wait(ident_t *loc, int gtid, long long *vec)
-{
- kmp_int32 shft, num_dims, i;
- kmp_uint32 flag;
- kmp_int64 iter_number; // iteration number of "collapsed" loop nest
- kmp_info_t *th = __kmp_threads[gtid];
- kmp_team_t *team = th->th.th_team;
- kmp_disp_t *pr_buf;
- kmp_int64 lo, up, st;
-
- KA_TRACE(20,("__kmpc_doacross_wait() enter: called T#%d\n", gtid));
- if( team->t.t_serialized ) {
- KA_TRACE(20,("__kmpc_doacross_wait() exit: serialized team\n"));
- return; // no dependencies if team is serialized
- }
-
- // calculate sequential iteration number and check out-of-bounds condition
- pr_buf = th->th.th_dispatch;
- KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
- num_dims = pr_buf->th_doacross_info[0];
- lo = pr_buf->th_doacross_info[2];
- up = pr_buf->th_doacross_info[3];
- st = pr_buf->th_doacross_info[4];
- if( st == 1 ) { // most common case
- if( vec[0] < lo || vec[0] > up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[0], lo, up));
- return;
- }
- iter_number = vec[0] - lo;
- } else if( st > 0 ) {
- if( vec[0] < lo || vec[0] > up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[0], lo, up));
- return;
- }
- iter_number = (kmp_uint64)(vec[0] - lo) / st;
- } else { // negative increment
- if( vec[0] > lo || vec[0] < up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[0], lo, up));
- return;
- }
- iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
- }
- for( i = 1; i < num_dims; ++i ) {
- kmp_int64 iter, ln;
- kmp_int32 j = i * 4;
- ln = pr_buf->th_doacross_info[j + 1];
- lo = pr_buf->th_doacross_info[j + 2];
- up = pr_buf->th_doacross_info[j + 3];
- st = pr_buf->th_doacross_info[j + 4];
- if( st == 1 ) {
- if( vec[i] < lo || vec[i] > up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[i], lo, up));
- return;
- }
- iter = vec[i] - lo;
- } else if( st > 0 ) {
- if( vec[i] < lo || vec[i] > up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[i], lo, up));
- return;
- }
- iter = (kmp_uint64)(vec[i] - lo) / st;
- } else { // st < 0
- if( vec[i] > lo || vec[i] < up ) {
- KA_TRACE(20,(
- "__kmpc_doacross_wait() exit: T#%d iter %lld is out of bounds [%lld,%lld]\n",
- gtid, vec[i], lo, up));
- return;
- }
- iter = (kmp_uint64)(lo - vec[i]) / (-st);
- }
- iter_number = iter + ln * iter_number;
- }
- shft = iter_number % 32; // use 32-bit granularity
- iter_number >>= 5; // divided by 32
- flag = 1 << shft;
- while( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 ) {
- KMP_YIELD(TRUE);
- }
- KA_TRACE(20,("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
- gtid, (iter_number<<5)+shft));
-}
-
-void
-__kmpc_doacross_post(ident_t *loc, int gtid, long long *vec)
-{
- kmp_int32 shft, num_dims, i;
- kmp_uint32 flag;
- kmp_int64 iter_number; // iteration number of "collapsed" loop nest
- kmp_info_t *th = __kmp_threads[gtid];
- kmp_team_t *team = th->th.th_team;
- kmp_disp_t *pr_buf;
- kmp_int64 lo, st;
-
- KA_TRACE(20,("__kmpc_doacross_post() enter: called T#%d\n", gtid));
- if( team->t.t_serialized ) {
- KA_TRACE(20,("__kmpc_doacross_post() exit: serialized team\n"));
- return; // no dependencies if team is serialized
- }
-
- // calculate sequential iteration number (same as in "wait" but no out-of-bounds checks)
- pr_buf = th->th.th_dispatch;
- KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
- num_dims = pr_buf->th_doacross_info[0];
- lo = pr_buf->th_doacross_info[2];
- st = pr_buf->th_doacross_info[4];
- if( st == 1 ) { // most common case
- iter_number = vec[0] - lo;
- } else if( st > 0 ) {
- iter_number = (kmp_uint64)(vec[0] - lo) / st;
- } else { // negative increment
- iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
- }
- for( i = 1; i < num_dims; ++i ) {
- kmp_int64 iter, ln;
- kmp_int32 j = i * 4;
- ln = pr_buf->th_doacross_info[j + 1];
- lo = pr_buf->th_doacross_info[j + 2];
- st = pr_buf->th_doacross_info[j + 4];
- if( st == 1 ) {
- iter = vec[i] - lo;
- } else if( st > 0 ) {
- iter = (kmp_uint64)(vec[i] - lo) / st;
- } else { // st < 0
- iter = (kmp_uint64)(lo - vec[i]) / (-st);
- }
- iter_number = iter + ln * iter_number;
- }
- shft = iter_number % 32; // use 32-bit granularity
- iter_number >>= 5; // divided by 32
- flag = 1 << shft;
- if( (flag & pr_buf->th_doacross_flags[iter_number]) == 0 )
- KMP_TEST_THEN_OR32( (kmp_int32*)&pr_buf->th_doacross_flags[iter_number], (kmp_int32)flag );
- KA_TRACE(20,("__kmpc_doacross_post() exit: T#%d iter %lld posted\n",
- gtid, (iter_number<<5)+shft));
-}
-
-void
-__kmpc_doacross_fini(ident_t *loc, int gtid)
-{
- kmp_int64 num_done;
- kmp_info_t *th = __kmp_threads[gtid];
- kmp_team_t *team = th->th.th_team;
- kmp_disp_t *pr_buf = th->th.th_dispatch;
-
- KA_TRACE(20,("__kmpc_doacross_fini() enter: called T#%d\n", gtid));
- if( team->t.t_serialized ) {
- KA_TRACE(20,("__kmpc_doacross_fini() exit: serialized team %p\n", team));
- return; // nothing to do
- }
- num_done = KMP_TEST_THEN_INC64((kmp_int64*)pr_buf->th_doacross_info[1]) + 1;
- if( num_done == th->th.th_team_nproc ) {
- // we are the last thread, need to free shared resources
- int idx = pr_buf->th_doacross_buf_idx - 1;
- dispatch_shared_info_t *sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
- KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] == (kmp_int64)&sh_buf->doacross_num_done);
- KMP_DEBUG_ASSERT(num_done == (kmp_int64)sh_buf->doacross_num_done);
- KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
- __kmp_thread_free(th, (void*)sh_buf->doacross_flags);
- sh_buf->doacross_flags = NULL;
- sh_buf->doacross_num_done = 0;
- sh_buf->doacross_buf_idx += __kmp_dispatch_num_buffers; // free buffer for future re-use
- }
- // free private resources (need to keep buffer index forever)
- __kmp_thread_free(th, (void*)pr_buf->th_doacross_info);
- pr_buf->th_doacross_info = NULL;
- KA_TRACE(20,("__kmpc_doacross_fini() exit: T#%d\n", gtid));
-}
-#endif
-
-// end of file //
-
Removed: openmp/trunk/runtime/src/kmp_debug.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_debug.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_debug.c (original)
+++ openmp/trunk/runtime/src/kmp_debug.c (removed)
@@ -1,142 +0,0 @@
-/*
- * kmp_debug.c -- debug utilities for the Guide library
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_debug.h" /* really necessary? */
-#include "kmp_i18n.h"
-#include "kmp_io.h"
-
-#ifdef KMP_DEBUG
-void
-__kmp_debug_printf_stdout( char const * format, ... )
-{
- va_list ap;
- va_start( ap, format );
-
- __kmp_vprintf( kmp_out, format, ap );
-
- va_end(ap);
-}
-#endif
-
-void
-__kmp_debug_printf( char const * format, ... )
-{
- va_list ap;
- va_start( ap, format );
-
- __kmp_vprintf( kmp_err, format, ap );
-
- va_end( ap );
-}
-
-#ifdef KMP_USE_ASSERT
- int
- __kmp_debug_assert(
- char const * msg,
- char const * file,
- int line
- ) {
-
- if ( file == NULL ) {
- file = KMP_I18N_STR( UnknownFile );
- } else {
- // Remove directories from path, leave only file name. File name is enough, there is no need
- // in bothering developers and customers with full paths.
- char const * slash = strrchr( file, '/' );
- if ( slash != NULL ) {
- file = slash + 1;
- }; // if
- }; // if
-
- #ifdef KMP_DEBUG
- __kmp_acquire_bootstrap_lock( & __kmp_stdio_lock );
- __kmp_debug_printf( "Assertion failure at %s(%d): %s.\n", file, line, msg );
- __kmp_release_bootstrap_lock( & __kmp_stdio_lock );
- #ifdef USE_ASSERT_BREAK
- #if KMP_OS_WINDOWS
- DebugBreak();
- #endif
- #endif // USE_ASSERT_BREAK
- #ifdef USE_ASSERT_STALL
- /* __kmp_infinite_loop(); */
- for(;;);
- #endif // USE_ASSERT_STALL
- #ifdef USE_ASSERT_SEG
- {
- int volatile * ZERO = (int*) 0;
- ++ (*ZERO);
- }
- #endif // USE_ASSERT_SEG
- #endif
-
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( AssertionFailure, file, line ),
- KMP_HNT( SubmitBugReport ),
- __kmp_msg_null
- );
-
- return 0;
-
- } // __kmp_debug_assert
-
-#endif // KMP_USE_ASSERT
-
-/* Dump debugging buffer to stderr */
-void
-__kmp_dump_debug_buffer( void )
-{
- if ( __kmp_debug_buffer != NULL ) {
- int i;
- int dc = __kmp_debug_count;
- char *db = & __kmp_debug_buffer[ (dc % __kmp_debug_buf_lines) * __kmp_debug_buf_chars ];
- char *db_end = & __kmp_debug_buffer[ __kmp_debug_buf_lines * __kmp_debug_buf_chars ];
- char *db2;
-
- __kmp_acquire_bootstrap_lock( & __kmp_stdio_lock );
- __kmp_printf_no_lock( "\nStart dump of debugging buffer (entry=%d):\n",
- dc % __kmp_debug_buf_lines );
-
- for ( i = 0; i < __kmp_debug_buf_lines; i++ ) {
-
- if ( *db != '\0' ) {
- /* Fix up where no carriage return before string termination char */
- for ( db2 = db + 1; db2 < db + __kmp_debug_buf_chars - 1; db2 ++) {
- if ( *db2 == '\0' ) {
- if ( *(db2-1) != '\n' ) { *db2 = '\n'; *(db2+1) = '\0'; }
- break;
- }
- }
- /* Handle case at end by shortening the printed message by one char if necessary */
- if ( db2 == db + __kmp_debug_buf_chars - 1 &&
- *db2 == '\0' && *(db2-1) != '\n' ) {
- *(db2-1) = '\n';
- }
-
- __kmp_printf_no_lock( "%4d: %.*s", i, __kmp_debug_buf_chars, db );
- *db = '\0'; /* only let it print once! */
- }
-
- db += __kmp_debug_buf_chars;
- if ( db >= db_end )
- db = __kmp_debug_buffer;
- }
-
- __kmp_printf_no_lock( "End dump of debugging buffer (entry=%d).\n\n",
- ( dc+i-1 ) % __kmp_debug_buf_lines );
- __kmp_release_bootstrap_lock( & __kmp_stdio_lock );
- }
-}
Removed: openmp/trunk/runtime/src/kmp_debugger.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_debugger.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_debugger.c (original)
+++ openmp/trunk/runtime/src/kmp_debugger.c (removed)
@@ -1,315 +0,0 @@
-#if USE_DEBUGGER
-/*
- * kmp_debugger.c -- debugger support.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_lock.h"
-#include "kmp_omp.h"
-#include "kmp_str.h"
-
-/*
- NOTE: All variable names are known to the debugger, do not change!
-*/
-
-#ifdef __cplusplus
- extern "C" {
- extern kmp_omp_struct_info_t __kmp_omp_debug_struct_info;
- } // extern "C"
-#endif // __cplusplus
-
-int __kmp_debugging = FALSE; // Boolean whether currently debugging OpenMP RTL.
-
-#define offset_and_size_of( structure, field ) \
- { \
- offsetof( structure, field ), \
- sizeof( ( (structure *) NULL)->field ) \
- }
-
-#define offset_and_size_not_available \
- { -1, -1 }
-
-#define addr_and_size_of( var ) \
- { \
- (kmp_uint64)( & var ), \
- sizeof( var ) \
- }
-
-#define nthr_buffer_size 1024
-static kmp_int32
-kmp_omp_nthr_info_buffer[ nthr_buffer_size ] =
- { nthr_buffer_size * sizeof( kmp_int32 ) };
-
-/* TODO: Check punctuation for various platforms here */
-static char func_microtask[] = "__kmp_invoke_microtask";
-static char func_fork[] = "__kmpc_fork_call";
-static char func_fork_teams[] = "__kmpc_fork_teams";
-
-
-// Various info about runtime structures: addresses, field offsets, sizes, etc.
-kmp_omp_struct_info_t
-__kmp_omp_debug_struct_info = {
-
- /* Change this only if you make a fundamental data structure change here */
- KMP_OMP_VERSION,
-
- /* sanity check. Only should be checked if versions are identical
- * This is also used for backward compatibility to get the runtime
- * structure size if it the runtime is older than the interface */
- sizeof( kmp_omp_struct_info_t ),
-
- /* OpenMP RTL version info. */
- addr_and_size_of( __kmp_version_major ),
- addr_and_size_of( __kmp_version_minor ),
- addr_and_size_of( __kmp_version_build ),
- addr_and_size_of( __kmp_openmp_version ),
- { (kmp_uint64)( __kmp_copyright ) + KMP_VERSION_MAGIC_LEN, 0 }, // Skip magic prefix.
-
- /* Various globals. */
- addr_and_size_of( __kmp_threads ),
- addr_and_size_of( __kmp_root ),
- addr_and_size_of( __kmp_threads_capacity ),
- addr_and_size_of( __kmp_monitor ),
-#if ! KMP_USE_DYNAMIC_LOCK
- addr_and_size_of( __kmp_user_lock_table ),
-#endif
- addr_and_size_of( func_microtask ),
- addr_and_size_of( func_fork ),
- addr_and_size_of( func_fork_teams ),
- addr_and_size_of( __kmp_team_counter ),
- addr_and_size_of( __kmp_task_counter ),
- addr_and_size_of( kmp_omp_nthr_info_buffer ),
- sizeof( void * ),
- OMP_LOCK_T_SIZE < sizeof(void *),
- bs_last_barrier,
- INITIAL_TASK_DEQUE_SIZE,
-
- // thread structure information
- sizeof( kmp_base_info_t ),
- offset_and_size_of( kmp_base_info_t, th_info ),
- offset_and_size_of( kmp_base_info_t, th_team ),
- offset_and_size_of( kmp_base_info_t, th_root ),
- offset_and_size_of( kmp_base_info_t, th_serial_team ),
- offset_and_size_of( kmp_base_info_t, th_ident ),
- offset_and_size_of( kmp_base_info_t, th_spin_here ),
- offset_and_size_of( kmp_base_info_t, th_next_waiting ),
- offset_and_size_of( kmp_base_info_t, th_task_team ),
- offset_and_size_of( kmp_base_info_t, th_current_task ),
- offset_and_size_of( kmp_base_info_t, th_task_state ),
- offset_and_size_of( kmp_base_info_t, th_bar ),
- offset_and_size_of( kmp_bstate_t, b_worker_arrived ),
-
-#if OMP_40_ENABLED
- // teams information
- offset_and_size_of( kmp_base_info_t, th_teams_microtask),
- offset_and_size_of( kmp_base_info_t, th_teams_level),
- offset_and_size_of( kmp_teams_size_t, nteams ),
- offset_and_size_of( kmp_teams_size_t, nth ),
-#endif
-
- // kmp_desc structure (for info field above)
- sizeof( kmp_desc_base_t ),
- offset_and_size_of( kmp_desc_base_t, ds_tid ),
- offset_and_size_of( kmp_desc_base_t, ds_gtid ),
- // On Windows* OS, ds_thread contains a thread /handle/, which is not usable, while thread /id/
- // is in ds_thread_id.
- #if KMP_OS_WINDOWS
- offset_and_size_of( kmp_desc_base_t, ds_thread_id),
- #else
- offset_and_size_of( kmp_desc_base_t, ds_thread),
- #endif
-
- // team structure information
- sizeof( kmp_base_team_t ),
- offset_and_size_of( kmp_base_team_t, t_master_tid ),
- offset_and_size_of( kmp_base_team_t, t_ident ),
- offset_and_size_of( kmp_base_team_t, t_parent ),
- offset_and_size_of( kmp_base_team_t, t_nproc ),
- offset_and_size_of( kmp_base_team_t, t_threads ),
- offset_and_size_of( kmp_base_team_t, t_serialized ),
- offset_and_size_of( kmp_base_team_t, t_id ),
- offset_and_size_of( kmp_base_team_t, t_pkfn ),
- offset_and_size_of( kmp_base_team_t, t_task_team ),
- offset_and_size_of( kmp_base_team_t, t_implicit_task_taskdata ),
-#if OMP_40_ENABLED
- offset_and_size_of( kmp_base_team_t, t_cancel_request ),
-#endif
- offset_and_size_of( kmp_base_team_t, t_bar ),
- offset_and_size_of( kmp_balign_team_t, b_master_arrived ),
- offset_and_size_of( kmp_balign_team_t, b_team_arrived ),
-
- // root structure information
- sizeof( kmp_base_root_t ),
- offset_and_size_of( kmp_base_root_t, r_root_team ),
- offset_and_size_of( kmp_base_root_t, r_hot_team ),
- offset_and_size_of( kmp_base_root_t, r_uber_thread ),
- offset_and_size_not_available,
-
- // ident structure information
- sizeof( ident_t ),
- offset_and_size_of( ident_t, psource ),
- offset_and_size_of( ident_t, flags ),
-
- // lock structure information
- sizeof( kmp_base_queuing_lock_t ),
- offset_and_size_of( kmp_base_queuing_lock_t, initialized ),
- offset_and_size_of( kmp_base_queuing_lock_t, location ),
- offset_and_size_of( kmp_base_queuing_lock_t, tail_id ),
- offset_and_size_of( kmp_base_queuing_lock_t, head_id ),
- offset_and_size_of( kmp_base_queuing_lock_t, next_ticket ),
- offset_and_size_of( kmp_base_queuing_lock_t, now_serving ),
- offset_and_size_of( kmp_base_queuing_lock_t, owner_id ),
- offset_and_size_of( kmp_base_queuing_lock_t, depth_locked ),
- offset_and_size_of( kmp_base_queuing_lock_t, flags ),
-
-#if ! KMP_USE_DYNAMIC_LOCK
- /* Lock table. */
- sizeof( kmp_lock_table_t ),
- offset_and_size_of( kmp_lock_table_t, used ),
- offset_and_size_of( kmp_lock_table_t, allocated ),
- offset_and_size_of( kmp_lock_table_t, table ),
-#endif
-
- // Task team structure information.
- sizeof( kmp_base_task_team_t ),
- offset_and_size_of( kmp_base_task_team_t, tt_threads_data ),
- offset_and_size_of( kmp_base_task_team_t, tt_found_tasks ),
- offset_and_size_of( kmp_base_task_team_t, tt_nproc ),
- offset_and_size_of( kmp_base_task_team_t, tt_unfinished_threads ),
- offset_and_size_of( kmp_base_task_team_t, tt_active ),
-
- // task_data_t.
- sizeof( kmp_taskdata_t ),
- offset_and_size_of( kmp_taskdata_t, td_task_id ),
- offset_and_size_of( kmp_taskdata_t, td_flags ),
- offset_and_size_of( kmp_taskdata_t, td_team ),
- offset_and_size_of( kmp_taskdata_t, td_parent ),
- offset_and_size_of( kmp_taskdata_t, td_level ),
- offset_and_size_of( kmp_taskdata_t, td_ident ),
- offset_and_size_of( kmp_taskdata_t, td_allocated_child_tasks ),
- offset_and_size_of( kmp_taskdata_t, td_incomplete_child_tasks ),
-
- offset_and_size_of( kmp_taskdata_t, td_taskwait_ident ),
- offset_and_size_of( kmp_taskdata_t, td_taskwait_counter ),
- offset_and_size_of( kmp_taskdata_t, td_taskwait_thread ),
-
-#if OMP_40_ENABLED
- offset_and_size_of( kmp_taskdata_t, td_taskgroup ),
- offset_and_size_of( kmp_taskgroup_t, count ),
- offset_and_size_of( kmp_taskgroup_t, cancel_request ),
-
- offset_and_size_of( kmp_taskdata_t, td_depnode ),
- offset_and_size_of( kmp_depnode_list_t, node ),
- offset_and_size_of( kmp_depnode_list_t, next ),
- offset_and_size_of( kmp_base_depnode_t, successors ),
- offset_and_size_of( kmp_base_depnode_t, task ),
- offset_and_size_of( kmp_base_depnode_t, npredecessors ),
- offset_and_size_of( kmp_base_depnode_t, nrefs ),
-#endif
- offset_and_size_of( kmp_task_t, routine ),
-
- // thread_data_t.
- sizeof( kmp_thread_data_t ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque_size ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque_head ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque_tail ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque_ntasks ),
- offset_and_size_of( kmp_base_thread_data_t, td_deque_last_stolen ),
-
- // The last field.
- KMP_OMP_VERSION,
-
-}; // __kmp_omp_debug_struct_info
-
-#undef offset_and_size_of
-#undef addr_and_size_of
-
-/*
- Intel compiler on IA-32 architecture issues a warning "conversion
- from "unsigned long long" to "char *" may lose significant bits"
- when 64-bit value is assigned to 32-bit pointer. Use this function
- to suppress the warning.
-*/
-static inline
-void *
-__kmp_convert_to_ptr(
- kmp_uint64 addr
-) {
- #if KMP_COMPILER_ICC
- #pragma warning( push )
- #pragma warning( disable: 810 ) // conversion from "unsigned long long" to "char *" may lose significant bits
- #pragma warning( disable: 1195 ) // conversion from integer to smaller pointer
- #endif // KMP_COMPILER_ICC
- return (void *) addr;
- #if KMP_COMPILER_ICC
- #pragma warning( pop )
- #endif // KMP_COMPILER_ICC
-} // __kmp_convert_to_ptr
-
-
-static int
-kmp_location_match(
- kmp_str_loc_t * loc,
- kmp_omp_nthr_item_t * item
-) {
-
- int file_match = 0;
- int func_match = 0;
- int line_match = 0;
-
- char * file = (char *) __kmp_convert_to_ptr( item->file );
- char * func = (char *) __kmp_convert_to_ptr( item->func );
- file_match = __kmp_str_fname_match( & loc->fname, file );
- func_match =
- item->func == 0 // If item->func is NULL, it allows any func name.
- ||
- strcmp( func, "*" ) == 0
- ||
- ( loc->func != NULL && strcmp( loc->func, func ) == 0 );
- line_match =
- item->begin <= loc->line
- &&
- ( item->end <= 0 || loc->line <= item->end ); // if item->end <= 0, it means "end of file".
-
- return ( file_match && func_match && line_match );
-
-} // kmp_location_match
-
-
-int
-__kmp_omp_num_threads(
- ident_t const * ident
-) {
-
- int num_threads = 0;
-
- kmp_omp_nthr_info_t * info =
- (kmp_omp_nthr_info_t *) __kmp_convert_to_ptr( __kmp_omp_debug_struct_info.nthr_info.addr );
- if ( info->num > 0 && info->array != 0 ) {
- kmp_omp_nthr_item_t * items = (kmp_omp_nthr_item_t *) __kmp_convert_to_ptr( info->array );
- kmp_str_loc_t loc = __kmp_str_loc_init( ident->psource, 1 );
- int i;
- for ( i = 0; i < info->num; ++ i ) {
- if ( kmp_location_match( & loc, & items[ i ] ) ) {
- num_threads = items[ i ].num_threads;
- }; // if
- }; // for
- __kmp_str_loc_free( & loc );
- }; // if
-
- return num_threads;;
-
-} // __kmp_omp_num_threads
-#endif /* USE_DEBUGGER */
Removed: openmp/trunk/runtime/src/kmp_environment.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_environment.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_environment.c (original)
+++ openmp/trunk/runtime/src/kmp_environment.c (removed)
@@ -1,595 +0,0 @@
-/*
- * kmp_environment.c -- Handle environment variables OS-independently.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*
- ------------------------------------------------------------------------------------------------
- We use GetEnvironmentVariable for Windows* OS instead of getenv because the act of
- loading a DLL on Windows* OS makes any user-set environment variables (i.e. with putenv())
- unavailable. getenv() apparently gets a clean copy of the env variables as they existed
- at the start of the run.
- JH 12/23/2002
- ------------------------------------------------------------------------------------------------
- On Windows* OS, there are two environments (at least, see below):
-
- 1. Environment maintained by Windows* OS on IA-32 architecture.
- Accessible through GetEnvironmentVariable(),
- SetEnvironmentVariable(), and GetEnvironmentStrings().
-
- 2. Environment maintained by C RTL. Accessible through getenv(), putenv().
-
- putenv() function updates both C and Windows* OS on IA-32 architecture. getenv() function
- search for variables in C RTL environment only. Windows* OS on IA-32 architecture functions work *only*
- with Windows* OS on IA-32 architecture.
-
- Windows* OS on IA-32 architecture maintained by OS, so there is always only one Windows* OS on
- IA-32 architecture per process. Changes in Windows* OS on IA-32 architecture are process-visible.
-
- C environment maintained by C RTL. Multiple copies of C RTL may be present in the process, and
- each C RTL maintains its own environment. :-(
-
- Thus, proper way to work with environment on Windows* OS is:
-
- 1. Set variables with putenv() function -- both C and Windows* OS on
- IA-32 architecture are being updated. Windows* OS on
- IA-32 architecture may be considered as primary target,
- while updating C RTL environment is a free bonus.
-
- 2. Get variables with GetEnvironmentVariable() -- getenv() does not
- search Windows* OS on IA-32 architecture, and can not see variables
- set with SetEnvironmentVariable().
-
- 2007-04-05 -- lev
- ------------------------------------------------------------------------------------------------
-*/
-
-#include "kmp_environment.h"
-
-#include "kmp_os.h" // KMP_OS_*.
-#include "kmp.h" //
-#include "kmp_str.h" // __kmp_str_*().
-#include "kmp_i18n.h"
-
-#if KMP_OS_UNIX
- #include <stdlib.h> // getenv, setenv, unsetenv.
- #include <string.h> // strlen, strcpy.
- #if KMP_OS_DARWIN
- #include <crt_externs.h>
- #define environ (*_NSGetEnviron())
- #else
- extern char * * environ;
- #endif
-#elif KMP_OS_WINDOWS
- #include <windows.h> // GetEnvironmentVariable, SetEnvironmentVariable, GetLastError.
-#else
- #error Unknown or unsupported OS.
-#endif
-
-
-// TODO: Eliminate direct memory allocations, use string operations instead.
-
-static inline
-void *
-allocate(
- size_t size
-) {
- void * ptr = KMP_INTERNAL_MALLOC( size );
- if ( ptr == NULL ) {
- KMP_FATAL( MemoryAllocFailed );
- }; // if
- return ptr;
-} // allocate
-
-
-char *
-__kmp_env_get( char const * name ) {
-
- char * result = NULL;
-
- #if KMP_OS_UNIX
- char const * value = getenv( name );
- if ( value != NULL ) {
- size_t len = KMP_STRLEN( value ) + 1;
- result = (char *) KMP_INTERNAL_MALLOC( len );
- if ( result == NULL ) {
- KMP_FATAL( MemoryAllocFailed );
- }; // if
- KMP_STRNCPY_S( result, len, value, len );
- }; // if
- #elif KMP_OS_WINDOWS
- /*
- We use GetEnvironmentVariable for Windows* OS instead of getenv because the act of
- loading a DLL on Windows* OS makes any user-set environment variables (i.e. with putenv())
- unavailable. getenv() apparently gets a clean copy of the env variables as they existed
- at the start of the run.
- JH 12/23/2002
- */
- DWORD rc;
- rc = GetEnvironmentVariable( name, NULL, 0 );
- if ( ! rc ) {
- DWORD error = GetLastError();
- if ( error != ERROR_ENVVAR_NOT_FOUND ) {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantGetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- // Variable is not found, it's ok, just continue.
- } else {
- DWORD len = rc;
- result = (char *) KMP_INTERNAL_MALLOC( len );
- if ( result == NULL ) {
- KMP_FATAL( MemoryAllocFailed );
- }; // if
- rc = GetEnvironmentVariable( name, result, len );
- if ( ! rc ) {
- // GetEnvironmentVariable() may return 0 if variable is empty.
- // In such a case GetLastError() returns ERROR_SUCCESS.
- DWORD error = GetLastError();
- if ( error != ERROR_SUCCESS ) {
- // Unexpected error. The variable should be in the environment,
- // and buffer should be large enough.
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantGetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- KMP_INTERNAL_FREE( (void *) result );
- result = NULL;
- }; // if
- }; // if
- }; // if
- #else
- #error Unknown or unsupported OS.
- #endif
-
- return result;
-
-} // func __kmp_env_get
-
-
-// TODO: Find and replace all regular free() with __kmp_env_free().
-
-void
-__kmp_env_free( char const * * value ) {
-
- KMP_DEBUG_ASSERT( value != NULL );
- KMP_INTERNAL_FREE( (void *) * value );
- * value = NULL;
-
-} // func __kmp_env_free
-
-
-
-int
-__kmp_env_exists( char const * name ) {
-
- #if KMP_OS_UNIX
- char const * value = getenv( name );
- return ( ( value == NULL ) ? ( 0 ) : ( 1 ) );
- #elif KMP_OS_WINDOWS
- DWORD rc;
- rc = GetEnvironmentVariable( name, NULL, 0 );
- if ( rc == 0 ) {
- DWORD error = GetLastError();
- if ( error != ERROR_ENVVAR_NOT_FOUND ) {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantGetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- return 0;
- }; // if
- return 1;
- #else
- #error Unknown or unsupported OS.
- #endif
-
-} // func __kmp_env_exists
-
-
-
-void
-__kmp_env_set( char const * name, char const * value, int overwrite ) {
-
- #if KMP_OS_UNIX
- int rc = setenv( name, value, overwrite );
- if ( rc != 0 ) {
- // Dead code. I tried to put too many variables into Linux* OS
- // environment on IA-32 architecture. When application consumes
- // more than ~2.5 GB of memory, entire system feels bad. Sometimes
- // application is killed (by OS?), sometimes system stops
- // responding... But this error message never appears. --ln
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantSetEnvVar, name ),
- KMP_HNT( NotEnoughMemory ),
- __kmp_msg_null
- );
- }; // if
- #elif KMP_OS_WINDOWS
- BOOL rc;
- if ( ! overwrite ) {
- rc = GetEnvironmentVariable( name, NULL, 0 );
- if ( rc ) {
- // Variable exists, do not overwrite.
- return;
- }; // if
- DWORD error = GetLastError();
- if ( error != ERROR_ENVVAR_NOT_FOUND ) {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantGetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- }; // if
- rc = SetEnvironmentVariable( name, value );
- if ( ! rc ) {
- DWORD error = GetLastError();
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantSetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- #else
- #error Unknown or unsupported OS.
- #endif
-
-} // func __kmp_env_set
-
-
-
-void
-__kmp_env_unset( char const * name ) {
-
- #if KMP_OS_UNIX
- unsetenv( name );
- #elif KMP_OS_WINDOWS
- BOOL rc = SetEnvironmentVariable( name, NULL );
- if ( ! rc ) {
- DWORD error = GetLastError();
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantSetEnvVar, name ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- #else
- #error Unknown or unsupported OS.
- #endif
-
-} // func __kmp_env_unset
-
-// -------------------------------------------------------------------------------------------------
-
-/*
- Intel OpenMP RTL string representation of environment: just a string of characters, variables
- are separated with vertical bars, e. g.:
-
- "KMP_WARNINGS=0|KMP_AFFINITY=compact|"
-
- Empty variables are allowed and ignored:
-
- "||KMP_WARNINGS=1||"
-
-*/
-
-static
-void
-___kmp_env_blk_parse_string(
- kmp_env_blk_t * block, // M: Env block to fill.
- char const * env // I: String to parse.
-) {
-
- char const chr_delimiter = '|';
- char const str_delimiter[] = { chr_delimiter, 0 };
-
- char * bulk = NULL;
- kmp_env_var_t * vars = NULL;
- int count = 0; // Number of used elements in vars array.
- int delimiters = 0; // Number of delimiters in input string.
-
- // Copy original string, we will modify the copy.
- bulk = __kmp_str_format( "%s", env );
-
- // Loop thru all the vars in environment block. Count delimiters (maximum number of variables
- // is number of delimiters plus one).
- {
- char const * ptr = bulk;
- for ( ; ; ) {
- ptr = strchr( ptr, chr_delimiter );
- if ( ptr == NULL ) {
- break;
- }; // if
- ++ delimiters;
- ptr += 1;
- }; // forever
- }
-
- // Allocate vars array.
- vars = (kmp_env_var_t *) allocate( ( delimiters + 1 ) * sizeof( kmp_env_var_t ) );
-
- // Loop thru all the variables.
- {
- char * var; // Pointer to variable (both name and value).
- char * name; // Pointer to name of variable.
- char * value; // Pointer to value.
- char * buf; // Buffer for __kmp_str_token() function.
- var = __kmp_str_token( bulk, str_delimiter, & buf ); // Get the first var.
- while ( var != NULL ) {
- // Save found variable in vars array.
- __kmp_str_split( var, '=', & name, & value );
- KMP_DEBUG_ASSERT( count < delimiters + 1 );
- vars[ count ].name = name;
- vars[ count ].value = value;
- ++ count;
- // Get the next var.
- var = __kmp_str_token( NULL, str_delimiter, & buf );
- }; // while
- }
-
- // Fill out result.
- block->bulk = bulk;
- block->vars = vars;
- block->count = count;
-
-}; // ___kmp_env_blk_parse_string
-
-
-
-/*
- Windows* OS (actually, DOS) environment block is a piece of memory with environment variables. Each
- variable is terminated with zero byte, entire block is terminated with one extra zero byte, so
- we have two zero bytes at the end of environment block, e. g.:
-
- "HOME=C:\\users\\lev\x00OS=Windows_NT\x00\x00"
-
- It is not clear how empty environment is represented. "\x00\x00"?
-*/
-
-#if KMP_OS_WINDOWS
-static
-void
-___kmp_env_blk_parse_windows(
- kmp_env_blk_t * block, // M: Env block to fill.
- char const * env // I: Pointer to Windows* OS (DOS) environment block.
-) {
-
- char * bulk = NULL;
- kmp_env_var_t * vars = NULL;
- int count = 0; // Number of used elements in vars array.
- int size = 0; // Size of bulk.
-
- char * name; // Pointer to name of variable.
- char * value; // Pointer to value.
-
- if ( env != NULL ) {
-
- // Loop thru all the vars in environment block. Count variables, find size of block.
- {
- char const * var; // Pointer to beginning of var.
- int len; // Length of variable.
- count = 0;
- var = env; // The first variable starts and beginning of environment block.
- len = KMP_STRLEN( var );
- while ( len != 0 ) {
- ++ count;
- size = size + len + 1;
- var = var + len + 1; // Move pointer to the beginning of the next variable.
- len = KMP_STRLEN( var );
- }; // while
- size = size + 1; // Total size of env block, including terminating zero byte.
- }
-
- // Copy original block to bulk, we will modify bulk, not original block.
- bulk = (char *) allocate( size );
- KMP_MEMCPY_S( bulk, size, env, size );
- // Allocate vars array.
- vars = (kmp_env_var_t *) allocate( count * sizeof( kmp_env_var_t ) );
-
- // Loop thru all the vars, now in bulk.
- {
- char * var; // Pointer to beginning of var.
- int len; // Length of variable.
- count = 0;
- var = bulk;
- len = KMP_STRLEN( var );
- while ( len != 0 ) {
- // Save variable in vars array.
- __kmp_str_split( var, '=', & name, & value );
- vars[ count ].name = name;
- vars[ count ].value = value;
- ++ count;
- // Get the next var.
- var = var + len + 1;
- len = KMP_STRLEN( var );
- }; // while
- }
-
- }; // if
-
- // Fill out result.
- block->bulk = bulk;
- block->vars = vars;
- block->count = count;
-
-}; // ___kmp_env_blk_parse_windows
-#endif
-
-
-/*
- Unix environment block is a array of pointers to variables, last pointer in array is NULL:
-
- { "HOME=/home/lev", "TERM=xterm", NULL }
-*/
-
-static
-void
-___kmp_env_blk_parse_unix(
- kmp_env_blk_t * block, // M: Env block to fill.
- char * * env // I: Unix environment to parse.
-) {
-
- char * bulk = NULL;
- kmp_env_var_t * vars = NULL;
- int count = 0;
- int size = 0; // Size of bulk.
-
- // Count number of variables and length of required bulk.
- {
- count = 0;
- size = 0;
- while ( env[ count ] != NULL ) {
- size += KMP_STRLEN( env[ count ] ) + 1;
- ++ count;
- }; // while
- }
-
- // Allocate memory.
- bulk = (char *) allocate( size );
- vars = (kmp_env_var_t *) allocate( count * sizeof( kmp_env_var_t ) );
-
- // Loop thru all the vars.
- {
- char * var; // Pointer to beginning of var.
- char * name; // Pointer to name of variable.
- char * value; // Pointer to value.
- int len; // Length of variable.
- int i;
- var = bulk;
- for ( i = 0; i < count; ++ i ) {
- // Copy variable to bulk.
- len = KMP_STRLEN( env[ i ] );
- KMP_MEMCPY_S( var, size, env[ i ], len + 1 );
- // Save found variable in vars array.
- __kmp_str_split( var, '=', & name, & value );
- vars[ i ].name = name;
- vars[ i ].value = value;
- // Move pointer.
- var += len + 1;
- }; // for
- }
-
- // Fill out result.
- block->bulk = bulk;
- block->vars = vars;
- block->count = count;
-
-}; // ___kmp_env_blk_parse_unix
-
-
-
-void
-__kmp_env_blk_init(
- kmp_env_blk_t * block, // M: Block to initialize.
- char const * bulk // I: Initialization string, or NULL.
-) {
-
- if ( bulk != NULL ) {
- ___kmp_env_blk_parse_string( block, bulk );
- } else {
- #if KMP_OS_UNIX
- ___kmp_env_blk_parse_unix( block, environ );
- #elif KMP_OS_WINDOWS
- {
- char * mem = GetEnvironmentStrings();
- if ( mem == NULL ) {
- DWORD error = GetLastError();
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantGetEnvironment ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }; // if
- ___kmp_env_blk_parse_windows( block, mem );
- FreeEnvironmentStrings( mem );
- }
- #else
- #error Unknown or unsupported OS.
- #endif
- }; // if
-
-} // __kmp_env_blk_init
-
-
-
-static
-int
-___kmp_env_var_cmp( // Comparison function for qsort().
- kmp_env_var_t const * lhs,
- kmp_env_var_t const * rhs
-) {
- return strcmp( lhs->name, rhs->name );
-}
-
-void
-__kmp_env_blk_sort(
- kmp_env_blk_t * block // M: Block of environment variables to sort.
-) {
-
- qsort(
- (void *) block->vars,
- block->count,
- sizeof( kmp_env_var_t ),
- ( int ( * )( void const *, void const * ) ) & ___kmp_env_var_cmp
- );
-
-} // __kmp_env_block_sort
-
-
-
-void
-__kmp_env_blk_free(
- kmp_env_blk_t * block // M: Block of environment variables to free.
-) {
-
- KMP_INTERNAL_FREE( (void *) block->vars );
- __kmp_str_free(&(block->bulk));
-
- block->count = 0;
- block->vars = NULL;
-
-} // __kmp_env_blk_free
-
-
-
-char const * // R: Value of variable or NULL if variable does not exist.
-__kmp_env_blk_var(
- kmp_env_blk_t * block, // I: Block of environment variables.
- char const * name // I: Name of variable to find.
-) {
-
- int i;
- for ( i = 0; i < block->count; ++ i ) {
- if ( strcmp( block->vars[ i ].name, name ) == 0 ) {
- return block->vars[ i ].value;
- }; // if
- }; // for
- return NULL;
-
-} // __kmp_env_block_var
-
-
-// end of file //
Removed: openmp/trunk/runtime/src/kmp_error.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_error.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_error.c (original)
+++ openmp/trunk/runtime/src/kmp_error.c (removed)
@@ -1,523 +0,0 @@
-/*
- * kmp_error.c -- KPTS functions for error checking at runtime
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_i18n.h"
-#include "kmp_str.h"
-#include "kmp_error.h"
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-#define MIN_STACK 100
-
-
-static char const * cons_text_c[] = {
- "(none)",
- "\"parallel\"",
- "work-sharing", /* this is not called "for" because of lowering of "sections" pragmas */
- "\"ordered\" work-sharing", /* this is not called "for ordered" because of lowering of "sections" pragmas */
- "\"sections\"",
- "work-sharing", /* this is not called "single" because of lowering of "sections" pragmas */
- "\"taskq\"",
- "\"taskq\"",
- "\"taskq ordered\"",
- "\"critical\"",
- "\"ordered\"", /* in PARALLEL */
- "\"ordered\"", /* in PDO */
- "\"ordered\"", /* in TASKQ */
- "\"master\"",
- "\"reduce\"",
- "\"barrier\""
-};
-
-#define get_src( ident ) ( (ident) == NULL ? NULL : (ident)->psource )
-
-#define PUSH_MSG( ct, ident ) \
- "\tpushing on stack: %s (%s)\n", cons_text_c[ (ct) ], get_src( (ident) )
-#define POP_MSG( p ) \
- "\tpopping off stack: %s (%s)\n", \
- cons_text_c[ (p)->stack_data[ tos ].type ], \
- get_src( (p)->stack_data[ tos ].ident )
-
-static int const cons_text_c_num = sizeof( cons_text_c ) / sizeof( char const * );
-
-/* ------------------------------------------------------------------------ */
-/* --------------- START OF STATIC LOCAL ROUTINES ------------------------- */
-/* ------------------------------------------------------------------------ */
-
-static void
-__kmp_check_null_func( void )
-{
- /* nothing to do */
-}
-
-static void
-__kmp_expand_cons_stack( int gtid, struct cons_header *p )
-{
- int i;
- struct cons_data *d;
-
- /* TODO for monitor perhaps? */
- if (gtid < 0)
- __kmp_check_null_func();
-
- KE_TRACE( 10, ("expand cons_stack (%d %d)\n", gtid, __kmp_get_gtid() ) );
-
- d = p->stack_data;
-
- p->stack_size = (p->stack_size * 2) + 100;
-
- /* TODO free the old data */
- p->stack_data = (struct cons_data *) __kmp_allocate( sizeof( struct cons_data ) * (p->stack_size+1) );
-
- for (i = p->stack_top; i >= 0; --i)
- p->stack_data[i] = d[i];
-
- /* NOTE: we do not free the old stack_data */
-}
-
-// NOTE: Function returns allocated memory, caller must free it!
-static char const *
-__kmp_pragma(
- int ct,
- ident_t const * ident
-) {
- char const * cons = NULL; // Construct name.
- char * file = NULL; // File name.
- char * func = NULL; // Function (routine) name.
- char * line = NULL; // Line number.
- kmp_str_buf_t buffer;
- kmp_msg_t prgm;
- __kmp_str_buf_init( & buffer );
- if ( 0 < ct && ct < cons_text_c_num ) {
- cons = cons_text_c[ ct ];
- } else {
- KMP_DEBUG_ASSERT( 0 );
- };
- if ( ident != NULL && ident->psource != NULL ) {
- char * tail = NULL;
- __kmp_str_buf_print( & buffer, "%s", ident->psource ); // Copy source to buffer.
- // Split string in buffer to file, func, and line.
- tail = buffer.str;
- __kmp_str_split( tail, ';', NULL, & tail );
- __kmp_str_split( tail, ';', & file, & tail );
- __kmp_str_split( tail, ';', & func, & tail );
- __kmp_str_split( tail, ';', & line, & tail );
- }; // if
- prgm = __kmp_msg_format( kmp_i18n_fmt_Pragma, cons, file, func, line );
- __kmp_str_buf_free( & buffer );
- return prgm.str;
-} // __kmp_pragma
-
-/* ------------------------------------------------------------------------ */
-/* ----------------- END OF STATIC LOCAL ROUTINES ------------------------- */
-/* ------------------------------------------------------------------------ */
-
-
-void
-__kmp_error_construct(
- kmp_i18n_id_t id, // Message identifier.
- enum cons_type ct, // Construct type.
- ident_t const * ident // Construct ident.
-) {
- char const * construct = __kmp_pragma( ct, ident );
- __kmp_msg( kmp_ms_fatal, __kmp_msg_format( id, construct ), __kmp_msg_null );
- KMP_INTERNAL_FREE( (void *) construct );
-}
-
-void
-__kmp_error_construct2(
- kmp_i18n_id_t id, // Message identifier.
- enum cons_type ct, // First construct type.
- ident_t const * ident, // First construct ident.
- struct cons_data const * cons // Second construct.
-) {
- char const * construct1 = __kmp_pragma( ct, ident );
- char const * construct2 = __kmp_pragma( cons->type, cons->ident );
- __kmp_msg( kmp_ms_fatal, __kmp_msg_format( id, construct1, construct2 ), __kmp_msg_null );
- KMP_INTERNAL_FREE( (void *) construct1 );
- KMP_INTERNAL_FREE( (void *) construct2 );
-}
-
-
-struct cons_header *
-__kmp_allocate_cons_stack( int gtid )
-{
- struct cons_header *p;
-
- /* TODO for monitor perhaps? */
- if ( gtid < 0 ) {
- __kmp_check_null_func();
- }; // if
- KE_TRACE( 10, ("allocate cons_stack (%d)\n", gtid ) );
- p = (struct cons_header *) __kmp_allocate( sizeof( struct cons_header ) );
- p->p_top = p->w_top = p->s_top = 0;
- p->stack_data = (struct cons_data *) __kmp_allocate( sizeof( struct cons_data ) * (MIN_STACK+1) );
- p->stack_size = MIN_STACK;
- p->stack_top = 0;
- p->stack_data[ 0 ].type = ct_none;
- p->stack_data[ 0 ].prev = 0;
- p->stack_data[ 0 ].ident = NULL;
- return p;
-}
-
-void
-__kmp_free_cons_stack( void * ptr ) {
- struct cons_header * p = (struct cons_header *) ptr;
- if ( p != NULL ) {
- if ( p->stack_data != NULL ) {
- __kmp_free( p->stack_data );
- p->stack_data = NULL;
- }; // if
- __kmp_free( p );
- }; // if
-}
-
-
-#if KMP_DEBUG
-static void
-dump_cons_stack( int gtid, struct cons_header * p ) {
- int i;
- int tos = p->stack_top;
- kmp_str_buf_t buffer;
- __kmp_str_buf_init( & buffer );
- __kmp_str_buf_print( & buffer, "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-\n" );
- __kmp_str_buf_print( & buffer, "Begin construct stack with %d items for thread %d\n", tos, gtid );
- __kmp_str_buf_print( & buffer, " stack_top=%d { P=%d, W=%d, S=%d }\n", tos, p->p_top, p->w_top, p->s_top );
- for ( i = tos; i > 0; i-- ) {
- struct cons_data * c = & ( p->stack_data[ i ] );
- __kmp_str_buf_print( & buffer, " stack_data[%2d] = { %s (%s) %d %p }\n", i, cons_text_c[ c->type ], get_src( c->ident ), c->prev, c->name );
- }; // for i
- __kmp_str_buf_print( & buffer, "End construct stack for thread %d\n", gtid );
- __kmp_str_buf_print( & buffer, "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-\n" );
- __kmp_debug_printf( "%s", buffer.str );
- __kmp_str_buf_free( & buffer );
-}
-#endif
-
-void
-__kmp_push_parallel( int gtid, ident_t const * ident )
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
-
- KMP_DEBUG_ASSERT( __kmp_threads[ gtid ]-> th.th_cons );
- KE_TRACE( 10, ("__kmp_push_parallel (%d %d)\n", gtid, __kmp_get_gtid() ) );
- KE_TRACE( 100, ( PUSH_MSG( ct_parallel, ident ) ) );
- if ( p->stack_top >= p->stack_size ) {
- __kmp_expand_cons_stack( gtid, p );
- }; // if
- tos = ++p->stack_top;
- p->stack_data[ tos ].type = ct_parallel;
- p->stack_data[ tos ].prev = p->p_top;
- p->stack_data[ tos ].ident = ident;
- p->stack_data[ tos ].name = NULL;
- p->p_top = tos;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
-}
-
-void
-__kmp_check_workshare( int gtid, enum cons_type ct, ident_t const * ident )
-{
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
-
- KMP_DEBUG_ASSERT( __kmp_threads[ gtid ]-> th.th_cons );
- KE_TRACE( 10, ("__kmp_check_workshare (%d %d)\n", gtid, __kmp_get_gtid() ) );
-
-
- if ( p->stack_top >= p->stack_size ) {
- __kmp_expand_cons_stack( gtid, p );
- }; // if
- if ( p->w_top > p->p_top &&
- !(IS_CONS_TYPE_TASKQ(p->stack_data[ p->w_top ].type) && IS_CONS_TYPE_TASKQ(ct))) {
- // We are already in a WORKSHARE construct for this PARALLEL region.
- __kmp_error_construct2( kmp_i18n_msg_CnsInvalidNesting, ct, ident, & p->stack_data[ p->w_top ] );
- }; // if
- if ( p->s_top > p->p_top ) {
- // We are already in a SYNC construct for this PARALLEL region.
- __kmp_error_construct2( kmp_i18n_msg_CnsInvalidNesting, ct, ident, & p->stack_data[ p->s_top ] );
- }; // if
-}
-
-void
-__kmp_push_workshare( int gtid, enum cons_type ct, ident_t const * ident )
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
- KE_TRACE( 10, ("__kmp_push_workshare (%d %d)\n", gtid, __kmp_get_gtid() ) );
- __kmp_check_workshare( gtid, ct, ident );
- KE_TRACE( 100, ( PUSH_MSG( ct, ident ) ) );
- tos = ++p->stack_top;
- p->stack_data[ tos ].type = ct;
- p->stack_data[ tos ].prev = p->w_top;
- p->stack_data[ tos ].ident = ident;
- p->stack_data[ tos ].name = NULL;
- p->w_top = tos;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
-}
-
-void
-#if KMP_USE_DYNAMIC_LOCK
-__kmp_check_sync( int gtid, enum cons_type ct, ident_t const * ident, kmp_user_lock_p lck, kmp_uint32 seq )
-#else
-__kmp_check_sync( int gtid, enum cons_type ct, ident_t const * ident, kmp_user_lock_p lck )
-#endif
-{
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
-
- KE_TRACE( 10, ("__kmp_check_sync (gtid=%d)\n", __kmp_get_gtid() ) );
-
- if (p->stack_top >= p->stack_size)
- __kmp_expand_cons_stack( gtid, p );
-
- if (ct == ct_ordered_in_parallel || ct == ct_ordered_in_pdo || ct == ct_ordered_in_taskq ) {
- if (p->w_top <= p->p_top) {
- /* we are not in a worksharing construct */
- #ifdef BUILD_PARALLEL_ORDERED
- /* do not report error messages for PARALLEL ORDERED */
- KMP_ASSERT( ct == ct_ordered_in_parallel );
- #else
- __kmp_error_construct( kmp_i18n_msg_CnsBoundToWorksharing, ct, ident );
- #endif /* BUILD_PARALLEL_ORDERED */
- } else {
- /* inside a WORKSHARING construct for this PARALLEL region */
- if (!IS_CONS_TYPE_ORDERED(p->stack_data[ p->w_top ].type)) {
- if (p->stack_data[ p->w_top ].type == ct_taskq) {
- __kmp_error_construct2(
- kmp_i18n_msg_CnsNotInTaskConstruct,
- ct, ident,
- & p->stack_data[ p->w_top ]
- );
- } else {
- __kmp_error_construct2(
- kmp_i18n_msg_CnsNoOrderedClause,
- ct, ident,
- & p->stack_data[ p->w_top ]
- );
- }
- }
- }
- if (p->s_top > p->p_top && p->s_top > p->w_top) {
- /* inside a sync construct which is inside a worksharing construct */
- int index = p->s_top;
- enum cons_type stack_type;
-
- stack_type = p->stack_data[ index ].type;
-
- if (stack_type == ct_critical ||
- ( ( stack_type == ct_ordered_in_parallel ||
- stack_type == ct_ordered_in_pdo ||
- stack_type == ct_ordered_in_taskq ) && /* C doesn't allow named ordered; ordered in ordered gets error */
- p->stack_data[ index ].ident != NULL &&
- (p->stack_data[ index ].ident->flags & KMP_IDENT_KMPC ))) {
- /* we are in ORDERED which is inside an ORDERED or CRITICAL construct */
- __kmp_error_construct2(
- kmp_i18n_msg_CnsInvalidNesting,
- ct, ident,
- & p->stack_data[ index ]
- );
- }
- }
- } else if ( ct == ct_critical ) {
-#if KMP_USE_DYNAMIC_LOCK
- if ( lck != NULL && __kmp_get_user_lock_owner( lck, seq ) == gtid ) { /* this same thread already has lock for this critical section */
-#else
- if ( lck != NULL && __kmp_get_user_lock_owner( lck ) == gtid ) { /* this same thread already has lock for this critical section */
-#endif
- int index = p->s_top;
- struct cons_data cons = { NULL, ct_critical, 0, NULL };
- /* walk up construct stack and try to find critical with matching name */
- while ( index != 0 && p->stack_data[ index ].name != lck ) {
- index = p->stack_data[ index ].prev;
- }
- if ( index != 0 ) {
- /* found match on the stack (may not always because of interleaved critical for Fortran) */
- cons = p->stack_data[ index ];
- }
- /* we are in CRITICAL which is inside a CRITICAL construct of the same name */
- __kmp_error_construct2( kmp_i18n_msg_CnsNestingSameName, ct, ident, & cons );
- }
- } else if ( ct == ct_master || ct == ct_reduce ) {
- if (p->w_top > p->p_top) {
- /* inside a WORKSHARING construct for this PARALLEL region */
- __kmp_error_construct2(
- kmp_i18n_msg_CnsInvalidNesting,
- ct, ident,
- & p->stack_data[ p->w_top ]
- );
- }
- if (ct == ct_reduce && p->s_top > p->p_top) {
- /* inside a another SYNC construct for this PARALLEL region */
- __kmp_error_construct2(
- kmp_i18n_msg_CnsInvalidNesting,
- ct, ident,
- & p->stack_data[ p->s_top ]
- );
- }; // if
- }; // if
-}
-
-void
-#if KMP_USE_DYNAMIC_LOCK
-__kmp_push_sync( int gtid, enum cons_type ct, ident_t const * ident, kmp_user_lock_p lck, kmp_uint32 seq )
-#else
-__kmp_push_sync( int gtid, enum cons_type ct, ident_t const * ident, kmp_user_lock_p lck )
-#endif
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
-
- KMP_ASSERT( gtid == __kmp_get_gtid() );
- KE_TRACE( 10, ("__kmp_push_sync (gtid=%d)\n", gtid ) );
-#if KMP_USE_DYNAMIC_LOCK
- __kmp_check_sync( gtid, ct, ident, lck, seq );
-#else
- __kmp_check_sync( gtid, ct, ident, lck );
-#endif
- KE_TRACE( 100, ( PUSH_MSG( ct, ident ) ) );
- tos = ++ p->stack_top;
- p->stack_data[ tos ].type = ct;
- p->stack_data[ tos ].prev = p->s_top;
- p->stack_data[ tos ].ident = ident;
- p->stack_data[ tos ].name = lck;
- p->s_top = tos;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
-}
-
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_pop_parallel( int gtid, ident_t const * ident )
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
- tos = p->stack_top;
- KE_TRACE( 10, ("__kmp_pop_parallel (%d %d)\n", gtid, __kmp_get_gtid() ) );
- if ( tos == 0 || p->p_top == 0 ) {
- __kmp_error_construct( kmp_i18n_msg_CnsDetectedEnd, ct_parallel, ident );
- }
- if ( tos != p->p_top || p->stack_data[ tos ].type != ct_parallel ) {
- __kmp_error_construct2(
- kmp_i18n_msg_CnsExpectedEnd,
- ct_parallel, ident,
- & p->stack_data[ tos ]
- );
- }
- KE_TRACE( 100, ( POP_MSG( p ) ) );
- p->p_top = p->stack_data[ tos ].prev;
- p->stack_data[ tos ].type = ct_none;
- p->stack_data[ tos ].ident = NULL;
- p->stack_top = tos - 1;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
-}
-
-enum cons_type
-__kmp_pop_workshare( int gtid, enum cons_type ct, ident_t const * ident )
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
-
- tos = p->stack_top;
- KE_TRACE( 10, ("__kmp_pop_workshare (%d %d)\n", gtid, __kmp_get_gtid() ) );
- if ( tos == 0 || p->w_top == 0 ) {
- __kmp_error_construct( kmp_i18n_msg_CnsDetectedEnd, ct, ident );
- }
-
- if ( tos != p->w_top ||
- ( p->stack_data[ tos ].type != ct &&
- /* below are two exceptions to the rule that construct types must match */
- ! ( p->stack_data[ tos ].type == ct_pdo_ordered && ct == ct_pdo ) &&
- ! ( p->stack_data[ tos ].type == ct_task_ordered && ct == ct_task )
- )
- ) {
- __kmp_check_null_func();
- __kmp_error_construct2(
- kmp_i18n_msg_CnsExpectedEnd,
- ct, ident,
- & p->stack_data[ tos ]
- );
- }
- KE_TRACE( 100, ( POP_MSG( p ) ) );
- p->w_top = p->stack_data[ tos ].prev;
- p->stack_data[ tos ].type = ct_none;
- p->stack_data[ tos ].ident = NULL;
- p->stack_top = tos - 1;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
- return p->stack_data[ p->w_top ].type;
-}
-
-void
-__kmp_pop_sync( int gtid, enum cons_type ct, ident_t const * ident )
-{
- int tos;
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
- tos = p->stack_top;
- KE_TRACE( 10, ("__kmp_pop_sync (%d %d)\n", gtid, __kmp_get_gtid() ) );
- if ( tos == 0 || p->s_top == 0 ) {
- __kmp_error_construct( kmp_i18n_msg_CnsDetectedEnd, ct, ident );
- };
- if ( tos != p->s_top || p->stack_data[ tos ].type != ct ) {
- __kmp_check_null_func();
- __kmp_error_construct2(
- kmp_i18n_msg_CnsExpectedEnd,
- ct, ident,
- & p->stack_data[ tos ]
- );
- };
- if ( gtid < 0 ) {
- __kmp_check_null_func();
- };
- KE_TRACE( 100, ( POP_MSG( p ) ) );
- p->s_top = p->stack_data[ tos ].prev;
- p->stack_data[ tos ].type = ct_none;
- p->stack_data[ tos ].ident = NULL;
- p->stack_top = tos - 1;
- KE_DUMP( 1000, dump_cons_stack( gtid, p ) );
-}
-
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_check_barrier( int gtid, enum cons_type ct, ident_t const * ident )
-{
- struct cons_header *p = __kmp_threads[ gtid ]->th.th_cons;
- KE_TRACE( 10, ("__kmp_check_barrier (loc: %p, gtid: %d %d)\n", ident, gtid, __kmp_get_gtid() ) );
- if ( ident != 0 ) {
- __kmp_check_null_func();
- }
- if ( p->w_top > p->p_top ) {
- /* we are already in a WORKSHARING construct for this PARALLEL region */
- __kmp_error_construct2(
- kmp_i18n_msg_CnsInvalidNesting,
- ct, ident,
- & p->stack_data[ p->w_top ]
- );
- }
- if (p->s_top > p->p_top) {
- /* we are already in a SYNC construct for this PARALLEL region */
- __kmp_error_construct2(
- kmp_i18n_msg_CnsInvalidNesting,
- ct, ident,
- & p->stack_data[ p->s_top ]
- );
- }
-}
-
-/* ------------------------------------------------------------------------ */
-
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
Removed: openmp/trunk/runtime/src/kmp_ftn_cdecl.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_ftn_cdecl.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_ftn_cdecl.c (original)
+++ openmp/trunk/runtime/src/kmp_ftn_cdecl.c (removed)
@@ -1,36 +0,0 @@
-/*
- * kmp_ftn_cdecl.c -- Fortran __cdecl linkage support for OpenMP.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_affinity.h"
-
-#if KMP_OS_WINDOWS
-# if defined KMP_WIN_CDECL || !defined KMP_DYNAMIC_LIB
-# define KMP_FTN_ENTRIES KMP_FTN_UPPER
-# endif
-#elif KMP_OS_UNIX
-# define KMP_FTN_ENTRIES KMP_FTN_PLAIN
-#endif
-
-// Note: This string is not printed when KMP_VERSION=1.
-char const __kmp_version_ftncdecl[] = KMP_VERSION_PREFIX "Fortran __cdecl OMP support: "
-#ifdef KMP_FTN_ENTRIES
- "yes";
-# define FTN_STDCALL /* no stdcall */
-# include "kmp_ftn_os.h"
-# include "kmp_ftn_entry.h"
-#else
- "no";
-#endif /* KMP_FTN_ENTRIES */
Removed: openmp/trunk/runtime/src/kmp_ftn_extra.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_ftn_extra.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_ftn_extra.c (original)
+++ openmp/trunk/runtime/src/kmp_ftn_extra.c (removed)
@@ -1,34 +0,0 @@
-/*
- * kmp_ftn_extra.c -- Fortran 'extra' linkage support for OpenMP.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_affinity.h"
-
-#if KMP_OS_WINDOWS
-# define KMP_FTN_ENTRIES KMP_FTN_PLAIN
-#elif KMP_OS_UNIX
-# define KMP_FTN_ENTRIES KMP_FTN_APPEND
-#endif
-
-// Note: This string is not printed when KMP_VERSION=1.
-char const __kmp_version_ftnextra[] = KMP_VERSION_PREFIX "Fortran \"extra\" OMP support: "
-#ifdef KMP_FTN_ENTRIES
- "yes";
-# define FTN_STDCALL /* nothing to do */
-# include "kmp_ftn_os.h"
-# include "kmp_ftn_entry.h"
-#else
- "no";
-#endif /* KMP_FTN_ENTRIES */
Removed: openmp/trunk/runtime/src/kmp_ftn_stdcall.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_ftn_stdcall.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_ftn_stdcall.c (original)
+++ openmp/trunk/runtime/src/kmp_ftn_stdcall.c (removed)
@@ -1,35 +0,0 @@
-/*
- * kmp_ftn_stdcall.c -- Fortran __stdcall linkage support for OpenMP.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-
-// Note: This string is not printed when KMP_VERSION=1.
-char const __kmp_version_ftnstdcall[] = KMP_VERSION_PREFIX "Fortran __stdcall OMP support: "
-#ifdef USE_FTN_STDCALL
- "yes";
-#else
- "no";
-#endif
-
-#ifdef USE_FTN_STDCALL
-
-#define FTN_STDCALL KMP_STDCALL
-#define KMP_FTN_ENTRIES USE_FTN_STDCALL
-
-#include "kmp_ftn_os.h"
-#include "kmp_ftn_entry.h"
-
-#endif /* USE_FTN_STDCALL */
-
Removed: openmp/trunk/runtime/src/kmp_global.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_global.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_global.c (original)
+++ openmp/trunk/runtime/src/kmp_global.c (removed)
@@ -1,497 +0,0 @@
-/*
- * kmp_global.c -- KPTS global variables for runtime support library
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_affinity.h"
-
-kmp_key_t __kmp_gtid_threadprivate_key;
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-kmp_cpuinfo_t __kmp_cpuinfo = { 0 }; // Not initialized
-#endif
-
-#if KMP_STATS_ENABLED
-#include "kmp_stats.h"
-// lock for modifying the global __kmp_stats_list
-kmp_tas_lock_t __kmp_stats_lock;
-
-// global list of per thread stats, the head is a sentinel node which accumulates all stats produced before __kmp_create_worker is called.
-kmp_stats_list* __kmp_stats_list;
-
-// thread local pointer to stats node within list
-__thread kmp_stats_list* __kmp_stats_thread_ptr = NULL;
-
-// gives reference tick for all events (considered the 0 tick)
-tsc_tick_count __kmp_stats_start_time;
-#endif
-
-/* ----------------------------------------------------- */
-/* INITIALIZATION VARIABLES */
-/* they are syncronized to write during init, but read anytime */
-volatile int __kmp_init_serial = FALSE;
-volatile int __kmp_init_gtid = FALSE;
-volatile int __kmp_init_common = FALSE;
-volatile int __kmp_init_middle = FALSE;
-volatile int __kmp_init_parallel = FALSE;
-#if KMP_USE_MONITOR
-volatile int __kmp_init_monitor = 0; /* 1 - launched, 2 - actually started (Windows* OS only) */
-#endif
-volatile int __kmp_init_user_locks = FALSE;
-
-/* list of address of allocated caches for commons */
-kmp_cached_addr_t *__kmp_threadpriv_cache_list = NULL;
-
-int __kmp_init_counter = 0;
-int __kmp_root_counter = 0;
-int __kmp_version = 0;
-
-volatile kmp_uint32 __kmp_team_counter = 0;
-volatile kmp_uint32 __kmp_task_counter = 0;
-
-unsigned int __kmp_init_wait = KMP_DEFAULT_INIT_WAIT; /* initial number of spin-tests */
-unsigned int __kmp_next_wait = KMP_DEFAULT_NEXT_WAIT; /* susequent number of spin-tests */
-
-size_t __kmp_stksize = KMP_DEFAULT_STKSIZE;
-#if KMP_USE_MONITOR
-size_t __kmp_monitor_stksize = 0; // auto adjust
-#endif
-size_t __kmp_stkoffset = KMP_DEFAULT_STKOFFSET;
-int __kmp_stkpadding = KMP_MIN_STKPADDING;
-
-size_t __kmp_malloc_pool_incr = KMP_DEFAULT_MALLOC_POOL_INCR;
-
-/* Barrier method defaults, settings, and strings */
-/* branch factor = 2^branch_bits (only relevant for tree and hyper barrier types) */
-#if KMP_ARCH_X86_64
-kmp_uint32 __kmp_barrier_gather_bb_dflt = 2; /* branch_factor = 4 */ /* hyper2: C78980 */
-kmp_uint32 __kmp_barrier_release_bb_dflt = 2; /* branch_factor = 4 */ /* hyper2: C78980 */
-#else
-kmp_uint32 __kmp_barrier_gather_bb_dflt = 2; /* branch_factor = 4 */ /* communication in core for MIC */
-kmp_uint32 __kmp_barrier_release_bb_dflt = 2; /* branch_factor = 4 */ /* communication in core for MIC */
-#endif // KMP_ARCH_X86_64
-#if KMP_ARCH_X86_64
-kmp_bar_pat_e __kmp_barrier_gather_pat_dflt = bp_hyper_bar; /* hyper2: C78980 */
-kmp_bar_pat_e __kmp_barrier_release_pat_dflt = bp_hyper_bar; /* hyper2: C78980 */
-#else
-kmp_bar_pat_e __kmp_barrier_gather_pat_dflt = bp_linear_bar;
-kmp_bar_pat_e __kmp_barrier_release_pat_dflt = bp_linear_bar;
-#endif
-kmp_uint32 __kmp_barrier_gather_branch_bits [ bs_last_barrier ] = { 0 };
-kmp_uint32 __kmp_barrier_release_branch_bits [ bs_last_barrier ] = { 0 };
-kmp_bar_pat_e __kmp_barrier_gather_pattern [ bs_last_barrier ] = { bp_linear_bar };
-kmp_bar_pat_e __kmp_barrier_release_pattern [ bs_last_barrier ] = { bp_linear_bar };
-char const *__kmp_barrier_branch_bit_env_name [ bs_last_barrier ] =
- { "KMP_PLAIN_BARRIER", "KMP_FORKJOIN_BARRIER"
- #if KMP_FAST_REDUCTION_BARRIER
- , "KMP_REDUCTION_BARRIER"
- #endif // KMP_FAST_REDUCTION_BARRIER
- };
-char const *__kmp_barrier_pattern_env_name [ bs_last_barrier ] =
- { "KMP_PLAIN_BARRIER_PATTERN", "KMP_FORKJOIN_BARRIER_PATTERN"
- #if KMP_FAST_REDUCTION_BARRIER
- , "KMP_REDUCTION_BARRIER_PATTERN"
- #endif // KMP_FAST_REDUCTION_BARRIER
- };
-char const *__kmp_barrier_type_name [ bs_last_barrier ] =
- { "plain", "forkjoin"
- #if KMP_FAST_REDUCTION_BARRIER
- , "reduction"
- #endif // KMP_FAST_REDUCTION_BARRIER
- };
-char const *__kmp_barrier_pattern_name[bp_last_bar] = {"linear","tree","hyper","hierarchical"};
-
-int __kmp_allThreadsSpecified = 0;
-size_t __kmp_align_alloc = CACHE_LINE;
-
-
-int __kmp_generate_warnings = kmp_warnings_low;
-int __kmp_reserve_warn = 0;
-int __kmp_xproc = 0;
-int __kmp_avail_proc = 0;
-size_t __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
-int __kmp_sys_max_nth = KMP_MAX_NTH;
-int __kmp_max_nth = 0;
-int __kmp_threads_capacity = 0;
-int __kmp_dflt_team_nth = 0;
-int __kmp_dflt_team_nth_ub = 0;
-int __kmp_tp_capacity = 0;
-int __kmp_tp_cached = 0;
-int __kmp_dflt_nested = FALSE;
-int __kmp_dispatch_num_buffers = KMP_DFLT_DISP_NUM_BUFF;
-int __kmp_dflt_max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT; /* max_active_levels limit */
-#if KMP_NESTED_HOT_TEAMS
-int __kmp_hot_teams_mode = 0; /* 0 - free extra threads when reduced */
- /* 1 - keep extra threads when reduced */
-int __kmp_hot_teams_max_level = 1; /* nesting level of hot teams */
-#endif
-enum library_type __kmp_library = library_none;
-enum sched_type __kmp_sched = kmp_sch_default; /* scheduling method for runtime scheduling */
-enum sched_type __kmp_static = kmp_sch_static_greedy; /* default static scheduling method */
-enum sched_type __kmp_guided = kmp_sch_guided_iterative_chunked; /* default guided scheduling method */
-enum sched_type __kmp_auto = kmp_sch_guided_analytical_chunked; /* default auto scheduling method */
-int __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
-#if KMP_USE_MONITOR
-int __kmp_monitor_wakeups = KMP_MIN_MONITOR_WAKEUPS;
-int __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( KMP_DEFAULT_BLOCKTIME, KMP_MIN_MONITOR_WAKEUPS );
-#endif
-#ifdef KMP_ADJUST_BLOCKTIME
-int __kmp_zero_bt = FALSE;
-#endif /* KMP_ADJUST_BLOCKTIME */
-#ifdef KMP_DFLT_NTH_CORES
-int __kmp_ncores = 0;
-#endif
-int __kmp_chunk = 0;
-int __kmp_abort_delay = 0;
-#if KMP_OS_LINUX && defined(KMP_TDATA_GTID)
-int __kmp_gtid_mode = 3; /* use __declspec(thread) TLS to store gtid */
-int __kmp_adjust_gtid_mode = FALSE;
-#elif KMP_OS_WINDOWS
-int __kmp_gtid_mode = 2; /* use TLS functions to store gtid */
-int __kmp_adjust_gtid_mode = FALSE;
-#else
-int __kmp_gtid_mode = 0; /* select method to get gtid based on #threads */
-int __kmp_adjust_gtid_mode = TRUE;
-#endif /* KMP_OS_LINUX && defined(KMP_TDATA_GTID) */
-#ifdef KMP_TDATA_GTID
-#if KMP_OS_WINDOWS
-__declspec(thread) int __kmp_gtid = KMP_GTID_DNE;
-#else
-__thread int __kmp_gtid = KMP_GTID_DNE;
-#endif /* KMP_OS_WINDOWS - workaround because Intel(R) Many Integrated Core compiler 20110316 doesn't accept __declspec */
-#endif /* KMP_TDATA_GTID */
-int __kmp_tls_gtid_min = INT_MAX;
-int __kmp_foreign_tp = TRUE;
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-int __kmp_inherit_fp_control = TRUE;
-kmp_int16 __kmp_init_x87_fpu_control_word = 0;
-kmp_uint32 __kmp_init_mxcsr = 0;
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-#ifdef USE_LOAD_BALANCE
-double __kmp_load_balance_interval = 1.0;
-#endif /* USE_LOAD_BALANCE */
-
-kmp_nested_nthreads_t __kmp_nested_nth = { NULL, 0, 0 };
-
-#if KMP_USE_ADAPTIVE_LOCKS
-
-kmp_adaptive_backoff_params_t __kmp_adaptive_backoff_params = { 1, 1024 }; // TODO: tune it!
-
-#if KMP_DEBUG_ADAPTIVE_LOCKS
-char * __kmp_speculative_statsfile = "-";
-#endif
-
-#endif // KMP_USE_ADAPTIVE_LOCKS
-
-#if OMP_40_ENABLED
-int __kmp_display_env = FALSE;
-int __kmp_display_env_verbose = FALSE;
-int __kmp_omp_cancellation = FALSE;
-#endif
-
-/* map OMP 3.0 schedule types with our internal schedule types */
-enum sched_type __kmp_sch_map[ kmp_sched_upper - kmp_sched_lower_ext + kmp_sched_upper_std - kmp_sched_lower - 2 ] = {
- kmp_sch_static_chunked, // ==> kmp_sched_static = 1
- kmp_sch_dynamic_chunked, // ==> kmp_sched_dynamic = 2
- kmp_sch_guided_chunked, // ==> kmp_sched_guided = 3
- kmp_sch_auto, // ==> kmp_sched_auto = 4
- kmp_sch_trapezoidal // ==> kmp_sched_trapezoidal = 101
- // will likely not used, introduced here just to debug the code
- // of public intel extension schedules
-};
-
-#if KMP_OS_LINUX
-enum clock_function_type __kmp_clock_function;
-int __kmp_clock_function_param;
-#endif /* KMP_OS_LINUX */
-
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
-enum mic_type __kmp_mic_type = non_mic;
-#endif
-
-#if KMP_AFFINITY_SUPPORTED
-
-KMPAffinity* __kmp_affinity_dispatch = NULL;
-
-# if KMP_USE_HWLOC
-int __kmp_hwloc_error = FALSE;
-hwloc_topology_t __kmp_hwloc_topology = NULL;
-# endif
-
-# if KMP_OS_WINDOWS
-# if KMP_GROUP_AFFINITY
-int __kmp_num_proc_groups = 1;
-# endif /* KMP_GROUP_AFFINITY */
-kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount = NULL;
-kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount = NULL;
-kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity = NULL;
-kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity = NULL;
-# endif /* KMP_OS_WINDOWS */
-
-size_t __kmp_affin_mask_size = 0;
-enum affinity_type __kmp_affinity_type = affinity_default;
-enum affinity_gran __kmp_affinity_gran = affinity_gran_default;
-int __kmp_affinity_gran_levels = -1;
-int __kmp_affinity_dups = TRUE;
-enum affinity_top_method __kmp_affinity_top_method = affinity_top_method_default;
-int __kmp_affinity_compact = 0;
-int __kmp_affinity_offset = 0;
-int __kmp_affinity_verbose = FALSE;
-int __kmp_affinity_warnings = TRUE;
-int __kmp_affinity_respect_mask = affinity_respect_mask_default;
-char * __kmp_affinity_proclist = NULL;
-kmp_affin_mask_t *__kmp_affinity_masks = NULL;
-unsigned __kmp_affinity_num_masks = 0;
-
-char const * __kmp_cpuinfo_file = NULL;
-
-#endif /* KMP_AFFINITY_SUPPORTED */
-
-#if OMP_40_ENABLED
-kmp_nested_proc_bind_t __kmp_nested_proc_bind = { NULL, 0, 0 };
-int __kmp_affinity_num_places = 0;
-#endif
-
-int __kmp_place_num_sockets = 0;
-int __kmp_place_socket_offset = 0;
-int __kmp_place_num_cores = 0;
-int __kmp_place_core_offset = 0;
-int __kmp_place_num_threads_per_core = 0;
-
-#if OMP_40_ENABLED
-kmp_int32 __kmp_default_device = 0;
-#endif
-
-kmp_tasking_mode_t __kmp_tasking_mode = tskm_task_teams;
-#if OMP_45_ENABLED
-kmp_int32 __kmp_max_task_priority = 0;
-#endif
-
-/* This check ensures that the compiler is passing the correct data type
- * for the flags formal parameter of the function kmpc_omp_task_alloc().
- * If the type is not a 4-byte type, then give an error message about
- * a non-positive length array pointing here. If that happens, the
- * kmp_tasking_flags_t structure must be redefined to have exactly 32 bits.
- */
-KMP_BUILD_ASSERT( sizeof(kmp_tasking_flags_t) == 4 );
-
-kmp_int32 __kmp_task_stealing_constraint = 1; /* Constrain task stealing by default */
-
-#ifdef DEBUG_SUSPEND
-int __kmp_suspend_count = 0;
-#endif
-
-int __kmp_settings = FALSE;
-int __kmp_duplicate_library_ok = 0;
-#if USE_ITT_BUILD
-int __kmp_forkjoin_frames = 1;
-int __kmp_forkjoin_frames_mode = 3;
-#endif
-PACKED_REDUCTION_METHOD_T __kmp_force_reduction_method = reduction_method_not_defined;
-int __kmp_determ_red = FALSE;
-
-#ifdef KMP_DEBUG
-int kmp_a_debug = 0;
-int kmp_b_debug = 0;
-int kmp_c_debug = 0;
-int kmp_d_debug = 0;
-int kmp_e_debug = 0;
-int kmp_f_debug = 0;
-int kmp_diag = 0;
-#endif
-
-/* For debug information logging using rotating buffer */
-int __kmp_debug_buf = FALSE; /* TRUE means use buffer, FALSE means print to stderr */
-int __kmp_debug_buf_lines = KMP_DEBUG_BUF_LINES_INIT; /* Lines of debug stored in buffer */
-int __kmp_debug_buf_chars = KMP_DEBUG_BUF_CHARS_INIT; /* Characters allowed per line in buffer */
-int __kmp_debug_buf_atomic = FALSE; /* TRUE means use atomic update of buffer entry pointer */
-
-char *__kmp_debug_buffer = NULL; /* Debug buffer itself */
-int __kmp_debug_count = 0; /* Counter for number of lines printed in buffer so far */
-int __kmp_debug_buf_warn_chars = 0; /* Keep track of char increase recommended in warnings */
-/* end rotating debug buffer */
-
-#ifdef KMP_DEBUG
-int __kmp_par_range; /* +1 => only go par for constructs in range */
- /* -1 => only go par for constructs outside range */
-char __kmp_par_range_routine[KMP_PAR_RANGE_ROUTINE_LEN] = { '\0' };
-char __kmp_par_range_filename[KMP_PAR_RANGE_FILENAME_LEN] = { '\0' };
-int __kmp_par_range_lb = 0;
-int __kmp_par_range_ub = INT_MAX;
-#endif /* KMP_DEBUG */
-
-/* For printing out dynamic storage map for threads and teams */
-int __kmp_storage_map = FALSE; /* True means print storage map for threads and teams */
-int __kmp_storage_map_verbose = FALSE; /* True means storage map includes placement info */
-int __kmp_storage_map_verbose_specified = FALSE;
-/* Initialize the library data structures when we fork a child process, defaults to TRUE */
-int __kmp_need_register_atfork = TRUE; /* At initialization, call pthread_atfork to install fork handler */
-int __kmp_need_register_atfork_specified = TRUE;
-
-int __kmp_env_chunk = FALSE; /* KMP_CHUNK specified? */
-int __kmp_env_stksize = FALSE; /* KMP_STACKSIZE specified? */
-int __kmp_env_omp_stksize = FALSE; /* OMP_STACKSIZE specified? */
-int __kmp_env_all_threads = FALSE;/* KMP_ALL_THREADS or KMP_MAX_THREADS specified? */
-int __kmp_env_omp_all_threads = FALSE;/* OMP_THREAD_LIMIT specified? */
-int __kmp_env_blocktime = FALSE; /* KMP_BLOCKTIME specified? */
-int __kmp_env_checks = FALSE; /* KMP_CHECKS specified? */
-int __kmp_env_consistency_check = FALSE; /* KMP_CONSISTENCY_CHECK specified? */
-
-kmp_uint32 __kmp_yield_init = KMP_INIT_WAIT;
-kmp_uint32 __kmp_yield_next = KMP_NEXT_WAIT;
-
-#if KMP_USE_MONITOR
-kmp_uint32 __kmp_yielding_on = 1;
-#if KMP_OS_CNK
-kmp_uint32 __kmp_yield_cycle = 0;
-#else
-kmp_uint32 __kmp_yield_cycle = 1; /* Yield-cycle is on by default */
-#endif
-kmp_int32 __kmp_yield_on_count = 10; /* By default, yielding is on for 10 monitor periods. */
-kmp_int32 __kmp_yield_off_count = 1; /* By default, yielding is off for 1 monitor periods. */
-#endif
-/* ----------------------------------------------------- */
-
-
-/* ------------------------------------------------------ */
-/* STATE mostly syncronized with global lock */
-/* data written to rarely by masters, read often by workers */
-/*
- * SHALL WE EDIT THE COMMENT BELOW IN SOME WAY?
- * TODO: None of this global padding stuff works consistently because
- * the order of declaration is not necessarily correlated to storage order.
- * To fix this, all the important globals must be put in a big structure
- * instead.
- */
-KMP_ALIGN_CACHE
- kmp_info_t **__kmp_threads = NULL;
- kmp_root_t **__kmp_root = NULL;
-
-/* data read/written to often by masters */
-KMP_ALIGN_CACHE
-volatile int __kmp_nth = 0;
-volatile int __kmp_all_nth = 0;
-int __kmp_thread_pool_nth = 0;
-volatile kmp_info_t *__kmp_thread_pool = NULL;
-volatile kmp_team_t *__kmp_team_pool = NULL;
-
-KMP_ALIGN_CACHE
-volatile int __kmp_thread_pool_active_nth = 0;
-
-/* -------------------------------------------------
- * GLOBAL/ROOT STATE */
-KMP_ALIGN_CACHE
-kmp_global_t __kmp_global = {{ 0 }};
-
-/* ----------------------------------------------- */
-/* GLOBAL SYNCHRONIZATION LOCKS */
-/* TODO verify the need for these locks and if they need to be global */
-
-#if KMP_USE_INTERNODE_ALIGNMENT
-/* Multinode systems have larger cache line granularity which can cause
- * false sharing if the alignment is not large enough for these locks */
-KMP_ALIGN_CACHE_INTERNODE
-
-kmp_bootstrap_lock_t __kmp_initz_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_initz_lock ); /* Control initializations */
-KMP_ALIGN_CACHE_INTERNODE
-kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
-KMP_ALIGN_CACHE_INTERNODE
-kmp_bootstrap_lock_t __kmp_exit_lock; /* exit() is not always thread-safe */
-#if KMP_USE_MONITOR
-KMP_ALIGN_CACHE_INTERNODE
-kmp_bootstrap_lock_t __kmp_monitor_lock; /* control monitor thread creation */
-#endif
-KMP_ALIGN_CACHE_INTERNODE
-kmp_bootstrap_lock_t __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and __kmp_threads expansion to co-exist */
-
-KMP_ALIGN_CACHE_INTERNODE
-kmp_lock_t __kmp_global_lock; /* Control OS/global access */
-KMP_ALIGN_CACHE_INTERNODE
-kmp_queuing_lock_t __kmp_dispatch_lock; /* Control dispatch access */
-KMP_ALIGN_CACHE_INTERNODE
-kmp_lock_t __kmp_debug_lock; /* Control I/O access for KMP_DEBUG */
-#else
-KMP_ALIGN_CACHE
-
-kmp_bootstrap_lock_t __kmp_initz_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_initz_lock ); /* Control initializations */
-kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */
-kmp_bootstrap_lock_t __kmp_exit_lock; /* exit() is not always thread-safe */
-#if KMP_USE_MONITOR
-kmp_bootstrap_lock_t __kmp_monitor_lock; /* control monitor thread creation */
-#endif
-kmp_bootstrap_lock_t __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and __kmp_threads expansion to co-exist */
-
-KMP_ALIGN(128)
-kmp_lock_t __kmp_global_lock; /* Control OS/global access */
-KMP_ALIGN(128)
-kmp_queuing_lock_t __kmp_dispatch_lock; /* Control dispatch access */
-KMP_ALIGN(128)
-kmp_lock_t __kmp_debug_lock; /* Control I/O access for KMP_DEBUG */
-#endif
-
-/* ----------------------------------------------- */
-
-#if KMP_HANDLE_SIGNALS
- /*
- Signal handling is disabled by default, because it confuses users: In case of sigsegv
- (or other trouble) in user code signal handler catches the signal, which then "appears" in
- the monitor thread (when the monitor executes raise() function). Users see signal in the
- monitor thread and blame OpenMP RTL.
-
- Grant said signal handling required on some older OSes (Irix?) supported by KAI, because
- bad applications hung but not aborted. Currently it is not a problem for Linux* OS, OS X* and
- Windows* OS.
-
- Grant: Found new hangs for EL4, EL5, and a Fedora Core machine. So I'm putting
- the default back for now to see if that fixes hangs on those machines.
-
- 2010-04013 Lev: It was a bug in Fortran RTL. Fortran RTL prints a kind of stack backtrace
- when program is aborting, but the code is not signal-safe. When multiple signals raised at
- the same time (which occurs in dynamic negative tests because all the worker threads detects
- the same error), Fortran RTL may hang. The bug finally fixed in Fortran RTL library provided
- by Steve R., and will be available soon.
- */
- int __kmp_handle_signals = FALSE;
-#endif
-
-/* ----------------------------------------------- */
-#ifdef BUILD_TV
-kmp_key_t __kmp_tv_key = 0;
-#endif
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-#ifdef DEBUG_SUSPEND
-int
-get_suspend_count_( void ) {
- int count = __kmp_suspend_count;
- __kmp_suspend_count = 0;
- return count;
-}
-void
-set_suspend_count_( int * value ) {
- __kmp_suspend_count = *value;
-}
-#endif
-
-// Symbols for MS mutual detection.
-int _You_must_link_with_exactly_one_OpenMP_library = 1;
-int _You_must_link_with_Intel_OpenMP_library = 1;
-#if KMP_OS_WINDOWS && ( KMP_VERSION_MAJOR > 4 )
- int _You_must_link_with_Microsoft_OpenMP_library = 1;
-#endif
-
-// end of file //
Removed: openmp/trunk/runtime/src/kmp_gsupport.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_gsupport.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_gsupport.c (original)
+++ openmp/trunk/runtime/src/kmp_gsupport.c (removed)
@@ -1,1621 +0,0 @@
-/*
- * kmp_gsupport.c
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_atomic.h"
-
-#if OMPT_SUPPORT
-#include "ompt-specific.h"
-#endif
-
-#ifdef __cplusplus
- extern "C" {
-#endif // __cplusplus
-
-#define MKLOC(loc,routine) \
- static ident_t (loc) = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;" };
-
-#include "kmp_ftn_os.h"
-
-void
-xexpand(KMP_API_NAME_GOMP_BARRIER)(void)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_barrier");
- KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid));
-#if OMPT_SUPPORT && OMPT_TRACE
- ompt_frame_t * ompt_frame;
- if (ompt_enabled ) {
- ompt_frame = __ompt_get_task_frame_internal(0);
- ompt_frame->reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
- __kmpc_barrier(&loc, gtid);
-}
-
-
-//
-// Mutual exclusion
-//
-
-//
-// The symbol that icc/ifort generates for unnamed for unnamed critical
-// sections - .gomp_critical_user_ - is defined using .comm in any objects
-// reference it. We can't reference it directly here in C code, as the
-// symbol contains a ".".
-//
-// The RTL contains an assembly language definition of .gomp_critical_user_
-// with another symbol __kmp_unnamed_critical_addr initialized with it's
-// address.
-//
-extern kmp_critical_name *__kmp_unnamed_critical_addr;
-
-
-void
-xexpand(KMP_API_NAME_GOMP_CRITICAL_START)(void)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_critical_start");
- KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid));
- __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_CRITICAL_END)(void)
-{
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_critical_end");
- KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid));
- __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_critical_name_start");
- KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid));
- __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr)
-{
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_critical_name_end");
- KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid));
- __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr);
-}
-
-
-//
-// The Gnu codegen tries to use locked operations to perform atomic updates
-// inline. If it can't, then it calls GOMP_atomic_start() before performing
-// the update and GOMP_atomic_end() afterward, regardless of the data type.
-//
-
-void
-xexpand(KMP_API_NAME_GOMP_ATOMIC_START)(void)
-{
- int gtid = __kmp_entry_gtid();
- KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid));
-
-#if OMPT_SUPPORT
- __ompt_thread_assign_wait_id(0);
-#endif
-
- __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_ATOMIC_END)(void)
-{
- int gtid = __kmp_get_gtid();
- KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid));
- __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
-}
-
-
-int
-xexpand(KMP_API_NAME_GOMP_SINGLE_START)(void)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_single_start");
- KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid));
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
- //
- // 3rd parameter == FALSE prevents kmp_enter_single from pushing a
- // workshare when USE_CHECKS is defined. We need to avoid the push,
- // as there is no corresponding GOMP_single_end() call.
- //
- return __kmp_enter_single(gtid, &loc, FALSE);
-}
-
-
-void *
-xexpand(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void)
-{
- void *retval;
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_single_copy_start");
- KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid));
-
- if (! TCR_4(__kmp_init_parallel))
- __kmp_parallel_initialize();
-
- //
- // If this is the first thread to enter, return NULL. The generated
- // code will then call GOMP_single_copy_end() for this thread only,
- // with the copyprivate data pointer as an argument.
- //
- if (__kmp_enter_single(gtid, &loc, FALSE))
- return NULL;
-
- //
- // Wait for the first thread to set the copyprivate data pointer,
- // and for all other threads to reach this point.
- //
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-
- //
- // Retrieve the value of the copyprivate data point, and wait for all
- // threads to do likewise, then return.
- //
- retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data;
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
- return retval;
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data)
-{
- int gtid = __kmp_get_gtid();
- KA_TRACE(20, ("GOMP_single_copy_end: T#%d\n", gtid));
-
- //
- // Set the copyprivate data pointer fo the team, then hit the barrier
- // so that the other threads will continue on and read it. Hit another
- // barrier before continuing, so that the know that the copyprivate
- // data pointer has been propagated to all threads before trying to
- // reuse the t_copypriv_data field.
- //
- __kmp_team_from_gtid(gtid)->t.t_copypriv_data = data;
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_ORDERED_START)(void)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_ordered_start");
- KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid));
- __kmpc_ordered(&loc, gtid);
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_ORDERED_END)(void)
-{
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_ordered_end");
- KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid));
- __kmpc_end_ordered(&loc, gtid);
-}
-
-
-//
-// Dispatch macro defs
-//
-// They come in two flavors: 64-bit unsigned, and either 32-bit signed
-// (IA-32 architecture) or 64-bit signed (Intel(R) 64).
-//
-
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS
-# define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4
-# define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4
-# define KMP_DISPATCH_NEXT __kmpc_dispatch_next_4
-#else
-# define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_8
-# define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_8
-# define KMP_DISPATCH_NEXT __kmpc_dispatch_next_8
-#endif /* KMP_ARCH_X86 */
-
-# define KMP_DISPATCH_INIT_ULL __kmp_aux_dispatch_init_8u
-# define KMP_DISPATCH_FINI_CHUNK_ULL __kmp_aux_dispatch_fini_chunk_8u
-# define KMP_DISPATCH_NEXT_ULL __kmpc_dispatch_next_8u
-
-
-//
-// The parallel contruct
-//
-
-#ifndef KMP_DEBUG
-static
-#endif /* KMP_DEBUG */
-void
-__kmp_GOMP_microtask_wrapper(int *gtid, int *npr, void (*task)(void *),
- void *data)
-{
-#if OMPT_SUPPORT
- kmp_info_t *thr;
- ompt_frame_t *ompt_frame;
- ompt_state_t enclosing_state;
-
- if (ompt_enabled) {
- // get pointer to thread data structure
- thr = __kmp_threads[*gtid];
-
- // save enclosing task state; set current state for task
- enclosing_state = thr->th.ompt_thread_info.state;
- thr->th.ompt_thread_info.state = ompt_state_work_parallel;
-
- // set task frame
- ompt_frame = __ompt_get_task_frame_internal(0);
- ompt_frame->exit_runtime_frame = __builtin_frame_address(0);
- }
-#endif
-
- task(data);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- // clear task frame
- ompt_frame->exit_runtime_frame = NULL;
-
- // restore enclosing state
- thr->th.ompt_thread_info.state = enclosing_state;
- }
-#endif
-}
-
-
-#ifndef KMP_DEBUG
-static
-#endif /* KMP_DEBUG */
-void
-__kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr,
- void (*task)(void *), void *data, unsigned num_threads, ident_t *loc,
- enum sched_type schedule, long start, long end, long incr, long chunk_size)
-{
- //
- // Intialize the loop worksharing construct.
- //
- KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size,
- schedule != kmp_sch_static);
-
-#if OMPT_SUPPORT
- kmp_info_t *thr;
- ompt_frame_t *ompt_frame;
- ompt_state_t enclosing_state;
-
- if (ompt_enabled) {
- thr = __kmp_threads[*gtid];
- // save enclosing task state; set current state for task
- enclosing_state = thr->th.ompt_thread_info.state;
- thr->th.ompt_thread_info.state = ompt_state_work_parallel;
-
- // set task frame
- ompt_frame = __ompt_get_task_frame_internal(0);
- ompt_frame->exit_runtime_frame = __builtin_frame_address(0);
- }
-#endif
-
- //
- // Now invoke the microtask.
- //
- task(data);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- // clear task frame
- ompt_frame->exit_runtime_frame = NULL;
-
- // reset enclosing state
- thr->th.ompt_thread_info.state = enclosing_state;
- }
-#endif
-}
-
-
-#ifndef KMP_DEBUG
-static
-#endif /* KMP_DEBUG */
-void
-__kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), microtask_t wrapper, int argc,...)
-{
- int rc;
- kmp_info_t *thr = __kmp_threads[gtid];
- kmp_team_t *team = thr->th.th_team;
- int tid = __kmp_tid_from_gtid(gtid);
-
- va_list ap;
- va_start(ap, argc);
-
- rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc,
-#if OMPT_SUPPORT
- VOLATILE_CAST(void *) unwrapped_task,
-#endif
- wrapper, __kmp_invoke_task_func,
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- &ap
-#else
- ap
-#endif
- );
-
- va_end(ap);
-
- if (rc) {
- __kmp_run_before_invoked_task(gtid, tid, thr, team);
- }
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
-#if OMPT_TRACE
- ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
-
- // implicit task callback
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- team_info->parallel_id, task_info->task_id);
- }
-#endif
- thr->th.ompt_thread_info.state = ompt_state_work_parallel;
- }
-#endif
-}
-
-static void
-__kmp_GOMP_serialized_parallel(ident_t *loc, kmp_int32 gtid, void (*task)(void *))
-{
-#if OMPT_SUPPORT
- ompt_parallel_id_t ompt_parallel_id;
- if (ompt_enabled) {
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
-
- ompt_parallel_id = __ompt_parallel_id_new(gtid);
-
- // parallel region callback
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) {
- int team_size = 1;
- ompt_callbacks.ompt_callback(ompt_event_parallel_begin)(
- task_info->task_id, &task_info->frame, ompt_parallel_id,
- team_size, (void *) task,
- OMPT_INVOKER(fork_context_gnu));
- }
- }
-#endif
-
- __kmp_serialized_parallel(loc, gtid);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- kmp_info_t *thr = __kmp_threads[gtid];
-
- ompt_task_id_t my_ompt_task_id = __ompt_task_id_new(gtid);
-
- // set up lightweight task
- ompt_lw_taskteam_t *lwt = (ompt_lw_taskteam_t *)
- __kmp_allocate(sizeof(ompt_lw_taskteam_t));
- __ompt_lw_taskteam_init(lwt, thr, gtid, (void *) task, ompt_parallel_id);
- lwt->ompt_task_info.task_id = my_ompt_task_id;
- __ompt_lw_taskteam_link(lwt, thr);
-
-#if OMPT_TRACE
- // implicit task callback
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- ompt_parallel_id, my_ompt_task_id);
- }
- thr->th.ompt_thread_info.state = ompt_state_work_parallel;
-#endif
- }
-#endif
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), void *data, unsigned num_threads)
-{
- int gtid = __kmp_entry_gtid();
-
-#if OMPT_SUPPORT
- ompt_frame_t *parent_frame, *frame;
-
- if (ompt_enabled) {
- parent_frame = __ompt_get_task_frame_internal(0);
- parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
-
- MKLOC(loc, "GOMP_parallel_start");
- KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid));
-
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
- if (num_threads != 0) {
- __kmp_push_num_threads(&loc, gtid, num_threads);
- }
- __kmp_GOMP_fork_call(&loc, gtid, task,
- (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, data);
- }
- else {
- __kmp_GOMP_serialized_parallel(&loc, gtid, task);
- }
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- frame = __ompt_get_task_frame_internal(0);
- frame->exit_runtime_frame = __builtin_frame_address(1);
- }
-#endif
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_PARALLEL_END)(void)
-{
- int gtid = __kmp_get_gtid();
- kmp_info_t *thr;
-
- thr = __kmp_threads[gtid];
-
- MKLOC(loc, "GOMP_parallel_end");
- KA_TRACE(20, ("GOMP_parallel_end: T#%d\n", gtid));
-
-
-#if OMPT_SUPPORT
- ompt_parallel_id_t parallel_id;
- ompt_task_id_t serialized_task_id;
- ompt_frame_t *ompt_frame = NULL;
-
- if (ompt_enabled) {
- ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
- parallel_id = team_info->parallel_id;
-
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
- serialized_task_id = task_info->task_id;
-
- // unlink if necessary. no-op if there is not a lightweight task.
- ompt_lw_taskteam_t *lwt = __ompt_lw_taskteam_unlink(thr);
- // GOMP allocates/frees lwt since it can't be kept on the stack
- if (lwt) {
- __kmp_free(lwt);
-
- }
- }
-#endif
-
- if (! thr->th.th_team->t.t_serialized) {
- __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr,
- thr->th.th_team);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- // Implicit task is finished here, in the barrier we might schedule deferred tasks,
- // these don't see the implicit task on the stack
- ompt_frame = __ompt_get_task_frame_internal(0);
- ompt_frame->exit_runtime_frame = NULL;
- }
-#endif
-
- __kmp_join_call(&loc, gtid
-#if OMPT_SUPPORT
- , fork_context_gnu
-#endif
- );
- }
- else {
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- parallel_id, serialized_task_id);
- }
-#endif
-
- __kmpc_end_serialized_parallel(&loc, gtid);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- // Record that we re-entered the runtime system in the frame that
- // created the parallel region.
- ompt_task_info_t *parent_task_info = __ompt_get_taskinfo(0);
-
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
- ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
- parallel_id, parent_task_info->task_id,
- OMPT_INVOKER(fork_context_gnu));
- }
-
- parent_task_info->frame.reenter_runtime_frame = NULL;
-
- thr->th.ompt_thread_info.state =
- (((thr->th.th_team)->t.t_serialized) ?
- ompt_state_work_serial : ompt_state_work_parallel);
- }
-#endif
- }
-}
-
-
-//
-// Loop worksharing constructs
-//
-
-//
-// The Gnu codegen passes in an exclusive upper bound for the overall range,
-// but the libguide dispatch code expects an inclusive upper bound, hence the
-// "end - incr" 5th argument to KMP_DISPATCH_INIT (and the " ub - str" 11th
-// argument to __kmp_GOMP_fork_call).
-//
-// Conversely, KMP_DISPATCH_NEXT returns and inclusive upper bound in *p_ub,
-// but the Gnu codegen expects an excluside upper bound, so the adjustment
-// "*p_ub += stride" compenstates for the discrepancy.
-//
-// Correction: the gnu codegen always adjusts the upper bound by +-1, not the
-// stride value. We adjust the dispatch parameters accordingly (by +-1), but
-// we still adjust p_ub by the actual stride value.
-//
-// The "runtime" versions do not take a chunk_sz parameter.
-//
-// The profile lib cannot support construct checking of unordered loops that
-// are predetermined by the compiler to be statically scheduled, as the gcc
-// codegen will not always emit calls to GOMP_loop_static_next() to get the
-// next iteration. Instead, it emits inline code to call omp_get_thread_num()
-// num and calculate the iteration space using the result. It doesn't do this
-// with ordered static loop, so they can be checked.
-//
-
-#define LOOP_START(func,schedule) \
- int func (long lb, long ub, long str, long chunk_sz, long *p_lb, \
- long *p_ub) \
- { \
- int status; \
- long stride; \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \
- gtid, lb, ub, str, chunk_sz )); \
- \
- if ((str > 0) ? (lb < ub) : (lb > ub)) { \
- KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \
- (schedule) != kmp_sch_static); \
- status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \
- (kmp_int *)p_ub, (kmp_int *)&stride); \
- if (status) { \
- KMP_DEBUG_ASSERT(stride == str); \
- *p_ub += (str > 0) ? 1 : -1; \
- } \
- } \
- else { \
- status = 0; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \
- gtid, *p_lb, *p_ub, status)); \
- return status; \
- }
-
-
-#define LOOP_RUNTIME_START(func,schedule) \
- int func (long lb, long ub, long str, long *p_lb, long *p_ub) \
- { \
- int status; \
- long stride; \
- long chunk_sz = 0; \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \
- gtid, lb, ub, str, chunk_sz )); \
- \
- if ((str > 0) ? (lb < ub) : (lb > ub)) { \
- KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \
- status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \
- (kmp_int *)p_ub, (kmp_int *)&stride); \
- if (status) { \
- KMP_DEBUG_ASSERT(stride == str); \
- *p_ub += (str > 0) ? 1 : -1; \
- } \
- } \
- else { \
- status = 0; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \
- gtid, *p_lb, *p_ub, status)); \
- return status; \
- }
-
-
-#define LOOP_NEXT(func,fini_code) \
- int func(long *p_lb, long *p_ub) \
- { \
- int status; \
- long stride; \
- int gtid = __kmp_get_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d\n", gtid)); \
- \
- fini_code \
- status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \
- (kmp_int *)p_ub, (kmp_int *)&stride); \
- if (status) { \
- *p_ub += (stride > 0) ? 1 : -1; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, stride 0x%lx, " \
- "returning %d\n", gtid, *p_lb, *p_ub, stride, status)); \
- return status; \
- }
-
-
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_STATIC_START), kmp_sch_static)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT), {})
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START), kmp_sch_dynamic_chunked)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT), {})
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_GUIDED_START), kmp_sch_guided_chunked)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT), {})
-LOOP_RUNTIME_START(xexpand(KMP_API_NAME_GOMP_LOOP_RUNTIME_START), kmp_sch_runtime)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {})
-
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START), kmp_ord_static)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START), kmp_ord_dynamic_chunked)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
-LOOP_START(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START), kmp_ord_guided_chunked)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
-LOOP_RUNTIME_START(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START), kmp_ord_runtime)
-LOOP_NEXT(xexpand(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); })
-
-
-void
-xexpand(KMP_API_NAME_GOMP_LOOP_END)(void)
-{
- int gtid = __kmp_get_gtid();
- KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid))
-
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-
- KA_TRACE(20, ("GOMP_loop_end exit: T#%d\n", gtid))
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_LOOP_END_NOWAIT)(void)
-{
- KA_TRACE(20, ("GOMP_loop_end_nowait: T#%d\n", __kmp_get_gtid()))
-}
-
-
-//
-// Unsigned long long loop worksharing constructs
-//
-// These are new with gcc 4.4
-//
-
-#define LOOP_START_ULL(func,schedule) \
- int func (int up, unsigned long long lb, unsigned long long ub, \
- unsigned long long str, unsigned long long chunk_sz, \
- unsigned long long *p_lb, unsigned long long *p_ub) \
- { \
- int status; \
- long long str2 = up ? ((long long)str) : -((long long)str); \
- long long stride; \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- \
- KA_TRACE(20, ( #func ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str 0x%llx, chunk_sz 0x%llx\n", \
- gtid, up, lb, ub, str, chunk_sz )); \
- \
- if ((str > 0) ? (lb < ub) : (lb > ub)) { \
- KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \
- (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \
- (schedule) != kmp_sch_static); \
- status = KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, \
- (kmp_uint64 *)p_lb, (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \
- if (status) { \
- KMP_DEBUG_ASSERT(stride == str2); \
- *p_ub += (str > 0) ? 1 : -1; \
- } \
- } \
- else { \
- status = 0; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \
- gtid, *p_lb, *p_ub, status)); \
- return status; \
- }
-
-
-#define LOOP_RUNTIME_START_ULL(func,schedule) \
- int func (int up, unsigned long long lb, unsigned long long ub, \
- unsigned long long str, unsigned long long *p_lb, \
- unsigned long long *p_ub) \
- { \
- int status; \
- long long str2 = up ? ((long long)str) : -((long long)str); \
- unsigned long long stride; \
- unsigned long long chunk_sz = 0; \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- \
- KA_TRACE(20, ( #func ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str 0x%llx, chunk_sz 0x%llx\n", \
- gtid, up, lb, ub, str, chunk_sz )); \
- \
- if ((str > 0) ? (lb < ub) : (lb > ub)) { \
- KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \
- (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, TRUE); \
- status = KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, \
- (kmp_uint64 *)p_lb, (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \
- if (status) { \
- KMP_DEBUG_ASSERT((long long)stride == str2); \
- *p_ub += (str > 0) ? 1 : -1; \
- } \
- } \
- else { \
- status = 0; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \
- gtid, *p_lb, *p_ub, status)); \
- return status; \
- }
-
-
-#define LOOP_NEXT_ULL(func,fini_code) \
- int func(unsigned long long *p_lb, unsigned long long *p_ub) \
- { \
- int status; \
- long long stride; \
- int gtid = __kmp_get_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d\n", gtid)); \
- \
- fini_code \
- status = KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \
- (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \
- if (status) { \
- *p_ub += (stride > 0) ? 1 : -1; \
- } \
- \
- KA_TRACE(20, ( #func " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, stride 0x%llx, " \
- "returning %d\n", gtid, *p_lb, *p_ub, stride, status)); \
- return status; \
- }
-
-
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START), kmp_sch_static)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT), {})
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START), kmp_sch_dynamic_chunked)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT), {})
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START), kmp_sch_guided_chunked)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT), {})
-LOOP_RUNTIME_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {})
-
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START), kmp_ord_static)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START), kmp_ord_dynamic_chunked)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
-LOOP_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START), kmp_ord_guided_chunked)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
-LOOP_RUNTIME_START_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START), kmp_ord_runtime)
-LOOP_NEXT_ULL(xexpand(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), \
- { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); })
-
-
-//
-// Combined parallel / loop worksharing constructs
-//
-// There are no ull versions (yet).
-//
-
-#define PARALLEL_LOOP_START(func, schedule, ompt_pre, ompt_post) \
- void func (void (*task) (void *), void *data, unsigned num_threads, \
- long lb, long ub, long str, long chunk_sz) \
- { \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \
- gtid, lb, ub, str, chunk_sz )); \
- \
- ompt_pre(); \
- \
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \
- if (num_threads != 0) { \
- __kmp_push_num_threads(&loc, gtid, num_threads); \
- } \
- __kmp_GOMP_fork_call(&loc, gtid, task, \
- (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, \
- task, data, num_threads, &loc, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \
- } \
- else { \
- __kmp_GOMP_serialized_parallel(&loc, gtid, task); \
- } \
- \
- KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \
- (schedule) != kmp_sch_static); \
- \
- ompt_post(); \
- \
- KA_TRACE(20, ( #func " exit: T#%d\n", gtid)); \
- }
-
-
-
-#if OMPT_SUPPORT
-
-#define OMPT_LOOP_PRE() \
- ompt_frame_t *parent_frame; \
- if (ompt_enabled) { \
- parent_frame = __ompt_get_task_frame_internal(0); \
- parent_frame->reenter_runtime_frame = __builtin_frame_address(1); \
- }
-
-
-#define OMPT_LOOP_POST() \
- if (ompt_enabled) { \
- parent_frame->reenter_runtime_frame = NULL; \
- }
-
-#else
-
-#define OMPT_LOOP_PRE()
-
-#define OMPT_LOOP_POST()
-
-#endif
-
-
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START),
- kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START),
- kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START),
- kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP_START(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START),
- kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST)
-
-
-//
-// Tasking constructs
-//
-
-void
-xexpand(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, void (*copy_func)(void *, void *),
- long arg_size, long arg_align, bool if_cond, unsigned gomp_flags)
-{
- MKLOC(loc, "GOMP_task");
- int gtid = __kmp_entry_gtid();
- kmp_int32 flags = 0;
- kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *) & flags;
-
- KA_TRACE(20, ("GOMP_task: T#%d\n", gtid));
-
- // The low-order bit is the "tied" flag
- if (gomp_flags & 1) {
- input_flags->tiedness = 1;
- }
- // The second low-order bit is the "final" flag
- if (gomp_flags & 2) {
- input_flags->final = 1;
- }
- input_flags->native = 1;
- // __kmp_task_alloc() sets up all other flags
-
- if (! if_cond) {
- arg_size = 0;
- }
-
- kmp_task_t *task = __kmp_task_alloc(&loc, gtid, input_flags,
- sizeof(kmp_task_t), arg_size ? arg_size + arg_align - 1 : 0,
- (kmp_routine_entry_t)func);
-
- if (arg_size > 0) {
- if (arg_align > 0) {
- task->shareds = (void *)((((size_t)task->shareds)
- + arg_align - 1) / arg_align * arg_align);
- }
- //else error??
-
- if (copy_func) {
- (*copy_func)(task->shareds, data);
- }
- else {
- KMP_MEMCPY(task->shareds, data, arg_size);
- }
- }
-
- if (if_cond) {
- __kmpc_omp_task(&loc, gtid, task);
- }
- else {
-#if OMPT_SUPPORT
- ompt_thread_info_t oldInfo;
- kmp_info_t *thread;
- kmp_taskdata_t *taskdata;
- if (ompt_enabled) {
- // Store the threads states and restore them after the task
- thread = __kmp_threads[ gtid ];
- taskdata = KMP_TASK_TO_TASKDATA(task);
- oldInfo = thread->th.ompt_thread_info;
- thread->th.ompt_thread_info.wait_id = 0;
- thread->th.ompt_thread_info.state = ompt_state_work_parallel;
- taskdata->ompt_task_info.frame.exit_runtime_frame =
- __builtin_frame_address(0);
- }
-#endif
-
- __kmpc_omp_task_begin_if0(&loc, gtid, task);
- func(data);
- __kmpc_omp_task_complete_if0(&loc, gtid, task);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- thread->th.ompt_thread_info = oldInfo;
- taskdata->ompt_task_info.frame.exit_runtime_frame = NULL;
- }
-#endif
- }
-
- KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid));
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_TASKWAIT)(void)
-{
- MKLOC(loc, "GOMP_taskwait");
- int gtid = __kmp_entry_gtid();
-
- KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid));
-
- __kmpc_omp_taskwait(&loc, gtid);
-
- KA_TRACE(20, ("GOMP_taskwait exit: T#%d\n", gtid));
-}
-
-
-//
-// Sections worksharing constructs
-//
-
-//
-// For the sections construct, we initialize a dynamically scheduled loop
-// worksharing construct with lb 1 and stride 1, and use the iteration #'s
-// that its returns as sections ids.
-//
-// There are no special entry points for ordered sections, so we always use
-// the dynamically scheduled workshare, even if the sections aren't ordered.
-//
-
-unsigned
-xexpand(KMP_API_NAME_GOMP_SECTIONS_START)(unsigned count)
-{
- int status;
- kmp_int lb, ub, stride;
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_sections_start");
- KA_TRACE(20, ("GOMP_sections_start: T#%d\n", gtid));
-
- KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
-
- status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride);
- if (status) {
- KMP_DEBUG_ASSERT(stride == 1);
- KMP_DEBUG_ASSERT(lb > 0);
- KMP_ASSERT(lb == ub);
- }
- else {
- lb = 0;
- }
-
- KA_TRACE(20, ("GOMP_sections_start exit: T#%d returning %u\n", gtid,
- (unsigned)lb));
- return (unsigned)lb;
-}
-
-
-unsigned
-xexpand(KMP_API_NAME_GOMP_SECTIONS_NEXT)(void)
-{
- int status;
- kmp_int lb, ub, stride;
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_sections_next");
- KA_TRACE(20, ("GOMP_sections_next: T#%d\n", gtid));
-
- status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride);
- if (status) {
- KMP_DEBUG_ASSERT(stride == 1);
- KMP_DEBUG_ASSERT(lb > 0);
- KMP_ASSERT(lb == ub);
- }
- else {
- lb = 0;
- }
-
- KA_TRACE(20, ("GOMP_sections_next exit: T#%d returning %u\n", gtid,
- (unsigned)lb));
- return (unsigned)lb;
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)(void (*task) (void *), void *data,
- unsigned num_threads, unsigned count)
-{
- int gtid = __kmp_entry_gtid();
-
-#if OMPT_SUPPORT
- ompt_frame_t *parent_frame;
-
- if (ompt_enabled) {
- parent_frame = __ompt_get_task_frame_internal(0);
- parent_frame->reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
-
- MKLOC(loc, "GOMP_parallel_sections_start");
- KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid));
-
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
- if (num_threads != 0) {
- __kmp_push_num_threads(&loc, gtid, num_threads);
- }
- __kmp_GOMP_fork_call(&loc, gtid, task,
- (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, task, data,
- num_threads, &loc, kmp_nm_dynamic_chunked, (kmp_int)1,
- (kmp_int)count, (kmp_int)1, (kmp_int)1);
- }
- else {
- __kmp_GOMP_serialized_parallel(&loc, gtid, task);
- }
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- parent_frame->reenter_runtime_frame = NULL;
- }
-#endif
-
- KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
-
- KA_TRACE(20, ("GOMP_parallel_sections_start exit: T#%d\n", gtid));
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_SECTIONS_END)(void)
-{
- int gtid = __kmp_get_gtid();
- KA_TRACE(20, ("GOMP_sections_end: T#%d\n", gtid))
-
- __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
-
- KA_TRACE(20, ("GOMP_sections_end exit: T#%d\n", gtid))
-}
-
-
-void
-xexpand(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT)(void)
-{
- KA_TRACE(20, ("GOMP_sections_end_nowait: T#%d\n", __kmp_get_gtid()))
-}
-
-// libgomp has an empty function for GOMP_taskyield as of 2013-10-10
-void
-xexpand(KMP_API_NAME_GOMP_TASKYIELD)(void)
-{
- KA_TRACE(20, ("GOMP_taskyield: T#%d\n", __kmp_get_gtid()))
- return;
-}
-
-#if OMP_40_ENABLED // these are new GOMP_4.0 entry points
-
-void
-xexpand(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), void *data, unsigned num_threads, unsigned int flags)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_parallel");
- KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid));
-
-#if OMPT_SUPPORT
- ompt_task_info_t *parent_task_info, *task_info;
- if (ompt_enabled) {
- parent_task_info = __ompt_get_taskinfo(0);
- parent_task_info->frame.reenter_runtime_frame = __builtin_frame_address(1);
- }
-#endif
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
- if (num_threads != 0) {
- __kmp_push_num_threads(&loc, gtid, num_threads);
- }
- if(flags != 0) {
- __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags);
- }
- __kmp_GOMP_fork_call(&loc, gtid, task,
- (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, data);
- }
- else {
- __kmp_GOMP_serialized_parallel(&loc, gtid, task);
- }
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- task_info = __ompt_get_taskinfo(0);
- task_info->frame.exit_runtime_frame = __builtin_frame_address(0);
- }
-#endif
- task(data);
- xexpand(KMP_API_NAME_GOMP_PARALLEL_END)();
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- task_info->frame.exit_runtime_frame = NULL;
- parent_task_info->frame.reenter_runtime_frame = NULL;
- }
-#endif
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task) (void *), void *data,
- unsigned num_threads, unsigned count, unsigned flags)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_parallel_sections");
- KA_TRACE(20, ("GOMP_parallel_sections: T#%d\n", gtid));
-
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) {
- if (num_threads != 0) {
- __kmp_push_num_threads(&loc, gtid, num_threads);
- }
- if(flags != 0) {
- __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags);
- }
- __kmp_GOMP_fork_call(&loc, gtid, task,
- (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, task, data,
- num_threads, &loc, kmp_nm_dynamic_chunked, (kmp_int)1,
- (kmp_int)count, (kmp_int)1, (kmp_int)1);
- }
- else {
- __kmp_GOMP_serialized_parallel(&loc, gtid, task);
- }
-
- KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE);
-
- task(data);
- xexpand(KMP_API_NAME_GOMP_PARALLEL_END)();
- KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid));
-}
-
-#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post) \
- void func (void (*task) (void *), void *data, unsigned num_threads, \
- long lb, long ub, long str, long chunk_sz, unsigned flags) \
- { \
- int gtid = __kmp_entry_gtid(); \
- MKLOC(loc, #func); \
- KA_TRACE(20, ( #func ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \
- gtid, lb, ub, str, chunk_sz )); \
- \
- ompt_pre(); \
- if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \
- if (num_threads != 0) { \
- __kmp_push_num_threads(&loc, gtid, num_threads); \
- } \
- if (flags != 0) { \
- __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); \
- } \
- __kmp_GOMP_fork_call(&loc, gtid, task, \
- (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, \
- task, data, num_threads, &loc, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \
- } \
- else { \
- __kmp_GOMP_serialized_parallel(&loc, gtid, task); \
- } \
- \
- KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \
- (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \
- (schedule) != kmp_sch_static); \
- task(data); \
- xexpand(KMP_API_NAME_GOMP_PARALLEL_END)(); \
- ompt_post(); \
- \
- KA_TRACE(20, ( #func " exit: T#%d\n", gtid)); \
- }
-
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), kmp_sch_static,
- OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), kmp_sch_dynamic_chunked,
- OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), kmp_sch_guided_chunked,
- OMPT_LOOP_PRE, OMPT_LOOP_POST)
-PARALLEL_LOOP(xexpand(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), kmp_sch_runtime,
- OMPT_LOOP_PRE, OMPT_LOOP_POST)
-
-
-void
-xexpand(KMP_API_NAME_GOMP_TASKGROUP_START)(void)
-{
- int gtid = __kmp_entry_gtid();
- MKLOC(loc, "GOMP_taskgroup_start");
- KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid));
-
- __kmpc_taskgroup(&loc, gtid);
-
- return;
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_TASKGROUP_END)(void)
-{
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_taskgroup_end");
- KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid));
-
- __kmpc_end_taskgroup(&loc, gtid);
-
- return;
-}
-
-#ifndef KMP_DEBUG
-static
-#endif /* KMP_DEBUG */
-kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) {
- kmp_int32 cncl_kind = 0;
- switch(gomp_kind) {
- case 1:
- cncl_kind = cancel_parallel;
- break;
- case 2:
- cncl_kind = cancel_loop;
- break;
- case 4:
- cncl_kind = cancel_sections;
- break;
- case 8:
- cncl_kind = cancel_taskgroup;
- break;
- }
- return cncl_kind;
-}
-
-bool
-xexpand(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which)
-{
- if(__kmp_omp_cancellation) {
- KMP_FATAL(NoGompCancellation);
- }
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_cancellation_point");
- KA_TRACE(20, ("GOMP_cancellation_point: T#%d\n", gtid));
-
- kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
-
- return __kmpc_cancellationpoint(&loc, gtid, cncl_kind);
-}
-
-bool
-xexpand(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void)
-{
- if(__kmp_omp_cancellation) {
- KMP_FATAL(NoGompCancellation);
- }
- KMP_FATAL(NoGompCancellation);
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_barrier_cancel");
- KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid));
-
- return __kmpc_cancel_barrier(&loc, gtid);
-}
-
-bool
-xexpand(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel)
-{
- if(__kmp_omp_cancellation) {
- KMP_FATAL(NoGompCancellation);
- } else {
- return FALSE;
- }
-
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_cancel");
- KA_TRACE(20, ("GOMP_cancel: T#%d\n", gtid));
-
- kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which);
-
- if(do_cancel == FALSE) {
- return xexpand(KMP_API_NAME_GOMP_CANCELLATION_POINT)(which);
- } else {
- return __kmpc_cancel(&loc, gtid, cncl_kind);
- }
-}
-
-bool
-xexpand(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void)
-{
- if(__kmp_omp_cancellation) {
- KMP_FATAL(NoGompCancellation);
- }
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_sections_end_cancel");
- KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid));
-
- return __kmpc_cancel_barrier(&loc, gtid);
-}
-
-bool
-xexpand(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void)
-{
- if(__kmp_omp_cancellation) {
- KMP_FATAL(NoGompCancellation);
- }
- int gtid = __kmp_get_gtid();
- MKLOC(loc, "GOMP_loop_end_cancel");
- KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid));
-
- return __kmpc_cancel_barrier(&loc, gtid);
-}
-
-// All target functions are empty as of 2014-05-29
-void
-xexpand(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn) (void *), const void *openmp_target,
- size_t mapnum, void **hostaddrs, size_t *sizes, unsigned char *kinds)
-{
- return;
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_TARGET_DATA)(int device, const void *openmp_target, size_t mapnum,
- void **hostaddrs, size_t *sizes, unsigned char *kinds)
-{
- return;
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_TARGET_END_DATA)(void)
-{
- return;
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_TARGET_UPDATE)(int device, const void *openmp_target, size_t mapnum,
- void **hostaddrs, size_t *sizes, unsigned char *kinds)
-{
- return;
-}
-
-void
-xexpand(KMP_API_NAME_GOMP_TEAMS)(unsigned int num_teams, unsigned int thread_limit)
-{
- return;
-}
-#endif // OMP_40_ENABLED
-
-
-/*
- The following sections of code create aliases for the GOMP_* functions,
- then create versioned symbols using the assembler directive .symver.
- This is only pertinent for ELF .so library
- xaliasify and xversionify are defined in kmp_ftn_os.h
-*/
-
-#ifdef KMP_USE_VERSION_SYMBOLS
-
-// GOMP_1.0 aliases
-xaliasify(KMP_API_NAME_GOMP_ATOMIC_END, 10);
-xaliasify(KMP_API_NAME_GOMP_ATOMIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_BARRIER, 10);
-xaliasify(KMP_API_NAME_GOMP_CRITICAL_END, 10);
-xaliasify(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10);
-xaliasify(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10);
-xaliasify(KMP_API_NAME_GOMP_CRITICAL_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_END, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_ORDERED_END, 10);
-xaliasify(KMP_API_NAME_GOMP_ORDERED_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_END, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_START, 10);
-xaliasify(KMP_API_NAME_GOMP_SECTIONS_END, 10);
-xaliasify(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10);
-xaliasify(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10);
-xaliasify(KMP_API_NAME_GOMP_SECTIONS_START, 10);
-xaliasify(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10);
-xaliasify(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10);
-xaliasify(KMP_API_NAME_GOMP_SINGLE_START, 10);
-
-// GOMP_2.0 aliases
-xaliasify(KMP_API_NAME_GOMP_TASK, 20);
-xaliasify(KMP_API_NAME_GOMP_TASKWAIT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20);
-xaliasify(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20);
-
-// GOMP_3.0 aliases
-xaliasify(KMP_API_NAME_GOMP_TASKYIELD, 30);
-
-// GOMP_4.0 aliases
-// The GOMP_parallel* entry points below aren't OpenMP 4.0 related.
-#if OMP_40_ENABLED
-xaliasify(KMP_API_NAME_GOMP_PARALLEL, 40);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40);
-xaliasify(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40);
-xaliasify(KMP_API_NAME_GOMP_TASKGROUP_START, 40);
-xaliasify(KMP_API_NAME_GOMP_TASKGROUP_END, 40);
-xaliasify(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40);
-xaliasify(KMP_API_NAME_GOMP_CANCEL, 40);
-xaliasify(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40);
-xaliasify(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40);
-xaliasify(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40);
-xaliasify(KMP_API_NAME_GOMP_TARGET, 40);
-xaliasify(KMP_API_NAME_GOMP_TARGET_DATA, 40);
-xaliasify(KMP_API_NAME_GOMP_TARGET_END_DATA, 40);
-xaliasify(KMP_API_NAME_GOMP_TARGET_UPDATE, 40);
-xaliasify(KMP_API_NAME_GOMP_TEAMS, 40);
-#endif
-
-// GOMP_1.0 versioned symbols
-xversionify(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_ATOMIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_BARRIER, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_CRITICAL_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_CRITICAL_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_ORDERED_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_ORDERED_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SECTIONS_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SECTIONS_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10, "GOMP_1.0");
-xversionify(KMP_API_NAME_GOMP_SINGLE_START, 10, "GOMP_1.0");
-
-// GOMP_2.0 versioned symbols
-xversionify(KMP_API_NAME_GOMP_TASK, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_TASKWAIT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20, "GOMP_2.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20, "GOMP_2.0");
-
-// GOMP_3.0 versioned symbols
-xversionify(KMP_API_NAME_GOMP_TASKYIELD, 30, "GOMP_3.0");
-
-// GOMP_4.0 versioned symbols
-#if OMP_40_ENABLED
-xversionify(KMP_API_NAME_GOMP_PARALLEL, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TASKGROUP_START, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TASKGROUP_END, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_CANCEL, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TARGET, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TARGET_DATA, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TARGET_END_DATA, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TARGET_UPDATE, 40, "GOMP_4.0");
-xversionify(KMP_API_NAME_GOMP_TEAMS, 40, "GOMP_4.0");
-#endif
-
-#endif // KMP_USE_VERSION_SYMBOLS
-
-#ifdef __cplusplus
- } //extern "C"
-#endif // __cplusplus
-
-
Removed: openmp/trunk/runtime/src/kmp_i18n.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_i18n.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_i18n.c (original)
+++ openmp/trunk/runtime/src/kmp_i18n.c (removed)
@@ -1,995 +0,0 @@
-/*
- * kmp_i18n.c
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-
-#include "kmp_i18n.h"
-
-#include "kmp_os.h"
-#include "kmp_debug.h"
-#include "kmp.h"
-#include "kmp_lock.h"
-#include "kmp_io.h" // __kmp_printf.
-
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <locale.h>
-#include <stdarg.h>
-
-#include "kmp_i18n_default.inc"
-#include "kmp_str.h"
-#include "kmp_environment.h"
-
-#undef KMP_I18N_OK
-
-#define get_section( id ) ( (id) >> 16 )
-#define get_number( id ) ( (id) & 0xFFFF )
-
-kmp_msg_t __kmp_msg_empty = { kmp_mt_dummy, 0, "", 0 };
-kmp_msg_t __kmp_msg_null = { kmp_mt_dummy, 0, NULL, 0 };
-static char const * no_message_available = "(No message available)";
-
-enum kmp_i18n_cat_status {
- KMP_I18N_CLOSED, // Not yet opened or closed.
- KMP_I18N_OPENED, // Opened successfully, ready to use.
- KMP_I18N_ABSENT // Opening failed, message catalog should not be used.
-}; // enum kmp_i18n_cat_status
-typedef enum kmp_i18n_cat_status kmp_i18n_cat_status_t;
-static volatile kmp_i18n_cat_status_t status = KMP_I18N_CLOSED;
-
-/*
- Message catalog is opened at first usage, so we have to synchronize opening to avoid race and
- multiple openings.
-
- Closing does not require synchronization, because catalog is closed very late at library
- shutting down, when no other threads are alive.
-*/
-
-static void __kmp_i18n_do_catopen();
-static kmp_bootstrap_lock_t lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( lock );
- // `lock' variable may be placed into __kmp_i18n_catopen function because it is used only by
- // that function. But we afraid a (buggy) compiler may treat it wrongly. So we put it outside of
- // function just in case.
-
-void
-__kmp_i18n_catopen(
-) {
- if ( status == KMP_I18N_CLOSED ) {
- __kmp_acquire_bootstrap_lock( & lock );
- if ( status == KMP_I18N_CLOSED ) {
- __kmp_i18n_do_catopen();
- }; // if
- __kmp_release_bootstrap_lock( & lock );
- }; // if
-} // func __kmp_i18n_catopen
-
-
-/*
- ================================================================================================
- Linux* OS and OS X* part.
- ================================================================================================
-*/
-
-#if KMP_OS_UNIX
-#define KMP_I18N_OK
-
-#include <nl_types.h>
-
-#define KMP_I18N_NULLCAT ((nl_catd)( -1 ))
-static nl_catd cat = KMP_I18N_NULLCAT; // !!! Shall it be volatile?
-static char const * name = ( KMP_VERSION_MAJOR == 4 ? "libguide.cat" : "libomp.cat" );
-
-/*
- Useful links:
- http://www.opengroup.org/onlinepubs/000095399/basedefs/xbd_chap08.html#tag_08_02
- http://www.opengroup.org/onlinepubs/000095399/functions/catopen.html
- http://www.opengroup.org/onlinepubs/000095399/functions/setlocale.html
-*/
-
-void
-__kmp_i18n_do_catopen(
-) {
- int english = 0;
- char * lang = __kmp_env_get( "LANG" );
- // TODO: What about LC_ALL or LC_MESSAGES?
-
- KMP_DEBUG_ASSERT( status == KMP_I18N_CLOSED );
- KMP_DEBUG_ASSERT( cat == KMP_I18N_NULLCAT );
-
- english =
- lang == NULL || // In all these cases English language is used.
- strcmp( lang, "" ) == 0 ||
- strcmp( lang, " " ) == 0 ||
- // Workaround for Fortran RTL bug DPD200137873 "Fortran runtime resets LANG env var
- // to space if it is not set".
- strcmp( lang, "C" ) == 0 ||
- strcmp( lang, "POSIX" ) == 0;
-
- if ( ! english ) { // English language is not yet detected, let us continue.
- // Format of LANG is: [language[_territory][.codeset][@modifier]]
- // Strip all parts except language.
- char * tail = NULL;
- __kmp_str_split( lang, '@', & lang, & tail );
- __kmp_str_split( lang, '.', & lang, & tail );
- __kmp_str_split( lang, '_', & lang, & tail );
- english = ( strcmp( lang, "en" ) == 0 );
- }; // if
-
- KMP_INTERNAL_FREE( lang );
-
- // Do not try to open English catalog because internal messages are
- // exact copy of messages in English catalog.
- if ( english ) {
- status = KMP_I18N_ABSENT; // mark catalog as absent so it will not be re-opened.
- return;
- }
-
- cat = catopen( name, 0 );
- // TODO: Why do we pass 0 in flags?
- status = ( cat == KMP_I18N_NULLCAT ? KMP_I18N_ABSENT : KMP_I18N_OPENED );
-
- if ( status == KMP_I18N_ABSENT ) {
- if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
- int error = errno; // Save errno immediately.
- char * nlspath = __kmp_env_get( "NLSPATH" );
- char * lang = __kmp_env_get( "LANG" );
-
- // Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
- // __kmp_i18n_catgets() will not try to open catalog, but will return default message.
- kmp_msg_t err_code = KMP_ERR( error );
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantOpenMessageCatalog, name ),
- err_code,
- KMP_HNT( CheckEnvVar, "NLSPATH", nlspath ),
- KMP_HNT( CheckEnvVar, "LANG", lang ),
- __kmp_msg_null
- );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
-
- KMP_INFORM( WillUseDefaultMessages );
- KMP_INTERNAL_FREE( nlspath );
- KMP_INTERNAL_FREE( lang );
- }
- } else { // status == KMP_I18N_OPENED
-
- int section = get_section( kmp_i18n_prp_Version );
- int number = get_number( kmp_i18n_prp_Version );
- char const * expected = __kmp_i18n_default_table.sect[ section ].str[ number ];
- // Expected version of the catalog.
- kmp_str_buf_t version; // Actual version of the catalog.
- __kmp_str_buf_init( & version );
- __kmp_str_buf_print( & version, "%s", catgets( cat, section, number, NULL ) );
-
- // String returned by catgets is invalid after closing the catalog, so copy it.
- if ( strcmp( version.str, expected ) != 0 ) {
- __kmp_i18n_catclose(); // Close bad catalog.
- status = KMP_I18N_ABSENT; // And mark it as absent.
- if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
- // And now print a warning using default messages.
- char const * name = "NLSPATH";
- char const * nlspath = __kmp_env_get( name );
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( WrongMessageCatalog, name, version.str, expected ),
- KMP_HNT( CheckEnvVar, name, nlspath ),
- __kmp_msg_null
- );
- KMP_INFORM( WillUseDefaultMessages );
- KMP_INTERNAL_FREE( (void *) nlspath );
- } // __kmp_generate_warnings
- }; // if
- __kmp_str_buf_free( & version );
-
- }; // if
-
-} // func __kmp_i18n_do_catopen
-
-
-void
-__kmp_i18n_catclose(
-) {
- if ( status == KMP_I18N_OPENED ) {
- KMP_DEBUG_ASSERT( cat != KMP_I18N_NULLCAT );
- catclose( cat );
- cat = KMP_I18N_NULLCAT;
- }; // if
- status = KMP_I18N_CLOSED;
-} // func __kmp_i18n_catclose
-
-
-char const *
-__kmp_i18n_catgets(
- kmp_i18n_id_t id
-) {
-
- int section = get_section( id );
- int number = get_number( id );
- char const * message = NULL;
-
- if ( 1 <= section && section <= __kmp_i18n_default_table.size ) {
- if ( 1 <= number && number <= __kmp_i18n_default_table.sect[ section ].size ) {
- if ( status == KMP_I18N_CLOSED ) {
- __kmp_i18n_catopen();
- }; // if
- if ( status == KMP_I18N_OPENED ) {
- message =
- catgets(
- cat,
- section, number,
- __kmp_i18n_default_table.sect[ section ].str[ number ]
- );
- }; // if
- if ( message == NULL ) {
- message = __kmp_i18n_default_table.sect[ section ].str[ number ];
- }; // if
- }; // if
- }; // if
- if ( message == NULL ) {
- message = no_message_available;
- }; // if
- return message;
-
-} // func __kmp_i18n_catgets
-
-
-#endif // KMP_OS_UNIX
-
-/*
- ================================================================================================
- Windows* OS part.
- ================================================================================================
-*/
-
-#if KMP_OS_WINDOWS
-#define KMP_I18N_OK
-
-#include "kmp_environment.h"
-#include <windows.h>
-
-#define KMP_I18N_NULLCAT NULL
-static HMODULE cat = KMP_I18N_NULLCAT; // !!! Shall it be volatile?
-static char const * name = ( KMP_VERSION_MAJOR == 4 ? "libguide40ui.dll" : "libompui.dll" );
-
-static kmp_i18n_table_t table = { 0, NULL };
- // Messages formatted by FormatMessage() should be freed, but catgets() interface assumes
- // user will not free messages. So we cache all the retrieved messages in the table, which
- // are freed at catclose().
-static UINT const default_code_page = CP_OEMCP;
-static UINT code_page = default_code_page;
-
-static char const * ___catgets( kmp_i18n_id_t id );
-static UINT get_code_page();
-static void kmp_i18n_table_free( kmp_i18n_table_t * table );
-
-
-static UINT
-get_code_page(
-) {
-
- UINT cp = default_code_page;
- char const * value = __kmp_env_get( "KMP_CODEPAGE" );
- if ( value != NULL ) {
- if ( _stricmp( value, "ANSI" ) == 0 ) {
- cp = CP_ACP;
- } else if ( _stricmp( value, "OEM" ) == 0 ) {
- cp = CP_OEMCP;
- } else if ( _stricmp( value, "UTF-8" ) == 0 || _stricmp( value, "UTF8" ) == 0 ) {
- cp = CP_UTF8;
- } else if ( _stricmp( value, "UTF-7" ) == 0 || _stricmp( value, "UTF7" ) == 0 ) {
- cp = CP_UTF7;
- } else {
- // !!! TODO: Issue a warning?
- }; // if
- }; // if
- KMP_INTERNAL_FREE( (void *) value );
- return cp;
-
-} // func get_code_page
-
-
-static void
-kmp_i18n_table_free(
- kmp_i18n_table_t * table
-) {
- int s;
- int m;
- for ( s = 0; s < table->size; ++ s ) {
- for ( m = 0; m < table->sect[ s ].size; ++ m ) {
- // Free message.
- KMP_INTERNAL_FREE( (void *) table->sect[ s ].str[ m ] );
- table->sect[ s ].str[ m ] = NULL;
- }; // for m
- table->sect[ s ].size = 0;
- // Free section itself.
- KMP_INTERNAL_FREE ( (void *) table->sect[ s ].str );
- table->sect[ s ].str = NULL;
- }; // for s
- table->size = 0;
- KMP_INTERNAL_FREE( (void *) table->sect );
- table->sect = NULL;
-} // kmp_i18n_table_free
-
-
-void
-__kmp_i18n_do_catopen(
-) {
-
- LCID locale_id = GetThreadLocale();
- WORD lang_id = LANGIDFROMLCID( locale_id );
- WORD primary_lang_id = PRIMARYLANGID( lang_id );
- kmp_str_buf_t path;
-
- KMP_DEBUG_ASSERT( status == KMP_I18N_CLOSED );
- KMP_DEBUG_ASSERT( cat == KMP_I18N_NULLCAT );
-
- __kmp_str_buf_init( & path );
-
- // Do not try to open English catalog because internal messages are
- // exact copy of messages in English catalog.
- if ( primary_lang_id == LANG_ENGLISH ) {
- status = KMP_I18N_ABSENT; // mark catalog as absent so it will not be re-opened.
- goto end;
- }; // if
-
- // Construct resource DLL name.
- /*
- Simple
- LoadLibrary( name )
- is not suitable due to security issue (see
- http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
- path to the message catalog.
- */
- {
-
- // Get handle of our DLL first.
- HMODULE handle;
- BOOL brc =
- GetModuleHandleEx(
- GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
- reinterpret_cast< LPCSTR >( & __kmp_i18n_do_catopen ),
- & handle
- );
- if ( ! brc ) { // Error occurred.
- status = KMP_I18N_ABSENT; // mark catalog as absent so it will not be re-opened.
- goto end;
- // TODO: Enable multiple messages (KMP_MSG) to be passed to __kmp_msg; and print
- // a proper warning.
- }; // if
-
- // Now get path to the our DLL.
- for ( ; ; ) {
- DWORD drc = GetModuleFileName( handle, path.str, path.size );
- if ( drc == 0 ) { // Error occurred.
- status = KMP_I18N_ABSENT;
- goto end;
- }; // if
- if ( drc < path.size ) {
- path.used = drc;
- break;
- }; // if
- __kmp_str_buf_reserve( & path, path.size * 2 );
- }; // forever
-
- // Now construct the name of message catalog.
- kmp_str_fname fname;
- __kmp_str_fname_init( & fname, path.str );
- __kmp_str_buf_clear( & path );
- __kmp_str_buf_print( & path, "%s%lu/%s", fname.dir, (unsigned long)( locale_id ), name );
- __kmp_str_fname_free( & fname );
-
- }
-
- // For security reasons, use LoadLibraryEx() and load message catalog as a data file.
- cat = LoadLibraryEx( path.str, NULL, LOAD_LIBRARY_AS_DATAFILE );
- status = ( cat == KMP_I18N_NULLCAT ? KMP_I18N_ABSENT : KMP_I18N_OPENED );
-
- if ( status == KMP_I18N_ABSENT ) {
- if (__kmp_generate_warnings > kmp_warnings_low) { // AC: only issue warning in case explicitly asked to
- DWORD error = GetLastError();
- // Infinite recursion will not occur -- status is KMP_I18N_ABSENT now, so
- // __kmp_i18n_catgets() will not try to open catalog but will return default message.
- /*
- If message catalog for another architecture found (e.g. OpenMP RTL
- for IA-32 architecture opens libompui.dll for Intel(R) 64)
- Windows* OS returns error 193 (ERROR_BAD_EXE_FORMAT). However,
- FormatMessage fails to return a message for this error, so user
- will see:
-
- OMP: Warning #2: Cannot open message catalog "1041\libompui.dll":
- OMP: System error #193: (No system error message available)
- OMP: Info #3: Default messages will be used.
-
- Issue a hint in this case to let cause of trouble more understandable.
- */
- kmp_msg_t err_code = KMP_SYSERRCODE(error);
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantOpenMessageCatalog, path.str ),
- err_code,
- ( error == ERROR_BAD_EXE_FORMAT ? KMP_HNT( BadExeFormat, path.str, KMP_ARCH_STR ) : __kmp_msg_null ),
- __kmp_msg_null
- );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
-
- KMP_INFORM( WillUseDefaultMessages );
- }
- } else { // status == KMP_I18N_OPENED
-
- int section = get_section( kmp_i18n_prp_Version );
- int number = get_number( kmp_i18n_prp_Version );
- char const * expected = __kmp_i18n_default_table.sect[ section ].str[ number ];
- kmp_str_buf_t version; // Actual version of the catalog.
- __kmp_str_buf_init( & version );
- __kmp_str_buf_print( & version, "%s", ___catgets( kmp_i18n_prp_Version ) );
- // String returned by catgets is invalid after closing the catalog, so copy it.
- if ( strcmp( version.str, expected ) != 0 ) {
- // Close bad catalog.
- __kmp_i18n_catclose();
- status = KMP_I18N_ABSENT; // And mark it as absent.
- if (__kmp_generate_warnings > kmp_warnings_low) {
- // And now print a warning using default messages.
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( WrongMessageCatalog, path.str, version.str, expected ),
- __kmp_msg_null
- );
- KMP_INFORM( WillUseDefaultMessages );
- } // __kmp_generate_warnings
- }; // if
- __kmp_str_buf_free( & version );
-
- }; // if
- code_page = get_code_page();
-
- end:
- __kmp_str_buf_free( & path );
- return;
-
-} // func __kmp_i18n_do_catopen
-
-
-void
-__kmp_i18n_catclose(
-) {
- if ( status == KMP_I18N_OPENED ) {
- KMP_DEBUG_ASSERT( cat != KMP_I18N_NULLCAT );
- kmp_i18n_table_free( & table );
- FreeLibrary( cat );
- cat = KMP_I18N_NULLCAT;
- }; // if
- code_page = default_code_page;
- status = KMP_I18N_CLOSED;
-} // func __kmp_i18n_catclose
-
-/*
- We use FormatMessage() to get strings from catalog, get system error messages, etc.
- FormatMessage() tends to return Windows* OS-style end-of-lines, "\r\n". When string is printed,
- printf() also replaces all the occurrences of "\n" with "\r\n" (again!), so sequences like
- "\r\r\r\n" appear in output. It is not too good.
-
- Additional mess comes from message catalog: Our catalog source en_US.mc file (generated by
- message-converter.pl) contains only "\n" characters, but en_US_msg_1033.bin file (produced by
- mc.exe) may contain "\r\n" or just "\n". This mess goes from en_US_msg_1033.bin file to
- message catalog, libompui.dll. For example, message
-
- Error
-
- (there is "\n" at the end) is compiled by mc.exe to "Error\r\n", while
-
- OMP: Error %1!d!: %2!s!\n
-
- (there is "\n" at the end as well) is compiled to "OMP: Error %1!d!: %2!s!\r\n\n".
-
- Thus, stripping all "\r" normalizes string and returns it to canonical form, so printf() will
- produce correct end-of-line sequences.
-
- ___strip_crs() serves for this purpose: it removes all the occurrences of "\r" in-place and
- returns new length of string.
-*/
-static
-int
-___strip_crs(
- char * str
-) {
- int in = 0; // Input character index.
- int out = 0; // Output character index.
- for ( ; ; ) {
- if ( str[ in ] != '\r' ) {
- str[ out ] = str[ in ];
- ++ out;
- }; // if
- if ( str[ in ] == 0 ) {
- break;
- }; // if
- ++ in;
- }; // forever
- return out - 1;
-} // func __strip_crs
-
-
-static
-char const *
-___catgets(
- kmp_i18n_id_t id
-) {
-
- char * result = NULL;
- PVOID addr = NULL;
- wchar_t * wmsg = NULL;
- DWORD wlen = 0;
- char * msg = NULL;
- int len = 0;
- int rc;
-
- KMP_DEBUG_ASSERT( cat != KMP_I18N_NULLCAT );
- wlen = // wlen does *not* include terminating null.
- FormatMessageW(
- FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_HMODULE |
- FORMAT_MESSAGE_IGNORE_INSERTS,
- cat,
- id,
- 0, // LangId
- (LPWSTR) & addr,
- 0, // Size in elements, not in bytes.
- NULL
- );
- if ( wlen <= 0 ) {
- goto end;
- }; // if
- wmsg = (wchar_t *) addr; // Warning: wmsg may be not nul-terminated!
-
- // Calculate length of multibyte message.
- len = // Since wlen does not include terminating null, len does not include it also.
- WideCharToMultiByte(
- code_page,
- 0, // Flags.
- wmsg, wlen, // Wide buffer and size.
- NULL, 0, // Buffer and size.
- NULL, NULL // Default char and used default char.
- );
- if ( len <= 0 ) {
- goto end;
- }; // if
-
- // Allocate memory.
- msg = (char *) KMP_INTERNAL_MALLOC( len + 1 );
-
- // Convert wide message to multibyte one.
- rc =
- WideCharToMultiByte(
- code_page,
- 0, // Flags.
- wmsg, wlen, // Wide buffer and size.
- msg, len, // Buffer and size.
- NULL, NULL // Default char and used default char.
- );
- if ( rc <= 0 || rc > len ) {
- goto end;
- }; // if
- KMP_DEBUG_ASSERT( rc == len );
- len = rc;
- msg[ len ] = 0; // Put terminating null to the end.
-
- // Stripping all "\r" before stripping last end-of-line simplifies the task.
- len = ___strip_crs( msg );
-
- // Every message in catalog is terminated with "\n". Strip it.
- if ( len >= 1 && msg[ len - 1 ] == '\n' ) {
- -- len;
- msg[ len ] = 0;
- }; // if
-
- // Everything looks ok.
- result = msg;
- msg = NULL;
-
- end:
-
- if ( msg != NULL ) {
- KMP_INTERNAL_FREE( msg );
- }; // if
- if ( wmsg != NULL ) {
- LocalFree( wmsg );
- }; // if
-
- return result;
-
-} // ___catgets
-
-
-char const *
-__kmp_i18n_catgets(
- kmp_i18n_id_t id
-) {
-
- int section = get_section( id );
- int number = get_number( id );
- char const * message = NULL;
-
- if ( 1 <= section && section <= __kmp_i18n_default_table.size ) {
- if ( 1 <= number && number <= __kmp_i18n_default_table.sect[ section ].size ) {
- if ( status == KMP_I18N_CLOSED ) {
- __kmp_i18n_catopen();
- }; // if
- if ( cat != KMP_I18N_NULLCAT ) {
- if ( table.size == 0 ) {
- table.sect = (kmp_i18n_section_t *)
- KMP_INTERNAL_CALLOC(
- ( __kmp_i18n_default_table.size + 2 ),
- sizeof( kmp_i18n_section_t )
- );
- table.size = __kmp_i18n_default_table.size;
- }; // if
- if ( table.sect[ section ].size == 0 ) {
- table.sect[ section ].str = (const char **)
- KMP_INTERNAL_CALLOC(
- __kmp_i18n_default_table.sect[ section ].size + 2,
- sizeof( char const * )
- );
- table.sect[ section ].size = __kmp_i18n_default_table.sect[ section ].size;
- }; // if
- if ( table.sect[ section ].str[ number ] == NULL ) {
- table.sect[ section ].str[ number ] = ___catgets( id );
- }; // if
- message = table.sect[ section ].str[ number ];
- }; // if
- if ( message == NULL ) {
- // Catalog is not opened or message is not found, return default message.
- message = __kmp_i18n_default_table.sect[ section ].str[ number ];
- }; // if
- }; // if
- }; // if
- if ( message == NULL ) {
- message = no_message_available;
- }; // if
- return message;
-
-} // func __kmp_i18n_catgets
-
-
-#endif // KMP_OS_WINDOWS
-
-// -------------------------------------------------------------------------------------------------
-
-#ifndef KMP_I18N_OK
- #error I18n support is not implemented for this OS.
-#endif // KMP_I18N_OK
-
-// -------------------------------------------------------------------------------------------------
-
-void
-__kmp_i18n_dump_catalog(
- kmp_str_buf_t * buffer
-) {
-
- struct kmp_i18n_id_range_t {
- kmp_i18n_id_t first;
- kmp_i18n_id_t last;
- }; // struct kmp_i18n_id_range_t
-
- static struct kmp_i18n_id_range_t ranges[] = {
- { kmp_i18n_prp_first, kmp_i18n_prp_last },
- { kmp_i18n_str_first, kmp_i18n_str_last },
- { kmp_i18n_fmt_first, kmp_i18n_fmt_last },
- { kmp_i18n_msg_first, kmp_i18n_msg_last },
- { kmp_i18n_hnt_first, kmp_i18n_hnt_last }
- }; // ranges
-
- int num_of_ranges = sizeof( ranges ) / sizeof( struct kmp_i18n_id_range_t );
- int range;
- kmp_i18n_id_t id;
-
- for ( range = 0; range < num_of_ranges; ++ range ) {
- __kmp_str_buf_print( buffer, "*** Set #%d ***\n", range + 1 );
- for ( id = (kmp_i18n_id_t)( ranges[ range ].first + 1 );
- id < ranges[ range ].last;
- id = (kmp_i18n_id_t)( id + 1 ) ) {
- __kmp_str_buf_print( buffer, "%d: <<%s>>\n", id, __kmp_i18n_catgets( id ) );
- }; // for id
- }; // for range
-
- __kmp_printf( "%s", buffer->str );
-
-} // __kmp_i18n_dump_catalog
-
-// -------------------------------------------------------------------------------------------------
-
-kmp_msg_t
-__kmp_msg_format(
- unsigned id_arg,
- ...
-) {
-
- kmp_msg_t msg;
- va_list args;
- kmp_str_buf_t buffer;
- __kmp_str_buf_init( & buffer );
-
- va_start( args, id_arg );
-
- // We use unsigned for the ID argument and explicitly cast it here to the
- // right enumerator because variadic functions are not compatible with
- // default promotions.
- kmp_i18n_id_t id = (kmp_i18n_id_t)id_arg;
-
- #if KMP_OS_UNIX
- // On Linux* OS and OS X*, printf() family functions process parameter numbers, for example:
- // "%2$s %1$s".
- __kmp_str_buf_vprint( & buffer, __kmp_i18n_catgets( id ), args );
- #elif KMP_OS_WINDOWS
- // On Winodws, printf() family functions does not recognize GNU style parameter numbers,
- // so we have to use FormatMessage() instead. It recognizes parameter numbers, e. g.:
- // "%2!s! "%1!s!".
- {
- LPTSTR str = NULL;
- int len;
- FormatMessage(
- FORMAT_MESSAGE_FROM_STRING | FORMAT_MESSAGE_ALLOCATE_BUFFER,
- __kmp_i18n_catgets( id ),
- 0, 0,
- (LPTSTR)( & str ),
- 0,
- & args
- );
- len = ___strip_crs( str );
- __kmp_str_buf_cat( & buffer, str, len );
- LocalFree( str );
- }
- #else
- #error
- #endif
- va_end( args );
- __kmp_str_buf_detach( & buffer );
-
- msg.type = (kmp_msg_type_t)( id >> 16 );
- msg.num = id & 0xFFFF;
- msg.str = buffer.str;
- msg.len = buffer.used;
-
- return msg;
-
-} // __kmp_msg_format
-
-// -------------------------------------------------------------------------------------------------
-
-static
-char *
-sys_error(
- int err
-) {
-
- char * message = NULL;
-
- #if KMP_OS_WINDOWS
-
- LPVOID buffer = NULL;
- int len;
- DWORD rc;
- rc =
- FormatMessage(
- FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
- NULL,
- err,
- MAKELANGID( LANG_NEUTRAL, SUBLANG_DEFAULT ), // Default language.
- (LPTSTR) & buffer,
- 0,
- NULL
- );
- if ( rc > 0 ) {
- // Message formatted. Copy it (so we can free it later with normal free().
- message = __kmp_str_format( "%s", (char *) buffer );
- len = ___strip_crs( message ); // Delete carriage returns if any.
- // Strip trailing newlines.
- while ( len > 0 && message[ len - 1 ] == '\n' ) {
- -- len;
- }; // while
- message[ len ] = 0;
- } else {
- // FormatMessage() failed to format system error message. GetLastError() would give us
- // error code, which we would convert to message... this it dangerous recursion, which
- // cannot clarify original error, so we will not even start it.
- }; // if
- if ( buffer != NULL ) {
- LocalFree( buffer );
- }; // if
-
- #else // Non-Windows* OS: Linux* OS or OS X*
-
- /*
- There are 2 incompatible versions of strerror_r:
-
- char * strerror_r( int, char *, size_t ); // GNU version
- int strerror_r( int, char *, size_t ); // XSI version
- */
-
- #if defined(__GLIBC__) && defined(_GNU_SOURCE)
-
- // GNU version of strerror_r.
-
- char buffer[ 2048 ];
- char * const err_msg = strerror_r( err, buffer, sizeof( buffer ) );
- // Do not eliminate this assignment to temporary variable, otherwise compiler would
- // not issue warning if strerror_r() returns `int' instead of expected `char *'.
- message = __kmp_str_format( "%s", err_msg );
-
- #else // OS X*, FreeBSD* etc.
-
- // XSI version of strerror_r.
-
- int size = 2048;
- char * buffer = (char *) KMP_INTERNAL_MALLOC( size );
- int rc;
- if (buffer == NULL) {
- KMP_FATAL(MemoryAllocFailed);
- }
- rc = strerror_r( err, buffer, size );
- if ( rc == -1 ) {
- rc = errno; // XSI version sets errno.
- }; // if
- while ( rc == ERANGE ) { // ERANGE means the buffer is too small.
- KMP_INTERNAL_FREE( buffer );
- size *= 2;
- buffer = (char *) KMP_INTERNAL_MALLOC( size );
- if (buffer == NULL) {
- KMP_FATAL(MemoryAllocFailed);
- }
- rc = strerror_r( err, buffer, size );
- if ( rc == -1 ) {
- rc = errno; // XSI version sets errno.
- }; // if
- }; // while
- if ( rc == 0 ) {
- message = buffer;
- } else {
- // Buffer is unused. Free it.
- KMP_INTERNAL_FREE( buffer );
- }; // if
-
- #endif
-
- #endif /* KMP_OS_WINDOWS */
-
- if ( message == NULL ) {
- // TODO: I18n this message.
- message = __kmp_str_format( "%s", "(No system error message available)" );
- }; // if
- return message;
-
-} // sys_error
-
-// -------------------------------------------------------------------------------------------------
-
-kmp_msg_t
-__kmp_msg_error_code(
- int code
-) {
-
- kmp_msg_t msg;
- msg.type = kmp_mt_syserr;
- msg.num = code;
- msg.str = sys_error( code );
- msg.len = KMP_STRLEN( msg.str );
- return msg;
-
-} // __kmp_msg_error_code
-
-// -------------------------------------------------------------------------------------------------
-
-kmp_msg_t
-__kmp_msg_error_mesg(
- char const * mesg
-) {
-
- kmp_msg_t msg;
- msg.type = kmp_mt_syserr;
- msg.num = 0;
- msg.str = __kmp_str_format( "%s", mesg );
- msg.len = KMP_STRLEN( msg.str );
- return msg;
-
-} // __kmp_msg_error_mesg
-
-// -------------------------------------------------------------------------------------------------
-
-void
-__kmp_msg(
- kmp_msg_severity_t severity,
- kmp_msg_t message,
- ...
-) {
-
- va_list args;
- kmp_i18n_id_t format; // format identifier
- kmp_msg_t fmsg; // formatted message
- kmp_str_buf_t buffer;
-
- if ( severity != kmp_ms_fatal && __kmp_generate_warnings == kmp_warnings_off )
- return; // no reason to form a string in order to not print it
-
- __kmp_str_buf_init( & buffer );
-
- // Format the primary message.
- switch ( severity ) {
- case kmp_ms_inform : {
- format = kmp_i18n_fmt_Info;
- } break;
- case kmp_ms_warning : {
- format = kmp_i18n_fmt_Warning;
- } break;
- case kmp_ms_fatal : {
- format = kmp_i18n_fmt_Fatal;
- } break;
- default : {
- KMP_DEBUG_ASSERT( 0 );
- };
- }; // switch
- fmsg = __kmp_msg_format( format, message.num, message.str );
- __kmp_str_free(&message.str);
- __kmp_str_buf_cat( & buffer, fmsg.str, fmsg.len );
- __kmp_str_free(&fmsg.str);
-
- // Format other messages.
- va_start( args, message );
- for ( ; ; ) {
- message = va_arg( args, kmp_msg_t );
- if ( message.type == kmp_mt_dummy && message.str == NULL ) {
- break;
- }; // if
- if ( message.type == kmp_mt_dummy && message.str == __kmp_msg_empty.str ) {
- continue;
- }; // if
- switch ( message.type ) {
- case kmp_mt_hint : {
- format = kmp_i18n_fmt_Hint;
- } break;
- case kmp_mt_syserr : {
- format = kmp_i18n_fmt_SysErr;
- } break;
- default : {
- KMP_DEBUG_ASSERT( 0 );
- };
- }; // switch
- fmsg = __kmp_msg_format( format, message.num, message.str );
- __kmp_str_free(&message.str);
- __kmp_str_buf_cat( & buffer, fmsg.str, fmsg.len );
- __kmp_str_free(&fmsg.str);
- }; // forever
- va_end( args );
-
- // Print formatted messages.
- // This lock prevents multiple fatal errors on the same problem.
- // __kmp_acquire_bootstrap_lock( & lock ); // GEH - This lock causing tests to hang on OS X*.
- __kmp_printf( "%s", buffer.str );
- __kmp_str_buf_free( & buffer );
-
- if ( severity == kmp_ms_fatal ) {
- #if KMP_OS_WINDOWS
- __kmp_thread_sleep( 500 ); /* Delay to give message a chance to appear before reaping */
- #endif
- __kmp_abort_process();
- }; // if
-
- // __kmp_release_bootstrap_lock( & lock ); // GEH - this lock causing tests to hang on OS X*.
-
-} // __kmp_msg
-
-// -------------------------------------------------------------------------------------------------
-
-// end of file //
Removed: openmp/trunk/runtime/src/kmp_import.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_import.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_import.c (original)
+++ openmp/trunk/runtime/src/kmp_import.c (removed)
@@ -1,42 +0,0 @@
-/*
- * kmp_import.c
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-/*
- ------------------------------------------------------------------------------------------------
- Object generated from this source file is linked to Windows* OS DLL import library (libompmd.lib)
- only! It is not a part of regular static or dynamic OpenMP RTL. Any code that just needs to go
- in the libompmd.lib (but not in libompmt.lib and libompmd.dll) should be placed in this
- file.
- ------------------------------------------------------------------------------------------------
-*/
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
- These symbols are required for mutual exclusion with Microsoft OpenMP RTL (and compatibility
- with MS Compiler).
-*/
-
-int _You_must_link_with_exactly_one_OpenMP_library = 1;
-int _You_must_link_with_Intel_OpenMP_library = 1;
-int _You_must_link_with_Microsoft_OpenMP_library = 1;
-
-#ifdef __cplusplus
-}
-#endif
-
-// end of file //
Removed: openmp/trunk/runtime/src/kmp_io.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_io.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_io.c (original)
+++ openmp/trunk/runtime/src/kmp_io.c (removed)
@@ -1,248 +0,0 @@
-/*
- * KMP_IO.c -- RTL IO
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdarg.h>
-#include <string.h>
-#ifndef __ABSOFT_WIN
-# include <sys/types.h>
-#endif
-
-#include "kmp_os.h"
-#include "kmp_lock.h"
-#include "kmp_str.h"
-#include "kmp_io.h"
-#include "kmp.h" // KMP_GTID_DNE, __kmp_debug_buf, etc
-
-#if KMP_OS_WINDOWS
-# pragma warning( push )
-# pragma warning( disable: 271 310 )
-# include <windows.h>
-# pragma warning( pop )
-#endif
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-kmp_bootstrap_lock_t __kmp_stdio_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_stdio_lock ); /* Control stdio functions */
-kmp_bootstrap_lock_t __kmp_console_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_console_lock ); /* Control console initialization */
-
-#if KMP_OS_WINDOWS
-
- # ifdef KMP_DEBUG
- /* __kmp_stdout is used only for dev build */
- static HANDLE __kmp_stdout = NULL;
- # endif
- static HANDLE __kmp_stderr = NULL;
- static int __kmp_console_exists = FALSE;
- static kmp_str_buf_t __kmp_console_buf;
-
- static int
- is_console( void )
- {
- char buffer[ 128 ];
- DWORD rc = 0;
- DWORD err = 0;
- // Try to get console title.
- SetLastError( 0 );
- // GetConsoleTitle does not reset last error in case of success or short buffer,
- // so we need to clear it explicitly.
- rc = GetConsoleTitle( buffer, sizeof( buffer ) );
- if ( rc == 0 ) {
- // rc == 0 means getting console title failed. Let us find out why.
- err = GetLastError();
- // err == 0 means buffer too short (we suppose console exists).
- // In Window applications we usually have err == 6 (invalid handle).
- }; // if
- return rc > 0 || err == 0;
- }
-
- void
- __kmp_close_console( void )
- {
- /* wait until user presses return before closing window */
- /* TODO only close if a window was opened */
- if( __kmp_console_exists ) {
- #ifdef KMP_DEBUG
- /* standard out is used only in dev build */
- __kmp_stdout = NULL;
- #endif
- __kmp_stderr = NULL;
- __kmp_str_buf_free( &__kmp_console_buf );
- __kmp_console_exists = FALSE;
- }
- }
-
- /* For windows, call this before stdout, stderr, or stdin are used.
- * It opens a console window and starts processing */
- static void
- __kmp_redirect_output( void )
- {
- __kmp_acquire_bootstrap_lock( &__kmp_console_lock );
-
- if( ! __kmp_console_exists ) {
- #ifdef KMP_DEBUG
- /* standard out is used only in dev build */
- HANDLE ho;
- #endif
- HANDLE he;
-
- __kmp_str_buf_init( &__kmp_console_buf );
-
- AllocConsole();
- // We do not check the result of AllocConsole because
- // 1. the call is harmless
- // 2. it is not clear how to communicate failue
- // 3. we will detect failure later when we get handle(s)
-
- #ifdef KMP_DEBUG
- ho = GetStdHandle( STD_OUTPUT_HANDLE );
- if ( ho == INVALID_HANDLE_VALUE || ho == NULL ) {
-
- DWORD err = GetLastError();
- // TODO: output error somehow (maybe message box)
- __kmp_stdout = NULL;
-
- } else {
-
- __kmp_stdout = ho; // temporary code, need new global for ho
-
- }
- #endif
- he = GetStdHandle( STD_ERROR_HANDLE );
- if ( he == INVALID_HANDLE_VALUE || he == NULL ) {
-
- DWORD err = GetLastError();
- // TODO: output error somehow (maybe message box)
- __kmp_stderr = NULL;
-
- } else {
-
- __kmp_stderr = he; // temporary code, need new global
- }
- __kmp_console_exists = TRUE;
- }
- __kmp_release_bootstrap_lock( &__kmp_console_lock );
- }
-
-#else
- #define __kmp_stderr (stderr)
-#endif /* KMP_OS_WINDOWS */
-
-void
-__kmp_vprintf( enum kmp_io __kmp_io, char const * format, va_list ap )
-{
- #if KMP_OS_WINDOWS
- if( !__kmp_console_exists ) {
- __kmp_redirect_output();
- }
- if( ! __kmp_stderr && __kmp_io == kmp_err ) {
- return;
- }
- #ifdef KMP_DEBUG
- if( ! __kmp_stdout && __kmp_io == kmp_out ) {
- return;
- }
- #endif
- #endif /* KMP_OS_WINDOWS */
-
- if ( __kmp_debug_buf && __kmp_debug_buffer != NULL ) {
-
- int dc = ( __kmp_debug_buf_atomic ?
- KMP_TEST_THEN_INC32( & __kmp_debug_count) : __kmp_debug_count++ )
- % __kmp_debug_buf_lines;
- char *db = & __kmp_debug_buffer[ dc * __kmp_debug_buf_chars ];
- int chars = 0;
-
- #ifdef KMP_DEBUG_PIDS
- chars = KMP_SNPRINTF( db, __kmp_debug_buf_chars, "pid=%d: ", (kmp_int32)getpid() );
- #endif
- chars += KMP_VSNPRINTF( db, __kmp_debug_buf_chars, format, ap );
-
- if ( chars + 1 > __kmp_debug_buf_chars ) {
- if ( chars + 1 > __kmp_debug_buf_warn_chars ) {
- #if KMP_OS_WINDOWS
- DWORD count;
- __kmp_str_buf_print( &__kmp_console_buf,
- "OMP warning: Debugging buffer overflow; increase KMP_DEBUG_BUF_CHARS to %d\n",
- chars + 1 );
- WriteFile( __kmp_stderr, __kmp_console_buf.str, __kmp_console_buf.used, &count, NULL );
- __kmp_str_buf_clear( &__kmp_console_buf );
- #else
- fprintf( __kmp_stderr,
- "OMP warning: Debugging buffer overflow; increase KMP_DEBUG_BUF_CHARS to %d\n",
- chars + 1 );
- fflush( __kmp_stderr );
- #endif
- __kmp_debug_buf_warn_chars = chars + 1;
- }
- /* terminate string if overflow occurred */
- db[ __kmp_debug_buf_chars - 2 ] = '\n';
- db[ __kmp_debug_buf_chars - 1 ] = '\0';
- }
- } else {
- #if KMP_OS_WINDOWS
- DWORD count;
- #ifdef KMP_DEBUG_PIDS
- __kmp_str_buf_print( &__kmp_console_buf, "pid=%d: ",
- (kmp_int32)getpid() );
- #endif
- __kmp_str_buf_vprint( &__kmp_console_buf, format, ap );
- WriteFile(
- __kmp_stderr,
- __kmp_console_buf.str,
- __kmp_console_buf.used,
- &count,
- NULL
- );
- __kmp_str_buf_clear( &__kmp_console_buf );
- #else
- #ifdef KMP_DEBUG_PIDS
- fprintf( __kmp_stderr, "pid=%d: ", (kmp_int32)getpid() );
- #endif
- vfprintf( __kmp_stderr, format, ap );
- fflush( __kmp_stderr );
- #endif
- }
-}
-
-void
-__kmp_printf( char const * format, ... )
-{
- va_list ap;
- va_start( ap, format );
-
- __kmp_acquire_bootstrap_lock( & __kmp_stdio_lock );
- __kmp_vprintf( kmp_err, format, ap );
- __kmp_release_bootstrap_lock( & __kmp_stdio_lock );
-
- va_end( ap );
-}
-
-void
-__kmp_printf_no_lock( char const * format, ... )
-{
- va_list ap;
- va_start( ap, format );
-
- __kmp_vprintf( kmp_err, format, ap );
-
- va_end( ap );
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
Removed: openmp/trunk/runtime/src/kmp_itt.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_itt.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_itt.c (original)
+++ openmp/trunk/runtime/src/kmp_itt.c (removed)
@@ -1,163 +0,0 @@
-#include "kmp_config.h"
-
-#if USE_ITT_BUILD
-/*
- * kmp_itt.c -- ITT Notify interface.
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp_itt.h"
-
-#if KMP_DEBUG
- #include "kmp_itt.inl"
-#endif
-
-
-#if USE_ITT_NOTIFY
-
- kmp_int32 __kmp_barrier_domain_count;
- kmp_int32 __kmp_region_domain_count;
- __itt_domain* __kmp_itt_barrier_domains[KMP_MAX_FRAME_DOMAINS];
- __itt_domain* __kmp_itt_region_domains[KMP_MAX_FRAME_DOMAINS];
- __itt_domain* __kmp_itt_imbalance_domains[KMP_MAX_FRAME_DOMAINS];
- kmp_int32 __kmp_itt_region_team_size[KMP_MAX_FRAME_DOMAINS];
- __itt_domain * metadata_domain = NULL;
- __itt_string_handle * string_handle_imbl = NULL;
- __itt_string_handle * string_handle_loop = NULL;
- __itt_string_handle * string_handle_sngl = NULL;
-
- #include "kmp_version.h"
- #include "kmp_i18n.h"
- #include "kmp_str.h"
-
- KMP_BUILD_ASSERT( sizeof( kmp_itt_mark_t ) == sizeof( __itt_mark_type ) );
-
- /*
- Previously used warnings:
-
- KMP_WARNING( IttAllNotifDisabled );
- KMP_WARNING( IttObjNotifDisabled );
- KMP_WARNING( IttMarkNotifDisabled );
- KMP_WARNING( IttUnloadLibFailed, libittnotify );
- */
-
-
- kmp_int32 __kmp_itt_prepare_delay = 0;
- kmp_bootstrap_lock_t __kmp_itt_debug_lock = KMP_BOOTSTRAP_LOCK_INITIALIZER( __kmp_itt_debug_lock );
-
-#endif // USE_ITT_NOTIFY
-
-void __kmp_itt_initialize() {
-
- // ITTNotify library is loaded and initialized at first call to any ittnotify function,
- // so we do not need to explicitly load it any more.
- // Jusr report OMP RTL version to ITTNotify.
-
- #if USE_ITT_NOTIFY
- // Report OpenMP RTL version.
- kmp_str_buf_t buf;
- __itt_mark_type version;
- __kmp_str_buf_init( & buf );
- __kmp_str_buf_print(
- & buf,
- "OMP RTL Version %d.%d.%d",
- __kmp_version_major,
- __kmp_version_minor,
- __kmp_version_build
- );
- if ( __itt_api_version_ptr != NULL ) {
- __kmp_str_buf_print( & buf, ":%s", __itt_api_version() );
- }; // if
- version = __itt_mark_create( buf.str );
- __itt_mark( version, NULL );
- __kmp_str_buf_free( & buf );
- #endif
-
-} // __kmp_itt_initialize
-
-
-void __kmp_itt_destroy() {
- #if USE_ITT_NOTIFY
- __kmp_itt_fini_ittlib();
- #endif
-} // __kmp_itt_destroy
-
-
-extern "C"
-void
-__itt_error_handler(
- __itt_error_code err,
- va_list args
-) {
-
- switch ( err ) {
- case __itt_error_no_module : {
- char const * library = va_arg( args, char const * );
-#if KMP_OS_WINDOWS
- int sys_err = va_arg( args, int );
- kmp_msg_t err_code = KMP_SYSERRCODE( sys_err );
- __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), err_code, __kmp_msg_null );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
-#else
- char const * sys_err = va_arg( args, char const * );
- kmp_msg_t err_code = KMP_SYSERRMESG( sys_err );
- __kmp_msg( kmp_ms_warning, KMP_MSG( IttLoadLibFailed, library ), err_code, __kmp_msg_null );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
-#endif
- } break;
- case __itt_error_no_symbol : {
- char const * library = va_arg( args, char const * );
- char const * symbol = va_arg( args, char const * );
- KMP_WARNING( IttLookupFailed, symbol, library );
- } break;
- case __itt_error_unknown_group : {
- char const * var = va_arg( args, char const * );
- char const * group = va_arg( args, char const * );
- KMP_WARNING( IttUnknownGroup, var, group );
- } break;
- case __itt_error_env_too_long : {
- char const * var = va_arg( args, char const * );
- size_t act_len = va_arg( args, size_t );
- size_t max_len = va_arg( args, size_t );
- KMP_WARNING( IttEnvVarTooLong, var, (unsigned long) act_len, (unsigned long) max_len );
- } break;
- case __itt_error_cant_read_env : {
- char const * var = va_arg( args, char const * );
- int sys_err = va_arg( args, int );
- kmp_msg_t err_code = KMP_ERR( sys_err );
- __kmp_msg( kmp_ms_warning, KMP_MSG( CantGetEnvVar, var ), err_code, __kmp_msg_null );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
- } break;
- case __itt_error_system : {
- char const * func = va_arg( args, char const * );
- int sys_err = va_arg( args, int );
- kmp_msg_t err_code = KMP_SYSERRCODE( sys_err );
- __kmp_msg( kmp_ms_warning, KMP_MSG( IttFunctionError, func ), err_code, __kmp_msg_null );
- if (__kmp_generate_warnings == kmp_warnings_off) {
- __kmp_str_free(&err_code.str);
- }
- } break;
- default : {
- KMP_WARNING( IttUnknownError, err );
- };
- }; // switch
-
-} // __itt_error_handler
-
-#endif /* USE_ITT_BUILD */
Removed: openmp/trunk/runtime/src/kmp_runtime.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_runtime.c?rev=289731&view=auto
==============================================================================
--- openmp/trunk/runtime/src/kmp_runtime.c (original)
+++ openmp/trunk/runtime/src/kmp_runtime.c (removed)
@@ -1,7683 +0,0 @@
-/*
- * kmp_runtime.c -- KPTS runtime support library
- */
-
-
-//===----------------------------------------------------------------------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.txt for details.
-//
-//===----------------------------------------------------------------------===//
-
-
-#include "kmp.h"
-#include "kmp_atomic.h"
-#include "kmp_wrapper_getpid.h"
-#include "kmp_environment.h"
-#include "kmp_itt.h"
-#include "kmp_str.h"
-#include "kmp_settings.h"
-#include "kmp_i18n.h"
-#include "kmp_io.h"
-#include "kmp_error.h"
-#include "kmp_stats.h"
-#include "kmp_wait_release.h"
-#include "kmp_affinity.h"
-
-#if OMPT_SUPPORT
-#include "ompt-specific.h"
-#endif
-
-/* these are temporary issues to be dealt with */
-#define KMP_USE_PRCTL 0
-
-#if KMP_OS_WINDOWS
-#include <process.h>
-#endif
-
-#include "tsan_annotations.h"
-
-#if defined(KMP_GOMP_COMPAT)
-char const __kmp_version_alt_comp[] = KMP_VERSION_PREFIX "alternative compiler support: yes";
-#endif /* defined(KMP_GOMP_COMPAT) */
-
-char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: "
-#if OMP_45_ENABLED
- "4.5 (201511)";
-#elif OMP_40_ENABLED
- "4.0 (201307)";
-#else
- "3.1 (201107)";
-#endif
-
-#ifdef KMP_DEBUG
-char const __kmp_version_lock[] = KMP_VERSION_PREFIX "lock type: run time selectable";
-#endif /* KMP_DEBUG */
-
-#define KMP_MIN( x, y ) ( (x) < (y) ? (x) : (y) )
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-kmp_info_t __kmp_monitor;
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* Forward declarations */
-
-void __kmp_cleanup( void );
-
-static void __kmp_initialize_info( kmp_info_t *, kmp_team_t *, int tid, int gtid );
-static void __kmp_initialize_team( kmp_team_t * team, int new_nproc, kmp_internal_control_t * new_icvs, ident_t * loc );
-#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
-static void __kmp_partition_places( kmp_team_t *team, int update_master_only=0 );
-#endif
-static void __kmp_do_serial_initialize( void );
-void __kmp_fork_barrier( int gtid, int tid );
-void __kmp_join_barrier( int gtid );
-void __kmp_setup_icv_copy( kmp_team_t *team, int new_nproc, kmp_internal_control_t * new_icvs, ident_t *loc );
-
-#ifdef USE_LOAD_BALANCE
-static int __kmp_load_balance_nproc( kmp_root_t * root, int set_nproc );
-#endif
-
-static int __kmp_expand_threads(int nWish, int nNeed);
-#if KMP_OS_WINDOWS
-static int __kmp_unregister_root_other_thread( int gtid );
-#endif
-static void __kmp_unregister_library( void ); // called by __kmp_internal_end()
-static void __kmp_reap_thread( kmp_info_t * thread, int is_root );
-static kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* Calculate the identifier of the current thread */
-/* fast (and somewhat portable) way to get unique */
-/* identifier of executing thread. */
-/* returns KMP_GTID_DNE if we haven't been assigned a gtid */
-
-int
-__kmp_get_global_thread_id( )
-{
- int i;
- kmp_info_t **other_threads;
- size_t stack_data;
- char *stack_addr;
- size_t stack_size;
- char *stack_base;
-
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
- __kmp_nth, __kmp_all_nth ));
-
- /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to a
- parallel region, made it return KMP_GTID_DNE to force serial_initialize by
- caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
- __kmp_init_gtid for this to work. */
-
- if ( !TCR_4(__kmp_init_gtid) ) return KMP_GTID_DNE;
-
-#ifdef KMP_TDATA_GTID
- if ( TCR_4(__kmp_gtid_mode) >= 3) {
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id: using TDATA\n" ));
- return __kmp_gtid;
- }
-#endif
- if ( TCR_4(__kmp_gtid_mode) >= 2) {
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id: using keyed TLS\n" ));
- return __kmp_gtid_get_specific();
- }
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id: using internal alg.\n" ));
-
- stack_addr = (char*) & stack_data;
- other_threads = __kmp_threads;
-
- /*
- ATT: The code below is a source of potential bugs due to unsynchronized access to
- __kmp_threads array. For example:
- 1. Current thread loads other_threads[i] to thr and checks it, it is non-NULL.
- 2. Current thread is suspended by OS.
- 3. Another thread unregisters and finishes (debug versions of free() may fill memory
- with something like 0xEF).
- 4. Current thread is resumed.
- 5. Current thread reads junk from *thr.
- TODO: Fix it.
- --ln
- */
-
- for( i = 0 ; i < __kmp_threads_capacity ; i++ ) {
-
- kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
- if( !thr ) continue;
-
- stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
- stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
-
- /* stack grows down -- search through all of the active threads */
-
- if( stack_addr <= stack_base ) {
- size_t stack_diff = stack_base - stack_addr;
-
- if( stack_diff <= stack_size ) {
- /* The only way we can be closer than the allocated */
- /* stack size is if we are running on this thread. */
- KMP_DEBUG_ASSERT( __kmp_gtid_get_specific() == i );
- return i;
- }
- }
- }
-
- /* get specific to try and determine our gtid */
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id: internal alg. failed to find "
- "thread, using TLS\n" ));
- i = __kmp_gtid_get_specific();
-
- /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
-
- /* if we havn't been assigned a gtid, then return code */
- if( i<0 ) return i;
-
- /* dynamically updated stack window for uber threads to avoid get_specific call */
- if( ! TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow) ) {
- KMP_FATAL( StackOverflow, i );
- }
-
- stack_base = (char *) other_threads[i]->th.th_info.ds.ds_stackbase;
- if( stack_addr > stack_base ) {
- TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
- TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
- other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr - stack_base);
- } else {
- TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize, stack_base - stack_addr);
- }
-
- /* Reprint stack bounds for ubermaster since they have been refined */
- if ( __kmp_storage_map ) {
- char *stack_end = (char *) other_threads[i]->th.th_info.ds.ds_stackbase;
- char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
- __kmp_print_storage_map_gtid( i, stack_beg, stack_end,
- other_threads[i]->th.th_info.ds.ds_stacksize,
- "th_%d stack (refinement)", i );
- }
- return i;
-}
-
-int
-__kmp_get_global_thread_id_reg( )
-{
- int gtid;
-
- if ( !__kmp_init_serial ) {
- gtid = KMP_GTID_DNE;
- } else
-#ifdef KMP_TDATA_GTID
- if ( TCR_4(__kmp_gtid_mode) >= 3 ) {
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id_reg: using TDATA\n" ));
- gtid = __kmp_gtid;
- } else
-#endif
- if ( TCR_4(__kmp_gtid_mode) >= 2 ) {
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id_reg: using keyed TLS\n" ));
- gtid = __kmp_gtid_get_specific();
- } else {
- KA_TRACE( 1000, ( "*** __kmp_get_global_thread_id_reg: using internal alg.\n" ));
- gtid = __kmp_get_global_thread_id();
- }
-
- /* we must be a new uber master sibling thread */
- if( gtid == KMP_GTID_DNE ) {
- KA_TRACE( 10, ( "__kmp_get_global_thread_id_reg: Encountered new root thread. "
- "Registering a new gtid.\n" ));
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
- if( !__kmp_init_serial ) {
- __kmp_do_serial_initialize();
- gtid = __kmp_gtid_get_specific();
- } else {
- gtid = __kmp_register_root(FALSE);
- }
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
- }
-
- KMP_DEBUG_ASSERT( gtid >=0 );
-
- return gtid;
-}
-
-/* caller must hold forkjoin_lock */
-void
-__kmp_check_stack_overlap( kmp_info_t *th )
-{
- int f;
- char *stack_beg = NULL;
- char *stack_end = NULL;
- int gtid;
-
- KA_TRACE(10,("__kmp_check_stack_overlap: called\n"));
- if ( __kmp_storage_map ) {
- stack_end = (char *) th->th.th_info.ds.ds_stackbase;
- stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
-
- gtid = __kmp_gtid_from_thread( th );
-
- if (gtid == KMP_GTID_MONITOR) {
- __kmp_print_storage_map_gtid( gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
- "th_%s stack (%s)", "mon",
- ( th->th.th_info.ds.ds_stackgrow ) ? "initial" : "actual" );
- } else {
- __kmp_print_storage_map_gtid( gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
- "th_%d stack (%s)", gtid,
- ( th->th.th_info.ds.ds_stackgrow ) ? "initial" : "actual" );
- }
- }
-
- /* No point in checking ubermaster threads since they use refinement and cannot overlap */
- gtid = __kmp_gtid_from_thread( th );
- if ( __kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid))
- {
- KA_TRACE(10,("__kmp_check_stack_overlap: performing extensive checking\n"));
- if ( stack_beg == NULL ) {
- stack_end = (char *) th->th.th_info.ds.ds_stackbase;
- stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
- }
-
- for( f=0 ; f < __kmp_threads_capacity ; f++ ) {
- kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
-
- if( f_th && f_th != th ) {
- char *other_stack_end = (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
- char *other_stack_beg = other_stack_end -
- (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
- if((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
- (stack_end > other_stack_beg && stack_end < other_stack_end)) {
-
- /* Print the other stack values before the abort */
- if ( __kmp_storage_map )
- __kmp_print_storage_map_gtid( -1, other_stack_beg, other_stack_end,
- (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
- "th_%d stack (overlapped)",
- __kmp_gtid_from_thread( f_th ) );
-
- __kmp_msg( kmp_ms_fatal, KMP_MSG( StackOverlap ), KMP_HNT( ChangeStackLimit ), __kmp_msg_null );
- }
- }
- }
- }
- KA_TRACE(10,("__kmp_check_stack_overlap: returning\n"));
-}
-
-
-/* ------------------------------------------------------------------------ */
-
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_infinite_loop( void )
-{
- static int done = FALSE;
-
- while (! done) {
- KMP_YIELD( 1 );
- }
-}
-
-#define MAX_MESSAGE 512
-
-void
-__kmp_print_storage_map_gtid( int gtid, void *p1, void *p2, size_t size, char const *format, ...) {
- char buffer[MAX_MESSAGE];
- va_list ap;
-
- va_start( ap, format);
- KMP_SNPRINTF( buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1, p2, (unsigned long) size, format );
- __kmp_acquire_bootstrap_lock( & __kmp_stdio_lock );
- __kmp_vprintf( kmp_err, buffer, ap );
-#if KMP_PRINT_DATA_PLACEMENT
- int node;
- if(gtid >= 0) {
- if(p1 <= p2 && (char*)p2 - (char*)p1 == size) {
- if( __kmp_storage_map_verbose ) {
- node = __kmp_get_host_node(p1);
- if(node < 0) /* doesn't work, so don't try this next time */
- __kmp_storage_map_verbose = FALSE;
- else {
- char *last;
- int lastNode;
- int localProc = __kmp_get_cpu_from_gtid(gtid);
-
- const int page_size = KMP_GET_PAGE_SIZE();
-
- p1 = (void *)( (size_t)p1 & ~((size_t)page_size - 1) );
- p2 = (void *)( ((size_t) p2 - 1) & ~((size_t)page_size - 1) );
- if(localProc >= 0)
- __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid, localProc>>1);
- else
- __kmp_printf_no_lock(" GTID %d\n", gtid);
-# if KMP_USE_PRCTL
-/* The more elaborate format is disabled for now because of the prctl hanging bug. */
- do {
- last = p1;
- lastNode = node;
- /* This loop collates adjacent pages with the same host node. */
- do {
- (char*)p1 += page_size;
- } while(p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
- __kmp_printf_no_lock(" %p-%p memNode %d\n", last,
- (char*)p1 - 1, lastNode);
- } while(p1 <= p2);
-# else
- __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
- (char*)p1 + (page_size - 1), __kmp_get_host_node(p1));
- if(p1 < p2) {
- __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
- (char*)p2 + (page_size - 1), __kmp_get_host_node(p2));
- }
-# endif
- }
- }
- } else
- __kmp_printf_no_lock(" %s\n", KMP_I18N_STR( StorageMapWarning ) );
- }
-#endif /* KMP_PRINT_DATA_PLACEMENT */
- __kmp_release_bootstrap_lock( & __kmp_stdio_lock );
-}
-
-void
-__kmp_warn( char const * format, ... )
-{
- char buffer[MAX_MESSAGE];
- va_list ap;
-
- if ( __kmp_generate_warnings == kmp_warnings_off ) {
- return;
- }
-
- va_start( ap, format );
-
- KMP_SNPRINTF( buffer, sizeof(buffer) , "OMP warning: %s\n", format );
- __kmp_acquire_bootstrap_lock( & __kmp_stdio_lock );
- __kmp_vprintf( kmp_err, buffer, ap );
- __kmp_release_bootstrap_lock( & __kmp_stdio_lock );
-
- va_end( ap );
-}
-
-void
-__kmp_abort_process()
-{
-
- // Later threads may stall here, but that's ok because abort() will kill them.
- __kmp_acquire_bootstrap_lock( & __kmp_exit_lock );
-
- if ( __kmp_debug_buf ) {
- __kmp_dump_debug_buffer();
- }; // if
-
- if ( KMP_OS_WINDOWS ) {
- // Let other threads know of abnormal termination and prevent deadlock
- // if abort happened during library initialization or shutdown
- __kmp_global.g.g_abort = SIGABRT;
-
- /*
- On Windows* OS by default abort() causes pop-up error box, which stalls nightly testing.
- Unfortunately, we cannot reliably suppress pop-up error boxes. _set_abort_behavior()
- works well, but this function is not available in VS7 (this is not problem for DLL, but
- it is a problem for static OpenMP RTL). SetErrorMode (and so, timelimit utility) does
- not help, at least in some versions of MS C RTL.
-
- It seems following sequence is the only way to simulate abort() and avoid pop-up error
- box.
- */
- raise( SIGABRT );
- _exit( 3 ); // Just in case, if signal ignored, exit anyway.
- } else {
- abort();
- }; // if
-
- __kmp_infinite_loop();
- __kmp_release_bootstrap_lock( & __kmp_exit_lock );
-
-} // __kmp_abort_process
-
-void
-__kmp_abort_thread( void )
-{
- // TODO: Eliminate g_abort global variable and this function.
- // In case of abort just call abort(), it will kill all the threads.
- __kmp_infinite_loop();
-} // __kmp_abort_thread
-
-/* ------------------------------------------------------------------------ */
-
-/*
- * Print out the storage map for the major kmp_info_t thread data structures
- * that are allocated together.
- */
-
-static void
-__kmp_print_thread_storage_map( kmp_info_t *thr, int gtid )
-{
- __kmp_print_storage_map_gtid( gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d", gtid );
-
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_info, &thr->th.th_team, sizeof(kmp_desc_t),
- "th_%d.th_info", gtid );
-
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_local, &thr->th.th_pri_head, sizeof(kmp_local_t),
- "th_%d.th_local", gtid );
-
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
- sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid );
-
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_bar[bs_plain_barrier],
- &thr->th.th_bar[bs_plain_barrier+1],
- sizeof(kmp_balign_t), "th_%d.th_bar[plain]", gtid);
-
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_bar[bs_forkjoin_barrier],
- &thr->th.th_bar[bs_forkjoin_barrier+1],
- sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]", gtid);
-
- #if KMP_FAST_REDUCTION_BARRIER
- __kmp_print_storage_map_gtid( gtid, &thr->th.th_bar[bs_reduction_barrier],
- &thr->th.th_bar[bs_reduction_barrier+1],
- sizeof(kmp_balign_t), "th_%d.th_bar[reduction]", gtid);
- #endif // KMP_FAST_REDUCTION_BARRIER
-}
-
-/*
- * Print out the storage map for the major kmp_team_t team data structures
- * that are allocated together.
- */
-
-static void
-__kmp_print_team_storage_map( const char *header, kmp_team_t *team, int team_id, int num_thr )
-{
- int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
- __kmp_print_storage_map_gtid( -1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
- header, team_id );
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_bar[0], &team->t.t_bar[bs_last_barrier],
- sizeof(kmp_balign_team_t) * bs_last_barrier, "%s_%d.t_bar", header, team_id );
-
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_bar[bs_plain_barrier], &team->t.t_bar[bs_plain_barrier+1],
- sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]", header, team_id );
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_bar[bs_forkjoin_barrier], &team->t.t_bar[bs_forkjoin_barrier+1],
- sizeof(kmp_balign_team_t), "%s_%d.t_bar[forkjoin]", header, team_id );
-
- #if KMP_FAST_REDUCTION_BARRIER
- __kmp_print_storage_map_gtid( -1, &team->t.t_bar[bs_reduction_barrier], &team->t.t_bar[bs_reduction_barrier+1],
- sizeof(kmp_balign_team_t), "%s_%d.t_bar[reduction]", header, team_id );
- #endif // KMP_FAST_REDUCTION_BARRIER
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
- sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id );
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
- sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id );
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_disp_buffer[0], &team->t.t_disp_buffer[num_disp_buff],
- sizeof(dispatch_shared_info_t) * num_disp_buff, "%s_%d.t_disp_buffer",
- header, team_id );
-
-
- __kmp_print_storage_map_gtid( -1, &team->t.t_taskq, &team->t.t_copypriv_data,
- sizeof(kmp_taskq_t), "%s_%d.t_taskq", header, team_id );
-}
-
-static void __kmp_init_allocator() {}
-static void __kmp_fini_allocator() {}
-
-/* ------------------------------------------------------------------------ */
-
-#ifdef KMP_DYNAMIC_LIB
-# if KMP_OS_WINDOWS
-
-static void
-__kmp_reset_lock( kmp_bootstrap_lock_t* lck ) {
- // TODO: Change to __kmp_break_bootstrap_lock().
- __kmp_init_bootstrap_lock( lck ); // make the lock released
-}
-
-static void
-__kmp_reset_locks_on_process_detach( int gtid_req ) {
- int i;
- int thread_count;
-
- // PROCESS_DETACH is expected to be called by a thread
- // that executes ProcessExit() or FreeLibrary().
- // OS terminates other threads (except the one calling ProcessExit or FreeLibrary).
- // So, it might be safe to access the __kmp_threads[] without taking the forkjoin_lock.
- // However, in fact, some threads can be still alive here, although being about to be terminated.
- // The threads in the array with ds_thread==0 are most suspicious.
- // Actually, it can be not safe to access the __kmp_threads[].
-
- // TODO: does it make sense to check __kmp_roots[] ?
-
- // Let's check that there are no other alive threads registered with the OMP lib.
- while( 1 ) {
- thread_count = 0;
- for( i = 0; i < __kmp_threads_capacity; ++i ) {
- if( !__kmp_threads ) continue;
- kmp_info_t* th = __kmp_threads[ i ];
- if( th == NULL ) continue;
- int gtid = th->th.th_info.ds.ds_gtid;
- if( gtid == gtid_req ) continue;
- if( gtid < 0 ) continue;
- DWORD exit_val;
- int alive = __kmp_is_thread_alive( th, &exit_val );
- if( alive ) {
- ++thread_count;
- }
- }
- if( thread_count == 0 ) break; // success
- }
-
- // Assume that I'm alone.
-
- // Now it might be probably safe to check and reset locks.
- // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
- __kmp_reset_lock( &__kmp_forkjoin_lock );
- #ifdef KMP_DEBUG
- __kmp_reset_lock( &__kmp_stdio_lock );
- #endif // KMP_DEBUG
-}
-
-BOOL WINAPI
-DllMain( HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved ) {
- //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
-
- switch( fdwReason ) {
-
- case DLL_PROCESS_ATTACH:
- KA_TRACE( 10, ("DllMain: PROCESS_ATTACH\n" ));
-
- return TRUE;
-
- case DLL_PROCESS_DETACH:
- KA_TRACE( 10, ("DllMain: PROCESS_DETACH T#%d\n",
- __kmp_gtid_get_specific() ));
-
- if( lpReserved != NULL )
- {
- // lpReserved is used for telling the difference:
- // lpReserved == NULL when FreeLibrary() was called,
- // lpReserved != NULL when the process terminates.
- // When FreeLibrary() is called, worker threads remain alive.
- // So they will release the forkjoin lock by themselves.
- // When the process terminates, worker threads disappear triggering
- // the problem of unreleased forkjoin lock as described below.
-
- // A worker thread can take the forkjoin lock.
- // The problem comes up if that worker thread becomes dead
- // before it releases the forkjoin lock.
- // The forkjoin lock remains taken, while the thread
- // executing DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below
- // will try to take the forkjoin lock and will always fail,
- // so that the application will never finish [normally].
- // This scenario is possible if __kmpc_end() has not been executed.
- // It looks like it's not a corner case, but common cases:
- // - the main function was compiled by an alternative compiler;
- // - the main function was compiled by icl but without /Qopenmp (application with plugins);
- // - application terminates by calling C exit(), Fortran CALL EXIT() or Fortran STOP.
- // - alive foreign thread prevented __kmpc_end from doing cleanup.
-
- // This is a hack to work around the problem.
- // TODO: !!! to figure out something better.
- __kmp_reset_locks_on_process_detach( __kmp_gtid_get_specific() );
- }
-
- __kmp_internal_end_library( __kmp_gtid_get_specific() );
-
- return TRUE;
-
- case DLL_THREAD_ATTACH:
- KA_TRACE( 10, ("DllMain: THREAD_ATTACH\n" ));
-
- /* if we wanted to register new siblings all the time here call
- * __kmp_get_gtid(); */
- return TRUE;
-
- case DLL_THREAD_DETACH:
- KA_TRACE( 10, ("DllMain: THREAD_DETACH T#%d\n",
- __kmp_gtid_get_specific() ));
-
- __kmp_internal_end_thread( __kmp_gtid_get_specific() );
- return TRUE;
- }
-
- return TRUE;
-}
-
-# endif /* KMP_OS_WINDOWS */
-#endif /* KMP_DYNAMIC_LIB */
-
-
-/* ------------------------------------------------------------------------ */
-
-/* Change the library type to "status" and return the old type */
-/* called from within initialization routines where __kmp_initz_lock is held */
-int
-__kmp_change_library( int status )
-{
- int old_status;
-
- old_status = __kmp_yield_init & 1; // check whether KMP_LIBRARY=throughput (even init count)
-
- if (status) {
- __kmp_yield_init |= 1; // throughput => turnaround (odd init count)
- }
- else {
- __kmp_yield_init &= ~1; // turnaround => throughput (even init count)
- }
-
- return old_status; // return previous setting of whether KMP_LIBRARY=throughput
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* __kmp_parallel_deo --
- * Wait until it's our turn.
- */
-void
-__kmp_parallel_deo( int *gtid_ref, int *cid_ref, ident_t *loc_ref )
-{
- int gtid = *gtid_ref;
-#ifdef BUILD_PARALLEL_ORDERED
- kmp_team_t *team = __kmp_team_from_gtid( gtid );
-#endif /* BUILD_PARALLEL_ORDERED */
-
- if( __kmp_env_consistency_check ) {
- if( __kmp_threads[gtid]->th.th_root->r.r_active )
-#if KMP_USE_DYNAMIC_LOCK
- __kmp_push_sync( gtid, ct_ordered_in_parallel, loc_ref, NULL, 0 );
-#else
- __kmp_push_sync( gtid, ct_ordered_in_parallel, loc_ref, NULL );
-#endif
- }
-#ifdef BUILD_PARALLEL_ORDERED
- if( !team->t.t_serialized ) {
- KMP_MB();
- KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid( gtid ), KMP_EQ, NULL);
- KMP_MB();
- }
-#endif /* BUILD_PARALLEL_ORDERED */
-}
-
-/* __kmp_parallel_dxo --
- * Signal the next task.
- */
-
-void
-__kmp_parallel_dxo( int *gtid_ref, int *cid_ref, ident_t *loc_ref )
-{
- int gtid = *gtid_ref;
-#ifdef BUILD_PARALLEL_ORDERED
- int tid = __kmp_tid_from_gtid( gtid );
- kmp_team_t *team = __kmp_team_from_gtid( gtid );
-#endif /* BUILD_PARALLEL_ORDERED */
-
- if( __kmp_env_consistency_check ) {
- if( __kmp_threads[gtid]->th.th_root->r.r_active )
- __kmp_pop_sync( gtid, ct_ordered_in_parallel, loc_ref );
- }
-#ifdef BUILD_PARALLEL_ORDERED
- if ( ! team->t.t_serialized ) {
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- /* use the tid of the next thread in this team */
- /* TODO repleace with general release procedure */
- team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc );
-
-#if OMPT_SUPPORT && OMPT_BLAME
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
- /* accept blame for "ordered" waiting */
- kmp_info_t *this_thread = __kmp_threads[gtid];
- ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
- this_thread->th.ompt_thread_info.wait_id);
- }
-#endif
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
- }
-#endif /* BUILD_PARALLEL_ORDERED */
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* The BARRIER for a SINGLE process section is always explicit */
-
-int
-__kmp_enter_single( int gtid, ident_t *id_ref, int push_ws )
-{
- int status;
- kmp_info_t *th;
- kmp_team_t *team;
-
- if( ! TCR_4(__kmp_init_parallel) )
- __kmp_parallel_initialize();
-
- th = __kmp_threads[ gtid ];
- team = th->th.th_team;
- status = 0;
-
- th->th.th_ident = id_ref;
-
- if ( team->t.t_serialized ) {
- status = 1;
- } else {
- kmp_int32 old_this = th->th.th_local.this_construct;
-
- ++th->th.th_local.this_construct;
- /* try to set team count to thread count--success means thread got the
- single block
- */
- /* TODO: Should this be acquire or release? */
- if (team->t.t_construct == old_this) {
- status = KMP_COMPARE_AND_STORE_ACQ32(&team->t.t_construct, old_this,
- th->th.th_local.this_construct);
- }
-#if USE_ITT_BUILD
- if ( __itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 && KMP_MASTER_GTID(gtid) &&
-#if OMP_40_ENABLED
- th->th.th_teams_microtask == NULL &&
-#endif
- team->t.t_active_level == 1 )
- { // Only report metadata by master of active team at level 1
- __kmp_itt_metadata_single( id_ref );
- }
-#endif /* USE_ITT_BUILD */
- }
-
- if( __kmp_env_consistency_check ) {
- if (status && push_ws) {
- __kmp_push_workshare( gtid, ct_psingle, id_ref );
- } else {
- __kmp_check_workshare( gtid, ct_psingle, id_ref );
- }
- }
-#if USE_ITT_BUILD
- if ( status ) {
- __kmp_itt_single_start( gtid );
- }
-#endif /* USE_ITT_BUILD */
- return status;
-}
-
-void
-__kmp_exit_single( int gtid )
-{
-#if USE_ITT_BUILD
- __kmp_itt_single_end( gtid );
-#endif /* USE_ITT_BUILD */
- if( __kmp_env_consistency_check )
- __kmp_pop_workshare( gtid, ct_psingle, NULL );
-}
-
-
-/*
- * determine if we can go parallel or must use a serialized parallel region and
- * how many threads we can use
- * set_nproc is the number of threads requested for the team
- * returns 0 if we should serialize or only use one thread,
- * otherwise the number of threads to use
- * The forkjoin lock is held by the caller.
- */
-static int
-__kmp_reserve_threads( kmp_root_t *root, kmp_team_t *parent_team,
- int master_tid, int set_nthreads
-#if OMP_40_ENABLED
- , int enter_teams
-#endif /* OMP_40_ENABLED */
-)
-{
- int capacity;
- int new_nthreads;
- KMP_DEBUG_ASSERT( __kmp_init_serial );
- KMP_DEBUG_ASSERT( root && parent_team );
-
- //
- // If dyn-var is set, dynamically adjust the number of desired threads,
- // according to the method specified by dynamic_mode.
- //
- new_nthreads = set_nthreads;
- if ( ! get__dynamic_2( parent_team, master_tid ) ) {
- ;
- }
-#ifdef USE_LOAD_BALANCE
- else if ( __kmp_global.g.g_dynamic_mode == dynamic_load_balance ) {
- new_nthreads = __kmp_load_balance_nproc( root, set_nthreads );
- if ( new_nthreads == 1 ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d load balance reduced reservation to 1 thread\n",
- master_tid ));
- return 1;
- }
- if ( new_nthreads < set_nthreads ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d load balance reduced reservation to %d threads\n",
- master_tid, new_nthreads ));
- }
- }
-#endif /* USE_LOAD_BALANCE */
- else if ( __kmp_global.g.g_dynamic_mode == dynamic_thread_limit ) {
- new_nthreads = __kmp_avail_proc - __kmp_nth + (root->r.r_active ? 1
- : root->r.r_hot_team->t.t_nproc);
- if ( new_nthreads <= 1 ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d thread limit reduced reservation to 1 thread\n",
- master_tid ));
- return 1;
- }
- if ( new_nthreads < set_nthreads ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d thread limit reduced reservation to %d threads\n",
- master_tid, new_nthreads ));
- }
- else {
- new_nthreads = set_nthreads;
- }
- }
- else if ( __kmp_global.g.g_dynamic_mode == dynamic_random ) {
- if ( set_nthreads > 2 ) {
- new_nthreads = __kmp_get_random( parent_team->t.t_threads[master_tid] );
- new_nthreads = ( new_nthreads % set_nthreads ) + 1;
- if ( new_nthreads == 1 ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d dynamic random reduced reservation to 1 thread\n",
- master_tid ));
- return 1;
- }
- if ( new_nthreads < set_nthreads ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d dynamic random reduced reservation to %d threads\n",
- master_tid, new_nthreads ));
- }
- }
- }
- else {
- KMP_ASSERT( 0 );
- }
-
- //
- // Respect KMP_ALL_THREADS, KMP_MAX_THREADS, OMP_THREAD_LIMIT.
- //
- if ( __kmp_nth + new_nthreads - ( root->r.r_active ? 1 :
- root->r.r_hot_team->t.t_nproc ) > __kmp_max_nth ) {
- int tl_nthreads = __kmp_max_nth - __kmp_nth + ( root->r.r_active ? 1 :
- root->r.r_hot_team->t.t_nproc );
- if ( tl_nthreads <= 0 ) {
- tl_nthreads = 1;
- }
-
- //
- // If dyn-var is false, emit a 1-time warning.
- //
- if ( ! get__dynamic_2( parent_team, master_tid )
- && ( ! __kmp_reserve_warn ) ) {
- __kmp_reserve_warn = 1;
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantFormThrTeam, set_nthreads, tl_nthreads ),
- KMP_HNT( Unset_ALL_THREADS ),
- __kmp_msg_null
- );
- }
- if ( tl_nthreads == 1 ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d KMP_ALL_THREADS reduced reservation to 1 thread\n",
- master_tid ));
- return 1;
- }
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d KMP_ALL_THREADS reduced reservation to %d threads\n",
- master_tid, tl_nthreads ));
- new_nthreads = tl_nthreads;
- }
-
- //
- // Check if the threads array is large enough, or needs expanding.
- //
- // See comment in __kmp_register_root() about the adjustment if
- // __kmp_threads[0] == NULL.
- //
- capacity = __kmp_threads_capacity;
- if ( TCR_PTR(__kmp_threads[0]) == NULL ) {
- --capacity;
- }
- if ( __kmp_nth + new_nthreads - ( root->r.r_active ? 1 :
- root->r.r_hot_team->t.t_nproc ) > capacity ) {
- //
- // Expand the threads array.
- //
- int slotsRequired = __kmp_nth + new_nthreads - ( root->r.r_active ? 1 :
- root->r.r_hot_team->t.t_nproc ) - capacity;
- int slotsAdded = __kmp_expand_threads(slotsRequired, slotsRequired);
- if ( slotsAdded < slotsRequired ) {
- //
- // The threads array was not expanded enough.
- //
- new_nthreads -= ( slotsRequired - slotsAdded );
- KMP_ASSERT( new_nthreads >= 1 );
-
- //
- // If dyn-var is false, emit a 1-time warning.
- //
- if ( ! get__dynamic_2( parent_team, master_tid )
- && ( ! __kmp_reserve_warn ) ) {
- __kmp_reserve_warn = 1;
- if ( __kmp_tp_cached ) {
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantFormThrTeam, set_nthreads, new_nthreads ),
- KMP_HNT( Set_ALL_THREADPRIVATE, __kmp_tp_capacity ),
- KMP_HNT( PossibleSystemLimitOnThreads ),
- __kmp_msg_null
- );
- }
- else {
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantFormThrTeam, set_nthreads, new_nthreads ),
- KMP_HNT( SystemLimitOnThreads ),
- __kmp_msg_null
- );
- }
- }
- }
- }
-
- if ( new_nthreads == 1 ) {
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d serializing team after reclaiming dead roots and rechecking; requested %d threads\n",
- __kmp_get_gtid(), set_nthreads ) );
- return 1;
- }
-
- KC_TRACE( 10, ( "__kmp_reserve_threads: T#%d allocating %d threads; requested %d threads\n",
- __kmp_get_gtid(), new_nthreads, set_nthreads ));
- return new_nthreads;
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* allocate threads from the thread pool and assign them to the new team */
-/* we are assured that there are enough threads available, because we
- * checked on that earlier within critical section forkjoin */
-
-static void
-__kmp_fork_team_threads( kmp_root_t *root, kmp_team_t *team,
- kmp_info_t *master_th, int master_gtid )
-{
- int i;
- int use_hot_team;
-
- KA_TRACE( 10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc ) );
- KMP_DEBUG_ASSERT( master_gtid == __kmp_get_gtid() );
- KMP_MB();
-
- /* first, let's setup the master thread */
- master_th->th.th_info.ds.ds_tid = 0;
- master_th->th.th_team = team;
- master_th->th.th_team_nproc = team->t.t_nproc;
- master_th->th.th_team_master = master_th;
- master_th->th.th_team_serialized = FALSE;
- master_th->th.th_dispatch = & team->t.t_dispatch[ 0 ];
-
- /* make sure we are not the optimized hot team */
-#if KMP_NESTED_HOT_TEAMS
- use_hot_team = 0;
- kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
- if( hot_teams ) { // hot teams array is not allocated if KMP_HOT_TEAMS_MAX_LEVEL=0
- int level = team->t.t_active_level - 1; // index in array of hot teams
- if( master_th->th.th_teams_microtask ) { // are we inside the teams?
- if( master_th->th.th_teams_size.nteams > 1 ) {
- ++level; // level was not increased in teams construct for team_of_masters
- }
- if( team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
- master_th->th.th_teams_level == team->t.t_level ) {
- ++level; // level was not increased in teams construct for team_of_workers before the parallel
- } // team->t.t_level will be increased inside parallel
- }
- if( level < __kmp_hot_teams_max_level ) {
- if( hot_teams[level].hot_team ) {
- // hot team has already been allocated for given level
- KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
- use_hot_team = 1; // the team is ready to use
- } else {
- use_hot_team = 0; // AC: threads are not allocated yet
- hot_teams[level].hot_team = team; // remember new hot team
- hot_teams[level].hot_team_nth = team->t.t_nproc;
- }
- } else {
- use_hot_team = 0;
- }
- }
-#else
- use_hot_team = team == root->r.r_hot_team;
-#endif
- if ( !use_hot_team ) {
-
- /* install the master thread */
- team->t.t_threads[ 0 ] = master_th;
- __kmp_initialize_info( master_th, team, 0, master_gtid );
-
- /* now, install the worker threads */
- for ( i=1 ; i < team->t.t_nproc ; i++ ) {
-
- /* fork or reallocate a new thread and install it in team */
- kmp_info_t *thr = __kmp_allocate_thread( root, team, i );
- team->t.t_threads[ i ] = thr;
- KMP_DEBUG_ASSERT( thr );
- KMP_DEBUG_ASSERT( thr->th.th_team == team );
- /* align team and thread arrived states */
- KA_TRACE( 20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived T#%d(%d:%d) join =%llu, plain=%llu\n",
- __kmp_gtid_from_tid( 0, team ), team->t.t_id, 0,
- __kmp_gtid_from_tid( i, team ), team->t.t_id, i,
- team->t.t_bar[ bs_forkjoin_barrier ].b_arrived,
- team->t.t_bar[ bs_plain_barrier ].b_arrived ) );
-#if OMP_40_ENABLED
- thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
- thr->th.th_teams_level = master_th->th.th_teams_level;
- thr->th.th_teams_size = master_th->th.th_teams_size;
-#endif
- { // Initialize threads' barrier data.
- int b;
- kmp_balign_t * balign = team->t.t_threads[ i ]->th.th_bar;
- for ( b = 0; b < bs_last_barrier; ++ b ) {
- balign[ b ].bb.b_arrived = team->t.t_bar[ b ].b_arrived;
- KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#if USE_DEBUGGER
- balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
-#endif
- }; // for b
- }
- }
-
-#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
- __kmp_partition_places( team );
-#endif
-
- }
-
- KMP_MB();
-}
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
-//
-// Propagate any changes to the floating point control registers out to the team
-// We try to avoid unnecessary writes to the relevant cache line in the team structure,
-// so we don't make changes unless they are needed.
-//
-inline static void
-propagateFPControl(kmp_team_t * team)
-{
- if ( __kmp_inherit_fp_control ) {
- kmp_int16 x87_fpu_control_word;
- kmp_uint32 mxcsr;
-
- // Get master values of FPU control flags (both X87 and vector)
- __kmp_store_x87_fpu_control_word( &x87_fpu_control_word );
- __kmp_store_mxcsr( &mxcsr );
- mxcsr &= KMP_X86_MXCSR_MASK;
-
- // There is no point looking at t_fp_control_saved here.
- // If it is TRUE, we still have to update the values if they are different from those we now have.
- // If it is FALSE we didn't save anything yet, but our objective is the same. We have to ensure
- // that the values in the team are the same as those we have.
- // So, this code achieves what we need whether or not t_fp_control_saved is true.
- // By checking whether the value needs updating we avoid unnecessary writes that would put the
- // cache-line into a written state, causing all threads in the team to have to read it again.
- KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
- KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
- // Although we don't use this value, other code in the runtime wants to know whether it should restore them.
- // So we must ensure it is correct.
- KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
- }
- else {
- // Similarly here. Don't write to this cache-line in the team structure unless we have to.
- KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
- }
-}
-
-// Do the opposite, setting the hardware registers to the updated values from the team.
-inline static void
-updateHWFPControl(kmp_team_t * team)
-{
- if ( __kmp_inherit_fp_control && team->t.t_fp_control_saved ) {
- //
- // Only reset the fp control regs if they have been changed in the team.
- // the parallel region that we are exiting.
- //
- kmp_int16 x87_fpu_control_word;
- kmp_uint32 mxcsr;
- __kmp_store_x87_fpu_control_word( &x87_fpu_control_word );
- __kmp_store_mxcsr( &mxcsr );
- mxcsr &= KMP_X86_MXCSR_MASK;
-
- if ( team->t.t_x87_fpu_control_word != x87_fpu_control_word ) {
- __kmp_clear_x87_fpu_status_word();
- __kmp_load_x87_fpu_control_word( &team->t.t_x87_fpu_control_word );
- }
-
- if ( team->t.t_mxcsr != mxcsr ) {
- __kmp_load_mxcsr( &team->t.t_mxcsr );
- }
- }
-}
-#else
-# define propagateFPControl(x) ((void)0)
-# define updateHWFPControl(x) ((void)0)
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-static void
-__kmp_alloc_argv_entries( int argc, kmp_team_t *team, int realloc ); // forward declaration
-
-/*
- * Run a parallel region that has been serialized, so runs only in a team of the single master thread.
- */
-void
-__kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
-{
- kmp_info_t *this_thr;
- kmp_team_t *serial_team;
-
- KC_TRACE( 10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid ) );
-
- /* Skip all this code for autopar serialized loops since it results in
- unacceptable overhead */
- if( loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR ) )
- return;
-
- if( ! TCR_4( __kmp_init_parallel ) )
- __kmp_parallel_initialize();
-
- this_thr = __kmp_threads[ global_tid ];
- serial_team = this_thr->th.th_serial_team;
-
- /* utilize the serialized team held by this thread */
- KMP_DEBUG_ASSERT( serial_team );
- KMP_MB();
-
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KMP_DEBUG_ASSERT(this_thr->th.th_task_team == this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
- KMP_DEBUG_ASSERT( serial_team->t.t_task_team[this_thr->th.th_task_state] == NULL );
- KA_TRACE( 20, ( "__kmpc_serialized_parallel: T#%d pushing task_team %p / team %p, new task_team = NULL\n",
- global_tid, this_thr->th.th_task_team, this_thr->th.th_team ) );
- this_thr->th.th_task_team = NULL;
- }
-
-#if OMP_40_ENABLED
- kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
- if ( this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false ) {
- proc_bind = proc_bind_false;
- }
- else if ( proc_bind == proc_bind_default ) {
- //
- // No proc_bind clause was specified, so use the current value
- // of proc-bind-var for this parallel region.
- //
- proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
- }
- //
- // Reset for next parallel region
- //
- this_thr->th.th_set_proc_bind = proc_bind_default;
-#endif /* OMP_40_ENABLED */
-
- if( this_thr->th.th_team != serial_team ) {
- // Nested level will be an index in the nested nthreads array
- int level = this_thr->th.th_team->t.t_level;
-
- if( serial_team->t.t_serialized ) {
- /* this serial team was already used
- * TODO increase performance by making this locks more specific */
- kmp_team_t *new_team;
-
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
-#if OMPT_SUPPORT
- ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid);
-#endif
-
- new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
-#if OMPT_SUPPORT
- ompt_parallel_id,
-#endif
-#if OMP_40_ENABLED
- proc_bind,
-#endif
- & this_thr->th.th_current_task->td_icvs,
- 0 USE_NESTED_HOT_ARG(NULL) );
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- KMP_ASSERT( new_team );
-
- /* setup new serialized team and install it */
- new_team->t.t_threads[0] = this_thr;
- new_team->t.t_parent = this_thr->th.th_team;
- serial_team = new_team;
- this_thr->th.th_serial_team = serial_team;
-
- KF_TRACE( 10, ( "__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
- global_tid, serial_team ) );
-
-
- /* TODO the above breaks the requirement that if we run out of
- * resources, then we can still guarantee that serialized teams
- * are ok, since we may need to allocate a new one */
- } else {
- KF_TRACE( 10, ( "__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
- global_tid, serial_team ) );
- }
-
- /* we have to initialize this serial team */
- KMP_DEBUG_ASSERT( serial_team->t.t_threads );
- KMP_DEBUG_ASSERT( serial_team->t.t_threads[0] == this_thr );
- KMP_DEBUG_ASSERT( this_thr->th.th_team != serial_team );
- serial_team->t.t_ident = loc;
- serial_team->t.t_serialized = 1;
- serial_team->t.t_nproc = 1;
- serial_team->t.t_parent = this_thr->th.th_team;
- serial_team->t.t_sched = this_thr->th.th_team->t.t_sched;
- this_thr->th.th_team = serial_team;
- serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
-
- KF_TRACE( 10, ( "__kmpc_serialized_parallel: T#d curtask=%p\n",
- global_tid, this_thr->th.th_current_task ) );
- KMP_ASSERT( this_thr->th.th_current_task->td_flags.executing == 1 );
- this_thr->th.th_current_task->td_flags.executing = 0;
-
- __kmp_push_current_task_to_thread( this_thr, serial_team, 0 );
-
- /* TODO: GEH: do the ICVs work for nested serialized teams? Don't we need an implicit task for
- each serialized task represented by team->t.t_serialized? */
- copy_icvs(
- & this_thr->th.th_current_task->td_icvs,
- & this_thr->th.th_current_task->td_parent->td_icvs );
-
- // Thread value exists in the nested nthreads array for the next nested level
- if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
- this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
- }
-
-#if OMP_40_ENABLED
- if ( __kmp_nested_proc_bind.used && ( level + 1 < __kmp_nested_proc_bind.used ) ) {
- this_thr->th.th_current_task->td_icvs.proc_bind
- = __kmp_nested_proc_bind.bind_types[ level + 1 ];
- }
-#endif /* OMP_40_ENABLED */
-
-#if USE_DEBUGGER
- serial_team->t.t_pkfn = (microtask_t)( ~0 ); // For the debugger.
-#endif
- this_thr->th.th_info.ds.ds_tid = 0;
-
- /* set thread cache values */
- this_thr->th.th_team_nproc = 1;
- this_thr->th.th_team_master = this_thr;
- this_thr->th.th_team_serialized = 1;
-
- serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
- serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
-
- propagateFPControl (serial_team);
-
- /* check if we need to allocate dispatch buffers stack */
- KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
- if ( !serial_team->t.t_dispatch->th_disp_buffer ) {
- serial_team->t.t_dispatch->th_disp_buffer = (dispatch_private_info_t *)
- __kmp_allocate( sizeof( dispatch_private_info_t ) );
- }
- this_thr->th.th_dispatch = serial_team->t.t_dispatch;
-
-#if OMPT_SUPPORT
- ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid);
- __ompt_team_assign_id(serial_team, ompt_parallel_id);
-#endif
-
- KMP_MB();
-
- } else {
- /* this serialized team is already being used,
- * that's fine, just add another nested level */
- KMP_DEBUG_ASSERT( this_thr->th.th_team == serial_team );
- KMP_DEBUG_ASSERT( serial_team->t.t_threads );
- KMP_DEBUG_ASSERT( serial_team->t.t_threads[0] == this_thr );
- ++ serial_team->t.t_serialized;
- this_thr->th.th_team_serialized = serial_team->t.t_serialized;
-
- // Nested level will be an index in the nested nthreads array
- int level = this_thr->th.th_team->t.t_level;
- // Thread value exists in the nested nthreads array for the next nested level
- if ( __kmp_nested_nth.used && ( level + 1 < __kmp_nested_nth.used ) ) {
- this_thr->th.th_current_task->td_icvs.nproc = __kmp_nested_nth.nth[ level + 1 ];
- }
- serial_team->t.t_level++;
- KF_TRACE( 10, ( "__kmpc_serialized_parallel: T#%d increasing nesting level of serial team %p to %d\n",
- global_tid, serial_team, serial_team->t.t_level ) );
-
- /* allocate/push dispatch buffers stack */
- KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
- {
- dispatch_private_info_t * disp_buffer = (dispatch_private_info_t *)
- __kmp_allocate( sizeof( dispatch_private_info_t ) );
- disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
- serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
- }
- this_thr->th.th_dispatch = serial_team->t.t_dispatch;
-
- KMP_MB();
- }
-
- if ( __kmp_env_consistency_check )
- __kmp_push_parallel( global_tid, NULL );
-
-}
-
-/* most of the work for a fork */
-/* return true if we really went parallel, false if serialized */
-int
-__kmp_fork_call(
- ident_t * loc,
- int gtid,
- enum fork_context_e call_context, // Intel, GNU, ...
- kmp_int32 argc,
-#if OMPT_SUPPORT
- void *unwrapped_task,
-#endif
- microtask_t microtask,
- launch_t invoker,
-/* TODO: revert workaround for Intel(R) 64 tracker #96 */
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- va_list * ap
-#else
- va_list ap
-#endif
- )
-{
- void **argv;
- int i;
- int master_tid;
- int master_this_cons;
- kmp_team_t *team;
- kmp_team_t *parent_team;
- kmp_info_t *master_th;
- kmp_root_t *root;
- int nthreads;
- int master_active;
- int master_set_numthreads;
- int level;
-#if OMP_40_ENABLED
- int active_level;
- int teams_level;
-#endif
-#if KMP_NESTED_HOT_TEAMS
- kmp_hot_team_ptr_t **p_hot_teams;
-#endif
- { // KMP_TIME_BLOCK
- KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
- KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
-
- KA_TRACE( 20, ("__kmp_fork_call: enter T#%d\n", gtid ));
- if ( __kmp_stkpadding > 0 && __kmp_root[gtid] != NULL ) {
- /* Some systems prefer the stack for the root thread(s) to start with */
- /* some gap from the parent stack to prevent false sharing. */
- void *dummy = KMP_ALLOCA(__kmp_stkpadding);
- /* These 2 lines below are so this does not get optimized out */
- if ( __kmp_stkpadding > KMP_MAX_STKPADDING )
- __kmp_stkpadding += (short)((kmp_int64)dummy);
- }
-
- /* initialize if needed */
- KMP_DEBUG_ASSERT( __kmp_init_serial ); // AC: potentially unsafe, not in sync with shutdown
- if( ! TCR_4(__kmp_init_parallel) )
- __kmp_parallel_initialize();
-
- /* setup current data */
- master_th = __kmp_threads[ gtid ]; // AC: potentially unsafe, not in sync with shutdown
- parent_team = master_th->th.th_team;
- master_tid = master_th->th.th_info.ds.ds_tid;
- master_this_cons = master_th->th.th_local.this_construct;
- root = master_th->th.th_root;
- master_active = root->r.r_active;
- master_set_numthreads = master_th->th.th_set_nproc;
-
-#if OMPT_SUPPORT
- ompt_parallel_id_t ompt_parallel_id;
- ompt_task_id_t ompt_task_id;
- ompt_frame_t *ompt_frame;
- ompt_task_id_t my_task_id;
- ompt_parallel_id_t my_parallel_id;
-
- if (ompt_enabled) {
- ompt_parallel_id = __ompt_parallel_id_new(gtid);
- ompt_task_id = __ompt_get_task_id_internal(0);
- ompt_frame = __ompt_get_task_frame_internal(0);
- }
-#endif
-
- // Nested level will be an index in the nested nthreads array
- level = parent_team->t.t_level;
- active_level = parent_team->t.t_active_level; // is used to launch non-serial teams even if nested is not allowed
-#if OMP_40_ENABLED
- teams_level = master_th->th.th_teams_level; // needed to check nesting inside the teams
-#endif
-#if KMP_NESTED_HOT_TEAMS
- p_hot_teams = &master_th->th.th_hot_teams;
- if( *p_hot_teams == NULL && __kmp_hot_teams_max_level > 0 ) {
- *p_hot_teams = (kmp_hot_team_ptr_t*)__kmp_allocate(
- sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
- (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
- (*p_hot_teams)[0].hot_team_nth = 1; // it is either actual or not needed (when active_level > 0)
- }
-#endif
-
-#if OMPT_SUPPORT
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) {
- int team_size = master_set_numthreads;
-
- ompt_callbacks.ompt_callback(ompt_event_parallel_begin)(
- ompt_task_id, ompt_frame, ompt_parallel_id,
- team_size, unwrapped_task, OMPT_INVOKER(call_context));
- }
-#endif
-
- master_th->th.th_ident = loc;
-
-#if OMP_40_ENABLED
- if ( master_th->th.th_teams_microtask &&
- ap && microtask != (microtask_t)__kmp_teams_master && level == teams_level ) {
- // AC: This is start of parallel that is nested inside teams construct.
- // The team is actual (hot), all workers are ready at the fork barrier.
- // No lock needed to initialize the team a bit, then free workers.
- parent_team->t.t_ident = loc;
- __kmp_alloc_argv_entries( argc, parent_team, TRUE );
- parent_team->t.t_argc = argc;
- argv = (void**)parent_team->t.t_argv;
- for( i=argc-1; i >= 0; --i )
-/* TODO: revert workaround for Intel(R) 64 tracker #96 */
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- *argv++ = va_arg( *ap, void * );
-#else
- *argv++ = va_arg( ap, void * );
-#endif
- /* Increment our nested depth levels, but not increase the serialization */
- if ( parent_team == master_th->th.th_serial_team ) {
- // AC: we are in serialized parallel
- __kmpc_serialized_parallel(loc, gtid);
- KMP_DEBUG_ASSERT( parent_team->t.t_serialized > 1 );
- parent_team->t.t_serialized--; // AC: need this in order enquiry functions
- // work correctly, will restore at join time
-
-#if OMPT_SUPPORT
- void *dummy;
- void **exit_runtime_p;
-
- ompt_lw_taskteam_t lw_taskteam;
-
- if (ompt_enabled) {
- __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
- unwrapped_task, ompt_parallel_id);
- lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
- exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
-
- __ompt_lw_taskteam_link(&lw_taskteam, master_th);
-
-#if OMPT_TRACE
- /* OMPT implicit task begin */
- my_task_id = lw_taskteam.ompt_task_info.task_id;
- my_parallel_id = parent_team->t.ompt_team_info.parallel_id;
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- my_parallel_id, my_task_id);
- }
-#endif
-
- /* OMPT state */
- master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
- } else {
- exit_runtime_p = &dummy;
- }
-#endif
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- __kmp_invoke_microtask( microtask, gtid, 0, argc, parent_team->t.t_argv
-#if OMPT_SUPPORT
- , exit_runtime_p
-#endif
- );
- }
-
-#if OMPT_SUPPORT
- *exit_runtime_p = NULL;
- if (ompt_enabled) {
-#if OMPT_TRACE
- lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
-
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- ompt_parallel_id, ompt_task_id);
- }
-
- __ompt_lw_taskteam_unlink(master_th);
- // reset clear the task id only after unlinking the task
- lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
-#endif
-
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
- ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
- ompt_parallel_id, ompt_task_id,
- OMPT_INVOKER(call_context));
- }
- master_th->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
- return TRUE;
- }
-
- parent_team->t.t_pkfn = microtask;
-#if OMPT_SUPPORT
- parent_team->t.ompt_team_info.microtask = unwrapped_task;
-#endif
- parent_team->t.t_invoke = invoker;
- KMP_TEST_THEN_INC32( (kmp_int32*) &root->r.r_in_parallel );
- parent_team->t.t_active_level ++;
- parent_team->t.t_level ++;
-
- /* Change number of threads in the team if requested */
- if ( master_set_numthreads ) { // The parallel has num_threads clause
- if ( master_set_numthreads < master_th->th.th_teams_size.nth ) {
- // AC: only can reduce the number of threads dynamically, cannot increase
- kmp_info_t **other_threads = parent_team->t.t_threads;
- parent_team->t.t_nproc = master_set_numthreads;
- for ( i = 0; i < master_set_numthreads; ++i ) {
- other_threads[i]->th.th_team_nproc = master_set_numthreads;
- }
- // Keep extra threads hot in the team for possible next parallels
- }
- master_th->th.th_set_nproc = 0;
- }
-
-#if USE_DEBUGGER
- if ( __kmp_debugging ) { // Let debugger override number of threads.
- int nth = __kmp_omp_num_threads( loc );
- if ( nth > 0 ) { // 0 means debugger does not want to change number of threads.
- master_set_numthreads = nth;
- }; // if
- }; // if
-#endif
-
- KF_TRACE( 10, ( "__kmp_fork_call: before internal fork: root=%p, team=%p, master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid ) );
- __kmp_internal_fork( loc, gtid, parent_team );
- KF_TRACE( 10, ( "__kmp_fork_call: after internal fork: root=%p, team=%p, master_th=%p, gtid=%d\n", root, parent_team, master_th, gtid ) );
-
- /* Invoke microtask for MASTER thread */
- KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n",
- gtid, parent_team->t.t_id, parent_team->t.t_pkfn ) );
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- if (! parent_team->t.t_invoke( gtid )) {
- KMP_ASSERT2( 0, "cannot invoke microtask for MASTER thread" );
- }
- }
- KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n",
- gtid, parent_team->t.t_id, parent_team->t.t_pkfn ) );
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- KA_TRACE( 20, ("__kmp_fork_call: parallel exit T#%d\n", gtid ));
-
- return TRUE;
- } // Parallel closely nested in teams construct
-#endif /* OMP_40_ENABLED */
-
-#if KMP_DEBUG
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KMP_DEBUG_ASSERT(master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]);
- }
-#endif
-
- if ( parent_team->t.t_active_level >= master_th->th.th_current_task->td_icvs.max_active_levels ) {
- nthreads = 1;
- } else {
-#if OMP_40_ENABLED
- int enter_teams = ((ap==NULL && active_level==0)||(ap && teams_level>0 && teams_level==level));
-#endif
- nthreads = master_set_numthreads ?
- master_set_numthreads : get__nproc_2( parent_team, master_tid ); // TODO: get nproc directly from current task
-
- // Check if we need to take forkjoin lock? (no need for serialized parallel out of teams construct).
- // This code moved here from __kmp_reserve_threads() to speedup nested serialized parallels.
- if (nthreads > 1) {
- if ( ( !get__nested(master_th) && (root->r.r_in_parallel
-#if OMP_40_ENABLED
- && !enter_teams
-#endif /* OMP_40_ENABLED */
- ) ) || ( __kmp_library == library_serial ) ) {
- KC_TRACE( 10, ( "__kmp_fork_call: T#%d serializing team; requested %d threads\n",
- gtid, nthreads ));
- nthreads = 1;
- }
- }
- if ( nthreads > 1 ) {
- /* determine how many new threads we can use */
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
- nthreads = __kmp_reserve_threads(root, parent_team, master_tid, nthreads
-#if OMP_40_ENABLED
-/* AC: If we execute teams from parallel region (on host), then teams should be created
- but each can only have 1 thread if nesting is disabled. If teams called from serial region,
- then teams and their threads should be created regardless of the nesting setting. */
- , enter_teams
-#endif /* OMP_40_ENABLED */
- );
- if ( nthreads == 1 ) {
- // Free lock for single thread execution here;
- // for multi-thread execution it will be freed later
- // after team of threads created and initialized
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- }
- }
- }
- KMP_DEBUG_ASSERT( nthreads > 0 );
-
- /* If we temporarily changed the set number of threads then restore it now */
- master_th->th.th_set_nproc = 0;
-
- /* create a serialized parallel region? */
- if ( nthreads == 1 ) {
- /* josh todo: hypothetical question: what do we do for OS X*? */
-#if KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
- void * args[ argc ];
-#else
- void * * args = (void**) KMP_ALLOCA( argc * sizeof( void * ) );
-#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) */
-
- KA_TRACE( 20, ("__kmp_fork_call: T#%d serializing parallel region\n", gtid ));
-
- __kmpc_serialized_parallel(loc, gtid);
-
- if ( call_context == fork_context_intel ) {
- /* TODO this sucks, use the compiler itself to pass args! :) */
- master_th->th.th_serial_team->t.t_ident = loc;
-#if OMP_40_ENABLED
- if ( !ap ) {
- // revert change made in __kmpc_serialized_parallel()
- master_th->th.th_serial_team->t.t_level--;
- // Get args from parent team for teams construct
-
-#if OMPT_SUPPORT
- void *dummy;
- void **exit_runtime_p;
-
- ompt_lw_taskteam_t lw_taskteam;
-
- if (ompt_enabled) {
- __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
- unwrapped_task, ompt_parallel_id);
- lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
- exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
-
- __ompt_lw_taskteam_link(&lw_taskteam, master_th);
-
-#if OMPT_TRACE
- my_task_id = lw_taskteam.ompt_task_info.task_id;
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- ompt_parallel_id, my_task_id);
- }
-#endif
-
- /* OMPT state */
- master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
- } else {
- exit_runtime_p = &dummy;
- }
-#endif
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- __kmp_invoke_microtask( microtask, gtid, 0, argc, parent_team->t.t_argv
-#if OMPT_SUPPORT
- , exit_runtime_p
-#endif
- );
- }
-
-#if OMPT_SUPPORT
- *exit_runtime_p = NULL;
- if (ompt_enabled) {
- lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
-
-#if OMPT_TRACE
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- ompt_parallel_id, ompt_task_id);
- }
-#endif
-
- __ompt_lw_taskteam_unlink(master_th);
- // reset clear the task id only after unlinking the task
- lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
-
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
- ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
- ompt_parallel_id, ompt_task_id,
- OMPT_INVOKER(call_context));
- }
- master_th->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
- } else if ( microtask == (microtask_t)__kmp_teams_master ) {
- KMP_DEBUG_ASSERT( master_th->th.th_team == master_th->th.th_serial_team );
- team = master_th->th.th_team;
- //team->t.t_pkfn = microtask;
- team->t.t_invoke = invoker;
- __kmp_alloc_argv_entries( argc, team, TRUE );
- team->t.t_argc = argc;
- argv = (void**) team->t.t_argv;
- if ( ap ) {
- for( i=argc-1; i >= 0; --i )
-// TODO: revert workaround for Intel(R) 64 tracker #96
-# if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- *argv++ = va_arg( *ap, void * );
-# else
- *argv++ = va_arg( ap, void * );
-# endif
- } else {
- for( i=0; i < argc; ++i )
- // Get args from parent team for teams construct
- argv[i] = parent_team->t.t_argv[i];
- }
- // AC: revert change made in __kmpc_serialized_parallel()
- // because initial code in teams should have level=0
- team->t.t_level--;
- // AC: call special invoker for outer "parallel" of the teams construct
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- invoker(gtid);
- }
- } else {
-#endif /* OMP_40_ENABLED */
- argv = args;
- for( i=argc-1; i >= 0; --i )
-// TODO: revert workaround for Intel(R) 64 tracker #96
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- *argv++ = va_arg( *ap, void * );
-#else
- *argv++ = va_arg( ap, void * );
-#endif
- KMP_MB();
-
-#if OMPT_SUPPORT
- void *dummy;
- void **exit_runtime_p;
-
- ompt_lw_taskteam_t lw_taskteam;
-
- if (ompt_enabled) {
- __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
- unwrapped_task, ompt_parallel_id);
- lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
- exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
-
- __ompt_lw_taskteam_link(&lw_taskteam, master_th);
-
-#if OMPT_TRACE
- /* OMPT implicit task begin */
- my_task_id = lw_taskteam.ompt_task_info.task_id;
- my_parallel_id = ompt_parallel_id;
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- my_parallel_id, my_task_id);
- }
-#endif
-
- /* OMPT state */
- master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
- } else {
- exit_runtime_p = &dummy;
- }
-#endif
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- __kmp_invoke_microtask( microtask, gtid, 0, argc, args
-#if OMPT_SUPPORT
- , exit_runtime_p
-#endif
- );
- }
-
-#if OMPT_SUPPORT
- *exit_runtime_p = NULL;
- if (ompt_enabled) {
-#if OMPT_TRACE
- lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
-
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- my_parallel_id, my_task_id);
- }
-#endif
-
- __ompt_lw_taskteam_unlink(master_th);
- // reset clear the task id only after unlinking the task
- lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
-
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
- ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
- ompt_parallel_id, ompt_task_id,
- OMPT_INVOKER(call_context));
- }
- master_th->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
-#if OMP_40_ENABLED
- }
-#endif /* OMP_40_ENABLED */
- }
- else if ( call_context == fork_context_gnu ) {
-#if OMPT_SUPPORT
- ompt_lw_taskteam_t *lwt = (ompt_lw_taskteam_t *)
- __kmp_allocate(sizeof(ompt_lw_taskteam_t));
- __ompt_lw_taskteam_init(lwt, master_th, gtid,
- unwrapped_task, ompt_parallel_id);
-
- lwt->ompt_task_info.task_id = __ompt_task_id_new(gtid);
- lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
- __ompt_lw_taskteam_link(lwt, master_th);
-#endif
-
- // we were called from GNU native code
- KA_TRACE( 20, ("__kmp_fork_call: T#%d serial exit\n", gtid ));
- return FALSE;
- }
- else {
- KMP_ASSERT2( call_context < fork_context_last, "__kmp_fork_call: unknown fork_context parameter" );
- }
-
-
- KA_TRACE( 20, ("__kmp_fork_call: T#%d serial exit\n", gtid ));
- KMP_MB();
- return FALSE;
- }
-
- // GEH: only modify the executing flag in the case when not serialized
- // serialized case is handled in kmpc_serialized_parallel
- KF_TRACE( 10, ( "__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, curtask=%p, curtask_max_aclevel=%d\n",
- parent_team->t.t_active_level, master_th, master_th->th.th_current_task,
- master_th->th.th_current_task->td_icvs.max_active_levels ) );
- // TODO: GEH - cannot do this assertion because root thread not set up as executing
- // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
- master_th->th.th_current_task->td_flags.executing = 0;
-
-#if OMP_40_ENABLED
- if ( !master_th->th.th_teams_microtask || level > teams_level )
-#endif /* OMP_40_ENABLED */
- {
- /* Increment our nested depth level */
- KMP_TEST_THEN_INC32( (kmp_int32*) &root->r.r_in_parallel );
- }
-
- // See if we need to make a copy of the ICVs.
- int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
- if ((level+1 < __kmp_nested_nth.used) && (__kmp_nested_nth.nth[level+1] != nthreads_icv)) {
- nthreads_icv = __kmp_nested_nth.nth[level+1];
- }
- else {
- nthreads_icv = 0; // don't update
- }
-
-#if OMP_40_ENABLED
- // Figure out the proc_bind_policy for the new team.
- kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
- kmp_proc_bind_t proc_bind_icv = proc_bind_default; // proc_bind_default means don't update
- if ( master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false ) {
- proc_bind = proc_bind_false;
- }
- else {
- if (proc_bind == proc_bind_default) {
- // No proc_bind clause specified; use current proc-bind-var for this parallel region
- proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
- }
- /* else: The proc_bind policy was specified explicitly on parallel clause. This
- overrides proc-bind-var for this parallel region, but does not change proc-bind-var. */
- // Figure the value of proc-bind-var for the child threads.
- if ((level+1 < __kmp_nested_proc_bind.used)
- && (__kmp_nested_proc_bind.bind_types[level+1] != master_th->th.th_current_task->td_icvs.proc_bind)) {
- proc_bind_icv = __kmp_nested_proc_bind.bind_types[level+1];
- }
- }
-
- // Reset for next parallel region
- master_th->th.th_set_proc_bind = proc_bind_default;
-#endif /* OMP_40_ENABLED */
-
- if ((nthreads_icv > 0)
-#if OMP_40_ENABLED
- || (proc_bind_icv != proc_bind_default)
-#endif /* OMP_40_ENABLED */
- ) {
- kmp_internal_control_t new_icvs;
- copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
- new_icvs.next = NULL;
- if (nthreads_icv > 0) {
- new_icvs.nproc = nthreads_icv;
- }
-
-#if OMP_40_ENABLED
- if (proc_bind_icv != proc_bind_default) {
- new_icvs.proc_bind = proc_bind_icv;
- }
-#endif /* OMP_40_ENABLED */
-
- /* allocate a new parallel team */
- KF_TRACE( 10, ( "__kmp_fork_call: before __kmp_allocate_team\n" ) );
- team = __kmp_allocate_team(root, nthreads, nthreads,
-#if OMPT_SUPPORT
- ompt_parallel_id,
-#endif
-#if OMP_40_ENABLED
- proc_bind,
-#endif
- &new_icvs, argc USE_NESTED_HOT_ARG(master_th) );
- } else {
- /* allocate a new parallel team */
- KF_TRACE( 10, ( "__kmp_fork_call: before __kmp_allocate_team\n" ) );
- team = __kmp_allocate_team(root, nthreads, nthreads,
-#if OMPT_SUPPORT
- ompt_parallel_id,
-#endif
-#if OMP_40_ENABLED
- proc_bind,
-#endif
- &master_th->th.th_current_task->td_icvs, argc
- USE_NESTED_HOT_ARG(master_th) );
- }
- KF_TRACE( 10, ( "__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team ) );
-
- /* setup the new team */
- KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
- KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
- KMP_CHECK_UPDATE(team->t.t_ident, loc);
- KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
- KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
-#if OMPT_SUPPORT
- KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.microtask, unwrapped_task);
-#endif
- KMP_CHECK_UPDATE(team->t.t_invoke, invoker); /* TODO move this to root, maybe */
- // TODO: parent_team->t.t_level == INT_MAX ???
-#if OMP_40_ENABLED
- if ( !master_th->th.th_teams_microtask || level > teams_level ) {
-#endif /* OMP_40_ENABLED */
- int new_level = parent_team->t.t_level + 1;
- KMP_CHECK_UPDATE(team->t.t_level, new_level);
- new_level = parent_team->t.t_active_level + 1;
- KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
-#if OMP_40_ENABLED
- } else {
- // AC: Do not increase parallel level at start of the teams construct
- int new_level = parent_team->t.t_level;
- KMP_CHECK_UPDATE(team->t.t_level, new_level);
- new_level = parent_team->t.t_active_level;
- KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
- }
-#endif /* OMP_40_ENABLED */
- kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
- if (team->t.t_sched.r_sched_type != new_sched.r_sched_type || team->t.t_sched.chunk != new_sched.chunk)
- team->t.t_sched = new_sched; // set master's schedule as new run-time schedule
-
-#if OMP_40_ENABLED
- KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
-#endif
-
- // Update the floating point rounding in the team if required.
- propagateFPControl(team);
-
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- // Set master's task team to team's task team. Unless this is hot team, it should be NULL.
-#if 0
- // Patch out an assertion that trips while the runtime seems to operate correctly.
- // Avoiding the preconditions that cause the assertion to trip has been promised as a forthcoming patch.
- KMP_DEBUG_ASSERT(master_th->th.th_task_team == parent_team->t.t_task_team[master_th->th.th_task_state]);
-#endif
- KA_TRACE( 20, ( "__kmp_fork_call: Master T#%d pushing task_team %p / team %p, new task_team %p / team %p\n",
- __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team,
- parent_team, team->t.t_task_team[master_th->th.th_task_state], team ) );
-
- if ( active_level || master_th->th.th_task_team ) {
- // Take a memo of master's task_state
- KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
- if (master_th->th.th_task_state_top >= master_th->th.th_task_state_stack_sz) { // increase size
- kmp_uint32 new_size = 2*master_th->th.th_task_state_stack_sz;
- kmp_uint8 *old_stack, *new_stack;
- kmp_uint32 i;
- new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
- for (i=0; i<master_th->th.th_task_state_stack_sz; ++i) {
- new_stack[i] = master_th->th.th_task_state_memo_stack[i];
- }
- for (i=master_th->th.th_task_state_stack_sz; i<new_size; ++i) { // zero-init rest of stack
- new_stack[i] = 0;
- }
- old_stack = master_th->th.th_task_state_memo_stack;
- master_th->th.th_task_state_memo_stack = new_stack;
- master_th->th.th_task_state_stack_sz = new_size;
- __kmp_free(old_stack);
- }
- // Store master's task_state on stack
- master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = master_th->th.th_task_state;
- master_th->th.th_task_state_top++;
-#if KMP_NESTED_HOT_TEAMS
- if (team == master_th->th.th_hot_teams[active_level].hot_team) { // Restore master's nested state if nested hot team
- master_th->th.th_task_state = master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top];
- }
- else {
-#endif
- master_th->th.th_task_state = 0;
-#if KMP_NESTED_HOT_TEAMS
- }
-#endif
- }
-#if !KMP_NESTED_HOT_TEAMS
- KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) || (team == root->r.r_hot_team));
-#endif
- }
-
- KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
- gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id, team->t.t_nproc ));
- KMP_DEBUG_ASSERT( team != root->r.r_hot_team ||
- ( team->t.t_master_tid == 0 &&
- ( team->t.t_parent == root->r.r_root_team || team->t.t_parent->t.t_serialized ) ));
- KMP_MB();
-
- /* now, setup the arguments */
- argv = (void**)team->t.t_argv;
-#if OMP_40_ENABLED
- if ( ap ) {
-#endif /* OMP_40_ENABLED */
- for ( i=argc-1; i >= 0; --i ) {
-// TODO: revert workaround for Intel(R) 64 tracker #96
-#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
- void *new_argv = va_arg(*ap, void *);
-#else
- void *new_argv = va_arg(ap, void *);
-#endif
- KMP_CHECK_UPDATE(*argv, new_argv);
- argv++;
- }
-#if OMP_40_ENABLED
- } else {
- for ( i=0; i < argc; ++i ) {
- // Get args from parent team for teams construct
- KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
- }
- }
-#endif /* OMP_40_ENABLED */
-
- /* now actually fork the threads */
- KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
- if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
- root->r.r_active = TRUE;
-
- __kmp_fork_team_threads( root, team, master_th, gtid );
- __kmp_setup_icv_copy( team, nthreads, &master_th->th.th_current_task->td_icvs, loc );
-
-#if OMPT_SUPPORT
- master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
-#endif
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
-
-#if USE_ITT_BUILD
- if ( team->t.t_active_level == 1 // only report frames at level 1
-# if OMP_40_ENABLED
- && !master_th->th.th_teams_microtask // not in teams construct
-# endif /* OMP_40_ENABLED */
- ) {
-#if USE_ITT_NOTIFY
- if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) &&
- ( __kmp_forkjoin_frames_mode == 3 || __kmp_forkjoin_frames_mode == 1 ) )
- {
- kmp_uint64 tmp_time = 0;
- if ( __itt_get_timestamp_ptr )
- tmp_time = __itt_get_timestamp();
- // Internal fork - report frame begin
- master_th->th.th_frame_time = tmp_time;
- if ( __kmp_forkjoin_frames_mode == 3 )
- team->t.t_region_time = tmp_time;
- } else // only one notification scheme (either "submit" or "forking/joined", not both)
-#endif /* USE_ITT_NOTIFY */
- if ( ( __itt_frame_begin_v3_ptr || KMP_ITT_DEBUG ) &&
- __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode )
- { // Mark start of "parallel" region for VTune.
- __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
- }
- }
-#endif /* USE_ITT_BUILD */
-
- /* now go on and do the work */
- KMP_DEBUG_ASSERT( team == __kmp_threads[gtid]->th.th_team );
- KMP_MB();
- KF_TRACE(10, ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
- root, team, master_th, gtid));
-
-#if USE_ITT_BUILD
- if ( __itt_stack_caller_create_ptr ) {
- team->t.t_stack_id = __kmp_itt_stack_caller_create(); // create new stack stitching id before entering fork barrier
- }
-#endif /* USE_ITT_BUILD */
-
-#if OMP_40_ENABLED
- if ( ap ) // AC: skip __kmp_internal_fork at teams construct, let only master threads execute
-#endif /* OMP_40_ENABLED */
- {
- __kmp_internal_fork( loc, gtid, team );
- KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, master_th=%p, gtid=%d\n",
- root, team, master_th, gtid));
- }
-
- if (call_context == fork_context_gnu) {
- KA_TRACE( 20, ("__kmp_fork_call: parallel exit T#%d\n", gtid ));
- return TRUE;
- }
-
- /* Invoke microtask for MASTER thread */
- KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n",
- gtid, team->t.t_id, team->t.t_pkfn ) );
- } // END of timer KMP_fork_call block
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- if (! team->t.t_invoke( gtid )) {
- KMP_ASSERT2( 0, "cannot invoke microtask for MASTER thread" );
- }
- }
- KA_TRACE( 20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n",
- gtid, team->t.t_id, team->t.t_pkfn ) );
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- KA_TRACE( 20, ("__kmp_fork_call: parallel exit T#%d\n", gtid ));
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- master_th->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
-
- return TRUE;
-}
-
-#if OMPT_SUPPORT
-static inline void
-__kmp_join_restore_state(
- kmp_info_t *thread,
- kmp_team_t *team)
-{
- // restore state outside the region
- thread->th.ompt_thread_info.state = ((team->t.t_serialized) ?
- ompt_state_work_serial : ompt_state_work_parallel);
-}
-
-static inline void
-__kmp_join_ompt(
- kmp_info_t *thread,
- kmp_team_t *team,
- ompt_parallel_id_t parallel_id,
- fork_context_e fork_context)
-{
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
- if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
- ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
- parallel_id, task_info->task_id, OMPT_INVOKER(fork_context));
- }
-
- task_info->frame.reenter_runtime_frame = NULL;
- __kmp_join_restore_state(thread,team);
-}
-#endif
-
-void
-__kmp_join_call(ident_t *loc, int gtid
-#if OMPT_SUPPORT
- , enum fork_context_e fork_context
-#endif
-#if OMP_40_ENABLED
- , int exit_teams
-#endif /* OMP_40_ENABLED */
-)
-{
- KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
- kmp_team_t *team;
- kmp_team_t *parent_team;
- kmp_info_t *master_th;
- kmp_root_t *root;
- int master_active;
- int i;
-
- KA_TRACE( 20, ("__kmp_join_call: enter T#%d\n", gtid ));
-
- /* setup current data */
- master_th = __kmp_threads[ gtid ];
- root = master_th->th.th_root;
- team = master_th->th.th_team;
- parent_team = team->t.t_parent;
-
- master_th->th.th_ident = loc;
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- master_th->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
-
-#if KMP_DEBUG
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KA_TRACE( 20, ( "__kmp_join_call: T#%d, old team = %p old task_team = %p, th_task_team = %p\n",
- __kmp_gtid_from_thread( master_th ), team,
- team->t.t_task_team[master_th->th.th_task_state], master_th->th.th_task_team) );
- KMP_DEBUG_ASSERT( master_th->th.th_task_team == team->t.t_task_team[master_th->th.th_task_state] );
- }
-#endif
-
- if( team->t.t_serialized ) {
-#if OMP_40_ENABLED
- if ( master_th->th.th_teams_microtask ) {
- // We are in teams construct
- int level = team->t.t_level;
- int tlevel = master_th->th.th_teams_level;
- if ( level == tlevel ) {
- // AC: we haven't incremented it earlier at start of teams construct,
- // so do it here - at the end of teams construct
- team->t.t_level++;
- } else if ( level == tlevel + 1 ) {
- // AC: we are exiting parallel inside teams, need to increment serialization
- // in order to restore it in the next call to __kmpc_end_serialized_parallel
- team->t.t_serialized++;
- }
- }
-#endif /* OMP_40_ENABLED */
- __kmpc_end_serialized_parallel( loc, gtid );
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- __kmp_join_restore_state(master_th, parent_team);
- }
-#endif
-
- return;
- }
-
- master_active = team->t.t_master_active;
-
-#if OMP_40_ENABLED
- if (!exit_teams)
-#endif /* OMP_40_ENABLED */
- {
- // AC: No barrier for internal teams at exit from teams construct.
- // But there is barrier for external team (league).
- __kmp_internal_join( loc, gtid, team );
- }
-#if OMP_40_ENABLED
- else {
- master_th->th.th_task_state = 0; // AC: no tasking in teams (out of any parallel)
- }
-#endif /* OMP_40_ENABLED */
-
- KMP_MB();
-
-#if OMPT_SUPPORT
- ompt_parallel_id_t parallel_id = team->t.ompt_team_info.parallel_id;
-#endif
-
-#if USE_ITT_BUILD
- if ( __itt_stack_caller_create_ptr ) {
- __kmp_itt_stack_caller_destroy( (__itt_caller)team->t.t_stack_id ); // destroy the stack stitching id after join barrier
- }
-
- // Mark end of "parallel" region for VTune.
- if ( team->t.t_active_level == 1
-# if OMP_40_ENABLED
- && !master_th->th.th_teams_microtask /* not in teams construct */
-# endif /* OMP_40_ENABLED */
- ) {
- master_th->th.th_ident = loc;
- // only one notification scheme (either "submit" or "forking/joined", not both)
- if ( ( __itt_frame_submit_v3_ptr || KMP_ITT_DEBUG ) && __kmp_forkjoin_frames_mode == 3 )
- __kmp_itt_frame_submit( gtid, team->t.t_region_time, master_th->th.th_frame_time,
- 0, loc, master_th->th.th_team_nproc, 1 );
- else if ( ( __itt_frame_end_v3_ptr || KMP_ITT_DEBUG ) &&
- ! __kmp_forkjoin_frames_mode && __kmp_forkjoin_frames )
- __kmp_itt_region_joined( gtid );
- } // active_level == 1
-#endif /* USE_ITT_BUILD */
-
-#if OMP_40_ENABLED
- if ( master_th->th.th_teams_microtask &&
- !exit_teams &&
- team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
- team->t.t_level == master_th->th.th_teams_level + 1 ) {
- // AC: We need to leave the team structure intact at the end
- // of parallel inside the teams construct, so that at the next
- // parallel same (hot) team works, only adjust nesting levels
-
- /* Decrement our nested depth level */
- team->t.t_level --;
- team->t.t_active_level --;
- KMP_TEST_THEN_DEC32( (kmp_int32*) &root->r.r_in_parallel );
-
- /* Restore number of threads in the team if needed */
- if ( master_th->th.th_team_nproc < master_th->th.th_teams_size.nth ) {
- int old_num = master_th->th.th_team_nproc;
- int new_num = master_th->th.th_teams_size.nth;
- kmp_info_t **other_threads = team->t.t_threads;
- team->t.t_nproc = new_num;
- for ( i = 0; i < old_num; ++i ) {
- other_threads[i]->th.th_team_nproc = new_num;
- }
- // Adjust states of non-used threads of the team
- for ( i = old_num; i < new_num; ++i ) {
- // Re-initialize thread's barrier data.
- int b;
- kmp_balign_t * balign = other_threads[i]->th.th_bar;
- for ( b = 0; b < bs_last_barrier; ++ b ) {
- balign[ b ].bb.b_arrived = team->t.t_bar[ b ].b_arrived;
- KMP_DEBUG_ASSERT(balign[ b ].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#if USE_DEBUGGER
- balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
-#endif
- }
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- // Synchronize thread's task state
- other_threads[i]->th.th_task_state = master_th->th.th_task_state;
- }
- }
- }
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context);
- }
-#endif
-
- return;
- }
-#endif /* OMP_40_ENABLED */
-
- /* do cleanup and restore the parent team */
- master_th->th.th_info .ds.ds_tid = team->t.t_master_tid;
- master_th->th.th_local.this_construct = team->t.t_master_this_cons;
-
- master_th->th.th_dispatch =
- & parent_team->t.t_dispatch[ team->t.t_master_tid ];
-
- /* jc: The following lock has instructions with REL and ACQ semantics,
- separating the parallel user code called in this parallel region
- from the serial user code called after this function returns.
- */
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
-#if OMP_40_ENABLED
- if ( !master_th->th.th_teams_microtask || team->t.t_level > master_th->th.th_teams_level )
-#endif /* OMP_40_ENABLED */
- {
- /* Decrement our nested depth level */
- KMP_TEST_THEN_DEC32( (kmp_int32*) &root->r.r_in_parallel );
- }
- KMP_DEBUG_ASSERT( root->r.r_in_parallel >= 0 );
-
-#if OMPT_SUPPORT && OMPT_TRACE
- if(ompt_enabled){
- ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- parallel_id, task_info->task_id);
- }
- task_info->frame.exit_runtime_frame = NULL;
- task_info->task_id = 0;
- }
-#endif
-
- KF_TRACE( 10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n",
- 0, master_th, team ) );
- __kmp_pop_current_task_from_thread( master_th );
-
-#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
- //
- // Restore master thread's partition.
- //
- master_th->th.th_first_place = team->t.t_first_place;
- master_th->th.th_last_place = team->t.t_last_place;
-#endif /* OMP_40_ENABLED */
-
- updateHWFPControl (team);
-
- if ( root->r.r_active != master_active )
- root->r.r_active = master_active;
-
- __kmp_free_team( root, team USE_NESTED_HOT_ARG(master_th) ); // this will free worker threads
-
- /* this race was fun to find. make sure the following is in the critical
- * region otherwise assertions may fail occasionally since the old team
- * may be reallocated and the hierarchy appears inconsistent. it is
- * actually safe to run and won't cause any bugs, but will cause those
- * assertion failures. it's only one deref&assign so might as well put this
- * in the critical region */
- master_th->th.th_team = parent_team;
- master_th->th.th_team_nproc = parent_team->t.t_nproc;
- master_th->th.th_team_master = parent_team->t.t_threads[0];
- master_th->th.th_team_serialized = parent_team->t.t_serialized;
-
- /* restore serialized team, if need be */
- if( parent_team->t.t_serialized &&
- parent_team != master_th->th.th_serial_team &&
- parent_team != root->r.r_root_team ) {
- __kmp_free_team( root, master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL) );
- master_th->th.th_serial_team = parent_team;
- }
-
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- if (master_th->th.th_task_state_top > 0) { // Restore task state from memo stack
- KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
- // Remember master's state if we re-use this nested hot team
- master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] = master_th->th.th_task_state;
- --master_th->th.th_task_state_top; // pop
- // Now restore state at this level
- master_th->th.th_task_state = master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top];
- }
- // Copy the task team from the parent team to the master thread
- master_th->th.th_task_team = parent_team->t.t_task_team[master_th->th.th_task_state];
- KA_TRACE( 20, ( "__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
- __kmp_gtid_from_thread( master_th ), master_th->th.th_task_team, parent_team ) );
- }
-
- // TODO: GEH - cannot do this assertion because root thread not set up as executing
- // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
- master_th->th.th_current_task->td_flags.executing = 1;
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context);
- }
-#endif
-
- KMP_MB();
- KA_TRACE( 20, ("__kmp_join_call: exit T#%d\n", gtid ));
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* Check whether we should push an internal control record onto the
- serial team stack. If so, do it. */
-void
-__kmp_save_internal_controls ( kmp_info_t * thread )
-{
-
- if ( thread->th.th_team != thread->th.th_serial_team ) {
- return;
- }
- if (thread->th.th_team->t.t_serialized > 1) {
- int push = 0;
-
- if (thread->th.th_team->t.t_control_stack_top == NULL) {
- push = 1;
- } else {
- if ( thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
- thread->th.th_team->t.t_serialized ) {
- push = 1;
- }
- }
- if (push) { /* push a record on the serial team's stack */
- kmp_internal_control_t * control = (kmp_internal_control_t *) __kmp_allocate(sizeof(kmp_internal_control_t));
-
- copy_icvs( control, & thread->th.th_current_task->td_icvs );
-
- control->serial_nesting_level = thread->th.th_team->t.t_serialized;
-
- control->next = thread->th.th_team->t.t_control_stack_top;
- thread->th.th_team->t.t_control_stack_top = control;
- }
- }
-}
-
-/* Changes set_nproc */
-void
-__kmp_set_num_threads( int new_nth, int gtid )
-{
- kmp_info_t *thread;
- kmp_root_t *root;
-
- KF_TRACE( 10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth ));
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- if (new_nth < 1)
- new_nth = 1;
- else if (new_nth > __kmp_max_nth)
- new_nth = __kmp_max_nth;
-
- KMP_COUNT_VALUE(OMP_set_numthreads, new_nth);
- thread = __kmp_threads[gtid];
-
- __kmp_save_internal_controls( thread );
-
- set__nproc( thread, new_nth );
-
- //
- // If this omp_set_num_threads() call will cause the hot team size to be
- // reduced (in the absence of a num_threads clause), then reduce it now,
- // rather than waiting for the next parallel region.
- //
- root = thread->th.th_root;
- if ( __kmp_init_parallel && ( ! root->r.r_active )
- && ( root->r.r_hot_team->t.t_nproc > new_nth )
-#if KMP_NESTED_HOT_TEAMS
- && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
-#endif
- ) {
- kmp_team_t *hot_team = root->r.r_hot_team;
- int f;
-
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
- // Release the extra threads we don't need any more.
- for ( f = new_nth; f < hot_team->t.t_nproc; f++ ) {
- KMP_DEBUG_ASSERT( hot_team->t.t_threads[f] != NULL );
- if ( __kmp_tasking_mode != tskm_immediate_exec) {
- // When decreasing team size, threads no longer in the team should unref task team.
- hot_team->t.t_threads[f]->th.th_task_team = NULL;
- }
- __kmp_free_thread( hot_team->t.t_threads[f] );
- hot_team->t.t_threads[f] = NULL;
- }
- hot_team->t.t_nproc = new_nth;
-#if KMP_NESTED_HOT_TEAMS
- if( thread->th.th_hot_teams ) {
- KMP_DEBUG_ASSERT( hot_team == thread->th.th_hot_teams[0].hot_team );
- thread->th.th_hot_teams[0].hot_team_nth = new_nth;
- }
-#endif
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
-
- //
- // Update the t_nproc field in the threads that are still active.
- //
- for( f=0 ; f < new_nth; f++ ) {
- KMP_DEBUG_ASSERT( hot_team->t.t_threads[f] != NULL );
- hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
- }
- // Special flag in case omp_set_num_threads() call
- hot_team->t.t_size_changed = -1;
- }
-}
-
-/* Changes max_active_levels */
-void
-__kmp_set_max_active_levels( int gtid, int max_active_levels )
-{
- kmp_info_t *thread;
-
- KF_TRACE( 10, ( "__kmp_set_max_active_levels: new max_active_levels for thread %d = (%d)\n", gtid, max_active_levels ) );
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- // validate max_active_levels
- if( max_active_levels < 0 ) {
- KMP_WARNING( ActiveLevelsNegative, max_active_levels );
- // We ignore this call if the user has specified a negative value.
- // The current setting won't be changed. The last valid setting will be used.
- // A warning will be issued (if warnings are allowed as controlled by the KMP_WARNINGS env var).
- KF_TRACE( 10, ( "__kmp_set_max_active_levels: the call is ignored: new max_active_levels for thread %d = (%d)\n", gtid, max_active_levels ) );
- return;
- }
- if( max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT ) {
- // it's OK, the max_active_levels is within the valid range: [ 0; KMP_MAX_ACTIVE_LEVELS_LIMIT ]
- // We allow a zero value. (implementation defined behavior)
- } else {
- KMP_WARNING( ActiveLevelsExceedLimit, max_active_levels, KMP_MAX_ACTIVE_LEVELS_LIMIT );
- max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
- // Current upper limit is MAX_INT. (implementation defined behavior)
- // If the input exceeds the upper limit, we correct the input to be the upper limit. (implementation defined behavior)
- // Actually, the flow should never get here until we use MAX_INT limit.
- }
- KF_TRACE( 10, ( "__kmp_set_max_active_levels: after validation: new max_active_levels for thread %d = (%d)\n", gtid, max_active_levels ) );
-
- thread = __kmp_threads[ gtid ];
-
- __kmp_save_internal_controls( thread );
-
- set__max_active_levels( thread, max_active_levels );
-
-}
-
-/* Gets max_active_levels */
-int
-__kmp_get_max_active_levels( int gtid )
-{
- kmp_info_t *thread;
-
- KF_TRACE( 10, ( "__kmp_get_max_active_levels: thread %d\n", gtid ) );
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- thread = __kmp_threads[ gtid ];
- KMP_DEBUG_ASSERT( thread->th.th_current_task );
- KF_TRACE( 10, ( "__kmp_get_max_active_levels: thread %d, curtask=%p, curtask_maxaclevel=%d\n",
- gtid, thread->th.th_current_task, thread->th.th_current_task->td_icvs.max_active_levels ) );
- return thread->th.th_current_task->td_icvs.max_active_levels;
-}
-
-/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
-void
-__kmp_set_schedule( int gtid, kmp_sched_t kind, int chunk )
-{
- kmp_info_t *thread;
-// kmp_team_t *team;
-
- KF_TRACE( 10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n", gtid, (int)kind, chunk ));
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- // Check if the kind parameter is valid, correct if needed.
- // Valid parameters should fit in one of two intervals - standard or extended:
- // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
- // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
- if ( kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
- ( kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std ) )
- {
- // TODO: Hint needs attention in case we change the default schedule.
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( ScheduleKindOutOfRange, kind ),
- KMP_HNT( DefaultScheduleKindUsed, "static, no chunk" ),
- __kmp_msg_null
- );
- kind = kmp_sched_default;
- chunk = 0; // ignore chunk value in case of bad kind
- }
-
- thread = __kmp_threads[ gtid ];
-
- __kmp_save_internal_controls( thread );
-
- if ( kind < kmp_sched_upper_std ) {
- if ( kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK ) {
- // differ static chunked vs. unchunked:
- // chunk should be invalid to indicate unchunked schedule (which is the default)
- thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
- } else {
- thread->th.th_current_task->td_icvs.sched.r_sched_type = __kmp_sch_map[ kind - kmp_sched_lower - 1 ];
- }
- } else {
- // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - kmp_sched_lower - 2 ];
- thread->th.th_current_task->td_icvs.sched.r_sched_type =
- __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std - kmp_sched_lower - 2 ];
- }
- if ( kind == kmp_sched_auto ) {
- // ignore parameter chunk for schedule auto
- thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
- } else {
- thread->th.th_current_task->td_icvs.sched.chunk = chunk;
- }
-}
-
-/* Gets def_sched_var ICV values */
-void
-__kmp_get_schedule( int gtid, kmp_sched_t * kind, int * chunk )
-{
- kmp_info_t *thread;
- enum sched_type th_type;
-
- KF_TRACE( 10, ("__kmp_get_schedule: thread %d\n", gtid ));
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- thread = __kmp_threads[ gtid ];
-
- th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
-
- switch ( th_type ) {
- case kmp_sch_static:
- case kmp_sch_static_greedy:
- case kmp_sch_static_balanced:
- *kind = kmp_sched_static;
- *chunk = 0; // chunk was not set, try to show this fact via zero value
- return;
- case kmp_sch_static_chunked:
- *kind = kmp_sched_static;
- break;
- case kmp_sch_dynamic_chunked:
- *kind = kmp_sched_dynamic;
- break;
- case kmp_sch_guided_chunked:
- case kmp_sch_guided_iterative_chunked:
- case kmp_sch_guided_analytical_chunked:
- *kind = kmp_sched_guided;
- break;
- case kmp_sch_auto:
- *kind = kmp_sched_auto;
- break;
- case kmp_sch_trapezoidal:
- *kind = kmp_sched_trapezoidal;
- break;
-#if KMP_STATIC_STEAL_ENABLED
- case kmp_sch_static_steal:
- *kind = kmp_sched_static_steal;
- break;
-#endif
- default:
- KMP_FATAL( UnknownSchedulingType, th_type );
- }
-
- *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
-}
-
-int
-__kmp_get_ancestor_thread_num( int gtid, int level ) {
-
- int ii, dd;
- kmp_team_t *team;
- kmp_info_t *thr;
-
- KF_TRACE( 10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level ));
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- // validate level
- if( level == 0 ) return 0;
- if( level < 0 ) return -1;
- thr = __kmp_threads[ gtid ];
- team = thr->th.th_team;
- ii = team->t.t_level;
- if( level > ii ) return -1;
-
-#if OMP_40_ENABLED
- if( thr->th.th_teams_microtask ) {
- // AC: we are in teams region where multiple nested teams have same level
- int tlevel = thr->th.th_teams_level; // the level of the teams construct
- if( level <= tlevel ) { // otherwise usual algorithm works (will not touch the teams)
- KMP_DEBUG_ASSERT( ii >= tlevel );
- // AC: As we need to pass by the teams league, we need to artificially increase ii
- if ( ii == tlevel ) {
- ii += 2; // three teams have same level
- } else {
- ii ++; // two teams have same level
- }
- }
- }
-#endif
-
- if( ii == level ) return __kmp_tid_from_gtid( gtid );
-
- dd = team->t.t_serialized;
- level++;
- while( ii > level )
- {
- for( dd = team->t.t_serialized; ( dd > 0 ) && ( ii > level ); dd--, ii-- )
- {
- }
- if( ( team->t.t_serialized ) && ( !dd ) ) {
- team = team->t.t_parent;
- continue;
- }
- if( ii > level ) {
- team = team->t.t_parent;
- dd = team->t.t_serialized;
- ii--;
- }
- }
-
- return ( dd > 1 ) ? ( 0 ) : ( team->t.t_master_tid );
-}
-
-int
-__kmp_get_team_size( int gtid, int level ) {
-
- int ii, dd;
- kmp_team_t *team;
- kmp_info_t *thr;
-
- KF_TRACE( 10, ("__kmp_get_team_size: thread %d %d\n", gtid, level ));
- KMP_DEBUG_ASSERT( __kmp_init_serial );
-
- // validate level
- if( level == 0 ) return 1;
- if( level < 0 ) return -1;
- thr = __kmp_threads[ gtid ];
- team = thr->th.th_team;
- ii = team->t.t_level;
- if( level > ii ) return -1;
-
-#if OMP_40_ENABLED
- if( thr->th.th_teams_microtask ) {
- // AC: we are in teams region where multiple nested teams have same level
- int tlevel = thr->th.th_teams_level; // the level of the teams construct
- if( level <= tlevel ) { // otherwise usual algorithm works (will not touch the teams)
- KMP_DEBUG_ASSERT( ii >= tlevel );
- // AC: As we need to pass by the teams league, we need to artificially increase ii
- if ( ii == tlevel ) {
- ii += 2; // three teams have same level
- } else {
- ii ++; // two teams have same level
- }
- }
- }
-#endif
-
- while( ii > level )
- {
- for( dd = team->t.t_serialized; ( dd > 0 ) && ( ii > level ); dd--, ii-- )
- {
- }
- if( team->t.t_serialized && ( !dd ) ) {
- team = team->t.t_parent;
- continue;
- }
- if( ii > level ) {
- team = team->t.t_parent;
- ii--;
- }
- }
-
- return team->t.t_nproc;
-}
-
-kmp_r_sched_t
-__kmp_get_schedule_global() {
-// This routine created because pairs (__kmp_sched, __kmp_chunk) and (__kmp_static, __kmp_guided)
-// may be changed by kmp_set_defaults independently. So one can get the updated schedule here.
-
- kmp_r_sched_t r_sched;
-
- // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static, __kmp_guided
- // __kmp_sched should keep original value, so that user can set KMP_SCHEDULE multiple times,
- // and thus have different run-time schedules in different roots (even in OMP 2.5)
- if ( __kmp_sched == kmp_sch_static ) {
- r_sched.r_sched_type = __kmp_static; // replace STATIC with more detailed schedule (balanced or greedy)
- } else if ( __kmp_sched == kmp_sch_guided_chunked ) {
- r_sched.r_sched_type = __kmp_guided; // replace GUIDED with more detailed schedule (iterative or analytical)
- } else {
- r_sched.r_sched_type = __kmp_sched; // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
- }
-
- if ( __kmp_chunk < KMP_DEFAULT_CHUNK ) { // __kmp_chunk may be wrong here (if it was not ever set)
- r_sched.chunk = KMP_DEFAULT_CHUNK;
- } else {
- r_sched.chunk = __kmp_chunk;
- }
-
- return r_sched;
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-
-/*
- * Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
- * at least argc number of *t_argv entries for the requested team.
- */
-static void
-__kmp_alloc_argv_entries( int argc, kmp_team_t *team, int realloc )
-{
-
- KMP_DEBUG_ASSERT( team );
- if( !realloc || argc > team->t.t_max_argc ) {
-
- KA_TRACE( 100, ( "__kmp_alloc_argv_entries: team %d: needed entries=%d, current entries=%d\n",
- team->t.t_id, argc, ( realloc ) ? team->t.t_max_argc : 0 ));
- /* if previously allocated heap space for args, free them */
- if ( realloc && team->t.t_argv != &team->t.t_inline_argv[0] )
- __kmp_free( (void *) team->t.t_argv );
-
- if ( argc <= KMP_INLINE_ARGV_ENTRIES ) {
- /* use unused space in the cache line for arguments */
- team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
- KA_TRACE( 100, ( "__kmp_alloc_argv_entries: team %d: inline allocate %d argv entries\n",
- team->t.t_id, team->t.t_max_argc ));
- team->t.t_argv = &team->t.t_inline_argv[0];
- if ( __kmp_storage_map ) {
- __kmp_print_storage_map_gtid( -1, &team->t.t_inline_argv[0],
- &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
- (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES),
- "team_%d.t_inline_argv",
- team->t.t_id );
- }
- } else {
- /* allocate space for arguments in the heap */
- team->t.t_max_argc = ( argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1 )) ?
- KMP_MIN_MALLOC_ARGV_ENTRIES : 2 * argc;
- KA_TRACE( 100, ( "__kmp_alloc_argv_entries: team %d: dynamic allocate %d argv entries\n",
- team->t.t_id, team->t.t_max_argc ));
- team->t.t_argv = (void**) __kmp_page_allocate( sizeof(void*) * team->t.t_max_argc );
- if ( __kmp_storage_map ) {
- __kmp_print_storage_map_gtid( -1, &team->t.t_argv[0], &team->t.t_argv[team->t.t_max_argc],
- sizeof(void *) * team->t.t_max_argc, "team_%d.t_argv",
- team->t.t_id );
- }
- }
- }
-}
-
-static void
-__kmp_allocate_team_arrays(kmp_team_t *team, int max_nth)
-{
- int i;
- int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
- team->t.t_threads = (kmp_info_t**) __kmp_allocate( sizeof(kmp_info_t*) * max_nth );
- team->t.t_disp_buffer = (dispatch_shared_info_t*)
- __kmp_allocate( sizeof(dispatch_shared_info_t) * num_disp_buff );
- team->t.t_dispatch = (kmp_disp_t*) __kmp_allocate( sizeof(kmp_disp_t) * max_nth );
- team->t.t_implicit_task_taskdata = (kmp_taskdata_t*) __kmp_allocate( sizeof(kmp_taskdata_t) * max_nth );
- team->t.t_max_nproc = max_nth;
-
- /* setup dispatch buffers */
- for(i = 0 ; i < num_disp_buff; ++i) {
- team->t.t_disp_buffer[i].buffer_index = i;
-#if OMP_45_ENABLED
- team->t.t_disp_buffer[i].doacross_buf_idx = i;
-#endif
- }
-}
-
-static void
-__kmp_free_team_arrays(kmp_team_t *team) {
- /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
- int i;
- for ( i = 0; i < team->t.t_max_nproc; ++ i ) {
- if ( team->t.t_dispatch[ i ].th_disp_buffer != NULL ) {
- __kmp_free( team->t.t_dispatch[ i ].th_disp_buffer );
- team->t.t_dispatch[ i ].th_disp_buffer = NULL;
- }; // if
- }; // for
- __kmp_free(team->t.t_threads);
- __kmp_free(team->t.t_disp_buffer);
- __kmp_free(team->t.t_dispatch);
- __kmp_free(team->t.t_implicit_task_taskdata);
- team->t.t_threads = NULL;
- team->t.t_disp_buffer = NULL;
- team->t.t_dispatch = NULL;
- team->t.t_implicit_task_taskdata = 0;
-}
-
-static void
-__kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
- kmp_info_t **oldThreads = team->t.t_threads;
-
- __kmp_free(team->t.t_disp_buffer);
- __kmp_free(team->t.t_dispatch);
- __kmp_free(team->t.t_implicit_task_taskdata);
- __kmp_allocate_team_arrays(team, max_nth);
-
- KMP_MEMCPY(team->t.t_threads, oldThreads, team->t.t_nproc * sizeof (kmp_info_t*));
-
- __kmp_free(oldThreads);
-}
-
-static kmp_internal_control_t
-__kmp_get_global_icvs( void ) {
-
- kmp_r_sched_t r_sched = __kmp_get_schedule_global(); // get current state of scheduling globals
-
-#if OMP_40_ENABLED
- KMP_DEBUG_ASSERT( __kmp_nested_proc_bind.used > 0 );
-#endif /* OMP_40_ENABLED */
-
- kmp_internal_control_t g_icvs = {
- 0, //int serial_nesting_level; //corresponds to the value of the th_team_serialized field
- (kmp_int8)__kmp_dflt_nested, //int nested; //internal control for nested parallelism (per thread)
- (kmp_int8)__kmp_global.g.g_dynamic, //internal control for dynamic adjustment of threads (per thread)
- (kmp_int8)__kmp_env_blocktime, //int bt_set; //internal control for whether blocktime is explicitly set
- __kmp_dflt_blocktime, //int blocktime; //internal control for blocktime
-#if KMP_USE_MONITOR
- __kmp_bt_intervals, //int bt_intervals; //internal control for blocktime intervals
-#endif
- __kmp_dflt_team_nth, //int nproc; //internal control for # of threads for next parallel region (per thread)
- // (use a max ub on value if __kmp_parallel_initialize not called yet)
- __kmp_dflt_max_active_levels, //int max_active_levels; //internal control for max_active_levels
- r_sched, //kmp_r_sched_t sched; //internal control for runtime schedule {sched,chunk} pair
-#if OMP_40_ENABLED
- __kmp_nested_proc_bind.bind_types[0],
- __kmp_default_device,
-#endif /* OMP_40_ENABLED */
- NULL //struct kmp_internal_control *next;
- };
-
- return g_icvs;
-}
-
-static kmp_internal_control_t
-__kmp_get_x_global_icvs( const kmp_team_t *team ) {
-
- kmp_internal_control_t gx_icvs;
- gx_icvs.serial_nesting_level = 0; // probably =team->t.t_serial like in save_inter_controls
- copy_icvs( & gx_icvs, & team->t.t_threads[0]->th.th_current_task->td_icvs );
- gx_icvs.next = NULL;
-
- return gx_icvs;
-}
-
-static void
-__kmp_initialize_root( kmp_root_t *root )
-{
- int f;
- kmp_team_t *root_team;
- kmp_team_t *hot_team;
- int hot_team_max_nth;
- kmp_r_sched_t r_sched = __kmp_get_schedule_global(); // get current state of scheduling globals
- kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
- KMP_DEBUG_ASSERT( root );
- KMP_ASSERT( ! root->r.r_begin );
-
- /* setup the root state structure */
- __kmp_init_lock( &root->r.r_begin_lock );
- root->r.r_begin = FALSE;
- root->r.r_active = FALSE;
- root->r.r_in_parallel = 0;
- root->r.r_blocktime = __kmp_dflt_blocktime;
- root->r.r_nested = __kmp_dflt_nested;
-
- /* setup the root team for this task */
- /* allocate the root team structure */
- KF_TRACE( 10, ( "__kmp_initialize_root: before root_team\n" ) );
-
- root_team =
- __kmp_allocate_team(
- root,
- 1, // new_nproc
- 1, // max_nproc
-#if OMPT_SUPPORT
- 0, // root parallel id
-#endif
-#if OMP_40_ENABLED
- __kmp_nested_proc_bind.bind_types[0],
-#endif
- &r_icvs,
- 0 // argc
- USE_NESTED_HOT_ARG(NULL) // master thread is unknown
- );
-#if USE_DEBUGGER
- // Non-NULL value should be assigned to make the debugger display the root team.
- TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)( ~ 0 ));
-#endif
-
- KF_TRACE( 10, ( "__kmp_initialize_root: after root_team = %p\n", root_team ) );
-
- root->r.r_root_team = root_team;
- root_team->t.t_control_stack_top = NULL;
-
- /* initialize root team */
- root_team->t.t_threads[0] = NULL;
- root_team->t.t_nproc = 1;
- root_team->t.t_serialized = 1;
- // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
- root_team->t.t_sched.r_sched_type = r_sched.r_sched_type;
- root_team->t.t_sched.chunk = r_sched.chunk;
- KA_TRACE( 20, ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
- root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
-
- /* setup the hot team for this task */
- /* allocate the hot team structure */
- KF_TRACE( 10, ( "__kmp_initialize_root: before hot_team\n" ) );
-
- hot_team =
- __kmp_allocate_team(
- root,
- 1, // new_nproc
- __kmp_dflt_team_nth_ub * 2, // max_nproc
-#if OMPT_SUPPORT
- 0, // root parallel id
-#endif
-#if OMP_40_ENABLED
- __kmp_nested_proc_bind.bind_types[0],
-#endif
- &r_icvs,
- 0 // argc
- USE_NESTED_HOT_ARG(NULL) // master thread is unknown
- );
- KF_TRACE( 10, ( "__kmp_initialize_root: after hot_team = %p\n", hot_team ) );
-
- root->r.r_hot_team = hot_team;
- root_team->t.t_control_stack_top = NULL;
-
- /* first-time initialization */
- hot_team->t.t_parent = root_team;
-
- /* initialize hot team */
- hot_team_max_nth = hot_team->t.t_max_nproc;
- for ( f = 0; f < hot_team_max_nth; ++ f ) {
- hot_team->t.t_threads[ f ] = NULL;
- }; // for
- hot_team->t.t_nproc = 1;
- // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
- hot_team->t.t_sched.r_sched_type = r_sched.r_sched_type;
- hot_team->t.t_sched.chunk = r_sched.chunk;
- hot_team->t.t_size_changed = 0;
-}
-
-#ifdef KMP_DEBUG
-
-
-typedef struct kmp_team_list_item {
- kmp_team_p const * entry;
- struct kmp_team_list_item * next;
-} kmp_team_list_item_t;
-typedef kmp_team_list_item_t * kmp_team_list_t;
-
-
-static void
-__kmp_print_structure_team_accum( // Add team to list of teams.
- kmp_team_list_t list, // List of teams.
- kmp_team_p const * team // Team to add.
-) {
-
- // List must terminate with item where both entry and next are NULL.
- // Team is added to the list only once.
- // List is sorted in ascending order by team id.
- // Team id is *not* a key.
-
- kmp_team_list_t l;
-
- KMP_DEBUG_ASSERT( list != NULL );
- if ( team == NULL ) {
- return;
- }; // if
-
- __kmp_print_structure_team_accum( list, team->t.t_parent );
- __kmp_print_structure_team_accum( list, team->t.t_next_pool );
-
- // Search list for the team.
- l = list;
- while ( l->next != NULL && l->entry != team ) {
- l = l->next;
- }; // while
- if ( l->next != NULL ) {
- return; // Team has been added before, exit.
- }; // if
-
- // Team is not found. Search list again for insertion point.
- l = list;
- while ( l->next != NULL && l->entry->t.t_id <= team->t.t_id ) {
- l = l->next;
- }; // while
-
- // Insert team.
- {
- kmp_team_list_item_t * item =
- (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( sizeof( kmp_team_list_item_t ) );
- * item = * l;
- l->entry = team;
- l->next = item;
- }
-
-}
-
-static void
-__kmp_print_structure_team(
- char const * title,
- kmp_team_p const * team
-
-) {
- __kmp_printf( "%s", title );
- if ( team != NULL ) {
- __kmp_printf( "%2x %p\n", team->t.t_id, team );
- } else {
- __kmp_printf( " - (nil)\n" );
- }; // if
-}
-
-static void
-__kmp_print_structure_thread(
- char const * title,
- kmp_info_p const * thread
-
-) {
- __kmp_printf( "%s", title );
- if ( thread != NULL ) {
- __kmp_printf( "%2d %p\n", thread->th.th_info.ds.ds_gtid, thread );
- } else {
- __kmp_printf( " - (nil)\n" );
- }; // if
-}
-
-void
-__kmp_print_structure(
- void
-) {
-
- kmp_team_list_t list;
-
- // Initialize list of teams.
- list = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC( sizeof( kmp_team_list_item_t ) );
- list->entry = NULL;
- list->next = NULL;
-
- __kmp_printf( "\n------------------------------\nGlobal Thread Table\n------------------------------\n" );
- {
- int gtid;
- for ( gtid = 0; gtid < __kmp_threads_capacity; ++ gtid ) {
- __kmp_printf( "%2d", gtid );
- if ( __kmp_threads != NULL ) {
- __kmp_printf( " %p", __kmp_threads[ gtid ] );
- }; // if
- if ( __kmp_root != NULL ) {
- __kmp_printf( " %p", __kmp_root[ gtid ] );
- }; // if
- __kmp_printf( "\n" );
- }; // for gtid
- }
-
- // Print out __kmp_threads array.
- __kmp_printf( "\n------------------------------\nThreads\n------------------------------\n" );
- if ( __kmp_threads != NULL ) {
- int gtid;
- for ( gtid = 0; gtid < __kmp_threads_capacity; ++ gtid ) {
- kmp_info_t const * thread = __kmp_threads[ gtid ];
- if ( thread != NULL ) {
- __kmp_printf( "GTID %2d %p:\n", gtid, thread );
- __kmp_printf( " Our Root: %p\n", thread->th.th_root );
- __kmp_print_structure_team( " Our Team: ", thread->th.th_team );
- __kmp_print_structure_team( " Serial Team: ", thread->th.th_serial_team );
- __kmp_printf( " Threads: %2d\n", thread->th.th_team_nproc );
- __kmp_print_structure_thread( " Master: ", thread->th.th_team_master );
- __kmp_printf( " Serialized?: %2d\n", thread->th.th_team_serialized );
- __kmp_printf( " Set NProc: %2d\n", thread->th.th_set_nproc );
-#if OMP_40_ENABLED
- __kmp_printf( " Set Proc Bind: %2d\n", thread->th.th_set_proc_bind );
-#endif
- __kmp_print_structure_thread( " Next in pool: ", thread->th.th_next_pool );
- __kmp_printf( "\n" );
- __kmp_print_structure_team_accum( list, thread->th.th_team );
- __kmp_print_structure_team_accum( list, thread->th.th_serial_team );
- }; // if
- }; // for gtid
- } else {
- __kmp_printf( "Threads array is not allocated.\n" );
- }; // if
-
- // Print out __kmp_root array.
- __kmp_printf( "\n------------------------------\nUbers\n------------------------------\n" );
- if ( __kmp_root != NULL ) {
- int gtid;
- for ( gtid = 0; gtid < __kmp_threads_capacity; ++ gtid ) {
- kmp_root_t const * root = __kmp_root[ gtid ];
- if ( root != NULL ) {
- __kmp_printf( "GTID %2d %p:\n", gtid, root );
- __kmp_print_structure_team( " Root Team: ", root->r.r_root_team );
- __kmp_print_structure_team( " Hot Team: ", root->r.r_hot_team );
- __kmp_print_structure_thread( " Uber Thread: ", root->r.r_uber_thread );
- __kmp_printf( " Active?: %2d\n", root->r.r_active );
- __kmp_printf( " Nested?: %2d\n", root->r.r_nested );
- __kmp_printf( " In Parallel: %2d\n", root->r.r_in_parallel );
- __kmp_printf( "\n" );
- __kmp_print_structure_team_accum( list, root->r.r_root_team );
- __kmp_print_structure_team_accum( list, root->r.r_hot_team );
- }; // if
- }; // for gtid
- } else {
- __kmp_printf( "Ubers array is not allocated.\n" );
- }; // if
-
- __kmp_printf( "\n------------------------------\nTeams\n------------------------------\n" );
- while ( list->next != NULL ) {
- kmp_team_p const * team = list->entry;
- int i;
- __kmp_printf( "Team %2x %p:\n", team->t.t_id, team );
- __kmp_print_structure_team( " Parent Team: ", team->t.t_parent );
- __kmp_printf( " Master TID: %2d\n", team->t.t_master_tid );
- __kmp_printf( " Max threads: %2d\n", team->t.t_max_nproc );
- __kmp_printf( " Levels of serial: %2d\n", team->t.t_serialized );
- __kmp_printf( " Number threads: %2d\n", team->t.t_nproc );
- for ( i = 0; i < team->t.t_nproc; ++ i ) {
- __kmp_printf( " Thread %2d: ", i );
- __kmp_print_structure_thread( "", team->t.t_threads[ i ] );
- }; // for i
- __kmp_print_structure_team( " Next in pool: ", team->t.t_next_pool );
- __kmp_printf( "\n" );
- list = list->next;
- }; // while
-
- // Print out __kmp_thread_pool and __kmp_team_pool.
- __kmp_printf( "\n------------------------------\nPools\n------------------------------\n" );
- __kmp_print_structure_thread( "Thread pool: ", (kmp_info_t *)__kmp_thread_pool );
- __kmp_print_structure_team( "Team pool: ", (kmp_team_t *)__kmp_team_pool );
- __kmp_printf( "\n" );
-
- // Free team list.
- while ( list != NULL ) {
- kmp_team_list_item_t * item = list;
- list = list->next;
- KMP_INTERNAL_FREE( item );
- }; // while
-
-}
-
-#endif
-
-
-//---------------------------------------------------------------------------
-// Stuff for per-thread fast random number generator
-// Table of primes
-
-static const unsigned __kmp_primes[] = {
- 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5,
- 0xba5703f5, 0xb495a877, 0xe1626741, 0x79695e6b,
- 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
- 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b,
- 0xbe4d6fe9, 0x5f15e201, 0x99afc3fd, 0xf3f16801,
- 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
- 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed,
- 0x085a3d61, 0x46eb5ea7, 0x3d9910ed, 0x2e687b5b,
- 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
- 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7,
- 0x54581edb, 0xf2480f45, 0x0bb9288f, 0xef1affc7,
- 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
- 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b,
- 0xfc411073, 0xc3749363, 0xb892d829, 0x3549366b,
- 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
- 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f
-};
-
-//---------------------------------------------------------------------------
-// __kmp_get_random: Get a random number using a linear congruential method.
-
-unsigned short
-__kmp_get_random( kmp_info_t * thread )
-{
- unsigned x = thread->th.th_x;
- unsigned short r = x>>16;
-
- thread->th.th_x = x*thread->th.th_a+1;
-
- KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",
- thread->th.th_info.ds.ds_tid, r) );
-
- return r;
-}
-//--------------------------------------------------------
-// __kmp_init_random: Initialize a random number generator
-
-void
-__kmp_init_random( kmp_info_t * thread )
-{
- unsigned seed = thread->th.th_info.ds.ds_tid;
-
- thread->th.th_a = __kmp_primes[seed%(sizeof(__kmp_primes)/sizeof(__kmp_primes[0]))];
- thread->th.th_x = (seed+1)*thread->th.th_a+1;
- KA_TRACE(30, ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a) );
-}
-
-
-#if KMP_OS_WINDOWS
-/* reclaim array entries for root threads that are already dead, returns number reclaimed */
-static int
-__kmp_reclaim_dead_roots(void) {
- int i, r = 0;
-
- for(i = 0; i < __kmp_threads_capacity; ++i) {
- if( KMP_UBER_GTID( i ) &&
- !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
- !__kmp_root[i]->r.r_active ) { // AC: reclaim only roots died in non-active state
- r += __kmp_unregister_root_other_thread(i);
- }
- }
- return r;
-}
-#endif
-
-/*
- This function attempts to create free entries in __kmp_threads and __kmp_root, and returns the number of
- free entries generated.
-
- For Windows* OS static library, the first mechanism used is to reclaim array entries for root threads that are
- already dead.
-
- On all platforms, expansion is attempted on the arrays __kmp_threads_ and __kmp_root, with appropriate
- update to __kmp_threads_capacity. Array capacity is increased by doubling with clipping to
- __kmp_tp_capacity, if threadprivate cache array has been created.
- Synchronization with __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
-
- After any dead root reclamation, if the clipping value allows array expansion to result in the generation
- of a total of nWish free slots, the function does that expansion. If not, but the clipping value allows
- array expansion to result in the generation of a total of nNeed free slots, the function does that expansion.
- Otherwise, nothing is done beyond the possible initial root thread reclamation. However, if nNeed is zero,
- a best-effort attempt is made to fulfil nWish as far as possible, i.e. the function will attempt to create
- as many free slots as possible up to nWish.
-
- If any argument is negative, the behavior is undefined.
-*/
-static int
-__kmp_expand_threads(int nWish, int nNeed) {
- int added = 0;
- int old_tp_cached;
- int __kmp_actual_max_nth;
-
- if(nNeed > nWish) /* normalize the arguments */
- nWish = nNeed;
-#if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB
-/* only for Windows static library */
- /* reclaim array entries for root threads that are already dead */
- added = __kmp_reclaim_dead_roots();
-
- if(nNeed) {
- nNeed -= added;
- if(nNeed < 0)
- nNeed = 0;
- }
- if(nWish) {
- nWish -= added;
- if(nWish < 0)
- nWish = 0;
- }
-#endif
- if(nWish <= 0)
- return added;
-
- while(1) {
- int nTarget;
- int minimumRequiredCapacity;
- int newCapacity;
- kmp_info_t **newThreads;
- kmp_root_t **newRoot;
-
- //
- // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth.
- // If __kmp_max_nth is set to some value less than __kmp_sys_max_nth
- // by the user via OMP_THREAD_LIMIT, then __kmp_threads_capacity may
- // become > __kmp_max_nth in one of two ways:
- //
- // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
- // may not be resused by another thread, so we may need to increase
- // __kmp_threads_capacity to __kmp_max_threads + 1.
- //
- // 2) New foreign root(s) are encountered. We always register new
- // foreign roots. This may cause a smaller # of threads to be
- // allocated at subsequent parallel regions, but the worker threads
- // hang around (and eventually go to sleep) and need slots in the
- // __kmp_threads[] array.
- //
- // Anyway, that is the reason for moving the check to see if
- // __kmp_max_threads was exceeded into __kmp_reseerve_threads()
- // instead of having it performed here. -BB
- //
- old_tp_cached = __kmp_tp_cached;
- __kmp_actual_max_nth = old_tp_cached ? __kmp_tp_capacity : __kmp_sys_max_nth;
- KMP_DEBUG_ASSERT(__kmp_actual_max_nth >= __kmp_threads_capacity);
-
- /* compute expansion headroom to check if we can expand and whether to aim for nWish or nNeed */
- nTarget = nWish;
- if(__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) {
- /* can't fulfil nWish, so try nNeed */
- if(nNeed) {
- nTarget = nNeed;
- if(__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) {
- /* possible expansion too small -- give up */
- break;
- }
- } else {
- /* best-effort */
- nTarget = __kmp_actual_max_nth - __kmp_threads_capacity;
- if(!nTarget) {
- /* can expand at all -- give up */
- break;
- }
- }
- }
- minimumRequiredCapacity = __kmp_threads_capacity + nTarget;
-
- newCapacity = __kmp_threads_capacity;
- do{
- newCapacity =
- newCapacity <= (__kmp_actual_max_nth >> 1) ?
- (newCapacity << 1) :
- __kmp_actual_max_nth;
- } while(newCapacity < minimumRequiredCapacity);
- newThreads = (kmp_info_t**) __kmp_allocate((sizeof(kmp_info_t*) + sizeof(kmp_root_t*)) * newCapacity + CACHE_LINE);
- newRoot = (kmp_root_t**) ((char*)newThreads + sizeof(kmp_info_t*) * newCapacity );
- KMP_MEMCPY(newThreads, __kmp_threads, __kmp_threads_capacity * sizeof(kmp_info_t*));
- KMP_MEMCPY(newRoot, __kmp_root, __kmp_threads_capacity * sizeof(kmp_root_t*));
- memset(newThreads + __kmp_threads_capacity, 0,
- (newCapacity - __kmp_threads_capacity) * sizeof(kmp_info_t*));
- memset(newRoot + __kmp_threads_capacity, 0,
- (newCapacity - __kmp_threads_capacity) * sizeof(kmp_root_t*));
-
- if(!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
- /* __kmp_tp_cached has changed, i.e. __kmpc_threadprivate_cached has allocated a threadprivate cache
- while we were allocating the expanded array, and our new capacity is larger than the threadprivate
- cache capacity, so we should deallocate the expanded arrays and try again. This is the first check
- of a double-check pair.
- */
- __kmp_free(newThreads);
- continue; /* start over and try again */
- }
- __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
- if(!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
- /* Same check as above, but this time with the lock so we can be sure if we can succeed. */
- __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
- __kmp_free(newThreads);
- continue; /* start over and try again */
- } else {
- /* success */
- // __kmp_free( __kmp_threads ); // ATT: It leads to crash. Need to be investigated.
- //
- *(kmp_info_t**volatile*)&__kmp_threads = newThreads;
- *(kmp_root_t**volatile*)&__kmp_root = newRoot;
- added += newCapacity - __kmp_threads_capacity;
- *(volatile int*)&__kmp_threads_capacity = newCapacity;
- __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
- break; /* succeeded, so we can exit the loop */
- }
- }
- return added;
-}
-
-/* register the current thread as a root thread and obtain our gtid */
-/* we must have the __kmp_initz_lock held at this point */
-/* Argument TRUE only if are the thread that calls from __kmp_do_serial_initialize() */
-int
-__kmp_register_root( int initial_thread )
-{
- kmp_info_t *root_thread;
- kmp_root_t *root;
- int gtid;
- int capacity;
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
- KA_TRACE( 20, ("__kmp_register_root: entered\n"));
- KMP_MB();
-
-
- /*
- 2007-03-02:
-
- If initial thread did not invoke OpenMP RTL yet, and this thread is not an initial one,
- "__kmp_all_nth >= __kmp_threads_capacity" condition does not work as expected -- it may
- return false (that means there is at least one empty slot in __kmp_threads array), but it
- is possible the only free slot is #0, which is reserved for initial thread and so cannot be
- used for this one. Following code workarounds this bug.
-
- However, right solution seems to be not reserving slot #0 for initial thread because:
- (1) there is no magic in slot #0,
- (2) we cannot detect initial thread reliably (the first thread which does serial
- initialization may be not a real initial thread).
- */
- capacity = __kmp_threads_capacity;
- if ( ! initial_thread && TCR_PTR(__kmp_threads[0]) == NULL ) {
- -- capacity;
- }; // if
-
- /* see if there are too many threads */
- if ( __kmp_all_nth >= capacity && !__kmp_expand_threads( 1, 1 ) ) {
- if ( __kmp_tp_cached ) {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantRegisterNewThread ),
- KMP_HNT( Set_ALL_THREADPRIVATE, __kmp_tp_capacity ),
- KMP_HNT( PossibleSystemLimitOnThreads ),
- __kmp_msg_null
- );
- }
- else {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( CantRegisterNewThread ),
- KMP_HNT( SystemLimitOnThreads ),
- __kmp_msg_null
- );
- }
- }; // if
-
- /* find an available thread slot */
- /* Don't reassign the zero slot since we need that to only be used by initial
- thread */
- for( gtid=(initial_thread ? 0 : 1) ; TCR_PTR(__kmp_threads[gtid]) != NULL ; gtid++ )
- ;
- KA_TRACE( 1, ("__kmp_register_root: found slot in threads array: T#%d\n", gtid ));
- KMP_ASSERT( gtid < __kmp_threads_capacity );
-
- /* update global accounting */
- __kmp_all_nth ++;
- TCW_4(__kmp_nth, __kmp_nth + 1);
-
- //
- // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search)
- // for low numbers of procs, and method #2 (keyed API call) for higher
- // numbers of procs.
- //
- if ( __kmp_adjust_gtid_mode ) {
- if ( __kmp_all_nth >= __kmp_tls_gtid_min ) {
- if ( TCR_4(__kmp_gtid_mode) != 2) {
- TCW_4(__kmp_gtid_mode, 2);
- }
- }
- else {
- if (TCR_4(__kmp_gtid_mode) != 1 ) {
- TCW_4(__kmp_gtid_mode, 1);
- }
- }
- }
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime to zero if necessary */
- /* Middle initialization might not have occurred yet */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- if ( __kmp_nth > __kmp_avail_proc ) {
- __kmp_zero_bt = TRUE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
- /* setup this new hierarchy */
- if( ! ( root = __kmp_root[gtid] )) {
- root = __kmp_root[gtid] = (kmp_root_t*) __kmp_allocate( sizeof(kmp_root_t) );
- KMP_DEBUG_ASSERT( ! root->r.r_root_team );
- }
-
-#if KMP_STATS_ENABLED
- // Initialize stats as soon as possible (right after gtid assignment).
- __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
- KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
- KMP_SET_THREAD_STATE(SERIAL_REGION);
- KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
-#endif
- __kmp_initialize_root( root );
-
- /* setup new root thread structure */
- if( root->r.r_uber_thread ) {
- root_thread = root->r.r_uber_thread;
- } else {
- root_thread = (kmp_info_t*) __kmp_allocate( sizeof(kmp_info_t) );
- if ( __kmp_storage_map ) {
- __kmp_print_thread_storage_map( root_thread, gtid );
- }
- root_thread->th.th_info .ds.ds_gtid = gtid;
- root_thread->th.th_root = root;
- if( __kmp_env_consistency_check ) {
- root_thread->th.th_cons = __kmp_allocate_cons_stack( gtid );
- }
- #if USE_FAST_MEMORY
- __kmp_initialize_fast_memory( root_thread );
- #endif /* USE_FAST_MEMORY */
-
- #if KMP_USE_BGET
- KMP_DEBUG_ASSERT( root_thread->th.th_local.bget_data == NULL );
- __kmp_initialize_bget( root_thread );
- #endif
- __kmp_init_random( root_thread ); // Initialize random number generator
- }
-
- /* setup the serial team held in reserve by the root thread */
- if( ! root_thread->th.th_serial_team ) {
- kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
- KF_TRACE( 10, ( "__kmp_register_root: before serial_team\n" ) );
-
- root_thread->th.th_serial_team = __kmp_allocate_team( root, 1, 1,
-#if OMPT_SUPPORT
- 0, // root parallel id
-#endif
-#if OMP_40_ENABLED
- proc_bind_default,
-#endif
- &r_icvs,
- 0 USE_NESTED_HOT_ARG(NULL) );
- }
- KMP_ASSERT( root_thread->th.th_serial_team );
- KF_TRACE( 10, ( "__kmp_register_root: after serial_team = %p\n",
- root_thread->th.th_serial_team ) );
-
- /* drop root_thread into place */
- TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
-
- root->r.r_root_team->t.t_threads[0] = root_thread;
- root->r.r_hot_team ->t.t_threads[0] = root_thread;
- root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
- root_thread->th.th_serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for execution (it is unused for now).
- root->r.r_uber_thread = root_thread;
-
- /* initialize the thread, get it ready to go */
- __kmp_initialize_info( root_thread, root->r.r_root_team, 0, gtid );
- TCW_4(__kmp_init_gtid, TRUE);
-
- /* prepare the master thread for get_gtid() */
- __kmp_gtid_set_specific( gtid );
-
-#if USE_ITT_BUILD
- __kmp_itt_thread_name( gtid );
-#endif /* USE_ITT_BUILD */
-
- #ifdef KMP_TDATA_GTID
- __kmp_gtid = gtid;
- #endif
- __kmp_create_worker( gtid, root_thread, __kmp_stksize );
- KMP_DEBUG_ASSERT( __kmp_gtid_get_specific() == gtid );
-
- KA_TRACE( 20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, plain=%u\n",
- gtid, __kmp_gtid_from_tid( 0, root->r.r_hot_team ),
- root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
- KMP_INIT_BARRIER_STATE ) );
- { // Initialize barrier data.
- int b;
- for ( b = 0; b < bs_last_barrier; ++ b ) {
- root_thread->th.th_bar[ b ].bb.b_arrived = KMP_INIT_BARRIER_STATE;
-#if USE_DEBUGGER
- root_thread->th.th_bar[ b ].bb.b_worker_arrived = 0;
-#endif
- }; // for
- }
- KMP_DEBUG_ASSERT( root->r.r_hot_team->t.t_bar[ bs_forkjoin_barrier ].b_arrived == KMP_INIT_BARRIER_STATE );
-
-#if KMP_AFFINITY_SUPPORTED
-# if OMP_40_ENABLED
- root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
- root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
- root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
- root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
-# endif
-
- if ( TCR_4(__kmp_init_middle) ) {
- __kmp_affinity_set_init_mask( gtid, TRUE );
- }
-#endif /* KMP_AFFINITY_SUPPORTED */
-
- __kmp_root_counter ++;
-
- KMP_MB();
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
-
- return gtid;
-}
-
-#if KMP_NESTED_HOT_TEAMS
-static int
-__kmp_free_hot_teams( kmp_root_t *root, kmp_info_t *thr, int level, const int max_level )
-{
- int i, n, nth;
- kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
- if( !hot_teams || !hot_teams[level].hot_team ) {
- return 0;
- }
- KMP_DEBUG_ASSERT( level < max_level );
- kmp_team_t *team = hot_teams[level].hot_team;
- nth = hot_teams[level].hot_team_nth;
- n = nth - 1; // master is not freed
- if( level < max_level - 1 ) {
- for( i = 0; i < nth; ++i ) {
- kmp_info_t *th = team->t.t_threads[i];
- n += __kmp_free_hot_teams( root, th, level + 1, max_level );
- if( i > 0 && th->th.th_hot_teams ) {
- __kmp_free( th->th.th_hot_teams );
- th->th.th_hot_teams = NULL;
- }
- }
- }
- __kmp_free_team( root, team, NULL );
- return n;
-}
-#endif
-
-/* Resets a root thread and clear its root and hot teams.
- Returns the number of __kmp_threads entries directly and indirectly freed.
-*/
-static int
-__kmp_reset_root(int gtid, kmp_root_t *root)
-{
- kmp_team_t * root_team = root->r.r_root_team;
- kmp_team_t * hot_team = root->r.r_hot_team;
- int n = hot_team->t.t_nproc;
- int i;
-
- KMP_DEBUG_ASSERT( ! root->r.r_active );
-
- root->r.r_root_team = NULL;
- root->r.r_hot_team = NULL;
- // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team before call
- // to __kmp_free_team().
- __kmp_free_team( root, root_team USE_NESTED_HOT_ARG(NULL) );
-#if KMP_NESTED_HOT_TEAMS
- if( __kmp_hot_teams_max_level > 0 ) { // need to free nested hot teams and their threads if any
- for( i = 0; i < hot_team->t.t_nproc; ++i ) {
- kmp_info_t *th = hot_team->t.t_threads[i];
- if( __kmp_hot_teams_max_level > 1 ) {
- n += __kmp_free_hot_teams( root, th, 1, __kmp_hot_teams_max_level );
- }
- if( th->th.th_hot_teams ) {
- __kmp_free( th->th.th_hot_teams );
- th->th.th_hot_teams = NULL;
- }
- }
- }
-#endif
- __kmp_free_team( root, hot_team USE_NESTED_HOT_ARG(NULL) );
-
- //
- // Before we can reap the thread, we need to make certain that all
- // other threads in the teams that had this root as ancestor have stopped trying to steal tasks.
- //
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- __kmp_wait_to_unref_task_teams();
- }
-
- #if KMP_OS_WINDOWS
- /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
- KA_TRACE( 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
- (LPVOID)&(root->r.r_uber_thread->th),
- root->r.r_uber_thread->th.th_info.ds.ds_thread ) );
- __kmp_free_handle( root->r.r_uber_thread->th.th_info.ds.ds_thread );
- #endif /* KMP_OS_WINDOWS */
-
-#if OMPT_SUPPORT
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
- int gtid = __kmp_get_gtid();
- __ompt_thread_end(ompt_thread_initial, gtid);
- }
-#endif
-
- TCW_4(__kmp_nth, __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
- __kmp_reap_thread( root->r.r_uber_thread, 1 );
-
- // We canot put root thread to __kmp_thread_pool, so we have to reap it istead of freeing.
- root->r.r_uber_thread = NULL;
- /* mark root as no longer in use */
- root->r.r_begin = FALSE;
-
- return n;
-}
-
-void
-__kmp_unregister_root_current_thread( int gtid )
-{
- KA_TRACE( 1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid ));
- /* this lock should be ok, since unregister_root_current_thread is never called during
- * and abort, only during a normal close. furthermore, if you have the
- * forkjoin lock, you should never try to get the initz lock */
-
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
- if( TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial ) {
- KC_TRACE( 10, ("__kmp_unregister_root_current_thread: already finished, exiting T#%d\n", gtid ));
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- return;
- }
- kmp_root_t *root = __kmp_root[gtid];
-
- KMP_DEBUG_ASSERT( __kmp_threads && __kmp_threads[gtid] );
- KMP_ASSERT( KMP_UBER_GTID( gtid ));
- KMP_ASSERT( root == __kmp_threads[gtid]->th.th_root );
- KMP_ASSERT( root->r.r_active == FALSE );
-
-
- KMP_MB();
-
-#if OMP_45_ENABLED
- kmp_info_t * thread = __kmp_threads[gtid];
- kmp_team_t * team = thread->th.th_team;
- kmp_task_team_t * task_team = thread->th.th_task_team;
-
- // we need to wait for the proxy tasks before finishing the thread
- if ( task_team != NULL && task_team->tt.tt_found_proxy_tasks ) {
-#if OMPT_SUPPORT
- // the runtime is shutting down so we won't report any events
- thread->th.ompt_thread_info.state = ompt_state_undefined;
-#endif
- __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
- }
-#endif
-
- __kmp_reset_root(gtid, root);
-
- /* free up this thread slot */
- __kmp_gtid_set_specific( KMP_GTID_DNE );
-#ifdef KMP_TDATA_GTID
- __kmp_gtid = KMP_GTID_DNE;
-#endif
-
- KMP_MB();
- KC_TRACE( 10, ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid ));
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
-}
-
-#if KMP_OS_WINDOWS
-/* __kmp_forkjoin_lock must be already held
- Unregisters a root thread that is not the current thread. Returns the number of
- __kmp_threads entries freed as a result.
- */
-static int
-__kmp_unregister_root_other_thread( int gtid )
-{
- kmp_root_t *root = __kmp_root[gtid];
- int r;
-
- KA_TRACE( 1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid ));
- KMP_DEBUG_ASSERT( __kmp_threads && __kmp_threads[gtid] );
- KMP_ASSERT( KMP_UBER_GTID( gtid ));
- KMP_ASSERT( root == __kmp_threads[gtid]->th.th_root );
- KMP_ASSERT( root->r.r_active == FALSE );
-
- r = __kmp_reset_root(gtid, root);
- KC_TRACE( 10, ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid ));
- return r;
-}
-#endif
-
-#if KMP_DEBUG
-void __kmp_task_info() {
-
- kmp_int32 gtid = __kmp_entry_gtid();
- kmp_int32 tid = __kmp_tid_from_gtid( gtid );
- kmp_info_t *this_thr = __kmp_threads[ gtid ];
- kmp_team_t *steam = this_thr->th.th_serial_team;
- kmp_team_t *team = this_thr->th.th_team;
-
- __kmp_printf( "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p curtask=%p ptask=%p\n",
- gtid, tid, this_thr, team, this_thr->th.th_current_task, team->t.t_implicit_task_taskdata[tid].td_parent );
-}
-#endif // KMP_DEBUG
-
-/* TODO optimize with one big memclr, take out what isn't needed,
- * split responsibility to workers as much as possible, and delay
- * initialization of features as much as possible */
-static void
-__kmp_initialize_info( kmp_info_t *this_thr, kmp_team_t *team, int tid, int gtid )
-{
- /* this_thr->th.th_info.ds.ds_gtid is setup in kmp_allocate_thread/create_worker
- * this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
- kmp_info_t *master = team->t.t_threads[0];
- KMP_DEBUG_ASSERT( this_thr != NULL );
- KMP_DEBUG_ASSERT( this_thr->th.th_serial_team );
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( team->t.t_threads );
- KMP_DEBUG_ASSERT( team->t.t_dispatch );
- KMP_DEBUG_ASSERT( master );
- KMP_DEBUG_ASSERT( master->th.th_root );
-
- KMP_MB();
-
- TCW_SYNC_PTR(this_thr->th.th_team, team);
-
- this_thr->th.th_info.ds.ds_tid = tid;
- this_thr->th.th_set_nproc = 0;
-#if OMP_40_ENABLED
- this_thr->th.th_set_proc_bind = proc_bind_default;
-# if KMP_AFFINITY_SUPPORTED
- this_thr->th.th_new_place = this_thr->th.th_current_place;
-# endif
-#endif
- this_thr->th.th_root = master->th.th_root;
-
- /* setup the thread's cache of the team structure */
- this_thr->th.th_team_nproc = team->t.t_nproc;
- this_thr->th.th_team_master = master;
- this_thr->th.th_team_serialized = team->t.t_serialized;
- TCW_PTR(this_thr->th.th_sleep_loc, NULL);
-
- KMP_DEBUG_ASSERT( team->t.t_implicit_task_taskdata );
-
- KF_TRACE( 10, ( "__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
- tid, gtid, this_thr, this_thr->th.th_current_task ) );
-
- __kmp_init_implicit_task( this_thr->th.th_team_master->th.th_ident, this_thr, team, tid, TRUE );
-
- KF_TRACE( 10, ( "__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
- tid, gtid, this_thr, this_thr->th.th_current_task ) );
- // TODO: Initialize ICVs from parent; GEH - isn't that already done in __kmp_initialize_team()?
-
- /* TODO no worksharing in speculative threads */
- this_thr->th.th_dispatch = &team->t.t_dispatch[ tid ];
-
- this_thr->th.th_local.this_construct = 0;
-
-#ifdef BUILD_TV
- this_thr->th.th_local.tv_data = 0;
-#endif
-
- if ( ! this_thr->th.th_pri_common ) {
- this_thr->th.th_pri_common = (struct common_table *) __kmp_allocate( sizeof(struct common_table) );
- if ( __kmp_storage_map ) {
- __kmp_print_storage_map_gtid(
- gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
- sizeof( struct common_table ), "th_%d.th_pri_common\n", gtid
- );
- }; // if
- this_thr->th.th_pri_head = NULL;
- }; // if
-
- /* Initialize dynamic dispatch */
- {
- volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
- /*
- * Use team max_nproc since this will never change for the team.
- */
- size_t disp_size = sizeof( dispatch_private_info_t ) *
- ( team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers );
- KD_TRACE( 10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid, team->t.t_max_nproc ) );
- KMP_ASSERT( dispatch );
- KMP_DEBUG_ASSERT( team->t.t_dispatch );
- KMP_DEBUG_ASSERT( dispatch == &team->t.t_dispatch[ tid ] );
-
- dispatch->th_disp_index = 0;
-#if OMP_45_ENABLED
- dispatch->th_doacross_buf_idx = 0;
-#endif
- if( ! dispatch->th_disp_buffer ) {
- dispatch->th_disp_buffer = (dispatch_private_info_t *) __kmp_allocate( disp_size );
-
- if ( __kmp_storage_map ) {
- __kmp_print_storage_map_gtid( gtid, &dispatch->th_disp_buffer[ 0 ],
- &dispatch->th_disp_buffer[ team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers ],
- disp_size, "th_%d.th_dispatch.th_disp_buffer "
- "(team_%d.t_dispatch[%d].th_disp_buffer)",
- gtid, team->t.t_id, gtid );
- }
- } else {
- memset( & dispatch->th_disp_buffer[0], '\0', disp_size );
- }
-
- dispatch->th_dispatch_pr_current = 0;
- dispatch->th_dispatch_sh_current = 0;
-
- dispatch->th_deo_fcn = 0; /* ORDERED */
- dispatch->th_dxo_fcn = 0; /* END ORDERED */
- }
-
- this_thr->th.th_next_pool = NULL;
-
- if (!this_thr->th.th_task_state_memo_stack) {
- size_t i;
- this_thr->th.th_task_state_memo_stack = (kmp_uint8 *) __kmp_allocate( 4*sizeof(kmp_uint8) );
- this_thr->th.th_task_state_top = 0;
- this_thr->th.th_task_state_stack_sz = 4;
- for (i=0; i<this_thr->th.th_task_state_stack_sz; ++i) // zero init the stack
- this_thr->th.th_task_state_memo_stack[i] = 0;
- }
-
- KMP_DEBUG_ASSERT( !this_thr->th.th_spin_here );
- KMP_DEBUG_ASSERT( this_thr->th.th_next_waiting == 0 );
-
- KMP_MB();
-}
-
-
-/* allocate a new thread for the requesting team. this is only called from within a
- * forkjoin critical section. we will first try to get an available thread from the
- * thread pool. if none is available, we will fork a new one assuming we are able
- * to create a new one. this should be assured, as the caller should check on this
- * first.
- */
-kmp_info_t *
-__kmp_allocate_thread( kmp_root_t *root, kmp_team_t *team, int new_tid )
-{
- kmp_team_t *serial_team;
- kmp_info_t *new_thr;
- int new_gtid;
-
- KA_TRACE( 20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid() ));
- KMP_DEBUG_ASSERT( root && team );
-#if !KMP_NESTED_HOT_TEAMS
- KMP_DEBUG_ASSERT( KMP_MASTER_GTID( __kmp_get_gtid() ));
-#endif
- KMP_MB();
-
- /* first, try to get one from the thread pool */
- if ( __kmp_thread_pool ) {
-
- new_thr = (kmp_info_t*)__kmp_thread_pool;
- __kmp_thread_pool = (volatile kmp_info_t *) new_thr->th.th_next_pool;
- if ( new_thr == __kmp_thread_pool_insert_pt ) {
- __kmp_thread_pool_insert_pt = NULL;
- }
- TCW_4(new_thr->th.th_in_pool, FALSE);
- //
- // Don't touch th_active_in_pool or th_active.
- // The worker thread adjusts those flags as it sleeps/awakens.
- //
- __kmp_thread_pool_nth--;
-
- KA_TRACE( 20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",
- __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid ));
- KMP_ASSERT( ! new_thr->th.th_team );
- KMP_DEBUG_ASSERT( __kmp_nth < __kmp_threads_capacity );
- KMP_DEBUG_ASSERT( __kmp_thread_pool_nth >= 0 );
-
- /* setup the thread structure */
- __kmp_initialize_info( new_thr, team, new_tid, new_thr->th.th_info.ds.ds_gtid );
- KMP_DEBUG_ASSERT( new_thr->th.th_serial_team );
-
- TCW_4(__kmp_nth, __kmp_nth + 1);
-
- new_thr->th.th_task_state = 0;
- new_thr->th.th_task_state_top = 0;
- new_thr->th.th_task_state_stack_sz = 4;
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime back to zero if necessar y */
- /* Middle initialization might not have occurred yet */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- if ( __kmp_nth > __kmp_avail_proc ) {
- __kmp_zero_bt = TRUE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
-#if KMP_DEBUG
- // If thread entered pool via __kmp_free_thread, wait_flag should != KMP_BARRIER_PARENT_FLAG.
- int b;
- kmp_balign_t * balign = new_thr->th.th_bar;
- for( b = 0; b < bs_last_barrier; ++ b )
- KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#endif
-
- KF_TRACE( 10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",
- __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid ));
-
- KMP_MB();
- return new_thr;
- }
-
-
- /* no, well fork a new one */
- KMP_ASSERT( __kmp_nth == __kmp_all_nth );
- KMP_ASSERT( __kmp_all_nth < __kmp_threads_capacity );
-
-#if KMP_USE_MONITOR
- //
- // If this is the first worker thread the RTL is creating, then also
- // launch the monitor thread. We try to do this as early as possible.
- //
- if ( ! TCR_4( __kmp_init_monitor ) ) {
- __kmp_acquire_bootstrap_lock( & __kmp_monitor_lock );
- if ( ! TCR_4( __kmp_init_monitor ) ) {
- KF_TRACE( 10, ( "before __kmp_create_monitor\n" ) );
- TCW_4( __kmp_init_monitor, 1 );
- __kmp_create_monitor( & __kmp_monitor );
- KF_TRACE( 10, ( "after __kmp_create_monitor\n" ) );
- #if KMP_OS_WINDOWS
- // AC: wait until monitor has started. This is a fix for CQ232808.
- // The reason is that if the library is loaded/unloaded in a loop with small (parallel)
- // work in between, then there is high probability that monitor thread started after
- // the library shutdown. At shutdown it is too late to cope with the problem, because
- // when the master is in DllMain (process detach) the monitor has no chances to start
- // (it is blocked), and master has no means to inform the monitor that the library has gone,
- // because all the memory which the monitor can access is going to be released/reset.
- while ( TCR_4(__kmp_init_monitor) < 2 ) {
- KMP_YIELD( TRUE );
- }
- KF_TRACE( 10, ( "after monitor thread has started\n" ) );
- #endif
- }
- __kmp_release_bootstrap_lock( & __kmp_monitor_lock );
- }
-#endif
-
- KMP_MB();
- for( new_gtid=1 ; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid ) {
- KMP_DEBUG_ASSERT( new_gtid < __kmp_threads_capacity );
- }
-
- /* allocate space for it. */
- new_thr = (kmp_info_t*) __kmp_allocate( sizeof(kmp_info_t) );
-
- TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
-
- if ( __kmp_storage_map ) {
- __kmp_print_thread_storage_map( new_thr, new_gtid );
- }
-
- /* add the reserve serialized team, initialized from the team's master thread */
- {
- kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs( team );
- KF_TRACE( 10, ( "__kmp_allocate_thread: before th_serial/serial_team\n" ) );
-
- new_thr->th.th_serial_team = serial_team =
- (kmp_team_t*) __kmp_allocate_team( root, 1, 1,
-#if OMPT_SUPPORT
- 0, // root parallel id
-#endif
-#if OMP_40_ENABLED
- proc_bind_default,
-#endif
- &r_icvs,
- 0 USE_NESTED_HOT_ARG(NULL) );
- }
- KMP_ASSERT ( serial_team );
- serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for execution (it is unused for now).
- serial_team->t.t_threads[0] = new_thr;
- KF_TRACE( 10, ( "__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
- new_thr ) );
-
- /* setup the thread structures */
- __kmp_initialize_info( new_thr, team, new_tid, new_gtid );
-
- #if USE_FAST_MEMORY
- __kmp_initialize_fast_memory( new_thr );
- #endif /* USE_FAST_MEMORY */
-
- #if KMP_USE_BGET
- KMP_DEBUG_ASSERT( new_thr->th.th_local.bget_data == NULL );
- __kmp_initialize_bget( new_thr );
- #endif
-
- __kmp_init_random( new_thr ); // Initialize random number generator
-
- /* Initialize these only once when thread is grabbed for a team allocation */
- KA_TRACE( 20, ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
- __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
-
- int b;
- kmp_balign_t * balign = new_thr->th.th_bar;
- for(b=0; b<bs_last_barrier; ++b) {
- balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
- balign[b].bb.team = NULL;
- balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
- balign[b].bb.use_oncore_barrier = 0;
- }
-
- new_thr->th.th_spin_here = FALSE;
- new_thr->th.th_next_waiting = 0;
-
-#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
- new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
- new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
- new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
- new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
-#endif
-
- TCW_4(new_thr->th.th_in_pool, FALSE);
- new_thr->th.th_active_in_pool = FALSE;
- TCW_4(new_thr->th.th_active, TRUE);
-
- /* adjust the global counters */
- __kmp_all_nth ++;
- __kmp_nth ++;
-
- //
- // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search)
- // for low numbers of procs, and method #2 (keyed API call) for higher
- // numbers of procs.
- //
- if ( __kmp_adjust_gtid_mode ) {
- if ( __kmp_all_nth >= __kmp_tls_gtid_min ) {
- if ( TCR_4(__kmp_gtid_mode) != 2) {
- TCW_4(__kmp_gtid_mode, 2);
- }
- }
- else {
- if (TCR_4(__kmp_gtid_mode) != 1 ) {
- TCW_4(__kmp_gtid_mode, 1);
- }
- }
- }
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime back to zero if necessary */
- /* Middle initialization might not have occurred yet */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- if ( __kmp_nth > __kmp_avail_proc ) {
- __kmp_zero_bt = TRUE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
- /* actually fork it and create the new worker thread */
- KF_TRACE( 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr ));
- __kmp_create_worker( new_gtid, new_thr, __kmp_stksize );
- KF_TRACE( 10, ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr ));
-
- KA_TRACE( 20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(), new_gtid ));
- KMP_MB();
- return new_thr;
-}
-
-/*
- * reinitialize team for reuse.
- *
- * The hot team code calls this case at every fork barrier, so EPCC barrier
- * test are extremely sensitive to changes in it, esp. writes to the team
- * struct, which cause a cache invalidation in all threads.
- *
- * IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!!
- */
-static void
-__kmp_reinitialize_team( kmp_team_t *team, kmp_internal_control_t *new_icvs, ident_t *loc ) {
- KF_TRACE( 10, ( "__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
- team->t.t_threads[0], team ) );
- KMP_DEBUG_ASSERT( team && new_icvs);
- KMP_DEBUG_ASSERT( ( ! TCR_4(__kmp_init_parallel) ) || new_icvs->nproc );
- KMP_CHECK_UPDATE(team->t.t_ident, loc);
-
- KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
-
- // Copy ICVs to the master thread's implicit taskdata
- __kmp_init_implicit_task( loc, team->t.t_threads[0], team, 0, FALSE );
- copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
-
- KF_TRACE( 10, ( "__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
- team->t.t_threads[0], team ) );
-}
-
-
-/* initialize the team data structure
- * this assumes the t_threads and t_max_nproc are already set
- * also, we don't touch the arguments */
-static void
-__kmp_initialize_team(
- kmp_team_t * team,
- int new_nproc,
- kmp_internal_control_t * new_icvs,
- ident_t * loc
-) {
- KF_TRACE( 10, ( "__kmp_initialize_team: enter: team=%p\n", team ) );
-
- /* verify */
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( new_nproc <= team->t.t_max_nproc );
- KMP_DEBUG_ASSERT( team->t.t_threads );
- KMP_MB();
-
- team->t.t_master_tid = 0; /* not needed */
- /* team->t.t_master_bar; not needed */
- team->t.t_serialized = new_nproc > 1 ? 0 : 1;
- team->t.t_nproc = new_nproc;
-
- /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
- team->t.t_next_pool = NULL;
- /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess up hot team */
-
- TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */
- team->t.t_invoke = NULL; /* not needed */
-
- // TODO???: team->t.t_max_active_levels = new_max_active_levels;
- team->t.t_sched = new_icvs->sched;
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
- team->t.t_fp_control_saved = FALSE; /* not needed */
- team->t.t_x87_fpu_control_word = 0; /* not needed */
- team->t.t_mxcsr = 0; /* not needed */
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
- team->t.t_construct = 0;
- __kmp_init_lock( & team->t.t_single_lock );
-
- team->t.t_ordered .dt.t_value = 0;
- team->t.t_master_active = FALSE;
-
- memset( & team->t.t_taskq, '\0', sizeof( kmp_taskq_t ));
-
-#ifdef KMP_DEBUG
- team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */
-#endif
- team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
-
- team->t.t_control_stack_top = NULL;
-
- __kmp_reinitialize_team( team, new_icvs, loc );
-
- KMP_MB();
- KF_TRACE( 10, ( "__kmp_initialize_team: exit: team=%p\n", team ) );
-}
-
-#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
-/* Sets full mask for thread and returns old mask, no changes to structures. */
-static void
-__kmp_set_thread_affinity_mask_full_tmp( kmp_affin_mask_t *old_mask )
-{
- if ( KMP_AFFINITY_CAPABLE() ) {
- int status;
- if ( old_mask != NULL ) {
- status = __kmp_get_system_affinity( old_mask, TRUE );
- int error = errno;
- if ( status != 0 ) {
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( ChangeThreadAffMaskError ),
- KMP_ERR( error ),
- __kmp_msg_null
- );
- }
- }
- __kmp_set_system_affinity( __kmp_affin_fullMask, TRUE );
- }
-}
-#endif
-
-#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
-
-//
-// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
-// It calculats the worker + master thread's partition based upon the parent
-// thread's partition, and binds each worker to a thread in their partition.
-// The master thread's partition should already include its current binding.
-//
-static void
-__kmp_partition_places( kmp_team_t *team, int update_master_only )
-{
- //
- // Copy the master thread's place partion to the team struct
- //
- kmp_info_t *master_th = team->t.t_threads[0];
- KMP_DEBUG_ASSERT( master_th != NULL );
- kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
- int first_place = master_th->th.th_first_place;
- int last_place = master_th->th.th_last_place;
- int masters_place = master_th->th.th_current_place;
- team->t.t_first_place = first_place;
- team->t.t_last_place = last_place;
-
- KA_TRACE( 20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) bound to place %d partition = [%d,%d]\n",
- proc_bind, __kmp_gtid_from_thread( team->t.t_threads[0] ), team->t.t_id,
- masters_place, first_place, last_place ) );
-
- switch ( proc_bind ) {
-
- case proc_bind_default:
- //
- // serial teams might have the proc_bind policy set to
- // proc_bind_default. It doesn't matter, as we don't
- // rebind the master thread for any proc_bind policy.
- //
- KMP_DEBUG_ASSERT( team->t.t_nproc == 1 );
- break;
-
- case proc_bind_master:
- {
- int f;
- int n_th = team->t.t_nproc;
- for ( f = 1; f < n_th; f++ ) {
- kmp_info_t *th = team->t.t_threads[f];
- KMP_DEBUG_ASSERT( th != NULL );
- th->th.th_first_place = first_place;
- th->th.th_last_place = last_place;
- th->th.th_new_place = masters_place;
-
- KA_TRACE( 100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d partition = [%d,%d]\n",
- __kmp_gtid_from_thread( team->t.t_threads[f] ),
- team->t.t_id, f, masters_place, first_place, last_place ) );
- }
- }
- break;
-
- case proc_bind_close:
- {
- int f;
- int n_th = team->t.t_nproc;
- int n_places;
- if ( first_place <= last_place ) {
- n_places = last_place - first_place + 1;
- }
- else {
- n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
- }
- if ( n_th <= n_places ) {
- int place = masters_place;
- for ( f = 1; f < n_th; f++ ) {
- kmp_info_t *th = team->t.t_threads[f];
- KMP_DEBUG_ASSERT( th != NULL );
-
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- th->th.th_first_place = first_place;
- th->th.th_last_place = last_place;
- th->th.th_new_place = place;
-
- KA_TRACE( 100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d partition = [%d,%d]\n",
- __kmp_gtid_from_thread( team->t.t_threads[f] ),
- team->t.t_id, f, place, first_place, last_place ) );
- }
- }
- else {
- int S, rem, gap, s_count;
- S = n_th / n_places;
- s_count = 0;
- rem = n_th - ( S * n_places );
- gap = rem > 0 ? n_places/rem : n_places;
- int place = masters_place;
- int gap_ct = gap;
- for ( f = 0; f < n_th; f++ ) {
- kmp_info_t *th = team->t.t_threads[f];
- KMP_DEBUG_ASSERT( th != NULL );
-
- th->th.th_first_place = first_place;
- th->th.th_last_place = last_place;
- th->th.th_new_place = place;
- s_count++;
-
- if ( (s_count == S) && rem && (gap_ct == gap) ) {
- // do nothing, add an extra thread to place on next iteration
- }
- else if ( (s_count == S+1) && rem && (gap_ct == gap) ) {
- // we added an extra thread to this place; move to next place
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- s_count = 0;
- gap_ct = 1;
- rem--;
- }
- else if (s_count == S) { // place full; don't add extra
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- gap_ct++;
- s_count = 0;
- }
-
- KA_TRACE( 100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d partition = [%d,%d]\n",
- __kmp_gtid_from_thread( team->t.t_threads[f] ),
- team->t.t_id, f, th->th.th_new_place, first_place,
- last_place ) );
- }
- KMP_DEBUG_ASSERT( place == masters_place );
- }
- }
- break;
-
- case proc_bind_spread:
- {
- int f;
- int n_th = team->t.t_nproc;
- int n_places;
- int thidx;
- if ( first_place <= last_place ) {
- n_places = last_place - first_place + 1;
- }
- else {
- n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
- }
- if ( n_th <= n_places ) {
- int place = masters_place;
- int S = n_places/n_th;
- int s_count, rem, gap, gap_ct;
- rem = n_places - n_th*S;
- gap = rem ? n_th/rem : 1;
- gap_ct = gap;
- thidx = n_th;
- if (update_master_only == 1)
- thidx = 1;
- for ( f = 0; f < thidx; f++ ) {
- kmp_info_t *th = team->t.t_threads[f];
- KMP_DEBUG_ASSERT( th != NULL );
-
- th->th.th_first_place = place;
- th->th.th_new_place = place;
- s_count = 1;
- while (s_count < S) {
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- s_count++;
- }
- if (rem && (gap_ct == gap)) {
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- rem--;
- gap_ct = 0;
- }
- th->th.th_last_place = place;
- gap_ct++;
-
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
-
- KA_TRACE( 100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d partition = [%d,%d]\n",
- __kmp_gtid_from_thread( team->t.t_threads[f] ),
- team->t.t_id, f, th->th.th_new_place,
- th->th.th_first_place, th->th.th_last_place ) );
- }
- KMP_DEBUG_ASSERT( update_master_only || place == masters_place );
- }
- else {
- int S, rem, gap, s_count;
- S = n_th / n_places;
- s_count = 0;
- rem = n_th - ( S * n_places );
- gap = rem > 0 ? n_places/rem : n_places;
- int place = masters_place;
- int gap_ct = gap;
- thidx = n_th;
- if (update_master_only == 1)
- thidx = 1;
- for ( f = 0; f < thidx; f++ ) {
- kmp_info_t *th = team->t.t_threads[f];
- KMP_DEBUG_ASSERT( th != NULL );
-
- th->th.th_first_place = place;
- th->th.th_last_place = place;
- th->th.th_new_place = place;
- s_count++;
-
- if ( (s_count == S) && rem && (gap_ct == gap) ) {
- // do nothing, add an extra thread to place on next iteration
- }
- else if ( (s_count == S+1) && rem && (gap_ct == gap) ) {
- // we added an extra thread to this place; move on to next place
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- s_count = 0;
- gap_ct = 1;
- rem--;
- }
- else if (s_count == S) { // place is full; don't add extra thread
- if ( place == last_place ) {
- place = first_place;
- }
- else if ( place == (int)(__kmp_affinity_num_masks - 1) ) {
- place = 0;
- }
- else {
- place++;
- }
- gap_ct++;
- s_count = 0;
- }
-
- KA_TRACE( 100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d partition = [%d,%d]\n",
- __kmp_gtid_from_thread( team->t.t_threads[f] ),
- team->t.t_id, f, th->th.th_new_place,
- th->th.th_first_place, th->th.th_last_place) );
- }
- KMP_DEBUG_ASSERT( update_master_only || place == masters_place );
- }
- }
- break;
-
- default:
- break;
- }
-
- KA_TRACE( 20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id ) );
-}
-
-#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
-
-/* allocate a new team data structure to use. take one off of the free pool if available */
-kmp_team_t *
-__kmp_allocate_team( kmp_root_t *root, int new_nproc, int max_nproc,
-#if OMPT_SUPPORT
- ompt_parallel_id_t ompt_parallel_id,
-#endif
-#if OMP_40_ENABLED
- kmp_proc_bind_t new_proc_bind,
-#endif
- kmp_internal_control_t *new_icvs,
- int argc USE_NESTED_HOT_ARG(kmp_info_t *master) )
-{
- KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
- int f;
- kmp_team_t *team;
- int use_hot_team = ! root->r.r_active;
- int level = 0;
-
- KA_TRACE( 20, ("__kmp_allocate_team: called\n"));
- KMP_DEBUG_ASSERT( new_nproc >=1 && argc >=0 );
- KMP_DEBUG_ASSERT( max_nproc >= new_nproc );
- KMP_MB();
-
-#if KMP_NESTED_HOT_TEAMS
- kmp_hot_team_ptr_t *hot_teams;
- if( master ) {
- team = master->th.th_team;
- level = team->t.t_active_level;
- if( master->th.th_teams_microtask ) { // in teams construct?
- if( master->th.th_teams_size.nteams > 1 && ( // #teams > 1
- team->t.t_pkfn == (microtask_t)__kmp_teams_master || // inner fork of the teams
- master->th.th_teams_level < team->t.t_level ) ) { // or nested parallel inside the teams
- ++level; // not increment if #teams==1, or for outer fork of the teams; increment otherwise
- }
- }
- hot_teams = master->th.th_hot_teams;
- if( level < __kmp_hot_teams_max_level && hot_teams && hot_teams[level].hot_team )
- { // hot team has already been allocated for given level
- use_hot_team = 1;
- } else {
- use_hot_team = 0;
- }
- }
-#endif
- // Optimization to use a "hot" team
- if( use_hot_team && new_nproc > 1 ) {
- KMP_DEBUG_ASSERT( new_nproc == max_nproc );
-#if KMP_NESTED_HOT_TEAMS
- team = hot_teams[level].hot_team;
-#else
- team = root->r.r_hot_team;
-#endif
-#if KMP_DEBUG
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KA_TRACE( 20, ("__kmp_allocate_team: hot team task_team[0] = %p task_team[1] = %p before reinit\n",
- team->t.t_task_team[0], team->t.t_task_team[1] ));
- }
-#endif
-
- // Has the number of threads changed?
- /* Let's assume the most common case is that the number of threads is unchanged, and
- put that case first. */
- if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
- KA_TRACE( 20, ("__kmp_allocate_team: reusing hot team\n" ));
- // This case can mean that omp_set_num_threads() was called and the hot team size
- // was already reduced, so we check the special flag
- if ( team->t.t_size_changed == -1 ) {
- team->t.t_size_changed = 1;
- } else {
- KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
- }
-
- // TODO???: team->t.t_max_active_levels = new_max_active_levels;
- kmp_r_sched_t new_sched = new_icvs->sched;
- if (team->t.t_sched.r_sched_type != new_sched.r_sched_type ||
- team->t.t_sched.chunk != new_sched.chunk)
- team->t.t_sched = new_sched; // set master's schedule as new run-time schedule
-
- __kmp_reinitialize_team( team, new_icvs, root->r.r_uber_thread->th.th_ident );
-
- KF_TRACE( 10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n",
- 0, team->t.t_threads[0], team ) );
- __kmp_push_current_task_to_thread( team->t.t_threads[ 0 ], team, 0 );
-
-#if OMP_40_ENABLED
-# if KMP_AFFINITY_SUPPORTED
- if ( ( team->t.t_size_changed == 0 )
- && ( team->t.t_proc_bind == new_proc_bind ) ) {
- if (new_proc_bind == proc_bind_spread) {
- __kmp_partition_places(team, 1); // add flag to update only master for spread
- }
- KA_TRACE( 200, ("__kmp_allocate_team: reusing hot team #%d bindings: proc_bind = %d, partition = [%d,%d]\n",
- team->t.t_id, new_proc_bind, team->t.t_first_place,
- team->t.t_last_place ) );
- }
- else {
- KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
- __kmp_partition_places( team );
- }
-# else
- KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
-# endif /* KMP_AFFINITY_SUPPORTED */
-#endif /* OMP_40_ENABLED */
- }
- else if( team->t.t_nproc > new_nproc ) {
- KA_TRACE( 20, ("__kmp_allocate_team: decreasing hot team thread count to %d\n", new_nproc ));
-
- team->t.t_size_changed = 1;
-#if KMP_NESTED_HOT_TEAMS
- if( __kmp_hot_teams_mode == 0 ) {
- // AC: saved number of threads should correspond to team's value in this mode,
- // can be bigger in mode 1, when hot team has some threads in reserve
- KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
- hot_teams[level].hot_team_nth = new_nproc;
-#endif // KMP_NESTED_HOT_TEAMS
- /* release the extra threads we don't need any more */
- for( f = new_nproc ; f < team->t.t_nproc ; f++ ) {
- KMP_DEBUG_ASSERT( team->t.t_threads[ f ] );
- if ( __kmp_tasking_mode != tskm_immediate_exec) {
- // When decreasing team size, threads no longer in the team should unref task team.
- team->t.t_threads[f]->th.th_task_team = NULL;
- }
- __kmp_free_thread( team->t.t_threads[ f ] );
- team->t.t_threads[ f ] = NULL;
- }
-#if KMP_NESTED_HOT_TEAMS
- } // (__kmp_hot_teams_mode == 0)
- else {
- // When keeping extra threads in team, switch threads to wait on own b_go flag
- for (f=new_nproc; f<team->t.t_nproc; ++f) {
- KMP_DEBUG_ASSERT(team->t.t_threads[f]);
- kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
- for (int b=0; b<bs_last_barrier; ++b) {
- if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
- balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
- }
- KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
- }
- }
- }
-#endif // KMP_NESTED_HOT_TEAMS
- team->t.t_nproc = new_nproc;
- // TODO???: team->t.t_max_active_levels = new_max_active_levels;
- if (team->t.t_sched.r_sched_type != new_icvs->sched.r_sched_type ||
- team->t.t_sched.chunk != new_icvs->sched.chunk)
- team->t.t_sched = new_icvs->sched;
- __kmp_reinitialize_team( team, new_icvs, root->r.r_uber_thread->th.th_ident );
-
- /* update the remaining threads */
- for(f = 0; f < new_nproc; ++f) {
- team->t.t_threads[f]->th.th_team_nproc = new_nproc;
- }
- // restore the current task state of the master thread: should be the implicit task
- KF_TRACE( 10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n",
- 0, team->t.t_threads[0], team ) );
-
- __kmp_push_current_task_to_thread( team->t.t_threads[ 0 ], team, 0 );
-
-#ifdef KMP_DEBUG
- for ( f = 0; f < team->t.t_nproc; f++ ) {
- KMP_DEBUG_ASSERT( team->t.t_threads[f] &&
- team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc );
- }
-#endif
-
-#if OMP_40_ENABLED
- KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
-# if KMP_AFFINITY_SUPPORTED
- __kmp_partition_places( team );
-# endif
-#endif
- }
- else { // team->t.t_nproc < new_nproc
-#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
- kmp_affin_mask_t *old_mask;
- if ( KMP_AFFINITY_CAPABLE() ) {
- KMP_CPU_ALLOC(old_mask);
- }
-#endif
-
- KA_TRACE( 20, ("__kmp_allocate_team: increasing hot team thread count to %d\n", new_nproc ));
-
- team->t.t_size_changed = 1;
-
-#if KMP_NESTED_HOT_TEAMS
- int avail_threads = hot_teams[level].hot_team_nth;
- if( new_nproc < avail_threads )
- avail_threads = new_nproc;
- kmp_info_t **other_threads = team->t.t_threads;
- for ( f = team->t.t_nproc; f < avail_threads; ++f ) {
- // Adjust barrier data of reserved threads (if any) of the team
- // Other data will be set in __kmp_initialize_info() below.
- int b;
- kmp_balign_t * balign = other_threads[f]->th.th_bar;
- for ( b = 0; b < bs_last_barrier; ++ b ) {
- balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
- KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#if USE_DEBUGGER
- balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
-#endif
- }
- }
- if( hot_teams[level].hot_team_nth >= new_nproc ) {
- // we have all needed threads in reserve, no need to allocate any
- // this only possible in mode 1, cannot have reserved threads in mode 0
- KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
- team->t.t_nproc = new_nproc; // just get reserved threads involved
- } else {
- // we may have some threads in reserve, but not enough
- team->t.t_nproc = hot_teams[level].hot_team_nth; // get reserved threads involved if any
- hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
-#endif // KMP_NESTED_HOT_TEAMS
- if(team->t.t_max_nproc < new_nproc) {
- /* reallocate larger arrays */
- __kmp_reallocate_team_arrays(team, new_nproc);
- __kmp_reinitialize_team( team, new_icvs, NULL );
- }
-
-#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
- /* Temporarily set full mask for master thread before
- creation of workers. The reason is that workers inherit
- the affinity from master, so if a lot of workers are
- created on the single core quickly, they don't get
- a chance to set their own affinity for a long time.
- */
- __kmp_set_thread_affinity_mask_full_tmp( old_mask );
-#endif
-
- /* allocate new threads for the hot team */
- for( f = team->t.t_nproc ; f < new_nproc ; f++ ) {
- kmp_info_t * new_worker = __kmp_allocate_thread( root, team, f );
- KMP_DEBUG_ASSERT( new_worker );
- team->t.t_threads[ f ] = new_worker;
-
- KA_TRACE( 20, ("__kmp_allocate_team: team %d init T#%d arrived: join=%llu, plain=%llu\n",
- team->t.t_id, __kmp_gtid_from_tid( f, team ), team->t.t_id, f,
- team->t.t_bar[bs_forkjoin_barrier].b_arrived,
- team->t.t_bar[bs_plain_barrier].b_arrived ) );
-
- { // Initialize barrier data for new threads.
- int b;
- kmp_balign_t * balign = new_worker->th.th_bar;
- for( b = 0; b < bs_last_barrier; ++ b ) {
- balign[ b ].bb.b_arrived = team->t.t_bar[ b ].b_arrived;
- KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#if USE_DEBUGGER
- balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
-#endif
- }
- }
- }
-
-#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
- if ( KMP_AFFINITY_CAPABLE() ) {
- /* Restore initial master thread's affinity mask */
- __kmp_set_system_affinity( old_mask, TRUE );
- KMP_CPU_FREE(old_mask);
- }
-#endif
-#if KMP_NESTED_HOT_TEAMS
- } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
-#endif // KMP_NESTED_HOT_TEAMS
- /* make sure everyone is syncronized */
- int old_nproc = team->t.t_nproc; // save old value and use to update only new threads below
- __kmp_initialize_team( team, new_nproc, new_icvs, root->r.r_uber_thread->th.th_ident );
-
- /* reinitialize the threads */
- KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
- for (f=0; f < team->t.t_nproc; ++f)
- __kmp_initialize_info( team->t.t_threads[ f ], team, f, __kmp_gtid_from_tid( f, team ) );
- if (level) { // set th_task_state for new threads in nested hot team
- // __kmp_initialize_info() no longer zeroes th_task_state, so we should only need to set the
- // th_task_state for the new threads. th_task_state for master thread will not be accurate until
- // after this in __kmp_fork_call(), so we look to the master's memo_stack to get the correct value.
- for (f=old_nproc; f < team->t.t_nproc; ++f)
- team->t.t_threads[f]->th.th_task_state = team->t.t_threads[0]->th.th_task_state_memo_stack[level];
- }
- else { // set th_task_state for new threads in non-nested hot team
- int old_state = team->t.t_threads[0]->th.th_task_state; // copy master's state
- for (f=old_nproc; f < team->t.t_nproc; ++f)
- team->t.t_threads[f]->th.th_task_state = old_state;
- }
-
-#ifdef KMP_DEBUG
- for ( f = 0; f < team->t.t_nproc; ++ f ) {
- KMP_DEBUG_ASSERT( team->t.t_threads[f] &&
- team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc );
- }
-#endif
-
-#if OMP_40_ENABLED
- KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
-# if KMP_AFFINITY_SUPPORTED
- __kmp_partition_places( team );
-# endif
-#endif
- } // Check changes in number of threads
-
-#if OMP_40_ENABLED
- kmp_info_t *master = team->t.t_threads[0];
- if( master->th.th_teams_microtask ) {
- for( f = 1; f < new_nproc; ++f ) {
- // propagate teams construct specific info to workers
- kmp_info_t *thr = team->t.t_threads[f];
- thr->th.th_teams_microtask = master->th.th_teams_microtask;
- thr->th.th_teams_level = master->th.th_teams_level;
- thr->th.th_teams_size = master->th.th_teams_size;
- }
- }
-#endif /* OMP_40_ENABLED */
-#if KMP_NESTED_HOT_TEAMS
- if( level ) {
- // Sync barrier state for nested hot teams, not needed for outermost hot team.
- for( f = 1; f < new_nproc; ++f ) {
- kmp_info_t *thr = team->t.t_threads[f];
- int b;
- kmp_balign_t * balign = thr->th.th_bar;
- for( b = 0; b < bs_last_barrier; ++ b ) {
- balign[ b ].bb.b_arrived = team->t.t_bar[ b ].b_arrived;
- KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
-#if USE_DEBUGGER
- balign[ b ].bb.b_worker_arrived = team->t.t_bar[ b ].b_team_arrived;
-#endif
- }
- }
- }
-#endif // KMP_NESTED_HOT_TEAMS
-
- /* reallocate space for arguments if necessary */
- __kmp_alloc_argv_entries( argc, team, TRUE );
- KMP_CHECK_UPDATE(team->t.t_argc, argc);
- //
- // The hot team re-uses the previous task team,
- // if untouched during the previous release->gather phase.
- //
-
- KF_TRACE( 10, ( " hot_team = %p\n", team ) );
-
-#if KMP_DEBUG
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- KA_TRACE( 20, ("__kmp_allocate_team: hot team task_team[0] = %p task_team[1] = %p after reinit\n",
- team->t.t_task_team[0], team->t.t_task_team[1] ));
- }
-#endif
-
-#if OMPT_SUPPORT
- __ompt_team_assign_id(team, ompt_parallel_id);
-#endif
-
- KMP_MB();
-
- return team;
- }
-
- /* next, let's try to take one from the team pool */
- KMP_MB();
- for( team = (kmp_team_t*) __kmp_team_pool ; (team) ; )
- {
- /* TODO: consider resizing undersized teams instead of reaping them, now that we have a resizing mechanism */
- if ( team->t.t_max_nproc >= max_nproc ) {
- /* take this team from the team pool */
- __kmp_team_pool = team->t.t_next_pool;
-
- /* setup the team for fresh use */
- __kmp_initialize_team( team, new_nproc, new_icvs, NULL );
-
- KA_TRACE( 20, ( "__kmp_allocate_team: setting task_team[0] %p and task_team[1] %p to NULL\n",
- &team->t.t_task_team[0], &team->t.t_task_team[1]) );
- team->t.t_task_team[0] = NULL;
- team->t.t_task_team[1] = NULL;
-
- /* reallocate space for arguments if necessary */
- __kmp_alloc_argv_entries( argc, team, TRUE );
- KMP_CHECK_UPDATE(team->t.t_argc, argc);
-
- KA_TRACE( 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
- team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
- { // Initialize barrier data.
- int b;
- for ( b = 0; b < bs_last_barrier; ++ b) {
- team->t.t_bar[ b ].b_arrived = KMP_INIT_BARRIER_STATE;
-#if USE_DEBUGGER
- team->t.t_bar[ b ].b_master_arrived = 0;
- team->t.t_bar[ b ].b_team_arrived = 0;
-#endif
- }
- }
-
-#if OMP_40_ENABLED
- team->t.t_proc_bind = new_proc_bind;
-#endif
-
- KA_TRACE( 20, ("__kmp_allocate_team: using team from pool %d.\n", team->t.t_id ));
-
-#if OMPT_SUPPORT
- __ompt_team_assign_id(team, ompt_parallel_id);
-#endif
-
- KMP_MB();
-
- return team;
- }
-
- /* reap team if it is too small, then loop back and check the next one */
- /* not sure if this is wise, but, will be redone during the hot-teams rewrite. */
- /* TODO: Use technique to find the right size hot-team, don't reap them */
- team = __kmp_reap_team( team );
- __kmp_team_pool = team;
- }
-
- /* nothing available in the pool, no matter, make a new team! */
- KMP_MB();
- team = (kmp_team_t*) __kmp_allocate( sizeof( kmp_team_t ) );
-
- /* and set it up */
- team->t.t_max_nproc = max_nproc;
- /* NOTE well, for some reason allocating one big buffer and dividing it
- * up seems to really hurt performance a lot on the P4, so, let's not use
- * this... */
- __kmp_allocate_team_arrays( team, max_nproc );
-
- KA_TRACE( 20, ( "__kmp_allocate_team: making a new team\n" ) );
- __kmp_initialize_team( team, new_nproc, new_icvs, NULL );
-
- KA_TRACE( 20, ( "__kmp_allocate_team: setting task_team[0] %p and task_team[1] %p to NULL\n",
- &team->t.t_task_team[0], &team->t.t_task_team[1] ) );
- team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes memory, no need to duplicate
- team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes memory, no need to duplicate
-
- if ( __kmp_storage_map ) {
- __kmp_print_team_storage_map( "team", team, team->t.t_id, new_nproc );
- }
-
- /* allocate space for arguments */
- __kmp_alloc_argv_entries( argc, team, FALSE );
- team->t.t_argc = argc;
-
- KA_TRACE( 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
- team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE ));
- { // Initialize barrier data.
- int b;
- for ( b = 0; b < bs_last_barrier; ++ b ) {
- team->t.t_bar[ b ].b_arrived = KMP_INIT_BARRIER_STATE;
-#if USE_DEBUGGER
- team->t.t_bar[ b ].b_master_arrived = 0;
- team->t.t_bar[ b ].b_team_arrived = 0;
-#endif
- }
- }
-
-#if OMP_40_ENABLED
- team->t.t_proc_bind = new_proc_bind;
-#endif
-
-#if OMPT_SUPPORT
- __ompt_team_assign_id(team, ompt_parallel_id);
- team->t.ompt_serialized_team_info = NULL;
-#endif
-
- KMP_MB();
-
- KA_TRACE( 20, ("__kmp_allocate_team: done creating a new team %d.\n", team->t.t_id ));
-
- return team;
-}
-
-/* TODO implement hot-teams at all levels */
-/* TODO implement lazy thread release on demand (disband request) */
-
-/* free the team. return it to the team pool. release all the threads
- * associated with it */
-void
-__kmp_free_team( kmp_root_t *root, kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master) )
-{
- int f;
- KA_TRACE( 20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(), team->t.t_id ));
-
- /* verify state */
- KMP_DEBUG_ASSERT( root );
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( team->t.t_nproc <= team->t.t_max_nproc );
- KMP_DEBUG_ASSERT( team->t.t_threads );
-
- int use_hot_team = team == root->r.r_hot_team;
-#if KMP_NESTED_HOT_TEAMS
- int level;
- kmp_hot_team_ptr_t *hot_teams;
- if( master ) {
- level = team->t.t_active_level - 1;
- if( master->th.th_teams_microtask ) { // in teams construct?
- if( master->th.th_teams_size.nteams > 1 ) {
- ++level; // level was not increased in teams construct for team_of_masters
- }
- if( team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
- master->th.th_teams_level == team->t.t_level ) {
- ++level; // level was not increased in teams construct for team_of_workers before the parallel
- } // team->t.t_level will be increased inside parallel
- }
- hot_teams = master->th.th_hot_teams;
- if( level < __kmp_hot_teams_max_level ) {
- KMP_DEBUG_ASSERT( team == hot_teams[level].hot_team );
- use_hot_team = 1;
- }
- }
-#endif // KMP_NESTED_HOT_TEAMS
-
- /* team is done working */
- TCW_SYNC_PTR(team->t.t_pkfn, NULL); // Important for Debugging Support Library.
- team->t.t_copyin_counter = 0; // init counter for possible reuse
- // Do not reset pointer to parent team to NULL for hot teams.
-
- /* if we are non-hot team, release our threads */
- if( ! use_hot_team ) {
- if ( __kmp_tasking_mode != tskm_immediate_exec ) {
- // Delete task teams
- int tt_idx;
- for (tt_idx=0; tt_idx<2; ++tt_idx) {
- kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
- if ( task_team != NULL ) {
- for (f=0; f<team->t.t_nproc; ++f) { // Have all threads unref task teams
- team->t.t_threads[f]->th.th_task_team = NULL;
- }
- KA_TRACE( 20, ( "__kmp_free_team: T#%d deactivating task_team %p on team %d\n", __kmp_get_gtid(), task_team, team->t.t_id ) );
-#if KMP_NESTED_HOT_TEAMS
- __kmp_free_task_team( master, task_team );
-#endif
- team->t.t_task_team[tt_idx] = NULL;
- }
- }
- }
-
- // Reset pointer to parent team only for non-hot teams.
- team->t.t_parent = NULL;
- team->t.t_level = 0;
- team->t.t_active_level = 0;
-
- /* free the worker threads */
- for ( f = 1; f < team->t.t_nproc; ++ f ) {
- KMP_DEBUG_ASSERT( team->t.t_threads[ f ] );
- __kmp_free_thread( team->t.t_threads[ f ] );
- team->t.t_threads[ f ] = NULL;
- }
-
- /* put the team back in the team pool */
- /* TODO limit size of team pool, call reap_team if pool too large */
- team->t.t_next_pool = (kmp_team_t*) __kmp_team_pool;
- __kmp_team_pool = (volatile kmp_team_t*) team;
- }
-
- KMP_MB();
-}
-
-
-/* reap the team. destroy it, reclaim all its resources and free its memory */
-kmp_team_t *
-__kmp_reap_team( kmp_team_t *team )
-{
- kmp_team_t *next_pool = team->t.t_next_pool;
-
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( team->t.t_dispatch );
- KMP_DEBUG_ASSERT( team->t.t_disp_buffer );
- KMP_DEBUG_ASSERT( team->t.t_threads );
- KMP_DEBUG_ASSERT( team->t.t_argv );
-
- /* TODO clean the threads that are a part of this? */
-
- /* free stuff */
-
- __kmp_free_team_arrays( team );
- if ( team->t.t_argv != &team->t.t_inline_argv[0] )
- __kmp_free( (void*) team->t.t_argv );
- __kmp_free( team );
-
- KMP_MB();
- return next_pool;
-}
-
-//
-// Free the thread. Don't reap it, just place it on the pool of available
-// threads.
-//
-// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
-// binding for the affinity mechanism to be useful.
-//
-// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
-// However, we want to avoid a potential performance problem by always
-// scanning through the list to find the correct point at which to insert
-// the thread (potential N**2 behavior). To do this we keep track of the
-// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
-// With single-level parallelism, threads will always be added to the tail
-// of the list, kept track of by __kmp_thread_pool_insert_pt. With nested
-// parallelism, all bets are off and we may need to scan through the entire
-// free list.
-//
-// This change also has a potentially large performance benefit, for some
-// applications. Previously, as threads were freed from the hot team, they
-// would be placed back on the free list in inverse order. If the hot team
-// grew back to it's original size, then the freed thread would be placed
-// back on the hot team in reverse order. This could cause bad cache
-// locality problems on programs where the size of the hot team regularly
-// grew and shrunk.
-//
-// Now, for single-level parallelism, the OMP tid is alway == gtid.
-//
-void
-__kmp_free_thread( kmp_info_t *this_th )
-{
- int gtid;
- kmp_info_t **scan;
-
- KA_TRACE( 20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
- __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid ));
-
- KMP_DEBUG_ASSERT( this_th );
-
- // When moving thread to pool, switch thread to wait on own b_go flag, and uninitialized (NULL team).
- int b;
- kmp_balign_t *balign = this_th->th.th_bar;
- for (b=0; b<bs_last_barrier; ++b) {
- if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
- balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
- balign[b].bb.team = NULL;
- balign[b].bb.leaf_kids = 0;
- }
- this_th->th.th_task_state = 0;
-
- /* put thread back on the free pool */
- TCW_PTR(this_th->th.th_team, NULL);
- TCW_PTR(this_th->th.th_root, NULL);
- TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */
-
- //
- // If the __kmp_thread_pool_insert_pt is already past the new insert
- // point, then we need to re-scan the entire list.
- //
- gtid = this_th->th.th_info.ds.ds_gtid;
- if ( __kmp_thread_pool_insert_pt != NULL ) {
- KMP_DEBUG_ASSERT( __kmp_thread_pool != NULL );
- if ( __kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid ) {
- __kmp_thread_pool_insert_pt = NULL;
- }
- }
-
- //
- // Scan down the list to find the place to insert the thread.
- // scan is the address of a link in the list, possibly the address of
- // __kmp_thread_pool itself.
- //
- // In the absence of nested parallism, the for loop will have 0 iterations.
- //
- if ( __kmp_thread_pool_insert_pt != NULL ) {
- scan = &( __kmp_thread_pool_insert_pt->th.th_next_pool );
- }
- else {
- scan = (kmp_info_t **)&__kmp_thread_pool;
- }
- for (; ( *scan != NULL ) && ( (*scan)->th.th_info.ds.ds_gtid < gtid );
- scan = &( (*scan)->th.th_next_pool ) );
-
- //
- // Insert the new element on the list, and set __kmp_thread_pool_insert_pt
- // to its address.
- //
- TCW_PTR(this_th->th.th_next_pool, *scan);
- __kmp_thread_pool_insert_pt = *scan = this_th;
- KMP_DEBUG_ASSERT( ( this_th->th.th_next_pool == NULL )
- || ( this_th->th.th_info.ds.ds_gtid
- < this_th->th.th_next_pool->th.th_info.ds.ds_gtid ) );
- TCW_4(this_th->th.th_in_pool, TRUE);
- __kmp_thread_pool_nth++;
-
- TCW_4(__kmp_nth, __kmp_nth - 1);
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime back to user setting or default if necessary */
- /* Middle initialization might never have occurred */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- KMP_DEBUG_ASSERT( __kmp_avail_proc > 0 );
- if ( __kmp_nth <= __kmp_avail_proc ) {
- __kmp_zero_bt = FALSE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
- KMP_MB();
-}
-
-
-/* ------------------------------------------------------------------------ */
-
-void *
-__kmp_launch_thread( kmp_info_t *this_thr )
-{
- int gtid = this_thr->th.th_info.ds.ds_gtid;
-/* void *stack_data;*/
- kmp_team_t *(*volatile pteam);
-
- KMP_MB();
- KA_TRACE( 10, ("__kmp_launch_thread: T#%d start\n", gtid ) );
-
- if( __kmp_env_consistency_check ) {
- this_thr->th.th_cons = __kmp_allocate_cons_stack( gtid ); // ATT: Memory leak?
- }
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- this_thr->th.ompt_thread_info.state = ompt_state_overhead;
- this_thr->th.ompt_thread_info.wait_id = 0;
- this_thr->th.ompt_thread_info.idle_frame = __builtin_frame_address(0);
- if (ompt_callbacks.ompt_callback(ompt_event_thread_begin)) {
- __ompt_thread_begin(ompt_thread_worker, gtid);
- }
- }
-#endif
-
- /* This is the place where threads wait for work */
- while( ! TCR_4(__kmp_global.g.g_done) ) {
- KMP_DEBUG_ASSERT( this_thr == __kmp_threads[ gtid ] );
- KMP_MB();
-
- /* wait for work to do */
- KA_TRACE( 20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid ));
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- this_thr->th.ompt_thread_info.state = ompt_state_idle;
- }
-#endif
-
- /* No tid yet since not part of a team */
- __kmp_fork_barrier( gtid, KMP_GTID_DNE );
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- this_thr->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
-
- pteam = (kmp_team_t *(*))(& this_thr->th.th_team);
-
- /* have we been allocated? */
- if ( TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done) ) {
-#if OMPT_SUPPORT
- ompt_task_info_t *task_info;
- ompt_parallel_id_t my_parallel_id;
- if (ompt_enabled) {
- task_info = __ompt_get_taskinfo(0);
- my_parallel_id = (*pteam)->t.ompt_team_info.parallel_id;
- }
-#endif
- /* we were just woken up, so run our new task */
- if ( TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL ) {
- int rc;
- KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
- gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam)->t.t_pkfn));
-
- updateHWFPControl (*pteam);
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
- // Initialize OMPT task id for implicit task.
- int tid = __kmp_tid_from_gtid(gtid);
- task_info->task_id = __ompt_task_id_new(tid);
- }
-#endif
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- rc = (*pteam)->t.t_invoke( gtid );
- }
- KMP_ASSERT( rc );
-
-#if OMPT_SUPPORT
- if (ompt_enabled) {
- /* no frame set while outside task */
- task_info->frame.exit_runtime_frame = NULL;
-
- this_thr->th.ompt_thread_info.state = ompt_state_overhead;
- }
-#endif
- KMP_MB();
- KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
- gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid), (*pteam)->t.t_pkfn));
- }
- /* join barrier after parallel region */
- __kmp_join_barrier( gtid );
-#if OMPT_SUPPORT && OMPT_TRACE
- if (ompt_enabled) {
- if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
- // don't access *pteam here: it may have already been freed
- // by the master thread behind the barrier (possible race)
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
- my_parallel_id, task_info->task_id);
- }
- task_info->frame.exit_runtime_frame = NULL;
- task_info->task_id = 0;
- }
-#endif
- }
- }
- TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
-
-#if OMPT_SUPPORT
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
- __ompt_thread_end(ompt_thread_worker, gtid);
- }
-#endif
-
- this_thr->th.th_task_team = NULL;
- /* run the destructors for the threadprivate data for this thread */
- __kmp_common_destroy_gtid( gtid );
-
- KA_TRACE( 10, ("__kmp_launch_thread: T#%d done\n", gtid ) );
- KMP_MB();
- return this_thr;
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_internal_end_dest( void *specific_gtid )
-{
- #if KMP_COMPILER_ICC
- #pragma warning( push )
- #pragma warning( disable: 810 ) // conversion from "void *" to "int" may lose significant bits
- #endif
- // Make sure no significant bits are lost
- int gtid = (kmp_intptr_t)specific_gtid - 1;
- #if KMP_COMPILER_ICC
- #pragma warning( pop )
- #endif
-
- KA_TRACE( 30, ("__kmp_internal_end_dest: T#%d\n", gtid));
- /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
- * this is because 0 is reserved for the nothing-stored case */
-
- /* josh: One reason for setting the gtid specific data even when it is being
- destroyed by pthread is to allow gtid lookup through thread specific data
- (__kmp_gtid_get_specific). Some of the code, especially stat code,
- that gets executed in the call to __kmp_internal_end_thread, actually
- gets the gtid through the thread specific data. Setting it here seems
- rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread
- to run smoothly.
- todo: get rid of this after we remove the dependence on
- __kmp_gtid_get_specific
- */
- if(gtid >= 0 && KMP_UBER_GTID(gtid))
- __kmp_gtid_set_specific( gtid );
- #ifdef KMP_TDATA_GTID
- __kmp_gtid = gtid;
- #endif
- __kmp_internal_end_thread( gtid );
-}
-
-#if KMP_OS_UNIX && KMP_DYNAMIC_LIB
-
-// 2009-09-08 (lev): It looks the destructor does not work. In simple test cases destructors work
-// perfectly, but in real libomp.so I have no evidence it is ever called. However, -fini linker
-// option in makefile.mk works fine.
-
-__attribute__(( destructor ))
-void
-__kmp_internal_end_dtor( void )
-{
- __kmp_internal_end_atexit();
-}
-
-void
-__kmp_internal_end_fini( void )
-{
- __kmp_internal_end_atexit();
-}
-
-#endif
-
-/* [Windows] josh: when the atexit handler is called, there may still be more than one thread alive */
-void
-__kmp_internal_end_atexit( void )
-{
- KA_TRACE( 30, ( "__kmp_internal_end_atexit\n" ) );
- /* [Windows]
- josh: ideally, we want to completely shutdown the library in this atexit handler, but
- stat code that depends on thread specific data for gtid fails because that data becomes
- unavailable at some point during the shutdown, so we call __kmp_internal_end_thread
- instead. We should eventually remove the dependency on __kmp_get_specific_gtid in the
- stat code and use __kmp_internal_end_library to cleanly shutdown the library.
-
-// TODO: Can some of this comment about GVS be removed?
- I suspect that the offending stat code is executed when the calling thread tries to
- clean up a dead root thread's data structures, resulting in GVS code trying to close
- the GVS structures for that thread, but since the stat code uses
- __kmp_get_specific_gtid to get the gtid with the assumption that the calling thread is
- cleaning up itself instead of another thread, it gets confused. This happens because
- allowing a thread to unregister and cleanup another thread is a recent modification for
- addressing an issue with Maxon Cinema4D. Based on the current design (20050722), a
- thread may end up trying to unregister another thread only if thread death does not
- trigger the calling of __kmp_internal_end_thread. For Linux* OS, there is the thread
- specific data destructor function to detect thread death. For Windows dynamic, there
- is DllMain(THREAD_DETACH). For Windows static, there is nothing. Thus, the
- workaround is applicable only for Windows static stat library.
- */
- __kmp_internal_end_library( -1 );
- #if KMP_OS_WINDOWS
- __kmp_close_console();
- #endif
-}
-
-static void
-__kmp_reap_thread(
- kmp_info_t * thread,
- int is_root
-) {
-
- // It is assumed __kmp_forkjoin_lock is acquired.
-
- int gtid;
-
- KMP_DEBUG_ASSERT( thread != NULL );
-
- gtid = thread->th.th_info.ds.ds_gtid;
-
- if ( ! is_root ) {
-
- if ( __kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ) {
- /* Assume the threads are at the fork barrier here */
- KA_TRACE( 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n", gtid ) );
- /* Need release fence here to prevent seg faults for tree forkjoin barrier (GEH) */
- ANNOTATE_HAPPENS_BEFORE(thread);
- kmp_flag_64 flag(&thread->th.th_bar[ bs_forkjoin_barrier ].bb.b_go, thread);
- __kmp_release_64(&flag);
- }; // if
-
- // Terminate OS thread.
- __kmp_reap_worker( thread );
-
- //
- // The thread was killed asynchronously. If it was actively
- // spinning in the thread pool, decrement the global count.
- //
- // There is a small timing hole here - if the worker thread was
- // just waking up after sleeping in the pool, had reset it's
- // th_active_in_pool flag but not decremented the global counter
- // __kmp_thread_pool_active_nth yet, then the global counter
- // might not get updated.
- //
- // Currently, this can only happen as the library is unloaded,
- // so there are no harmful side effects.
- //
- if ( thread->th.th_active_in_pool ) {
- thread->th.th_active_in_pool = FALSE;
- KMP_TEST_THEN_DEC32(
- (kmp_int32 *) &__kmp_thread_pool_active_nth );
- KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
- }
-
- // Decrement # of [worker] threads in the pool.
- KMP_DEBUG_ASSERT( __kmp_thread_pool_nth > 0 );
- --__kmp_thread_pool_nth;
- }; // if
-
- __kmp_free_implicit_task(thread);
-
- // Free the fast memory for tasking
- #if USE_FAST_MEMORY
- __kmp_free_fast_memory( thread );
- #endif /* USE_FAST_MEMORY */
-
- __kmp_suspend_uninitialize_thread( thread );
-
- KMP_DEBUG_ASSERT( __kmp_threads[ gtid ] == thread );
- TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
-
- -- __kmp_all_nth;
- // __kmp_nth was decremented when thread is added to the pool.
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime back to user setting or default if necessary */
- /* Middle initialization might never have occurred */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- KMP_DEBUG_ASSERT( __kmp_avail_proc > 0 );
- if ( __kmp_nth <= __kmp_avail_proc ) {
- __kmp_zero_bt = FALSE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
- /* free the memory being used */
- if( __kmp_env_consistency_check ) {
- if ( thread->th.th_cons ) {
- __kmp_free_cons_stack( thread->th.th_cons );
- thread->th.th_cons = NULL;
- }; // if
- }
-
- if ( thread->th.th_pri_common != NULL ) {
- __kmp_free( thread->th.th_pri_common );
- thread->th.th_pri_common = NULL;
- }; // if
-
- if (thread->th.th_task_state_memo_stack != NULL) {
- __kmp_free(thread->th.th_task_state_memo_stack);
- thread->th.th_task_state_memo_stack = NULL;
- }
-
- #if KMP_USE_BGET
- if ( thread->th.th_local.bget_data != NULL ) {
- __kmp_finalize_bget( thread );
- }; // if
- #endif
-
-#if KMP_AFFINITY_SUPPORTED
- if ( thread->th.th_affin_mask != NULL ) {
- KMP_CPU_FREE( thread->th.th_affin_mask );
- thread->th.th_affin_mask = NULL;
- }; // if
-#endif /* KMP_AFFINITY_SUPPORTED */
-
- __kmp_reap_team( thread->th.th_serial_team );
- thread->th.th_serial_team = NULL;
- __kmp_free( thread );
-
- KMP_MB();
-
-} // __kmp_reap_thread
-
-static void
-__kmp_internal_end(void)
-{
- int i;
-
- /* First, unregister the library */
- __kmp_unregister_library();
-
- #if KMP_OS_WINDOWS
- /* In Win static library, we can't tell when a root actually dies, so we
- reclaim the data structures for any root threads that have died but not
- unregistered themselves, in order to shut down cleanly.
- In Win dynamic library we also can't tell when a thread dies.
- */
- __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of dead roots
- #endif
-
- for( i=0 ; i<__kmp_threads_capacity ; i++ )
- if( __kmp_root[i] )
- if( __kmp_root[i]->r.r_active )
- break;
- KMP_MB(); /* Flush all pending memory write invalidates. */
- TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
-
- if ( i < __kmp_threads_capacity ) {
-#if KMP_USE_MONITOR
- // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- //
- // Need to check that monitor was initialized before reaping it.
- // If we are called form __kmp_atfork_child (which sets
- // __kmp_init_parallel = 0), then __kmp_monitor will appear to
- // contain valid data, but it is only valid in the parent process,
- // not the child.
- //
- // New behavior (201008): instead of keying off of the flag
- // __kmp_init_parallel, the monitor thread creation is keyed off
- // of the new flag __kmp_init_monitor.
- //
- __kmp_acquire_bootstrap_lock( & __kmp_monitor_lock );
- if ( TCR_4( __kmp_init_monitor ) ) {
- __kmp_reap_monitor( & __kmp_monitor );
- TCW_4( __kmp_init_monitor, 0 );
- }
- __kmp_release_bootstrap_lock( & __kmp_monitor_lock );
- KA_TRACE( 10, ("__kmp_internal_end: monitor reaped\n" ) );
-#endif // KMP_USE_MONITOR
- } else {
- /* TODO move this to cleanup code */
- #ifdef KMP_DEBUG
- /* make sure that everything has properly ended */
- for ( i = 0; i < __kmp_threads_capacity; i++ ) {
- if( __kmp_root[i] ) {
-// KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC: there can be uber threads alive here
- KMP_ASSERT( ! __kmp_root[i]->r.r_active ); // TODO: can they be active?
- }
- }
- #endif
-
- KMP_MB();
-
- // Reap the worker threads.
- // This is valid for now, but be careful if threads are reaped sooner.
- while ( __kmp_thread_pool != NULL ) { // Loop thru all the thread in the pool.
- // Get the next thread from the pool.
- kmp_info_t * thread = (kmp_info_t *) __kmp_thread_pool;
- __kmp_thread_pool = thread->th.th_next_pool;
- // Reap it.
- thread->th.th_next_pool = NULL;
- thread->th.th_in_pool = FALSE;
- __kmp_reap_thread( thread, 0 );
- }; // while
- __kmp_thread_pool_insert_pt = NULL;
-
- // Reap teams.
- while ( __kmp_team_pool != NULL ) { // Loop thru all the teams in the pool.
- // Get the next team from the pool.
- kmp_team_t * team = (kmp_team_t *) __kmp_team_pool;
- __kmp_team_pool = team->t.t_next_pool;
- // Reap it.
- team->t.t_next_pool = NULL;
- __kmp_reap_team( team );
- }; // while
-
- __kmp_reap_task_teams( );
-
- for ( i = 0; i < __kmp_threads_capacity; ++ i ) {
- // TBD: Add some checking...
- // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
- }
-
- /* Make sure all threadprivate destructors get run by joining with all worker
- threads before resetting this flag */
- TCW_SYNC_4(__kmp_init_common, FALSE);
-
- KA_TRACE( 10, ("__kmp_internal_end: all workers reaped\n" ) );
- KMP_MB();
-
-#if KMP_USE_MONITOR
- //
- // See note above: One of the possible fixes for CQ138434 / CQ140126
- //
- // FIXME: push both code fragments down and CSE them?
- // push them into __kmp_cleanup() ?
- //
- __kmp_acquire_bootstrap_lock( & __kmp_monitor_lock );
- if ( TCR_4( __kmp_init_monitor ) ) {
- __kmp_reap_monitor( & __kmp_monitor );
- TCW_4( __kmp_init_monitor, 0 );
- }
- __kmp_release_bootstrap_lock( & __kmp_monitor_lock );
- KA_TRACE( 10, ("__kmp_internal_end: monitor reaped\n" ) );
-#endif
- } /* else !__kmp_global.t_active */
- TCW_4(__kmp_init_gtid, FALSE);
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- __kmp_cleanup();
-#if OMPT_SUPPORT
- ompt_fini();
-#endif
-}
-
-void
-__kmp_internal_end_library( int gtid_req )
-{
- /* if we have already cleaned up, don't try again, it wouldn't be pretty */
- /* this shouldn't be a race condition because __kmp_internal_end() is the
- * only place to clear __kmp_serial_init */
- /* we'll check this later too, after we get the lock */
- // 2009-09-06: We do not set g_abort without setting g_done. This check looks redundaant,
- // because the next check will work in any case.
- if( __kmp_global.g.g_abort ) {
- KA_TRACE( 11, ("__kmp_internal_end_library: abort, exiting\n" ));
- /* TODO abort? */
- return;
- }
- if( TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial ) {
- KA_TRACE( 10, ("__kmp_internal_end_library: already finished\n" ));
- return;
- }
-
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- /* find out who we are and what we should do */
- {
- int gtid = (gtid_req>=0) ? gtid_req : __kmp_gtid_get_specific();
- KA_TRACE( 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req ));
- if( gtid == KMP_GTID_SHUTDOWN ) {
- KA_TRACE( 10, ("__kmp_internal_end_library: !__kmp_init_runtime, system already shutdown\n" ));
- return;
- } else if( gtid == KMP_GTID_MONITOR ) {
- KA_TRACE( 10, ("__kmp_internal_end_library: monitor thread, gtid not registered, or system shutdown\n" ));
- return;
- } else if( gtid == KMP_GTID_DNE ) {
- KA_TRACE( 10, ("__kmp_internal_end_library: gtid not registered or system shutdown\n" ));
- /* we don't know who we are, but we may still shutdown the library */
- } else if( KMP_UBER_GTID( gtid )) {
- /* unregister ourselves as an uber thread. gtid is no longer valid */
- if( __kmp_root[gtid]->r.r_active ) {
- __kmp_global.g.g_abort = -1;
- TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
- KA_TRACE( 10, ("__kmp_internal_end_library: root still active, abort T#%d\n", gtid ));
- return;
- } else {
- KA_TRACE( 10, ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid ));
- __kmp_unregister_root_current_thread( gtid );
- }
- } else {
- /* worker threads may call this function through the atexit handler, if they call exit() */
- /* For now, skip the usual subsequent processing and just dump the debug buffer.
- TODO: do a thorough shutdown instead
- */
- #ifdef DUMP_DEBUG_ON_EXIT
- if ( __kmp_debug_buf )
- __kmp_dump_debug_buffer( );
- #endif
- return;
- }
- }
- /* synchronize the termination process */
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
-
- /* have we already finished */
- if( __kmp_global.g.g_abort ) {
- KA_TRACE( 10, ("__kmp_internal_end_library: abort, exiting\n" ));
- /* TODO abort? */
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
- if( TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial ) {
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
-
- /* We need this lock to enforce mutex between this reading of
- __kmp_threads_capacity and the writing by __kmp_register_root.
- Alternatively, we can use a counter of roots that is
- atomically updated by __kmp_get_global_thread_id_reg,
- __kmp_do_serial_initialize and __kmp_internal_end_*.
- */
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
- /* now we can safely conduct the actual termination */
- __kmp_internal_end();
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-
- KA_TRACE( 10, ("__kmp_internal_end_library: exit\n" ) );
-
- #ifdef DUMP_DEBUG_ON_EXIT
- if ( __kmp_debug_buf )
- __kmp_dump_debug_buffer();
- #endif
-
- #if KMP_OS_WINDOWS
- __kmp_close_console();
- #endif
-
- __kmp_fini_allocator();
-
-} // __kmp_internal_end_library
-
-void
-__kmp_internal_end_thread( int gtid_req )
-{
- int i;
-
- /* if we have already cleaned up, don't try again, it wouldn't be pretty */
- /* this shouldn't be a race condition because __kmp_internal_end() is the
- * only place to clear __kmp_serial_init */
- /* we'll check this later too, after we get the lock */
- // 2009-09-06: We do not set g_abort without setting g_done. This check looks redundant,
- // because the next check will work in any case.
- if( __kmp_global.g.g_abort ) {
- KA_TRACE( 11, ("__kmp_internal_end_thread: abort, exiting\n" ));
- /* TODO abort? */
- return;
- }
- if( TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: already finished\n" ));
- return;
- }
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- /* find out who we are and what we should do */
- {
- int gtid = (gtid_req>=0) ? gtid_req : __kmp_gtid_get_specific();
- KA_TRACE( 10, ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req ));
- if( gtid == KMP_GTID_SHUTDOWN ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system already shutdown\n" ));
- return;
- } else if( gtid == KMP_GTID_MONITOR ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: monitor thread, gtid not registered, or system shutdown\n" ));
- return;
- } else if( gtid == KMP_GTID_DNE ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: gtid not registered or system shutdown\n" ));
- return;
- /* we don't know who we are */
- } else if( KMP_UBER_GTID( gtid )) {
- /* unregister ourselves as an uber thread. gtid is no longer valid */
- if( __kmp_root[gtid]->r.r_active ) {
- __kmp_global.g.g_abort = -1;
- TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
- KA_TRACE( 10, ("__kmp_internal_end_thread: root still active, abort T#%d\n", gtid ));
- return;
- } else {
- KA_TRACE( 10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n", gtid ));
- __kmp_unregister_root_current_thread( gtid );
- }
- } else {
- /* just a worker thread, let's leave */
- KA_TRACE( 10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid ));
-
- if ( gtid >= 0 ) {
- __kmp_threads[gtid]->th.th_task_team = NULL;
- }
-
- KA_TRACE( 10, ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n", gtid ));
- return;
- }
- }
- #if defined KMP_DYNAMIC_LIB
- // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber thread,
- // because we will better shutdown later in the library destructor.
- // The reason of this change is performance problem when non-openmp thread
- // in a loop forks and joins many openmp threads. We can save a lot of time
- // keeping worker threads alive until the program shutdown.
- // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966) and
- // Windows(DPD200287443) that occurs when using critical sections from foreign threads.
- KA_TRACE( 10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req) );
- return;
- #endif
- /* synchronize the termination process */
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
-
- /* have we already finished */
- if( __kmp_global.g.g_abort ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: abort, exiting\n" ));
- /* TODO abort? */
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
- if( TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial ) {
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
-
- /* We need this lock to enforce mutex between this reading of
- __kmp_threads_capacity and the writing by __kmp_register_root.
- Alternatively, we can use a counter of roots that is
- atomically updated by __kmp_get_global_thread_id_reg,
- __kmp_do_serial_initialize and __kmp_internal_end_*.
- */
-
- /* should we finish the run-time? are all siblings done? */
- __kmp_acquire_bootstrap_lock( &__kmp_forkjoin_lock );
-
- for ( i = 0; i < __kmp_threads_capacity; ++ i ) {
- if ( KMP_UBER_GTID( i ) ) {
- KA_TRACE( 10, ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i ));
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- };
- }
-
- /* now we can safely conduct the actual termination */
-
- __kmp_internal_end();
-
- __kmp_release_bootstrap_lock( &__kmp_forkjoin_lock );
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-
- KA_TRACE( 10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req ) );
-
- #ifdef DUMP_DEBUG_ON_EXIT
- if ( __kmp_debug_buf )
- __kmp_dump_debug_buffer();
- #endif
-} // __kmp_internal_end_thread
-
-// -------------------------------------------------------------------------------------------------
-// Library registration stuff.
-
-static long __kmp_registration_flag = 0;
- // Random value used to indicate library initialization.
-static char * __kmp_registration_str = NULL;
- // Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
-
-
-static inline
-char *
-__kmp_reg_status_name() {
- /*
- On RHEL 3u5 if linked statically, getpid() returns different values in each thread.
- If registration and unregistration go in different threads (omp_misc_other_root_exit.cpp test case),
- the name of registered_lib_env env var can not be found, because the name will contain different pid.
- */
- return __kmp_str_format( "__KMP_REGISTERED_LIB_%d", (int) getpid() );
-} // __kmp_reg_status_get
-
-
-void
-__kmp_register_library_startup(
- void
-) {
-
- char * name = __kmp_reg_status_name(); // Name of the environment variable.
- int done = 0;
- union {
- double dtime;
- long ltime;
- } time;
- #if KMP_ARCH_X86 || KMP_ARCH_X86_64
- __kmp_initialize_system_tick();
- #endif
- __kmp_read_system_time( & time.dtime );
- __kmp_registration_flag = 0xCAFE0000L | ( time.ltime & 0x0000FFFFL );
- __kmp_registration_str =
- __kmp_str_format(
- "%p-%lx-%s",
- & __kmp_registration_flag,
- __kmp_registration_flag,
- KMP_LIBRARY_FILE
- );
-
- KA_TRACE( 50, ( "__kmp_register_library_startup: %s=\"%s\"\n", name, __kmp_registration_str ) );
-
- while ( ! done ) {
-
- char * value = NULL; // Actual value of the environment variable.
-
- // Set environment variable, but do not overwrite if it is exist.
- __kmp_env_set( name, __kmp_registration_str, 0 );
- // Check the variable is written.
- value = __kmp_env_get( name );
- if ( value != NULL && strcmp( value, __kmp_registration_str ) == 0 ) {
-
- done = 1; // Ok, environment variable set successfully, exit the loop.
-
- } else {
-
- // Oops. Write failed. Another copy of OpenMP RTL is in memory.
- // Check whether it alive or dead.
- int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
- char * tail = value;
- char * flag_addr_str = NULL;
- char * flag_val_str = NULL;
- char const * file_name = NULL;
- __kmp_str_split( tail, '-', & flag_addr_str, & tail );
- __kmp_str_split( tail, '-', & flag_val_str, & tail );
- file_name = tail;
- if ( tail != NULL ) {
- long * flag_addr = 0;
- long flag_val = 0;
- KMP_SSCANF( flag_addr_str, "%p", & flag_addr );
- KMP_SSCANF( flag_val_str, "%lx", & flag_val );
- if ( flag_addr != 0 && flag_val != 0 && strcmp( file_name, "" ) != 0 ) {
- // First, check whether environment-encoded address is mapped into addr space.
- // If so, dereference it to see if it still has the right value.
-
- if ( __kmp_is_address_mapped( flag_addr ) && * flag_addr == flag_val ) {
- neighbor = 1;
- } else {
- // If not, then we know the other copy of the library is no longer running.
- neighbor = 2;
- }; // if
- }; // if
- }; // if
- switch ( neighbor ) {
- case 0 : // Cannot parse environment variable -- neighbor status unknown.
- // Assume it is the incompatible format of future version of the library.
- // Assume the other library is alive.
- // WARN( ... ); // TODO: Issue a warning.
- file_name = "unknown library";
- // Attention! Falling to the next case. That's intentional.
- case 1 : { // Neighbor is alive.
- // Check it is allowed.
- char * duplicate_ok = __kmp_env_get( "KMP_DUPLICATE_LIB_OK" );
- if ( ! __kmp_str_match_true( duplicate_ok ) ) {
- // That's not allowed. Issue fatal error.
- __kmp_msg(
- kmp_ms_fatal,
- KMP_MSG( DuplicateLibrary, KMP_LIBRARY_FILE, file_name ),
- KMP_HNT( DuplicateLibrary ),
- __kmp_msg_null
- );
- }; // if
- KMP_INTERNAL_FREE( duplicate_ok );
- __kmp_duplicate_library_ok = 1;
- done = 1; // Exit the loop.
- } break;
- case 2 : { // Neighbor is dead.
- // Clear the variable and try to register library again.
- __kmp_env_unset( name );
- } break;
- default : {
- KMP_DEBUG_ASSERT( 0 );
- } break;
- }; // switch
-
- }; // if
- KMP_INTERNAL_FREE( (void *) value );
-
- }; // while
- KMP_INTERNAL_FREE( (void *) name );
-
-} // func __kmp_register_library_startup
-
-
-void
-__kmp_unregister_library( void ) {
-
- char * name = __kmp_reg_status_name();
- char * value = __kmp_env_get( name );
-
- KMP_DEBUG_ASSERT( __kmp_registration_flag != 0 );
- KMP_DEBUG_ASSERT( __kmp_registration_str != NULL );
- if ( value != NULL && strcmp( value, __kmp_registration_str ) == 0 ) {
- // Ok, this is our variable. Delete it.
- __kmp_env_unset( name );
- }; // if
-
- KMP_INTERNAL_FREE( __kmp_registration_str );
- KMP_INTERNAL_FREE( value );
- KMP_INTERNAL_FREE( name );
-
- __kmp_registration_flag = 0;
- __kmp_registration_str = NULL;
-
-} // __kmp_unregister_library
-
-
-// End of Library registration stuff.
-// -------------------------------------------------------------------------------------------------
-
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
-
-static void __kmp_check_mic_type()
-{
- kmp_cpuid_t cpuid_state = {0};
- kmp_cpuid_t * cs_p = &cpuid_state;
- __kmp_x86_cpuid(1, 0, cs_p);
- // We don't support mic1 at the moment
- if( (cs_p->eax & 0xff0) == 0xB10 ) {
- __kmp_mic_type = mic2;
- } else if( (cs_p->eax & 0xf0ff0) == 0x50670 ) {
- __kmp_mic_type = mic3;
- } else {
- __kmp_mic_type = non_mic;
- }
-}
-
-#endif /* KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS) */
-
-static void
-__kmp_do_serial_initialize( void )
-{
- int i, gtid;
- int size;
-
- KA_TRACE( 10, ("__kmp_do_serial_initialize: enter\n" ) );
-
- KMP_DEBUG_ASSERT( sizeof( kmp_int32 ) == 4 );
- KMP_DEBUG_ASSERT( sizeof( kmp_uint32 ) == 4 );
- KMP_DEBUG_ASSERT( sizeof( kmp_int64 ) == 8 );
- KMP_DEBUG_ASSERT( sizeof( kmp_uint64 ) == 8 );
- KMP_DEBUG_ASSERT( sizeof( kmp_intptr_t ) == sizeof( void * ) );
-
-#if OMPT_SUPPORT
- ompt_pre_init();
-#endif
-
- __kmp_validate_locks();
-
- /* Initialize internal memory allocator */
- __kmp_init_allocator();
-
- /* Register the library startup via an environment variable
- and check to see whether another copy of the library is already
- registered. */
-
- __kmp_register_library_startup( );
-
- /* TODO reinitialization of library */
- if( TCR_4(__kmp_global.g.g_done) ) {
- KA_TRACE( 10, ("__kmp_do_serial_initialize: reinitialization of library\n" ) );
- }
-
- __kmp_global.g.g_abort = 0;
- TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
-
- /* initialize the locks */
-#if KMP_USE_ADAPTIVE_LOCKS
-#if KMP_DEBUG_ADAPTIVE_LOCKS
- __kmp_init_speculative_stats();
-#endif
-#endif
-#if KMP_STATS_ENABLED
- __kmp_stats_init();
-#endif
- __kmp_init_lock( & __kmp_global_lock );
- __kmp_init_queuing_lock( & __kmp_dispatch_lock );
- __kmp_init_lock( & __kmp_debug_lock );
- __kmp_init_atomic_lock( & __kmp_atomic_lock );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_1i );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_2i );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_4i );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_4r );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_8i );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_8r );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_8c );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_10r );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_16r );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_16c );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_20c );
- __kmp_init_atomic_lock( & __kmp_atomic_lock_32c );
- __kmp_init_bootstrap_lock( & __kmp_forkjoin_lock );
- __kmp_init_bootstrap_lock( & __kmp_exit_lock );
-#if KMP_USE_MONITOR
- __kmp_init_bootstrap_lock( & __kmp_monitor_lock );
-#endif
- __kmp_init_bootstrap_lock( & __kmp_tp_cached_lock );
-
- /* conduct initialization and initial setup of configuration */
-
- __kmp_runtime_initialize();
-
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
- __kmp_check_mic_type();
-#endif
-
- // Some global variable initialization moved here from kmp_env_initialize()
-#ifdef KMP_DEBUG
- kmp_diag = 0;
-#endif
- __kmp_abort_delay = 0;
-
- // From __kmp_init_dflt_team_nth()
- /* assume the entire machine will be used */
- __kmp_dflt_team_nth_ub = __kmp_xproc;
- if( __kmp_dflt_team_nth_ub < KMP_MIN_NTH ) {
- __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
- }
- if( __kmp_dflt_team_nth_ub > __kmp_sys_max_nth ) {
- __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
- }
- __kmp_max_nth = __kmp_sys_max_nth;
-
- // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME" part
- __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
-#if KMP_USE_MONITOR
- __kmp_monitor_wakeups = KMP_WAKEUPS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
- __kmp_bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME( __kmp_dflt_blocktime, __kmp_monitor_wakeups );
-#endif
- // From "KMP_LIBRARY" part of __kmp_env_initialize()
- __kmp_library = library_throughput;
- // From KMP_SCHEDULE initialization
- __kmp_static = kmp_sch_static_balanced;
- // AC: do not use analytical here, because it is non-monotonous
- //__kmp_guided = kmp_sch_guided_iterative_chunked;
- //__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no need to repeate assignment
- // Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch bit control and barrier method
- // control parts
- #if KMP_FAST_REDUCTION_BARRIER
- #define kmp_reduction_barrier_gather_bb ((int)1)
- #define kmp_reduction_barrier_release_bb ((int)1)
- #define kmp_reduction_barrier_gather_pat bp_hyper_bar
- #define kmp_reduction_barrier_release_pat bp_hyper_bar
- #endif // KMP_FAST_REDUCTION_BARRIER
- for ( i=bs_plain_barrier; i<bs_last_barrier; i++ ) {
- __kmp_barrier_gather_branch_bits [ i ] = __kmp_barrier_gather_bb_dflt;
- __kmp_barrier_release_branch_bits[ i ] = __kmp_barrier_release_bb_dflt;
- __kmp_barrier_gather_pattern [ i ] = __kmp_barrier_gather_pat_dflt;
- __kmp_barrier_release_pattern[ i ] = __kmp_barrier_release_pat_dflt;
- #if KMP_FAST_REDUCTION_BARRIER
- if( i == bs_reduction_barrier ) { // tested and confirmed on ALTIX only ( lin_64 ): hyper,1
- __kmp_barrier_gather_branch_bits [ i ] = kmp_reduction_barrier_gather_bb;
- __kmp_barrier_release_branch_bits[ i ] = kmp_reduction_barrier_release_bb;
- __kmp_barrier_gather_pattern [ i ] = kmp_reduction_barrier_gather_pat;
- __kmp_barrier_release_pattern[ i ] = kmp_reduction_barrier_release_pat;
- }
- #endif // KMP_FAST_REDUCTION_BARRIER
- }
- #if KMP_FAST_REDUCTION_BARRIER
- #undef kmp_reduction_barrier_release_pat
- #undef kmp_reduction_barrier_gather_pat
- #undef kmp_reduction_barrier_release_bb
- #undef kmp_reduction_barrier_gather_bb
- #endif // KMP_FAST_REDUCTION_BARRIER
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
- if (__kmp_mic_type == mic2) { // KNC
- // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
- __kmp_barrier_gather_branch_bits [ bs_plain_barrier ] = 3; // plain gather
- __kmp_barrier_release_branch_bits[ bs_forkjoin_barrier ] = 1; // forkjoin release
- __kmp_barrier_gather_pattern [ bs_forkjoin_barrier ] = bp_hierarchical_bar;
- __kmp_barrier_release_pattern[ bs_forkjoin_barrier ] = bp_hierarchical_bar;
- }
-#if KMP_FAST_REDUCTION_BARRIER
- if (__kmp_mic_type == mic2) { // KNC
- __kmp_barrier_gather_pattern [ bs_reduction_barrier ] = bp_hierarchical_bar;
- __kmp_barrier_release_pattern[ bs_reduction_barrier ] = bp_hierarchical_bar;
- }
-#endif
-#endif
-
- // From KMP_CHECKS initialization
-#ifdef KMP_DEBUG
- __kmp_env_checks = TRUE; /* development versions have the extra checks */
-#else
- __kmp_env_checks = FALSE; /* port versions do not have the extra checks */
-#endif
-
- // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
- __kmp_foreign_tp = TRUE;
-
- __kmp_global.g.g_dynamic = FALSE;
- __kmp_global.g.g_dynamic_mode = dynamic_default;
-
- __kmp_env_initialize( NULL );
-
- // Print all messages in message catalog for testing purposes.
- #ifdef KMP_DEBUG
- char const * val = __kmp_env_get( "KMP_DUMP_CATALOG" );
- if ( __kmp_str_match_true( val ) ) {
- kmp_str_buf_t buffer;
- __kmp_str_buf_init( & buffer );
- __kmp_i18n_dump_catalog( & buffer );
- __kmp_printf( "%s", buffer.str );
- __kmp_str_buf_free( & buffer );
- }; // if
- __kmp_env_free( & val );
- #endif
-
- __kmp_threads_capacity = __kmp_initial_threads_capacity( __kmp_dflt_team_nth_ub );
- // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
- __kmp_tp_capacity = __kmp_default_tp_capacity(__kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
-
- // If the library is shut down properly, both pools must be NULL. Just in case, set them
- // to NULL -- some memory may leak, but subsequent code will work even if pools are not freed.
- KMP_DEBUG_ASSERT( __kmp_thread_pool == NULL );
- KMP_DEBUG_ASSERT( __kmp_thread_pool_insert_pt == NULL );
- KMP_DEBUG_ASSERT( __kmp_team_pool == NULL );
- __kmp_thread_pool = NULL;
- __kmp_thread_pool_insert_pt = NULL;
- __kmp_team_pool = NULL;
-
- /* Allocate all of the variable sized records */
- /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are expandable */
- /* Since allocation is cache-aligned, just add extra padding at the end */
- size = (sizeof(kmp_info_t*) + sizeof(kmp_root_t*))*__kmp_threads_capacity + CACHE_LINE;
- __kmp_threads = (kmp_info_t**) __kmp_allocate( size );
- __kmp_root = (kmp_root_t**) ((char*)__kmp_threads + sizeof(kmp_info_t*) * __kmp_threads_capacity );
-
- /* init thread counts */
- KMP_DEBUG_ASSERT( __kmp_all_nth == 0 ); // Asserts fail if the library is reinitializing and
- KMP_DEBUG_ASSERT( __kmp_nth == 0 ); // something was wrong in termination.
- __kmp_all_nth = 0;
- __kmp_nth = 0;
-
- /* setup the uber master thread and hierarchy */
- gtid = __kmp_register_root( TRUE );
- KA_TRACE( 10, ("__kmp_do_serial_initialize T#%d\n", gtid ));
- KMP_ASSERT( KMP_UBER_GTID( gtid ) );
- KMP_ASSERT( KMP_INITIAL_GTID( gtid ) );
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- __kmp_common_initialize();
-
- #if KMP_OS_UNIX
- /* invoke the child fork handler */
- __kmp_register_atfork();
- #endif
-
- #if ! defined KMP_DYNAMIC_LIB
- {
- /* Invoke the exit handler when the program finishes, only for static library.
- For dynamic library, we already have _fini and DllMain.
- */
- int rc = atexit( __kmp_internal_end_atexit );
- if ( rc != 0 ) {
- __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "atexit()" ), KMP_ERR( rc ), __kmp_msg_null );
- }; // if
- }
- #endif
-
- #if KMP_HANDLE_SIGNALS
- #if KMP_OS_UNIX
- /* NOTE: make sure that this is called before the user installs
- * their own signal handlers so that the user handlers
- * are called first. this way they can return false,
- * not call our handler, avoid terminating the library,
- * and continue execution where they left off. */
- __kmp_install_signals( FALSE );
- #endif /* KMP_OS_UNIX */
- #if KMP_OS_WINDOWS
- __kmp_install_signals( TRUE );
- #endif /* KMP_OS_WINDOWS */
- #endif
-
- /* we have finished the serial initialization */
- __kmp_init_counter ++;
-
- __kmp_init_serial = TRUE;
-
- if (__kmp_settings) {
- __kmp_env_print();
- }
-
-#if OMP_40_ENABLED
- if (__kmp_display_env || __kmp_display_env_verbose) {
- __kmp_env_print_2();
- }
-#endif // OMP_40_ENABLED
-
-#if OMPT_SUPPORT
- ompt_post_init();
-#endif
-
- KMP_MB();
-
- KA_TRACE( 10, ("__kmp_do_serial_initialize: exit\n" ) );
-}
-
-void
-__kmp_serial_initialize( void )
-{
- if ( __kmp_init_serial ) {
- return;
- }
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
- if ( __kmp_init_serial ) {
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
- __kmp_do_serial_initialize();
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-}
-
-static void
-__kmp_do_middle_initialize( void )
-{
- int i, j;
- int prev_dflt_team_nth;
-
- if( !__kmp_init_serial ) {
- __kmp_do_serial_initialize();
- }
-
- KA_TRACE( 10, ("__kmp_middle_initialize: enter\n" ) );
-
- //
- // Save the previous value for the __kmp_dflt_team_nth so that
- // we can avoid some reinitialization if it hasn't changed.
- //
- prev_dflt_team_nth = __kmp_dflt_team_nth;
-
-#if KMP_AFFINITY_SUPPORTED
- //
- // __kmp_affinity_initialize() will try to set __kmp_ncores to the
- // number of cores on the machine.
- //
- __kmp_affinity_initialize();
-
- //
- // Run through the __kmp_threads array and set the affinity mask
- // for each root thread that is currently registered with the RTL.
- //
- for ( i = 0; i < __kmp_threads_capacity; i++ ) {
- if ( TCR_PTR( __kmp_threads[ i ] ) != NULL ) {
- __kmp_affinity_set_init_mask( i, TRUE );
- }
- }
-#endif /* KMP_AFFINITY_SUPPORTED */
-
- KMP_ASSERT( __kmp_xproc > 0 );
- if ( __kmp_avail_proc == 0 ) {
- __kmp_avail_proc = __kmp_xproc;
- }
-
- // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3), correct them now
- j = 0;
- while ( ( j < __kmp_nested_nth.used ) && ! __kmp_nested_nth.nth[ j ] ) {
- __kmp_nested_nth.nth[ j ] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub = __kmp_avail_proc;
- j++;
- }
-
- if ( __kmp_dflt_team_nth == 0 ) {
-#ifdef KMP_DFLT_NTH_CORES
- //
- // Default #threads = #cores
- //
- __kmp_dflt_team_nth = __kmp_ncores;
- KA_TRACE( 20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = __kmp_ncores (%d)\n",
- __kmp_dflt_team_nth ) );
-#else
- //
- // Default #threads = #available OS procs
- //
- __kmp_dflt_team_nth = __kmp_avail_proc;
- KA_TRACE( 20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = __kmp_avail_proc(%d)\n",
- __kmp_dflt_team_nth ) );
-#endif /* KMP_DFLT_NTH_CORES */
- }
-
- if ( __kmp_dflt_team_nth < KMP_MIN_NTH ) {
- __kmp_dflt_team_nth = KMP_MIN_NTH;
- }
- if( __kmp_dflt_team_nth > __kmp_sys_max_nth ) {
- __kmp_dflt_team_nth = __kmp_sys_max_nth;
- }
-
- //
- // There's no harm in continuing if the following check fails,
- // but it indicates an error in the previous logic.
- //
- KMP_DEBUG_ASSERT( __kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub );
-
- if ( __kmp_dflt_team_nth != prev_dflt_team_nth ) {
- //
- // Run through the __kmp_threads array and set the num threads icv
- // for each root thread that is currently registered with the RTL
- // (which has not already explicitly set its nthreads-var with a
- // call to omp_set_num_threads()).
- //
- for ( i = 0; i < __kmp_threads_capacity; i++ ) {
- kmp_info_t *thread = __kmp_threads[ i ];
- if ( thread == NULL ) continue;
- if ( thread->th.th_current_task->td_icvs.nproc != 0 ) continue;
-
- set__nproc( __kmp_threads[ i ], __kmp_dflt_team_nth );
- }
- }
- KA_TRACE( 20, ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
- __kmp_dflt_team_nth) );
-
-#ifdef KMP_ADJUST_BLOCKTIME
- /* Adjust blocktime to zero if necessary */
- /* now that __kmp_avail_proc is set */
- if ( !__kmp_env_blocktime && ( __kmp_avail_proc > 0 ) ) {
- KMP_DEBUG_ASSERT( __kmp_avail_proc > 0 );
- if ( __kmp_nth > __kmp_avail_proc ) {
- __kmp_zero_bt = TRUE;
- }
- }
-#endif /* KMP_ADJUST_BLOCKTIME */
-
- /* we have finished middle initialization */
- TCW_SYNC_4(__kmp_init_middle, TRUE);
-
- KA_TRACE( 10, ("__kmp_do_middle_initialize: exit\n" ) );
-}
-
-void
-__kmp_middle_initialize( void )
-{
- if ( __kmp_init_middle ) {
- return;
- }
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
- if ( __kmp_init_middle ) {
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
- return;
- }
- __kmp_do_middle_initialize();
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-}
-
-void
-__kmp_parallel_initialize( void )
-{
- int gtid = __kmp_entry_gtid(); // this might be a new root
-
- /* synchronize parallel initialization (for sibling) */
- if( TCR_4(__kmp_init_parallel) ) return;
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
- if( TCR_4(__kmp_init_parallel) ) { __kmp_release_bootstrap_lock( &__kmp_initz_lock ); return; }
-
- /* TODO reinitialization after we have already shut down */
- if( TCR_4(__kmp_global.g.g_done) ) {
- KA_TRACE( 10, ("__kmp_parallel_initialize: attempt to init while shutting down\n" ) );
- __kmp_infinite_loop();
- }
-
- /* jc: The lock __kmp_initz_lock is already held, so calling __kmp_serial_initialize
- would cause a deadlock. So we call __kmp_do_serial_initialize directly.
- */
- if( !__kmp_init_middle ) {
- __kmp_do_middle_initialize();
- }
-
- /* begin initialization */
- KA_TRACE( 10, ("__kmp_parallel_initialize: enter\n" ) );
- KMP_ASSERT( KMP_UBER_GTID( gtid ) );
-
-#if KMP_ARCH_X86 || KMP_ARCH_X86_64
- //
- // Save the FP control regs.
- // Worker threads will set theirs to these values at thread startup.
- //
- __kmp_store_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
- __kmp_store_mxcsr( &__kmp_init_mxcsr );
- __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
-#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
-
-#if KMP_OS_UNIX
-# if KMP_HANDLE_SIGNALS
- /* must be after __kmp_serial_initialize */
- __kmp_install_signals( TRUE );
-# endif
-#endif
-
- __kmp_suspend_initialize();
-
-#if defined(USE_LOAD_BALANCE)
- if ( __kmp_global.g.g_dynamic_mode == dynamic_default ) {
- __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
- }
-#else
- if ( __kmp_global.g.g_dynamic_mode == dynamic_default ) {
- __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
- }
-#endif
-
- if ( __kmp_version ) {
- __kmp_print_version_2();
- }
-
- /* we have finished parallel initialization */
- TCW_SYNC_4(__kmp_init_parallel, TRUE);
-
- KMP_MB();
- KA_TRACE( 10, ("__kmp_parallel_initialize: exit\n" ) );
-
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-}
-
-
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_run_before_invoked_task( int gtid, int tid, kmp_info_t *this_thr,
- kmp_team_t *team )
-{
- kmp_disp_t *dispatch;
-
- KMP_MB();
-
- /* none of the threads have encountered any constructs, yet. */
- this_thr->th.th_local.this_construct = 0;
-#if KMP_CACHE_MANAGE
- KMP_CACHE_PREFETCH( &this_thr->th.th_bar[ bs_forkjoin_barrier ].bb.b_arrived );
-#endif /* KMP_CACHE_MANAGE */
- dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
- KMP_DEBUG_ASSERT( dispatch );
- KMP_DEBUG_ASSERT( team->t.t_dispatch );
- //KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[ this_thr->th.th_info.ds.ds_tid ] );
-
- dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
-#if OMP_45_ENABLED
- dispatch->th_doacross_buf_idx = 0; /* reset the doacross dispatch buffer counter */
-#endif
- if( __kmp_env_consistency_check )
- __kmp_push_parallel( gtid, team->t.t_ident );
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
-}
-
-void
-__kmp_run_after_invoked_task( int gtid, int tid, kmp_info_t *this_thr,
- kmp_team_t *team )
-{
- if( __kmp_env_consistency_check )
- __kmp_pop_parallel( gtid, team->t.t_ident );
-
- __kmp_finish_implicit_task(this_thr);
-}
-
-int
-__kmp_invoke_task_func( int gtid )
-{
- int rc;
- int tid = __kmp_tid_from_gtid( gtid );
- kmp_info_t *this_thr = __kmp_threads[ gtid ];
- kmp_team_t *team = this_thr->th.th_team;
-
- __kmp_run_before_invoked_task( gtid, tid, this_thr, team );
-#if USE_ITT_BUILD
- if ( __itt_stack_caller_create_ptr ) {
- __kmp_itt_stack_callee_enter( (__itt_caller)team->t.t_stack_id ); // inform ittnotify about entering user's code
- }
-#endif /* USE_ITT_BUILD */
-#if INCLUDE_SSC_MARKS
- SSC_MARK_INVOKING();
-#endif
-
-#if OMPT_SUPPORT
- void *dummy;
- void **exit_runtime_p;
- ompt_task_id_t my_task_id;
- ompt_parallel_id_t my_parallel_id;
-
- if (ompt_enabled) {
- exit_runtime_p = &(team->t.t_implicit_task_taskdata[tid].
- ompt_task_info.frame.exit_runtime_frame);
- } else {
- exit_runtime_p = &dummy;
- }
-
-#if OMPT_TRACE
- my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id;
- my_parallel_id = team->t.ompt_team_info.parallel_id;
- if (ompt_enabled &&
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
- ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
- my_parallel_id, my_task_id);
- }
-#endif
-#endif
-
- {
- KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
- KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
- rc = __kmp_invoke_microtask( (microtask_t) TCR_SYNC_PTR(team->t.t_pkfn),
- gtid, tid, (int) team->t.t_argc, (void **) team->t.t_argv
-#if OMPT_SUPPORT
- , exit_runtime_p
-#endif
- );
-#if OMPT_SUPPORT
- *exit_runtime_p = NULL;
-#endif
- }
-
-#if USE_ITT_BUILD
- if ( __itt_stack_caller_create_ptr ) {
- __kmp_itt_stack_callee_leave( (__itt_caller)team->t.t_stack_id ); // inform ittnotify about leaving user's code
- }
-#endif /* USE_ITT_BUILD */
- __kmp_run_after_invoked_task( gtid, tid, this_thr, team );
-
- return rc;
-}
-
-#if OMP_40_ENABLED
-void
-__kmp_teams_master( int gtid )
-{
- // This routine is called by all master threads in teams construct
- kmp_info_t *thr = __kmp_threads[ gtid ];
- kmp_team_t *team = thr->th.th_team;
- ident_t *loc = team->t.t_ident;
- thr->th.th_set_nproc = thr->th.th_teams_size.nth;
- KMP_DEBUG_ASSERT( thr->th.th_teams_microtask );
- KMP_DEBUG_ASSERT( thr->th.th_set_nproc );
- KA_TRACE( 20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n",
- gtid, __kmp_tid_from_gtid( gtid ), thr->th.th_teams_microtask ) );
- // Launch league of teams now, but not let workers execute
- // (they hang on fork barrier until next parallel)
-#if INCLUDE_SSC_MARKS
- SSC_MARK_FORKING();
-#endif
- __kmp_fork_call( loc, gtid, fork_context_intel,
- team->t.t_argc,
-#if OMPT_SUPPORT
- (void *)thr->th.th_teams_microtask, // "unwrapped" task
-#endif
- (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
- VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
- NULL );
-#if INCLUDE_SSC_MARKS
- SSC_MARK_JOINING();
-#endif
-
- // AC: last parameter "1" eliminates join barrier which won't work because
- // worker threads are in a fork barrier waiting for more parallel regions
- __kmp_join_call( loc, gtid
-#if OMPT_SUPPORT
- , fork_context_intel
-#endif
- , 1 );
-}
-
-int
-__kmp_invoke_teams_master( int gtid )
-{
- kmp_info_t *this_thr = __kmp_threads[ gtid ];
- kmp_team_t *team = this_thr->th.th_team;
- #if KMP_DEBUG
- if ( !__kmp_threads[gtid]-> th.th_team->t.t_serialized )
- KMP_DEBUG_ASSERT( (void*)__kmp_threads[gtid]-> th.th_team->t.t_pkfn == (void*)__kmp_teams_master );
- #endif
- __kmp_run_before_invoked_task( gtid, 0, this_thr, team );
- __kmp_teams_master( gtid );
- __kmp_run_after_invoked_task( gtid, 0, this_thr, team );
- return 1;
-}
-#endif /* OMP_40_ENABLED */
-
-/* this sets the requested number of threads for the next parallel region
- * encountered by this team */
-/* since this should be enclosed in the forkjoin critical section it
- * should avoid race conditions with assymmetrical nested parallelism */
-
-void
-__kmp_push_num_threads( ident_t *id, int gtid, int num_threads )
-{
- kmp_info_t *thr = __kmp_threads[gtid];
-
- if( num_threads > 0 )
- thr->th.th_set_nproc = num_threads;
-}
-
-#if OMP_40_ENABLED
-
-/* this sets the requested number of teams for the teams region and/or
- * the number of threads for the next parallel region encountered */
-void
-__kmp_push_num_teams( ident_t *id, int gtid, int num_teams, int num_threads )
-{
- kmp_info_t *thr = __kmp_threads[gtid];
- KMP_DEBUG_ASSERT(num_teams >= 0);
- KMP_DEBUG_ASSERT(num_threads >= 0);
-
- if( num_teams == 0 )
- num_teams = 1; // default number of teams is 1.
- if( num_teams > __kmp_max_nth ) { // if too many teams requested?
- if ( !__kmp_reserve_warn ) {
- __kmp_reserve_warn = 1;
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantFormThrTeam, num_teams, __kmp_max_nth ),
- KMP_HNT( Unset_ALL_THREADS ),
- __kmp_msg_null
- );
- }
- num_teams = __kmp_max_nth;
- }
- // Set number of teams (number of threads in the outer "parallel" of the teams)
- thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
-
- // Remember the number of threads for inner parallel regions
- if( num_threads == 0 ) {
- if( !TCR_4(__kmp_init_middle) )
- __kmp_middle_initialize(); // get __kmp_avail_proc calculated
- num_threads = __kmp_avail_proc / num_teams;
- if( num_teams * num_threads > __kmp_max_nth ) {
- // adjust num_threads w/o warning as it is not user setting
- num_threads = __kmp_max_nth / num_teams;
- }
- } else {
- if( num_teams * num_threads > __kmp_max_nth ) {
- int new_threads = __kmp_max_nth / num_teams;
- if ( !__kmp_reserve_warn ) { // user asked for too many threads
- __kmp_reserve_warn = 1; // that conflicts with OMP_THREAD_LIMIT
- __kmp_msg(
- kmp_ms_warning,
- KMP_MSG( CantFormThrTeam, num_threads, new_threads ),
- KMP_HNT( Unset_ALL_THREADS ),
- __kmp_msg_null
- );
- }
- num_threads = new_threads;
- }
- }
- thr->th.th_teams_size.nth = num_threads;
-}
-
-
-//
-// Set the proc_bind var to use in the following parallel region.
-//
-void
-__kmp_push_proc_bind( ident_t *id, int gtid, kmp_proc_bind_t proc_bind )
-{
- kmp_info_t *thr = __kmp_threads[gtid];
- thr->th.th_set_proc_bind = proc_bind;
-}
-
-#endif /* OMP_40_ENABLED */
-
-/* Launch the worker threads into the microtask. */
-
-void
-__kmp_internal_fork( ident_t *id, int gtid, kmp_team_t *team )
-{
- kmp_info_t *this_thr = __kmp_threads[gtid];
-
-#ifdef KMP_DEBUG
- int f;
-#endif /* KMP_DEBUG */
-
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( this_thr->th.th_team == team );
- KMP_ASSERT( KMP_MASTER_GTID(gtid) );
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- team->t.t_construct = 0; /* no single directives seen yet */
- team->t.t_ordered.dt.t_value = 0; /* thread 0 enters the ordered section first */
-
- /* Reset the identifiers on the dispatch buffer */
- KMP_DEBUG_ASSERT( team->t.t_disp_buffer );
- if ( team->t.t_max_nproc > 1 ) {
- int i;
- for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
- team->t.t_disp_buffer[ i ].buffer_index = i;
-#if OMP_45_ENABLED
- team->t.t_disp_buffer[i].doacross_buf_idx = i;
-#endif
- }
- } else {
- team->t.t_disp_buffer[ 0 ].buffer_index = 0;
-#if OMP_45_ENABLED
- team->t.t_disp_buffer[0].doacross_buf_idx = 0;
-#endif
- }
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
- KMP_ASSERT( this_thr->th.th_team == team );
-
-#ifdef KMP_DEBUG
- for( f=0 ; f<team->t.t_nproc ; f++ ) {
- KMP_DEBUG_ASSERT( team->t.t_threads[f] &&
- team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc );
- }
-#endif /* KMP_DEBUG */
-
- /* release the worker threads so they may begin working */
- __kmp_fork_barrier( gtid, 0 );
-}
-
-
-void
-__kmp_internal_join( ident_t *id, int gtid, kmp_team_t *team )
-{
- kmp_info_t *this_thr = __kmp_threads[gtid];
-
- KMP_DEBUG_ASSERT( team );
- KMP_DEBUG_ASSERT( this_thr->th.th_team == team );
- KMP_ASSERT( KMP_MASTER_GTID(gtid) );
- KMP_MB(); /* Flush all pending memory write invalidates. */
-
- /* Join barrier after fork */
-
-#ifdef KMP_DEBUG
- if (__kmp_threads[gtid] && __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc ) {
- __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n",gtid, gtid, __kmp_threads[gtid]);
- __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, team->t.t_nproc=%d\n",
- gtid, __kmp_threads[gtid]->th.th_team_nproc, team, team->t.t_nproc);
- __kmp_print_structure();
- }
- KMP_DEBUG_ASSERT( __kmp_threads[gtid] &&
- __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc );
-#endif /* KMP_DEBUG */
-
- __kmp_join_barrier( gtid ); /* wait for everyone */
-
- KMP_MB(); /* Flush all pending memory write invalidates. */
- KMP_ASSERT( this_thr->th.th_team == team );
-}
-
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-#ifdef USE_LOAD_BALANCE
-
-//
-// Return the worker threads actively spinning in the hot team, if we
-// are at the outermost level of parallelism. Otherwise, return 0.
-//
-static int
-__kmp_active_hot_team_nproc( kmp_root_t *root )
-{
- int i;
- int retval;
- kmp_team_t *hot_team;
-
- if ( root->r.r_active ) {
- return 0;
- }
- hot_team = root->r.r_hot_team;
- if ( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
- return hot_team->t.t_nproc - 1; // Don't count master thread
- }
-
- //
- // Skip the master thread - it is accounted for elsewhere.
- //
- retval = 0;
- for ( i = 1; i < hot_team->t.t_nproc; i++ ) {
- if ( hot_team->t.t_threads[i]->th.th_active ) {
- retval++;
- }
- }
- return retval;
-}
-
-//
-// Perform an automatic adjustment to the number of
-// threads used by the next parallel region.
-//
-static int
-__kmp_load_balance_nproc( kmp_root_t *root, int set_nproc )
-{
- int retval;
- int pool_active;
- int hot_team_active;
- int team_curr_active;
- int system_active;
-
- KB_TRACE( 20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n",
- root, set_nproc ) );
- KMP_DEBUG_ASSERT( root );
- KMP_DEBUG_ASSERT( root->r.r_root_team->t.t_threads[0]->th.th_current_task->td_icvs.dynamic == TRUE );
- KMP_DEBUG_ASSERT( set_nproc > 1 );
-
- if ( set_nproc == 1) {
- KB_TRACE( 20, ("__kmp_load_balance_nproc: serial execution.\n" ) );
- return 1;
- }
-
- //
- // Threads that are active in the thread pool, active in the hot team
- // for this particular root (if we are at the outer par level), and
- // the currently executing thread (to become the master) are available
- // to add to the new team, but are currently contributing to the system
- // load, and must be accounted for.
- //
- pool_active = TCR_4(__kmp_thread_pool_active_nth);
- hot_team_active = __kmp_active_hot_team_nproc( root );
- team_curr_active = pool_active + hot_team_active + 1;
-
- //
- // Check the system load.
- //
- system_active = __kmp_get_load_balance( __kmp_avail_proc + team_curr_active );
- KB_TRACE( 30, ("__kmp_load_balance_nproc: system active = %d pool active = %d hot team active = %d\n",
- system_active, pool_active, hot_team_active ) );
-
- if ( system_active < 0 ) {
- //
- // There was an error reading the necessary info from /proc,
- // so use the thread limit algorithm instead. Once we set
- // __kmp_global.g.g_dynamic_mode = dynamic_thread_limit,
- // we shouldn't wind up getting back here.
- //
- __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
- KMP_WARNING( CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit" );
-
- //
- // Make this call behave like the thread limit algorithm.
- //
- retval = __kmp_avail_proc - __kmp_nth + (root->r.r_active ? 1
- : root->r.r_hot_team->t.t_nproc);
- if ( retval > set_nproc ) {
- retval = set_nproc;
- }
- if ( retval < KMP_MIN_NTH ) {
- retval = KMP_MIN_NTH;
- }
-
- KB_TRACE( 20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n", retval ) );
- return retval;
- }
-
- //
- // There is a slight delay in the load balance algorithm in detecting
- // new running procs. The real system load at this instant should be
- // at least as large as the #active omp thread that are available to
- // add to the team.
- //
- if ( system_active < team_curr_active ) {
- system_active = team_curr_active;
- }
- retval = __kmp_avail_proc - system_active + team_curr_active;
- if ( retval > set_nproc ) {
- retval = set_nproc;
- }
- if ( retval < KMP_MIN_NTH ) {
- retval = KMP_MIN_NTH;
- }
-
- KB_TRACE( 20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval ) );
- return retval;
-} // __kmp_load_balance_nproc()
-
-#endif /* USE_LOAD_BALANCE */
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-/* NOTE: this is called with the __kmp_init_lock held */
-void
-__kmp_cleanup( void )
-{
- int f;
-
- KA_TRACE( 10, ("__kmp_cleanup: enter\n" ) );
-
- if (TCR_4(__kmp_init_parallel)) {
-#if KMP_HANDLE_SIGNALS
- __kmp_remove_signals();
-#endif
- TCW_4(__kmp_init_parallel, FALSE);
- }
-
- if (TCR_4(__kmp_init_middle)) {
-#if KMP_AFFINITY_SUPPORTED
- __kmp_affinity_uninitialize();
-#endif /* KMP_AFFINITY_SUPPORTED */
- __kmp_cleanup_hierarchy();
- TCW_4(__kmp_init_middle, FALSE);
- }
-
- KA_TRACE( 10, ("__kmp_cleanup: go serial cleanup\n" ) );
-
- if (__kmp_init_serial) {
- __kmp_runtime_destroy();
- __kmp_init_serial = FALSE;
- }
-
- for ( f = 0; f < __kmp_threads_capacity; f++ ) {
- if ( __kmp_root[ f ] != NULL ) {
- __kmp_free( __kmp_root[ f ] );
- __kmp_root[ f ] = NULL;
- }
- }
- __kmp_free( __kmp_threads );
- // __kmp_threads and __kmp_root were allocated at once, as single block, so there is no need in
- // freeing __kmp_root.
- __kmp_threads = NULL;
- __kmp_root = NULL;
- __kmp_threads_capacity = 0;
-
-#if KMP_USE_DYNAMIC_LOCK
- __kmp_cleanup_indirect_user_locks();
-#else
- __kmp_cleanup_user_locks();
-#endif
-
- #if KMP_AFFINITY_SUPPORTED
- KMP_INTERNAL_FREE( (void *) __kmp_cpuinfo_file );
- __kmp_cpuinfo_file = NULL;
- #endif /* KMP_AFFINITY_SUPPORTED */
-
- #if KMP_USE_ADAPTIVE_LOCKS
- #if KMP_DEBUG_ADAPTIVE_LOCKS
- __kmp_print_speculative_stats();
- #endif
- #endif
- KMP_INTERNAL_FREE( __kmp_nested_nth.nth );
- __kmp_nested_nth.nth = NULL;
- __kmp_nested_nth.size = 0;
- __kmp_nested_nth.used = 0;
-
- __kmp_i18n_catclose();
-
-#if KMP_STATS_ENABLED
- __kmp_stats_fini();
-#endif
-
- KA_TRACE( 10, ("__kmp_cleanup: exit\n" ) );
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-int
-__kmp_ignore_mppbeg( void )
-{
- char *env;
-
- if ((env = getenv( "KMP_IGNORE_MPPBEG" )) != NULL) {
- if (__kmp_str_match_false( env ))
- return FALSE;
- }
- // By default __kmpc_begin() is no-op.
- return TRUE;
-}
-
-int
-__kmp_ignore_mppend( void )
-{
- char *env;
-
- if ((env = getenv( "KMP_IGNORE_MPPEND" )) != NULL) {
- if (__kmp_str_match_false( env ))
- return FALSE;
- }
- // By default __kmpc_end() is no-op.
- return TRUE;
-}
-
-void
-__kmp_internal_begin( void )
-{
- int gtid;
- kmp_root_t *root;
-
- /* this is a very important step as it will register new sibling threads
- * and assign these new uber threads a new gtid */
- gtid = __kmp_entry_gtid();
- root = __kmp_threads[ gtid ]->th.th_root;
- KMP_ASSERT( KMP_UBER_GTID( gtid ));
-
- if( root->r.r_begin ) return;
- __kmp_acquire_lock( &root->r.r_begin_lock, gtid );
- if( root->r.r_begin ) {
- __kmp_release_lock( & root->r.r_begin_lock, gtid );
- return;
- }
-
- root->r.r_begin = TRUE;
-
- __kmp_release_lock( & root->r.r_begin_lock, gtid );
-}
-
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_user_set_library (enum library_type arg)
-{
- int gtid;
- kmp_root_t *root;
- kmp_info_t *thread;
-
- /* first, make sure we are initialized so we can get our gtid */
-
- gtid = __kmp_entry_gtid();
- thread = __kmp_threads[ gtid ];
-
- root = thread->th.th_root;
-
- KA_TRACE( 20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg, library_serial ));
- if (root->r.r_in_parallel) { /* Must be called in serial section of top-level thread */
- KMP_WARNING( SetLibraryIncorrectCall );
- return;
- }
-
- switch ( arg ) {
- case library_serial :
- thread->th.th_set_nproc = 0;
- set__nproc( thread, 1 );
- break;
- case library_turnaround :
- thread->th.th_set_nproc = 0;
- set__nproc( thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub );
- break;
- case library_throughput :
- thread->th.th_set_nproc = 0;
- set__nproc( thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth : __kmp_dflt_team_nth_ub );
- break;
- default:
- KMP_FATAL( UnknownLibraryType, arg );
- }
-
- __kmp_aux_set_library ( arg );
-}
-
-void
-__kmp_aux_set_stacksize( size_t arg )
-{
- if (! __kmp_init_serial)
- __kmp_serial_initialize();
-
-#if KMP_OS_DARWIN
- if (arg & (0x1000 - 1)) {
- arg &= ~(0x1000 - 1);
- if(arg + 0x1000) /* check for overflow if we round up */
- arg += 0x1000;
- }
-#endif
- __kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
-
- /* only change the default stacksize before the first parallel region */
- if (! TCR_4(__kmp_init_parallel)) {
- size_t value = arg; /* argument is in bytes */
-
- if (value < __kmp_sys_min_stksize )
- value = __kmp_sys_min_stksize ;
- else if (value > KMP_MAX_STKSIZE)
- value = KMP_MAX_STKSIZE;
-
- __kmp_stksize = value;
-
- __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */
- }
-
- __kmp_release_bootstrap_lock( &__kmp_initz_lock );
-}
-
-/* set the behaviour of the runtime library */
-/* TODO this can cause some odd behaviour with sibling parallelism... */
-void
-__kmp_aux_set_library (enum library_type arg)
-{
- __kmp_library = arg;
-
- switch ( __kmp_library ) {
- case library_serial :
- {
- KMP_INFORM( LibraryIsSerial );
- (void) __kmp_change_library( TRUE );
- }
- break;
- case library_turnaround :
- (void) __kmp_change_library( TRUE );
- break;
- case library_throughput :
- (void) __kmp_change_library( FALSE );
- break;
- default:
- KMP_FATAL( UnknownLibraryType, arg );
- }
-}
-
-/* ------------------------------------------------------------------------ */
-/* ------------------------------------------------------------------------ */
-
-void
-__kmp_aux_set_blocktime (int arg, kmp_info_t *thread, int tid)
-{
- int blocktime = arg; /* argument is in milliseconds */
-#if KMP_USE_MONITOR
- int bt_intervals;
-#endif
- int bt_set;
-
- __kmp_save_internal_controls( thread );
-
- /* Normalize and set blocktime for the teams */
- if (blocktime < KMP_MIN_BLOCKTIME)
- blocktime = KMP_MIN_BLOCKTIME;
- else if (blocktime > KMP_MAX_BLOCKTIME)
- blocktime = KMP_MAX_BLOCKTIME;
-
- set__blocktime_team( thread->th.th_team, tid, blocktime );
- set__blocktime_team( thread->th.th_serial_team, 0, blocktime );
-
-#if KMP_USE_MONITOR
- /* Calculate and set blocktime intervals for the teams */
- bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
-
- set__bt_intervals_team( thread->th.th_team, tid, bt_intervals );
- set__bt_intervals_team( thread->th.th_serial_team, 0, bt_intervals );
-#endif
-
- /* Set whether blocktime has been set to "TRUE" */
- bt_set = TRUE;
-
- set__bt_set_team( thread->th.th_team, tid, bt_set );
- set__bt_set_team( thread->th.th_serial_team, 0, bt_set );
-#if KMP_USE_MONITOR
- KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
- "bt_intervals=%d, monitor_updates=%d\n",
- __kmp_gtid_from_tid(tid, thread->th.th_team),
- thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
- __kmp_monitor_wakeups));
-#else
- KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
- __kmp_gtid_from_tid(tid, thread->th.th_team),
- thread->th.th_team->t.t_id, tid, blocktime));
-#endif
-}
-
-void
-__kmp_aux_set_defaults(
- char const * str,
- int len
-) {
- if ( ! __kmp_init_serial ) {
- __kmp_serial_initialize();
- };
- __kmp_env_initialize( str );
-
- if (__kmp_settings
-#if OMP_40_ENABLED
- || __kmp_display_env || __kmp_display_env_verbose
-#endif // OMP_40_ENABLED
- ) {
- __kmp_env_print();
- }
-} // __kmp_aux_set_defaults
-
-/* ------------------------------------------------------------------------ */
-
-/*
- * internal fast reduction routines
- */
-
-PACKED_REDUCTION_METHOD_T
-__kmp_determine_reduction_method( ident_t *loc, kmp_int32 global_tid,
- kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
- kmp_critical_name *lck )
-{
-
- // Default reduction method: critical construct ( lck != NULL, like in current PAROPT )
- // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method can be selected by RTL
- // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method can be selected by RTL
- // Finally, it's up to OpenMP RTL to make a decision on which method to select among generated by PAROPT.
-
- PACKED_REDUCTION_METHOD_T retval;
-
- int team_size;
-
- KMP_DEBUG_ASSERT( loc ); // it would be nice to test ( loc != 0 )
- KMP_DEBUG_ASSERT( lck ); // it would be nice to test ( lck != 0 )
-
- #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED ( ( loc->flags & ( KMP_IDENT_ATOMIC_REDUCE ) ) == ( KMP_IDENT_ATOMIC_REDUCE ) )
- #define FAST_REDUCTION_TREE_METHOD_GENERATED ( ( reduce_data ) && ( reduce_func ) )
-
- retval = critical_reduce_block;
-
- team_size = __kmp_get_team_num_threads( global_tid ); // another choice of getting a team size ( with 1 dynamic deference ) is slower
-
- if( team_size == 1 ) {
-
- retval = empty_reduce_block;
-
- } else {
-
- int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
- int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
-
- #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
-
- #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN
-
- int teamsize_cutoff = 4;
-
-#if KMP_ARCH_X86_64 && (KMP_OS_LINUX || KMP_OS_WINDOWS)
- if( __kmp_mic_type != non_mic ) {
- teamsize_cutoff = 8;
- }
-#endif
- if( tree_available ) {
- if( team_size <= teamsize_cutoff ) {
- if ( atomic_available ) {
- retval = atomic_reduce_block;
- }
- } else {
- retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
- }
- } else if ( atomic_available ) {
- retval = atomic_reduce_block;
- }
- #else
- #error "Unknown or unsupported OS"
- #endif // KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN
-
- #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
-
- #if KMP_OS_LINUX || KMP_OS_WINDOWS
-
- // basic tuning
-
- if( atomic_available ) {
- if( num_vars <= 2 ) { // && ( team_size <= 8 ) due to false-sharing ???
- retval = atomic_reduce_block;
- }
- } // otherwise: use critical section
-
- #elif KMP_OS_DARWIN
-
- if( atomic_available && ( num_vars <= 3 ) ) {
- retval = atomic_reduce_block;
- } else if( tree_available ) {
- if( ( reduce_size > ( 9 * sizeof( kmp_real64 ) ) ) && ( reduce_size < ( 2000 * sizeof( kmp_real64 ) ) ) ) {
- retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
- }
- } // otherwise: use critical section
-
- #else
- #error "Unknown or unsupported OS"
- #endif
-
- #else
- #error "Unknown or unsupported architecture"
- #endif
-
- }
-
- // KMP_FORCE_REDUCTION
-
- // If the team is serialized (team_size == 1), ignore the forced reduction
- // method and stay with the unsynchronized method (empty_reduce_block)
- if( __kmp_force_reduction_method != reduction_method_not_defined && team_size != 1) {
-
- PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
-
- int atomic_available, tree_available;
-
- switch( ( forced_retval = __kmp_force_reduction_method ) )
- {
- case critical_reduce_block:
- KMP_ASSERT( lck ); // lck should be != 0
- break;
-
- case atomic_reduce_block:
- atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
- if( ! atomic_available ) {
- KMP_WARNING(RedMethodNotSupported, "atomic");
- forced_retval = critical_reduce_block;
- }
- break;
-
- case tree_reduce_block:
- tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
- if( ! tree_available ) {
- KMP_WARNING(RedMethodNotSupported, "tree");
- forced_retval = critical_reduce_block;
- } else {
- #if KMP_FAST_REDUCTION_BARRIER
- forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
- #endif
- }
- break;
-
- default:
- KMP_ASSERT( 0 ); // "unsupported method specified"
- }
-
- retval = forced_retval;
- }
-
- KA_TRACE(10, ( "reduction method selected=%08x\n", retval ) );
-
- #undef FAST_REDUCTION_TREE_METHOD_GENERATED
- #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
-
- return ( retval );
-}
-
-// this function is for testing set/get/determine reduce method
-kmp_int32
-__kmp_get_reduce_method( void ) {
- return ( ( __kmp_entry_thread()->th.th_local.packed_reduction_method ) >> 8 );
-}
-
-/* ------------------------------------------------------------------------ */
More information about the Openmp-commits
mailing list