[Openmp-commits] [openmp] r255373 - Hinted lock (OpenMP 4.5 feature) Updates/Fixes
Jonathan Peyton via Openmp-commits
openmp-commits at lists.llvm.org
Fri Dec 11 13:49:08 PST 2015
Author: jlpeyton
Date: Fri Dec 11 15:49:08 2015
New Revision: 255373
URL: http://llvm.org/viewvc/llvm-project?rev=255373&view=rev
Log:
Hinted lock (OpenMP 4.5 feature) Updates/Fixes
There are going to be two more patches which bring this feature up to date and in line with OpenMP 4.5.
* Renamed jump tables for the lock functions (and some clean up).
* Renamed some macros to be in KMP_ namespace.
* Return type of unset functions changed from void to int.
* Enabled use of _xebgin() et al. intrinsics for accessing TSX instructions.
Differential Revision: http://reviews.llvm.org/D15199
Modified:
openmp/trunk/runtime/src/kmp_csupport.c
openmp/trunk/runtime/src/kmp_lock.cpp
openmp/trunk/runtime/src/kmp_lock.h
Modified: openmp/trunk/runtime/src/kmp_csupport.c
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_csupport.c?rev=255373&r1=255372&r2=255373&view=diff
==============================================================================
--- openmp/trunk/runtime/src/kmp_csupport.c (original)
+++ openmp/trunk/runtime/src/kmp_csupport.c Fri Dec 11 15:49:08 2015
@@ -2028,7 +2028,7 @@ __kmpc_set_lock( ident_t * loc, kmp_int3
} else
# endif
{
- __kmp_direct_set_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+ __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
}
# if USE_ITT_BUILD
__kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
@@ -2146,7 +2146,7 @@ __kmpc_unset_lock( ident_t *loc, kmp_int
} else
# endif
{
- __kmp_direct_unset_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+ __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
}
#else // KMP_USE_DYNAMIC_LOCK
@@ -2286,7 +2286,7 @@ __kmpc_test_lock( ident_t *loc, kmp_int3
} else
# endif
{
- rc = __kmp_direct_test_ops[tag]((kmp_dyna_lock_t *)user_lock, gtid);
+ rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
}
if (rc) {
# if USE_ITT_BUILD
Modified: openmp/trunk/runtime/src/kmp_lock.cpp
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_lock.cpp?rev=255373&r1=255372&r2=255373&view=diff
==============================================================================
--- openmp/trunk/runtime/src/kmp_lock.cpp (original)
+++ openmp/trunk/runtime/src/kmp_lock.cpp Fri Dec 11 15:49:08 2015
@@ -1881,8 +1881,12 @@ __kmp_set_queuing_lock_flags( kmp_queuin
RTM Adaptive locks
*/
-// TODO: Use the header for intrinsics below with the compiler 13.0
-//#include <immintrin.h>
+#if KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300
+
+#include <immintrin.h>
+#define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
+
+#else
// Values from the status register after failed speculation.
#define _XBEGIN_STARTED (~0u)
@@ -1986,6 +1990,8 @@ static __inline void _xend()
__asm__ volatile (".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG) :::"memory");
#endif
+#endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300
+
//
// Statistics is collected for testing purpose
//
@@ -2235,7 +2241,7 @@ __kmp_test_adaptive_lock_only( kmp_adapt
// Lock is now visibly acquired, so someone beat us to it.
// Abort the transaction so we'll restart from _xbegin with the
// failure status.
- _xabort(0x01)
+ _xabort(0x01);
KMP_ASSERT2( 0, "should not get here" );
}
return 1; // Lock has been acquired (speculatively)
@@ -3004,7 +3010,7 @@ __kmp_set_drdpa_lock_flags( kmp_drdpa_lo
#if KMP_USE_DYNAMIC_LOCK
-// Definitions of lock hints.
+// Definitions of lock hints - can't include omp.h because of other name clashes.
# ifndef __OMP_H
typedef enum kmp_lock_hint_t {
kmp_lock_hint_none = 0,
@@ -3017,22 +3023,15 @@ typedef enum kmp_lock_hint_t {
# endif
// Direct lock initializers. It simply writes a tag to the low 8 bits of the lock word.
-#define expand_init_lock(l, a) \
-static void init_##l##_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq) { \
- *lck = KMP_LOCK_FREE(l); \
- KA_TRACE(20, ("Initialized direct lock, tag = %x\n", *lck)); \
+static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck, kmp_dyna_lockseq_t seq)
+{
+ TCW_4(*lck, KMP_GET_D_TAG(seq));
+ KA_TRACE(20, ("__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
}
-FOREACH_D_LOCK(expand_init_lock, 0)
-#undef expand_init_lock
#if KMP_HAS_HLE
// HLE lock functions - imported from the testbed runtime.
-#if KMP_MIC
-# define machine_pause() _mm_delay_32(10) // TODO: find the right argument
-#else
-# define machine_pause() _mm_pause()
-#endif
#define HLE_ACQUIRE ".byte 0xf2;"
#define HLE_RELEASE ".byte 0xf3;"
@@ -3049,7 +3048,7 @@ swap4(kmp_uint32 volatile *p, kmp_uint32
static void
__kmp_destroy_hle_lock(kmp_dyna_lock_t *lck)
{
- *lck = 0;
+ TCW_4(*lck, 0);
}
static void
@@ -3061,7 +3060,7 @@ __kmp_acquire_hle_lock(kmp_dyna_lock_t *
do {
while (*(kmp_uint32 volatile *)lck != KMP_LOCK_FREE(hle)) {
for (int i = delay; i != 0; --i)
- machine_pause();
+ KMP_CPU_PAUSE();
delay = ((delay << 1) | 1) & 7;
}
} while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
@@ -3074,19 +3073,20 @@ __kmp_acquire_hle_lock_with_checks(kmp_d
__kmp_acquire_hle_lock(lck, gtid); // TODO: add checks
}
-static void
+static int
__kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid)
{
__asm__ volatile(HLE_RELEASE "movl %1,%0"
: "=m"(*lck)
: "r"(KMP_LOCK_FREE(hle))
: "memory");
+ return KMP_LOCK_RELEASED;
}
-static void
+static int
__kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck, kmp_int32 gtid)
{
- __kmp_release_hle_lock(lck, gtid); // TODO: add checks
+ return __kmp_release_hle_lock(lck, gtid); // TODO: add checks
}
static int
@@ -3107,74 +3107,88 @@ __kmp_test_hle_lock_with_checks(kmp_dyna
static void __kmp_init_indirect_lock(kmp_dyna_lock_t * l, kmp_dyna_lockseq_t tag);
static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t * lock);
static void __kmp_set_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
-static void __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
+static int __kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
static int __kmp_test_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32);
static void __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
-static void __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
+static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32);
//
// Jump tables for the indirect lock functions.
// Only fill in the odd entries, that avoids the need to shift out the low bit.
//
-#define expand_func0(l, op) 0,op##_##l##_##lock,
-void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
- = { __kmp_init_indirect_lock, 0, FOREACH_D_LOCK(expand_func0, init) };
-
-#define expand_func1(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_##lock,
-void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *)
- = { __kmp_destroy_indirect_lock, 0, FOREACH_D_LOCK(expand_func1, destroy) };
-
-// Differentiates *lock and *lock_with_checks.
-#define expand_func2(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func2c(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static void (*direct_set_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
- = { { __kmp_set_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, acquire) },
- { __kmp_set_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, acquire) } };
-static void (*direct_unset_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
- = { { __kmp_unset_indirect_lock, 0, FOREACH_D_LOCK(expand_func2, release) },
- { __kmp_unset_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func2c, release) } };
-
-#define expand_func3(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func3c(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static int (*direct_test_tab[][KMP_NUM_D_LOCKS*2+2])(kmp_dyna_lock_t *, kmp_int32)
- = { { __kmp_test_indirect_lock, 0, FOREACH_D_LOCK(expand_func3, test) },
- { __kmp_test_indirect_lock_with_checks, 0, FOREACH_D_LOCK(expand_func3c, test) } };
+
+// init functions
+#define expand(l, op) 0,__kmp_init_direct_lock,
+void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t)
+ = { __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init) };
+#undef expand
+
+// destroy functions
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
+void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *)
+ = { __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy) };
+#undef expand
+
+// set/acquire functions
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
+static void (*direct_set[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
+#undef expand
+#define expand(l, op) 0,(void (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
+static void (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_set_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, acquire) };
+#undef expand
+
+// unset/release and test functions
+#define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
+static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release) };
+static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test) };
+#undef expand
+#define expand(l, op) 0,(int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
+static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_unset_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, release) };
+static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32)
+ = { __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test) };
+#undef expand
// Exposes only one set of jump tables (*lock or *lock_with_checks).
-void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
-void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
-int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32) = 0;
+void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
+int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
+int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
//
// Jump tables for the indirect lock functions.
//
-#define expand_func4(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
-void (*__kmp_indirect_init_ops[])(kmp_user_lock_p)
- = { FOREACH_I_LOCK(expand_func4, init) };
-void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p)
- = { FOREACH_I_LOCK(expand_func4, destroy) };
-
-// Differentiates *lock and *lock_with_checks.
-#define expand_func5(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func5c(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static void (*indirect_set_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
- = { { FOREACH_I_LOCK(expand_func5, acquire) },
- { FOREACH_I_LOCK(expand_func5c, acquire) } };
-static void (*indirect_unset_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
- = { { FOREACH_I_LOCK(expand_func5, release) },
- { FOREACH_I_LOCK(expand_func5c, release) } };
-
-#define expand_func6(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
-#define expand_func6c(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
-static int (*indirect_test_tab[][KMP_NUM_I_LOCKS])(kmp_user_lock_p, kmp_int32)
- = { { FOREACH_I_LOCK(expand_func6, test) },
- { FOREACH_I_LOCK(expand_func6c, test) } };
-
-// Exposes only one set of jump tables (*lock or *lock_with_checks).
-void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32) = 0;
-void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32) = 0;
-int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32) = 0;
+#define expand(l, op) (void (*)(kmp_user_lock_p))__kmp_##op##_##l##_##lock,
+void (*__kmp_indirect_init[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, init) };
+void (*__kmp_indirect_destroy[])(kmp_user_lock_p) = { KMP_FOREACH_I_LOCK(expand, destroy) };
+#undef expand
+
+// set/acquire functions
+#define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
+static void (*indirect_set[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
+#undef expand
+#define expand(l, op) (void (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
+static void (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, acquire) };
+#undef expand
+
+// unset/release and test functions
+#define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock,
+static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
+static int (*indirect_test[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
+#undef expand
+#define expand(l, op) (int (*)(kmp_user_lock_p, kmp_int32))__kmp_##op##_##l##_##lock_with_checks,
+static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, release) };
+static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = { KMP_FOREACH_I_LOCK(expand, test) };
+#undef expand
+
+// Exposes only one jump tables (*lock or *lock_with_checks).
+void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
+int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
+int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
// Lock index table.
kmp_indirect_lock_t **__kmp_indirect_lock_table;
@@ -3334,11 +3348,11 @@ __kmp_set_indirect_lock(kmp_dyna_lock_t
KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
}
-static void
+static int
__kmp_unset_indirect_lock(kmp_dyna_lock_t * lock, kmp_int32 gtid)
{
kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
- KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
+ return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
}
static int
@@ -3355,11 +3369,11 @@ __kmp_set_indirect_lock_with_checks(kmp_
KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
}
-static void
+static int
__kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t * lock, kmp_int32 gtid)
{
kmp_indirect_lock_t *l = __kmp_lookup_indirect_lock((void **)lock, "omp_unset_lock");
- KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
+ return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
}
static int
@@ -3469,15 +3483,6 @@ __kmp_init_nest_lock_hinted(void **lock,
#endif
}
-// Initializes the lock table for indirect locks.
-static void
-__kmp_init_indirect_lock_table()
-{
- __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(sizeof(kmp_indirect_lock_t *)*1024);
- __kmp_indirect_lock_table_size = 1024;
- __kmp_indirect_lock_table_next = 0;
-}
-
#if KMP_USE_ADAPTIVE_LOCKS
# define init_lock_func(table, expand) { \
table[locktag_ticket] = expand(ticket); \
@@ -3503,15 +3508,28 @@ __kmp_init_indirect_lock_table()
void
__kmp_init_dynamic_user_locks()
{
- // Initialize jump table location
- int offset = (__kmp_env_consistency_check)? 1: 0;
- __kmp_direct_set_ops = direct_set_tab[offset];
- __kmp_direct_unset_ops = direct_unset_tab[offset];
- __kmp_direct_test_ops = direct_test_tab[offset];
- __kmp_indirect_set_ops = indirect_set_tab[offset];
- __kmp_indirect_unset_ops = indirect_unset_tab[offset];
- __kmp_indirect_test_ops = indirect_test_tab[offset];
- __kmp_init_indirect_lock_table();
+ // Initialize jump table for the lock functions
+ if (__kmp_env_consistency_check) {
+ __kmp_direct_set = direct_set_check;
+ __kmp_direct_unset = direct_unset_check;
+ __kmp_direct_test = direct_test_check;
+ __kmp_indirect_set = indirect_set_check;
+ __kmp_indirect_unset = indirect_unset_check;
+ __kmp_indirect_test = indirect_test_check;
+ }
+ else {
+ __kmp_direct_set = direct_set;
+ __kmp_direct_unset = direct_unset;
+ __kmp_direct_test = direct_test;
+ __kmp_indirect_set = indirect_set;
+ __kmp_indirect_unset = indirect_unset;
+ __kmp_indirect_test = indirect_test;
+ }
+
+ // Initialize lock index table
+ __kmp_indirect_lock_table = (kmp_indirect_lock_t **)__kmp_allocate(sizeof(kmp_indirect_lock_t *)*1024);
+ __kmp_indirect_lock_table_size = 1024;
+ __kmp_indirect_lock_table_next = 0;
// Initialize lock accessor/modifier
// Could have used designated initializer, but -TP /Qstd=c99 did not work with icl.exe.
Modified: openmp/trunk/runtime/src/kmp_lock.h
URL: http://llvm.org/viewvc/llvm-project/openmp/trunk/runtime/src/kmp_lock.h?rev=255373&r1=255372&r2=255373&view=diff
==============================================================================
--- openmp/trunk/runtime/src/kmp_lock.h (original)
+++ openmp/trunk/runtime/src/kmp_lock.h Fri Dec 11 15:49:08 2015
@@ -1040,45 +1040,46 @@ extern void __kmp_cleanup_user_locks();
// All nested locks are indirect lock types.
#if KMP_HAS_FUTEX
# if KMP_HAS_HLE
-# define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
-# define KMP_LAST_D_LOCK_SEQ lockseq_hle
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a)
+# define KMP_LAST_D_LOCK lockseq_hle
# else
-# define FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
-# define KMP_LAST_D_LOCK_SEQ lockseq_futex
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
+# define KMP_LAST_D_LOCK lockseq_futex
# endif // KMP_HAS_HLE
# if KMP_USE_ADAPTIVE_LOCKS
-# define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
m(nested_queuing, a) m(nested_drdpa, a)
# else
-# define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
m(nested_queuing, a) m(nested_drdpa, a)
# endif // KMP_USE_ADAPTIVE_LOCKS
#else
# if KMP_HAS_HLE
-# define FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a)
-# define KMP_LAST_D_LOCK_SEQ lockseq_hle
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a)
+# define KMP_LAST_D_LOCK lockseq_hle
# else
-# define FOREACH_D_LOCK(m, a) m(tas, a)
-# define KMP_LAST_D_LOCK_SEQ lockseq_tas
+# define KMP_FOREACH_D_LOCK(m, a) m(tas, a)
+# define KMP_LAST_D_LOCK lockseq_tas
# endif // KMP_HAS_HLE
# if KMP_USE_ADAPTIVE_LOCKS
-# define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) \
m(nested_tas, a) m(nested_ticket, a) \
m(nested_queuing, a) m(nested_drdpa, a)
# else
-# define FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
+# define KMP_FOREACH_I_LOCK(m, a) m(ticket, a) m(queuing, a) m(drdpa, a) \
m(nested_tas, a) m(nested_ticket, a) \
m(nested_queuing, a) m(nested_drdpa, a)
# endif // KMP_USE_ADAPTIVE_LOCKS
#endif // KMP_HAS_FUTEX
// Information used in dynamic dispatch
-#define KMP_LOCK_VALUE_SHIFT 8
-#define KMP_LOCK_TYPE_MASK ((1<<KMP_LOCK_VALUE_SHIFT)-1)
-#define KMP_NUM_D_LOCKS KMP_LAST_D_LOCK_SEQ
-#define KMP_NUM_I_LOCKS (locktag_nested_drdpa+1)
+#define KMP_LOCK_SHIFT 8 // number of low bits to be used as tag for direct locks
+#define KMP_FIRST_D_LOCK lockseq_tas
+#define KMP_FIRST_I_LOCK lockseq_ticket
+#define KMP_LAST_I_LOCK lockseq_nested_drdpa
+#define KMP_NUM_I_LOCKS (locktag_nested_drdpa+1) // number of indirect lock types
// Base type for dynamic locks.
typedef kmp_uint32 kmp_dyna_lock_t;
@@ -1088,28 +1089,28 @@ typedef kmp_uint32 kmp_dyna_lock_t;
typedef enum {
lockseq_indirect = 0,
#define expand_seq(l,a) lockseq_##l,
- FOREACH_D_LOCK(expand_seq, 0)
- FOREACH_I_LOCK(expand_seq, 0)
+ KMP_FOREACH_D_LOCK(expand_seq, 0)
+ KMP_FOREACH_I_LOCK(expand_seq, 0)
#undef expand_seq
} kmp_dyna_lockseq_t;
// Enumerates indirect lock tags.
typedef enum {
#define expand_tag(l,a) locktag_##l,
- FOREACH_I_LOCK(expand_tag, 0)
+ KMP_FOREACH_I_LOCK(expand_tag, 0)
#undef expand_tag
} kmp_indirect_locktag_t;
// Utility macros that extract information from lock sequences.
-#define KMP_IS_D_LOCK(seq) (seq >= lockseq_tas && seq <= KMP_LAST_D_LOCK_SEQ)
-#define KMP_IS_I_LOCK(seq) (seq >= lockseq_ticket && seq <= lockseq_nested_drdpa)
-#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)(seq - lockseq_ticket)
-#define KMP_GET_D_TAG(seq) (seq<<1 | 1)
+#define KMP_IS_D_LOCK(seq) ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK)
+#define KMP_IS_I_LOCK(seq) ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK)
+#define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq) - KMP_FIRST_I_LOCK)
+#define KMP_GET_D_TAG(seq) ((seq)<<1 | 1)
// Enumerates direct lock tags starting from indirect tag.
typedef enum {
#define expand_tag(l,a) locktag_##l = KMP_GET_D_TAG(lockseq_##l),
- FOREACH_D_LOCK(expand_tag, 0)
+ KMP_FOREACH_D_LOCK(expand_tag, 0)
#undef expand_tag
} kmp_direct_locktag_t;
@@ -1120,45 +1121,45 @@ typedef struct {
} kmp_indirect_lock_t;
// Function tables for direct locks. Set/unset/test differentiate functions with/without consistency checking.
-extern void (*__kmp_direct_init_ops[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
-extern void (*__kmp_direct_destroy_ops[])(kmp_dyna_lock_t *);
-extern void (*(*__kmp_direct_set_ops))(kmp_dyna_lock_t *, kmp_int32);
-extern void (*(*__kmp_direct_unset_ops))(kmp_dyna_lock_t *, kmp_int32);
-extern int (*(*__kmp_direct_test_ops))(kmp_dyna_lock_t *, kmp_int32);
+extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
+extern void (*__kmp_direct_destroy[])(kmp_dyna_lock_t *);
+extern void (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32);
+extern int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32);
+extern int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32);
// Function tables for indirect locks. Set/unset/test differentiate functions with/withuot consistency checking.
-extern void (*__kmp_indirect_init_ops[])(kmp_user_lock_p);
-extern void (*__kmp_indirect_destroy_ops[])(kmp_user_lock_p);
-extern void (*(*__kmp_indirect_set_ops))(kmp_user_lock_p, kmp_int32);
-extern void (*(*__kmp_indirect_unset_ops))(kmp_user_lock_p, kmp_int32);
-extern int (*(*__kmp_indirect_test_ops))(kmp_user_lock_p, kmp_int32);
+extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
+extern void (*__kmp_indirect_destroy[])(kmp_user_lock_p);
+extern void (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32);
+extern int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32);
+extern int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32);
// Extracts direct lock tag from a user lock pointer
-#define KMP_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & KMP_LOCK_TYPE_MASK & -(*((kmp_dyna_lock_t *)(l)) & 1))
+#define KMP_EXTRACT_D_TAG(l) (*((kmp_dyna_lock_t *)(l)) & ((1<<KMP_LOCK_SHIFT)-1) & -(*((kmp_dyna_lock_t *)(l)) & 1))
// Extracts indirect lock index from a user lock pointer
#define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1)
// Returns function pointer to the direct lock function with l (kmp_dyna_lock_t *) and op (operation type).
-#define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op##_ops[KMP_EXTRACT_D_TAG(l)]
+#define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)]
// Returns function pointer to the indirect lock function with l (kmp_indirect_lock_t *) and op (operation type).
-#define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op##_ops[((kmp_indirect_lock_t *)(l))->type]
+#define KMP_I_LOCK_FUNC(l, op) __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type]
// Initializes a direct lock with the given lock pointer and lock sequence.
-#define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init_ops[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
+#define KMP_INIT_D_LOCK(l, seq) __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
// Initializes an indirect lock with the given lock pointer and lock sequence.
-#define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init_ops[0]((kmp_dyna_lock_t *)(l), seq)
+#define KMP_INIT_I_LOCK(l, seq) __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq)
// Returns "free" lock value for the given lock type.
#define KMP_LOCK_FREE(type) (locktag_##type)
// Returns "busy" lock value for the given lock teyp.
-#define KMP_LOCK_BUSY(v, type) ((v)<<KMP_LOCK_VALUE_SHIFT | locktag_##type)
+#define KMP_LOCK_BUSY(v, type) ((v)<<KMP_LOCK_SHIFT | locktag_##type)
// Returns lock value after removing (shifting) lock tag.
-#define KMP_LOCK_STRIP(v) ((v)>>KMP_LOCK_VALUE_SHIFT)
+#define KMP_LOCK_STRIP(v) ((v)>>KMP_LOCK_SHIFT)
// Updates __kmp_user_lock_seq with the give lock type.
#define KMP_STORE_LOCK_SEQ(type) (__kmp_user_lock_seq = lockseq_##type)
More information about the Openmp-commits
mailing list