[compiler-rt] r369289 - [TSan] Rename file to make it clear that it defines interceptors
Julian Lettner via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 19 11:41:20 PDT 2019
Author: yln
Date: Mon Aug 19 11:41:20 2019
New Revision: 369289
URL: http://llvm.org/viewvc/llvm-project?rev=369289&view=rev
Log:
[TSan] Rename file to make it clear that it defines interceptors
Rename file `tsan_libdispatch.cpp -> tsan_libdispatch_interceptors.cpp`
to make it clear that it's main purpose is defining interceptors.
Added:
compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch_interceptors.cpp
Removed:
compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch.cpp
Modified:
compiler-rt/trunk/lib/tsan/CMakeLists.txt
Modified: compiler-rt/trunk/lib/tsan/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/CMakeLists.txt?rev=369289&r1=369288&r2=369289&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/CMakeLists.txt (original)
+++ compiler-rt/trunk/lib/tsan/CMakeLists.txt Mon Aug 19 11:41:20 2019
@@ -76,7 +76,7 @@ endif()
if(COMPILER_RT_INTERCEPT_LIBDISPATCH)
list(APPEND TSAN_SOURCES
- rtl/tsan_libdispatch.cpp
+ rtl/tsan_libdispatch_interceptors.cpp
)
list(APPEND TSAN_RTL_CFLAGS ${COMPILER_RT_LIBDISPATCH_CFLAGS})
endif()
Removed: compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch.cpp?rev=369288&view=auto
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch.cpp (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch.cpp (removed)
@@ -1,782 +0,0 @@
-//===-- tsan_libdispatch.cpp ----------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file is a part of ThreadSanitizer (TSan), a race detector.
-//
-// Support for intercepting libdispatch (GCD).
-//===----------------------------------------------------------------------===//
-
-#include "sanitizer_common/sanitizer_common.h"
-#include "interception/interception.h"
-#include "tsan_interceptors.h"
-#include "tsan_rtl.h"
-
-#include "BlocksRuntime/Block.h"
-#include "tsan_dispatch_defs.h"
-
-namespace __tsan {
- typedef u16 uint16_t;
-
-typedef struct {
- dispatch_queue_t queue;
- void *orig_context;
- dispatch_function_t orig_work;
- bool free_context_in_callback;
- bool submitted_synchronously;
- bool is_barrier_block;
- uptr non_queue_sync_object;
-} block_context_t;
-
-// The offsets of different fields of the dispatch_queue_t structure, exported
-// by libdispatch.dylib.
-extern "C" struct dispatch_queue_offsets_s {
- const uint16_t dqo_version;
- const uint16_t dqo_label;
- const uint16_t dqo_label_size;
- const uint16_t dqo_flags;
- const uint16_t dqo_flags_size;
- const uint16_t dqo_serialnum;
- const uint16_t dqo_serialnum_size;
- const uint16_t dqo_width;
- const uint16_t dqo_width_size;
- const uint16_t dqo_running;
- const uint16_t dqo_running_size;
- const uint16_t dqo_suspend_cnt;
- const uint16_t dqo_suspend_cnt_size;
- const uint16_t dqo_target_queue;
- const uint16_t dqo_target_queue_size;
- const uint16_t dqo_priority;
- const uint16_t dqo_priority_size;
-} dispatch_queue_offsets;
-
-static bool IsQueueSerial(dispatch_queue_t q) {
- CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
- uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
- CHECK_NE(width, 0);
- return width == 1;
-}
-
-static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
- CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
- dispatch_queue_t tq = *(
- dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
- return tq;
-}
-
-static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
- dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
- CHECK_NE(tq, 0);
- return tq;
-}
-
-static block_context_t *AllocContext(ThreadState *thr, uptr pc,
- dispatch_queue_t queue, void *orig_context,
- dispatch_function_t orig_work) {
- block_context_t *new_context =
- (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));
- new_context->queue = queue;
- new_context->orig_context = orig_context;
- new_context->orig_work = orig_work;
- new_context->free_context_in_callback = true;
- new_context->submitted_synchronously = false;
- new_context->is_barrier_block = false;
- new_context->non_queue_sync_object = 0;
- return new_context;
-}
-
-#define GET_QUEUE_SYNC_VARS(context, q) \
- bool is_queue_serial = q && IsQueueSerial(q); \
- uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
- uptr serial_sync = (uptr)sync_ptr; \
- uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
- bool serial_task = context->is_barrier_block || is_queue_serial
-
-static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
- block_context_t *context) {
- uptr submit_sync = (uptr)context;
- Acquire(thr, pc, submit_sync);
-
- dispatch_queue_t q = context->queue;
- do {
- GET_QUEUE_SYNC_VARS(context, q);
- if (serial_sync) Acquire(thr, pc, serial_sync);
- if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
-
- if (q) q = GetTargetQueueFromQueue(q);
- } while (q);
-}
-
-static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
- block_context_t *context) {
- uptr submit_sync = (uptr)context;
- if (context->submitted_synchronously) Release(thr, pc, submit_sync);
-
- dispatch_queue_t q = context->queue;
- do {
- GET_QUEUE_SYNC_VARS(context, q);
- if (serial_task && serial_sync) Release(thr, pc, serial_sync);
- if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
-
- if (q) q = GetTargetQueueFromQueue(q);
- } while (q);
-}
-
-static void dispatch_callback_wrap(void *param) {
- SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
- block_context_t *context = (block_context_t *)param;
-
- dispatch_sync_pre_execute(thr, pc, context);
-
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- context->orig_work(context->orig_context);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
-
- dispatch_sync_post_execute(thr, pc, context);
-
- if (context->free_context_in_callback) user_free(thr, pc, context);
-}
-
-static void invoke_block(void *param) {
- dispatch_block_t block = (dispatch_block_t)param;
- block();
-}
-
-static void invoke_and_release_block(void *param) {
- dispatch_block_t block = (dispatch_block_t)param;
- block();
- Block_release(block);
-}
-
-#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \
- TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
- SCOPED_TSAN_INTERCEPTOR(name, q, block); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
- dispatch_block_t heap_block = Block_copy(block); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
- block_context_t *new_context = \
- AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
- new_context->is_barrier_block = barrier; \
- Release(thr, pc, (uptr)new_context); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
- REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
- }
-
-#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
- TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
- DISPATCH_NOESCAPE dispatch_block_t block) { \
- SCOPED_TSAN_INTERCEPTOR(name, q, block); \
- block_context_t new_context = { \
- q, block, &invoke_block, false, true, barrier, 0}; \
- Release(thr, pc, (uptr)&new_context); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
- REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
- Acquire(thr, pc, (uptr)&new_context); \
- }
-
-#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \
- TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
- dispatch_function_t work) { \
- SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
- block_context_t *new_context = \
- AllocContext(thr, pc, q, context, work); \
- new_context->is_barrier_block = barrier; \
- Release(thr, pc, (uptr)new_context); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
- REAL(name)(q, new_context, dispatch_callback_wrap); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
- }
-
-#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
- TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
- dispatch_function_t work) { \
- SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
- block_context_t new_context = { \
- q, context, work, false, true, barrier, 0}; \
- Release(thr, pc, (uptr)&new_context); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
- REAL(name)(q, &new_context, dispatch_callback_wrap); \
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
- Acquire(thr, pc, (uptr)&new_context); \
- }
-
-#define DISPATCH_INTERCEPT(name, barrier) \
- DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
- DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \
- DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \
- DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
-
-// We wrap dispatch_async, dispatch_sync and friends where we allocate a new
-// context, which is used to synchronize (we release the context before
-// submitting, and the callback acquires it before executing the original
-// callback).
-DISPATCH_INTERCEPT(dispatch, false)
-DISPATCH_INTERCEPT(dispatch_barrier, true)
-
-DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
- dispatch_queue_t queue, void *context, dispatch_function_t work)
-
-TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
- dispatch_queue_t queue, dispatch_block_t block) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- dispatch_block_t heap_block = Block_copy(block);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- block_context_t *new_context =
- AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
- Release(thr, pc, (uptr)new_context);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
-}
-
-TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
- dispatch_queue_t queue, void *context,
- dispatch_function_t work) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
- WRAP(dispatch_after)(when, queue, ^(void) {
- work(context);
- });
-}
-
-// GCD's dispatch_once implementation has a fast path that contains a racy read
-// and it's inlined into user's code. Furthermore, this fast path doesn't
-// establish a proper happens-before relations between the initialization and
-// code following the call to dispatch_once. We could deal with this in
-// instrumented code, but there's not much we can do about it in system
-// libraries. Let's disable the fast path (by never storing the value ~0 to
-// predicate), so the interceptor is always called, and let's add proper release
-// and acquire semantics. Since TSan does not see its own atomic stores, the
-// race on predicate won't be reported - the only accesses to it that TSan sees
-// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
-// both a macro and a real function, we want to intercept the function, so we
-// need to undefine the macro.
-#undef dispatch_once
-TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
- DISPATCH_NOESCAPE dispatch_block_t block) {
- SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
- atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
- u32 v = atomic_load(a, memory_order_acquire);
- if (v == 0 &&
- atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- block();
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- Release(thr, pc, (uptr)a);
- atomic_store(a, 2, memory_order_release);
- } else {
- while (v != 2) {
- internal_sched_yield();
- v = atomic_load(a, memory_order_acquire);
- }
- Acquire(thr, pc, (uptr)a);
- }
-}
-
-#undef dispatch_once_f
-TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
- void *context, dispatch_function_t function) {
- SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- WRAP(dispatch_once)(predicate, ^(void) {
- function(context);
- });
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
-}
-
-TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
- dispatch_semaphore_t dsema) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
- Release(thr, pc, (uptr)dsema);
- return REAL(dispatch_semaphore_signal)(dsema);
-}
-
-TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
- dispatch_time_t timeout) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
- long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
- if (result == 0) Acquire(thr, pc, (uptr)dsema);
- return result;
-}
-
-TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
- dispatch_time_t timeout) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
- long_t result = REAL(dispatch_group_wait)(group, timeout);
- if (result == 0) Acquire(thr, pc, (uptr)group);
- return result;
-}
-
-// Used, but not intercepted.
-extern "C" void dispatch_group_enter(dispatch_group_t group);
-
-TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
- // Acquired in the group notification callback in dispatch_group_notify[_f].
- Release(thr, pc, (uptr)group);
- REAL(dispatch_group_leave)(group);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
- dispatch_queue_t queue, dispatch_block_t block) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
- dispatch_retain(group);
- dispatch_group_enter(group);
- __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
- WRAP(dispatch_async)(queue, ^(void) {
- block_copy();
- Block_release(block_copy);
- WRAP(dispatch_group_leave)(group);
- dispatch_release(group);
- });
-}
-
-TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
- dispatch_queue_t queue, void *context,
- dispatch_function_t work) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
- dispatch_retain(group);
- dispatch_group_enter(group);
- WRAP(dispatch_async)(queue, ^(void) {
- work(context);
- WRAP(dispatch_group_leave)(group);
- dispatch_release(group);
- });
-}
-
-DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
- dispatch_queue_t q, void *context, dispatch_function_t work)
-
-TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
- dispatch_queue_t q, dispatch_block_t block) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
-
- // To make sure the group is still available in the callback (otherwise
- // it can be already destroyed). Will be released in the callback.
- dispatch_retain(group);
-
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- dispatch_block_t heap_block = Block_copy(^(void) {
- {
- SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
- // Released when leaving the group (dispatch_group_leave).
- Acquire(thr, pc, (uptr)group);
- }
- dispatch_release(group);
- block();
- });
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- block_context_t *new_context =
- AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
- new_context->is_barrier_block = true;
- Release(thr, pc, (uptr)new_context);
- REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
- dispatch_queue_t q, void *context, dispatch_function_t work) {
- WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
- dispatch_source_t source, dispatch_block_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_event_handler)(source, nullptr);
- dispatch_queue_t q = GetTargetQueueFromSource(source);
- __block block_context_t new_context = {
- q, handler, &invoke_block, false, false, false, 0 };
- dispatch_block_t new_handler = Block_copy(^(void) {
- new_context.orig_context = handler; // To explicitly capture "handler".
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_source_set_event_handler)(source, new_handler);
- Block_release(new_handler);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
- dispatch_source_t source, dispatch_function_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_event_handler)(source, nullptr);
- dispatch_block_t block = ^(void) {
- handler(dispatch_get_context(source));
- };
- WRAP(dispatch_source_set_event_handler)(source, block);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
- dispatch_source_t source, dispatch_block_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
- dispatch_queue_t q = GetTargetQueueFromSource(source);
- __block block_context_t new_context = {
- q, handler, &invoke_block, false, false, false, 0};
- dispatch_block_t new_handler = Block_copy(^(void) {
- new_context.orig_context = handler; // To explicitly capture "handler".
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_source_set_cancel_handler)(source, new_handler);
- Block_release(new_handler);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
- dispatch_source_t source, dispatch_function_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
- handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
- dispatch_block_t block = ^(void) {
- handler(dispatch_get_context(source));
- };
- WRAP(dispatch_source_set_cancel_handler)(source, block);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
- dispatch_source_t source, dispatch_block_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
- handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_registration_handler)(source, nullptr);
- dispatch_queue_t q = GetTargetQueueFromSource(source);
- __block block_context_t new_context = {
- q, handler, &invoke_block, false, false, false, 0};
- dispatch_block_t new_handler = Block_copy(^(void) {
- new_context.orig_context = handler; // To explicitly capture "handler".
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_source_set_registration_handler)(source, new_handler);
- Block_release(new_handler);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
- dispatch_source_t source, dispatch_function_t handler) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
- handler);
- if (handler == nullptr)
- return REAL(dispatch_source_set_registration_handler)(source, nullptr);
- dispatch_block_t block = ^(void) {
- handler(dispatch_get_context(source));
- };
- WRAP(dispatch_source_set_registration_handler)(source, block);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
- dispatch_queue_t queue,
- DISPATCH_NOESCAPE void (^block)(size_t)) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
-
- u8 sync1, sync2;
- uptr parent_to_child_sync = (uptr)&sync1;
- uptr child_to_parent_sync = (uptr)&sync2;
-
- Release(thr, pc, parent_to_child_sync);
- void (^new_block)(size_t) = ^(size_t iteration) {
- SCOPED_INTERCEPTOR_RAW(dispatch_apply);
- Acquire(thr, pc, parent_to_child_sync);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- block(iteration);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- Release(thr, pc, child_to_parent_sync);
- };
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- REAL(dispatch_apply)(iterations, queue, new_block);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- Acquire(thr, pc, child_to_parent_sync);
-}
-
-static void invoke_block_iteration(void *param, size_t iteration) {
- auto block = (void (^)(size_t)) param;
- block(iteration);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
- dispatch_queue_t queue, void *context,
- void (*work)(void *, size_t)) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
-
- // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
- // implements dispatch_apply in terms of dispatch_apply_f.
- u8 sync1, sync2;
- uptr parent_to_child_sync = (uptr)&sync1;
- uptr child_to_parent_sync = (uptr)&sync2;
-
- Release(thr, pc, parent_to_child_sync);
- void (^new_block)(size_t) = ^(size_t iteration) {
- SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
- Acquire(thr, pc, parent_to_child_sync);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- work(context, iteration);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- Release(thr, pc, child_to_parent_sync);
- };
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- Acquire(thr, pc, child_to_parent_sync);
-}
-
-DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
-DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
-
-TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
- size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
- if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
- return REAL(dispatch_data_create)(buffer, size, q, destructor);
-
- if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
- destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
- else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
- destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
-
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
- dispatch_block_t heap_block = Block_copy(destructor);
- SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
- block_context_t *new_context =
- AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
- uptr submit_sync = (uptr)new_context;
- Release(thr, pc, submit_sync);
- return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
- dispatch_callback_wrap(new_context);
- });
-}
-
-typedef void (^fd_handler_t)(dispatch_data_t data, int error);
-typedef void (^cleanup_handler_t)(int error);
-
-TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
- dispatch_queue_t q, fd_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
- new_context.orig_context = ^(void) {
- h(data, error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_read)(fd, length, q, new_h);
- Block_release(new_h);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
- dispatch_queue_t q, fd_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
- new_context.orig_context = ^(void) {
- h(data, error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_write)(fd, data, q, new_h);
- Block_release(new_h);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
- size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- dispatch_io_handler_t new_h =
- Block_copy(^(bool done, dispatch_data_t data, int error) {
- new_context.orig_context = ^(void) {
- h(done, data, error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_io_read)(channel, offset, length, q, new_h);
- Block_release(new_h);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
- dispatch_data_t data, dispatch_queue_t q,
- dispatch_io_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- dispatch_io_handler_t new_h =
- Block_copy(^(bool done, dispatch_data_t data, int error) {
- new_context.orig_context = ^(void) {
- h(done, data, error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_io_write)(channel, offset, data, q, new_h);
- Block_release(new_h);
-}
-
-TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
- dispatch_block_t barrier) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
- __block block_context_t new_context = {
- nullptr, nullptr, &invoke_block, false, false, false, 0};
- new_context.non_queue_sync_object = (uptr)channel;
- new_context.is_barrier_block = true;
- dispatch_block_t new_block = Block_copy(^(void) {
- new_context.orig_context = ^(void) {
- barrier();
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- REAL(dispatch_io_barrier)(channel, new_block);
- Block_release(new_block);
-}
-
-TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
- dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
- __block dispatch_io_t new_channel = nullptr;
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- cleanup_handler_t new_h = Block_copy(^(int error) {
- {
- SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
- Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
- }
- new_context.orig_context = ^(void) {
- h(error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
- Block_release(new_h);
- return new_channel;
-}
-
-TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
- dispatch_io_type_t type, const char *path, int oflag,
- mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
- q, h);
- __block dispatch_io_t new_channel = nullptr;
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- cleanup_handler_t new_h = Block_copy(^(int error) {
- {
- SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
- Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
- }
- new_context.orig_context = ^(void) {
- h(error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- new_channel =
- REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
- Block_release(new_h);
- return new_channel;
-}
-
-TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
- dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
- cleanup_handler_t h) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
- __block dispatch_io_t new_channel = nullptr;
- __block block_context_t new_context = {
- q, nullptr, &invoke_block, false, false, false, 0};
- cleanup_handler_t new_h = Block_copy(^(int error) {
- {
- SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
- Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
- }
- new_context.orig_context = ^(void) {
- h(error);
- };
- dispatch_callback_wrap(&new_context);
- });
- uptr submit_sync = (uptr)&new_context;
- Release(thr, pc, submit_sync);
- new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
- Block_release(new_h);
- return new_channel;
-}
-
-TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
- dispatch_io_close_flags_t flags) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
- Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
- return REAL(dispatch_io_close)(channel, flags);
-}
-
-// Resuming a suspended queue needs to synchronize with all subsequent
-// executions of blocks in that queue.
-TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
- SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
- Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync
- // in dispatch_sync_pre_execute
- return REAL(dispatch_resume)(o);
-}
-
-void InitializeLibdispatchInterceptors() {
- INTERCEPT_FUNCTION(dispatch_async);
- INTERCEPT_FUNCTION(dispatch_async_f);
- INTERCEPT_FUNCTION(dispatch_sync);
- INTERCEPT_FUNCTION(dispatch_sync_f);
- INTERCEPT_FUNCTION(dispatch_barrier_async);
- INTERCEPT_FUNCTION(dispatch_barrier_async_f);
- INTERCEPT_FUNCTION(dispatch_barrier_sync);
- INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
- INTERCEPT_FUNCTION(dispatch_after);
- INTERCEPT_FUNCTION(dispatch_after_f);
- INTERCEPT_FUNCTION(dispatch_once);
- INTERCEPT_FUNCTION(dispatch_once_f);
- INTERCEPT_FUNCTION(dispatch_semaphore_signal);
- INTERCEPT_FUNCTION(dispatch_semaphore_wait);
- INTERCEPT_FUNCTION(dispatch_group_wait);
- INTERCEPT_FUNCTION(dispatch_group_leave);
- INTERCEPT_FUNCTION(dispatch_group_async);
- INTERCEPT_FUNCTION(dispatch_group_async_f);
- INTERCEPT_FUNCTION(dispatch_group_notify);
- INTERCEPT_FUNCTION(dispatch_group_notify_f);
- INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
- INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
- INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
- INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
- INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
- INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
- INTERCEPT_FUNCTION(dispatch_apply);
- INTERCEPT_FUNCTION(dispatch_apply_f);
- INTERCEPT_FUNCTION(dispatch_data_create);
- INTERCEPT_FUNCTION(dispatch_read);
- INTERCEPT_FUNCTION(dispatch_write);
- INTERCEPT_FUNCTION(dispatch_io_read);
- INTERCEPT_FUNCTION(dispatch_io_write);
- INTERCEPT_FUNCTION(dispatch_io_barrier);
- INTERCEPT_FUNCTION(dispatch_io_create);
- INTERCEPT_FUNCTION(dispatch_io_create_with_path);
- INTERCEPT_FUNCTION(dispatch_io_create_with_io);
- INTERCEPT_FUNCTION(dispatch_io_close);
- INTERCEPT_FUNCTION(dispatch_resume);
-}
-
-} // namespace __tsan
Added: compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch_interceptors.cpp
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch_interceptors.cpp?rev=369289&view=auto
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch_interceptors.cpp (added)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_libdispatch_interceptors.cpp Mon Aug 19 11:41:20 2019
@@ -0,0 +1,782 @@
+//===-- tsan_libdispatch_interceptors.cpp ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of ThreadSanitizer (TSan), a race detector.
+//
+// Support for intercepting libdispatch (GCD).
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_common/sanitizer_common.h"
+#include "interception/interception.h"
+#include "tsan_interceptors.h"
+#include "tsan_rtl.h"
+
+#include "BlocksRuntime/Block.h"
+#include "tsan_dispatch_defs.h"
+
+namespace __tsan {
+ typedef u16 uint16_t;
+
+typedef struct {
+ dispatch_queue_t queue;
+ void *orig_context;
+ dispatch_function_t orig_work;
+ bool free_context_in_callback;
+ bool submitted_synchronously;
+ bool is_barrier_block;
+ uptr non_queue_sync_object;
+} block_context_t;
+
+// The offsets of different fields of the dispatch_queue_t structure, exported
+// by libdispatch.dylib.
+extern "C" struct dispatch_queue_offsets_s {
+ const uint16_t dqo_version;
+ const uint16_t dqo_label;
+ const uint16_t dqo_label_size;
+ const uint16_t dqo_flags;
+ const uint16_t dqo_flags_size;
+ const uint16_t dqo_serialnum;
+ const uint16_t dqo_serialnum_size;
+ const uint16_t dqo_width;
+ const uint16_t dqo_width_size;
+ const uint16_t dqo_running;
+ const uint16_t dqo_running_size;
+ const uint16_t dqo_suspend_cnt;
+ const uint16_t dqo_suspend_cnt_size;
+ const uint16_t dqo_target_queue;
+ const uint16_t dqo_target_queue_size;
+ const uint16_t dqo_priority;
+ const uint16_t dqo_priority_size;
+} dispatch_queue_offsets;
+
+static bool IsQueueSerial(dispatch_queue_t q) {
+ CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
+ uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
+ CHECK_NE(width, 0);
+ return width == 1;
+}
+
+static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {
+ CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);
+ dispatch_queue_t tq = *(
+ dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);
+ return tq;
+}
+
+static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {
+ dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);
+ CHECK_NE(tq, 0);
+ return tq;
+}
+
+static block_context_t *AllocContext(ThreadState *thr, uptr pc,
+ dispatch_queue_t queue, void *orig_context,
+ dispatch_function_t orig_work) {
+ block_context_t *new_context =
+ (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));
+ new_context->queue = queue;
+ new_context->orig_context = orig_context;
+ new_context->orig_work = orig_work;
+ new_context->free_context_in_callback = true;
+ new_context->submitted_synchronously = false;
+ new_context->is_barrier_block = false;
+ new_context->non_queue_sync_object = 0;
+ return new_context;
+}
+
+#define GET_QUEUE_SYNC_VARS(context, q) \
+ bool is_queue_serial = q && IsQueueSerial(q); \
+ uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \
+ uptr serial_sync = (uptr)sync_ptr; \
+ uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \
+ bool serial_task = context->is_barrier_block || is_queue_serial
+
+static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,
+ block_context_t *context) {
+ uptr submit_sync = (uptr)context;
+ Acquire(thr, pc, submit_sync);
+
+ dispatch_queue_t q = context->queue;
+ do {
+ GET_QUEUE_SYNC_VARS(context, q);
+ if (serial_sync) Acquire(thr, pc, serial_sync);
+ if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);
+
+ if (q) q = GetTargetQueueFromQueue(q);
+ } while (q);
+}
+
+static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,
+ block_context_t *context) {
+ uptr submit_sync = (uptr)context;
+ if (context->submitted_synchronously) Release(thr, pc, submit_sync);
+
+ dispatch_queue_t q = context->queue;
+ do {
+ GET_QUEUE_SYNC_VARS(context, q);
+ if (serial_task && serial_sync) Release(thr, pc, serial_sync);
+ if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);
+
+ if (q) q = GetTargetQueueFromQueue(q);
+ } while (q);
+}
+
+static void dispatch_callback_wrap(void *param) {
+ SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);
+ block_context_t *context = (block_context_t *)param;
+
+ dispatch_sync_pre_execute(thr, pc, context);
+
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ context->orig_work(context->orig_context);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+
+ dispatch_sync_post_execute(thr, pc, context);
+
+ if (context->free_context_in_callback) user_free(thr, pc, context);
+}
+
+static void invoke_block(void *param) {
+ dispatch_block_t block = (dispatch_block_t)param;
+ block();
+}
+
+static void invoke_and_release_block(void *param) {
+ dispatch_block_t block = (dispatch_block_t)param;
+ block();
+ Block_release(block);
+}
+
+#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \
+ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
+ SCOPED_TSAN_INTERCEPTOR(name, q, block); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
+ dispatch_block_t heap_block = Block_copy(block); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
+ block_context_t *new_context = \
+ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \
+ new_context->is_barrier_block = barrier; \
+ Release(thr, pc, (uptr)new_context); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
+ REAL(name##_f)(q, new_context, dispatch_callback_wrap); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
+ }
+
+#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \
+ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \
+ DISPATCH_NOESCAPE dispatch_block_t block) { \
+ SCOPED_TSAN_INTERCEPTOR(name, q, block); \
+ block_context_t new_context = { \
+ q, block, &invoke_block, false, true, barrier, 0}; \
+ Release(thr, pc, (uptr)&new_context); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
+ REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
+ Acquire(thr, pc, (uptr)&new_context); \
+ }
+
+#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \
+ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
+ dispatch_function_t work) { \
+ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
+ block_context_t *new_context = \
+ AllocContext(thr, pc, q, context, work); \
+ new_context->is_barrier_block = barrier; \
+ Release(thr, pc, (uptr)new_context); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
+ REAL(name)(q, new_context, dispatch_callback_wrap); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
+ }
+
+#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \
+ TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
+ dispatch_function_t work) { \
+ SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \
+ block_context_t new_context = { \
+ q, context, work, false, true, barrier, 0}; \
+ Release(thr, pc, (uptr)&new_context); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
+ REAL(name)(q, &new_context, dispatch_callback_wrap); \
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
+ Acquire(thr, pc, (uptr)&new_context); \
+ }
+
+#define DISPATCH_INTERCEPT(name, barrier) \
+ DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \
+ DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \
+ DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \
+ DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)
+
+// We wrap dispatch_async, dispatch_sync and friends where we allocate a new
+// context, which is used to synchronize (we release the context before
+// submitting, and the callback acquires it before executing the original
+// callback).
+DISPATCH_INTERCEPT(dispatch, false)
+DISPATCH_INTERCEPT(dispatch_barrier, true)
+
+DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,
+ dispatch_queue_t queue, void *context, dispatch_function_t work)
+
+TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,
+ dispatch_queue_t queue, dispatch_block_t block) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ dispatch_block_t heap_block = Block_copy(block);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ block_context_t *new_context =
+ AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);
+ Release(thr, pc, (uptr)new_context);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+}
+
+TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
+ dispatch_queue_t queue, void *context,
+ dispatch_function_t work) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);
+ WRAP(dispatch_after)(when, queue, ^(void) {
+ work(context);
+ });
+}
+
+// GCD's dispatch_once implementation has a fast path that contains a racy read
+// and it's inlined into user's code. Furthermore, this fast path doesn't
+// establish a proper happens-before relations between the initialization and
+// code following the call to dispatch_once. We could deal with this in
+// instrumented code, but there's not much we can do about it in system
+// libraries. Let's disable the fast path (by never storing the value ~0 to
+// predicate), so the interceptor is always called, and let's add proper release
+// and acquire semantics. Since TSan does not see its own atomic stores, the
+// race on predicate won't be reported - the only accesses to it that TSan sees
+// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
+// both a macro and a real function, we want to intercept the function, so we
+// need to undefine the macro.
+#undef dispatch_once
+TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
+ DISPATCH_NOESCAPE dispatch_block_t block) {
+ SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);
+ atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
+ u32 v = atomic_load(a, memory_order_acquire);
+ if (v == 0 &&
+ atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ block();
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ Release(thr, pc, (uptr)a);
+ atomic_store(a, 2, memory_order_release);
+ } else {
+ while (v != 2) {
+ internal_sched_yield();
+ v = atomic_load(a, memory_order_acquire);
+ }
+ Acquire(thr, pc, (uptr)a);
+ }
+}
+
+#undef dispatch_once_f
+TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
+ void *context, dispatch_function_t function) {
+ SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ WRAP(dispatch_once)(predicate, ^(void) {
+ function(context);
+ });
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
+ dispatch_semaphore_t dsema) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
+ Release(thr, pc, (uptr)dsema);
+ return REAL(dispatch_semaphore_signal)(dsema);
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
+ dispatch_time_t timeout) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
+ long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
+ if (result == 0) Acquire(thr, pc, (uptr)dsema);
+ return result;
+}
+
+TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
+ dispatch_time_t timeout) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
+ long_t result = REAL(dispatch_group_wait)(group, timeout);
+ if (result == 0) Acquire(thr, pc, (uptr)group);
+ return result;
+}
+
+// Used, but not intercepted.
+extern "C" void dispatch_group_enter(dispatch_group_t group);
+
+TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
+ // Acquired in the group notification callback in dispatch_group_notify[_f].
+ Release(thr, pc, (uptr)group);
+ REAL(dispatch_group_leave)(group);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
+ dispatch_queue_t queue, dispatch_block_t block) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
+ dispatch_retain(group);
+ dispatch_group_enter(group);
+ __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);
+ WRAP(dispatch_async)(queue, ^(void) {
+ block_copy();
+ Block_release(block_copy);
+ WRAP(dispatch_group_leave)(group);
+ dispatch_release(group);
+ });
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
+ dispatch_queue_t queue, void *context,
+ dispatch_function_t work) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
+ dispatch_retain(group);
+ dispatch_group_enter(group);
+ WRAP(dispatch_async)(queue, ^(void) {
+ work(context);
+ WRAP(dispatch_group_leave)(group);
+ dispatch_release(group);
+ });
+}
+
+DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,
+ dispatch_queue_t q, void *context, dispatch_function_t work)
+
+TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
+ dispatch_queue_t q, dispatch_block_t block) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
+
+ // To make sure the group is still available in the callback (otherwise
+ // it can be already destroyed). Will be released in the callback.
+ dispatch_retain(group);
+
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ dispatch_block_t heap_block = Block_copy(^(void) {
+ {
+ SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);
+ // Released when leaving the group (dispatch_group_leave).
+ Acquire(thr, pc, (uptr)group);
+ }
+ dispatch_release(group);
+ block();
+ });
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ block_context_t *new_context =
+ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
+ new_context->is_barrier_block = true;
+ Release(thr, pc, (uptr)new_context);
+ REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
+ dispatch_queue_t q, void *context, dispatch_function_t work) {
+ WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,
+ dispatch_source_t source, dispatch_block_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_event_handler)(source, nullptr);
+ dispatch_queue_t q = GetTargetQueueFromSource(source);
+ __block block_context_t new_context = {
+ q, handler, &invoke_block, false, false, false, 0 };
+ dispatch_block_t new_handler = Block_copy(^(void) {
+ new_context.orig_context = handler; // To explicitly capture "handler".
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_source_set_event_handler)(source, new_handler);
+ Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,
+ dispatch_source_t source, dispatch_function_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_event_handler)(source, nullptr);
+ dispatch_block_t block = ^(void) {
+ handler(dispatch_get_context(source));
+ };
+ WRAP(dispatch_source_set_event_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,
+ dispatch_source_t source, dispatch_block_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
+ dispatch_queue_t q = GetTargetQueueFromSource(source);
+ __block block_context_t new_context = {
+ q, handler, &invoke_block, false, false, false, 0};
+ dispatch_block_t new_handler = Block_copy(^(void) {
+ new_context.orig_context = handler; // To explicitly capture "handler".
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_source_set_cancel_handler)(source, new_handler);
+ Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,
+ dispatch_source_t source, dispatch_function_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,
+ handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_cancel_handler)(source, nullptr);
+ dispatch_block_t block = ^(void) {
+ handler(dispatch_get_context(source));
+ };
+ WRAP(dispatch_source_set_cancel_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,
+ dispatch_source_t source, dispatch_block_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,
+ handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_registration_handler)(source, nullptr);
+ dispatch_queue_t q = GetTargetQueueFromSource(source);
+ __block block_context_t new_context = {
+ q, handler, &invoke_block, false, false, false, 0};
+ dispatch_block_t new_handler = Block_copy(^(void) {
+ new_context.orig_context = handler; // To explicitly capture "handler".
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_source_set_registration_handler)(source, new_handler);
+ Block_release(new_handler);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,
+ dispatch_source_t source, dispatch_function_t handler) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,
+ handler);
+ if (handler == nullptr)
+ return REAL(dispatch_source_set_registration_handler)(source, nullptr);
+ dispatch_block_t block = ^(void) {
+ handler(dispatch_get_context(source));
+ };
+ WRAP(dispatch_source_set_registration_handler)(source, block);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,
+ dispatch_queue_t queue,
+ DISPATCH_NOESCAPE void (^block)(size_t)) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);
+
+ u8 sync1, sync2;
+ uptr parent_to_child_sync = (uptr)&sync1;
+ uptr child_to_parent_sync = (uptr)&sync2;
+
+ Release(thr, pc, parent_to_child_sync);
+ void (^new_block)(size_t) = ^(size_t iteration) {
+ SCOPED_INTERCEPTOR_RAW(dispatch_apply);
+ Acquire(thr, pc, parent_to_child_sync);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ block(iteration);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ Release(thr, pc, child_to_parent_sync);
+ };
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ REAL(dispatch_apply)(iterations, queue, new_block);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ Acquire(thr, pc, child_to_parent_sync);
+}
+
+static void invoke_block_iteration(void *param, size_t iteration) {
+ auto block = (void (^)(size_t)) param;
+ block(iteration);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,
+ dispatch_queue_t queue, void *context,
+ void (*work)(void *, size_t)) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);
+
+ // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch
+ // implements dispatch_apply in terms of dispatch_apply_f.
+ u8 sync1, sync2;
+ uptr parent_to_child_sync = (uptr)&sync1;
+ uptr child_to_parent_sync = (uptr)&sync2;
+
+ Release(thr, pc, parent_to_child_sync);
+ void (^new_block)(size_t) = ^(size_t iteration) {
+ SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);
+ Acquire(thr, pc, parent_to_child_sync);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ work(context, iteration);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ Release(thr, pc, child_to_parent_sync);
+ };
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ Acquire(thr, pc, child_to_parent_sync);
+}
+
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz)
+
+TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,
+ size_t size, dispatch_queue_t q, dispatch_block_t destructor) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);
+ if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))
+ return REAL(dispatch_data_create)(buffer, size, q, destructor);
+
+ if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)
+ destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };
+ else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)
+ destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };
+
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
+ dispatch_block_t heap_block = Block_copy(destructor);
+ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
+ block_context_t *new_context =
+ AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
+ uptr submit_sync = (uptr)new_context;
+ Release(thr, pc, submit_sync);
+ return REAL(dispatch_data_create)(buffer, size, q, ^(void) {
+ dispatch_callback_wrap(new_context);
+ });
+}
+
+typedef void (^fd_handler_t)(dispatch_data_t data, int error);
+typedef void (^cleanup_handler_t)(int error);
+
+TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,
+ dispatch_queue_t q, fd_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
+ new_context.orig_context = ^(void) {
+ h(data, error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_read)(fd, length, q, new_h);
+ Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,
+ dispatch_queue_t q, fd_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {
+ new_context.orig_context = ^(void) {
+ h(data, error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_write)(fd, data, q, new_h);
+ Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,
+ size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ dispatch_io_handler_t new_h =
+ Block_copy(^(bool done, dispatch_data_t data, int error) {
+ new_context.orig_context = ^(void) {
+ h(done, data, error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_io_read)(channel, offset, length, q, new_h);
+ Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,
+ dispatch_data_t data, dispatch_queue_t q,
+ dispatch_io_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ dispatch_io_handler_t new_h =
+ Block_copy(^(bool done, dispatch_data_t data, int error) {
+ new_context.orig_context = ^(void) {
+ h(done, data, error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_io_write)(channel, offset, data, q, new_h);
+ Block_release(new_h);
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,
+ dispatch_block_t barrier) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);
+ __block block_context_t new_context = {
+ nullptr, nullptr, &invoke_block, false, false, false, 0};
+ new_context.non_queue_sync_object = (uptr)channel;
+ new_context.is_barrier_block = true;
+ dispatch_block_t new_block = Block_copy(^(void) {
+ new_context.orig_context = ^(void) {
+ barrier();
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ REAL(dispatch_io_barrier)(channel, new_block);
+ Block_release(new_block);
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,
+ dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);
+ __block dispatch_io_t new_channel = nullptr;
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ cleanup_handler_t new_h = Block_copy(^(int error) {
+ {
+ SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+ Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
+ }
+ new_context.orig_context = ^(void) {
+ h(error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);
+ Block_release(new_h);
+ return new_channel;
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,
+ dispatch_io_type_t type, const char *path, int oflag,
+ mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,
+ q, h);
+ __block dispatch_io_t new_channel = nullptr;
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ cleanup_handler_t new_h = Block_copy(^(int error) {
+ {
+ SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+ Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
+ }
+ new_context.orig_context = ^(void) {
+ h(error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ new_channel =
+ REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);
+ Block_release(new_h);
+ return new_channel;
+}
+
+TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,
+ dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,
+ cleanup_handler_t h) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);
+ __block dispatch_io_t new_channel = nullptr;
+ __block block_context_t new_context = {
+ q, nullptr, &invoke_block, false, false, false, 0};
+ cleanup_handler_t new_h = Block_copy(^(int error) {
+ {
+ SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);
+ Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.
+ }
+ new_context.orig_context = ^(void) {
+ h(error);
+ };
+ dispatch_callback_wrap(&new_context);
+ });
+ uptr submit_sync = (uptr)&new_context;
+ Release(thr, pc, submit_sync);
+ new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);
+ Block_release(new_h);
+ return new_channel;
+}
+
+TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,
+ dispatch_io_close_flags_t flags) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);
+ Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].
+ return REAL(dispatch_io_close)(channel, flags);
+}
+
+// Resuming a suspended queue needs to synchronize with all subsequent
+// executions of blocks in that queue.
+TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {
+ SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);
+ Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync
+ // in dispatch_sync_pre_execute
+ return REAL(dispatch_resume)(o);
+}
+
+void InitializeLibdispatchInterceptors() {
+ INTERCEPT_FUNCTION(dispatch_async);
+ INTERCEPT_FUNCTION(dispatch_async_f);
+ INTERCEPT_FUNCTION(dispatch_sync);
+ INTERCEPT_FUNCTION(dispatch_sync_f);
+ INTERCEPT_FUNCTION(dispatch_barrier_async);
+ INTERCEPT_FUNCTION(dispatch_barrier_async_f);
+ INTERCEPT_FUNCTION(dispatch_barrier_sync);
+ INTERCEPT_FUNCTION(dispatch_barrier_sync_f);
+ INTERCEPT_FUNCTION(dispatch_after);
+ INTERCEPT_FUNCTION(dispatch_after_f);
+ INTERCEPT_FUNCTION(dispatch_once);
+ INTERCEPT_FUNCTION(dispatch_once_f);
+ INTERCEPT_FUNCTION(dispatch_semaphore_signal);
+ INTERCEPT_FUNCTION(dispatch_semaphore_wait);
+ INTERCEPT_FUNCTION(dispatch_group_wait);
+ INTERCEPT_FUNCTION(dispatch_group_leave);
+ INTERCEPT_FUNCTION(dispatch_group_async);
+ INTERCEPT_FUNCTION(dispatch_group_async_f);
+ INTERCEPT_FUNCTION(dispatch_group_notify);
+ INTERCEPT_FUNCTION(dispatch_group_notify_f);
+ INTERCEPT_FUNCTION(dispatch_source_set_event_handler);
+ INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);
+ INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);
+ INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);
+ INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);
+ INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);
+ INTERCEPT_FUNCTION(dispatch_apply);
+ INTERCEPT_FUNCTION(dispatch_apply_f);
+ INTERCEPT_FUNCTION(dispatch_data_create);
+ INTERCEPT_FUNCTION(dispatch_read);
+ INTERCEPT_FUNCTION(dispatch_write);
+ INTERCEPT_FUNCTION(dispatch_io_read);
+ INTERCEPT_FUNCTION(dispatch_io_write);
+ INTERCEPT_FUNCTION(dispatch_io_barrier);
+ INTERCEPT_FUNCTION(dispatch_io_create);
+ INTERCEPT_FUNCTION(dispatch_io_create_with_path);
+ INTERCEPT_FUNCTION(dispatch_io_create_with_io);
+ INTERCEPT_FUNCTION(dispatch_io_close);
+ INTERCEPT_FUNCTION(dispatch_resume);
+}
+
+} // namespace __tsan
More information about the llvm-commits
mailing list