[compiler-rt] r353401 - tsan: Optimize performance of Thread sanitizer memory access functions
Dmitry Vyukov via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 7 04:43:10 PST 2019
Author: dvyukov
Date: Thu Feb 7 04:43:09 2019
New Revision: 353401
URL: http://llvm.org/viewvc/llvm-project?rev=353401&view=rev
Log:
tsan: Optimize performance of Thread sanitizer memory access functions
- Manually unwind code in MemoryAccessImpl1() because clang do not optimize it
- Check for .rodata section only in read operations
- Place LIKELY/UNLIKELY on fast paths
This speeds up synthetic memory access benchmarks by 10-20%.
[dvyukov: fixed up consts in check_analyze.sh]
Author: yuri (Yuri Per)
Reviewed in: https://reviews.llvm.org/D57882
Context: https://reviews.llvm.org/D54889
Modified:
compiler-rt/trunk/lib/tsan/check_analyze.sh
compiler-rt/trunk/lib/tsan/rtl/tsan_interface_atomic.cc
compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cc
compiler-rt/trunk/lib/tsan/rtl/tsan_update_shadow_word_inl.h
Modified: compiler-rt/trunk/lib/tsan/check_analyze.sh
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/check_analyze.sh?rev=353401&r1=353400&r2=353401&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/check_analyze.sh (original)
+++ compiler-rt/trunk/lib/tsan/check_analyze.sh Thu Feb 7 04:43:09 2019
@@ -34,16 +34,22 @@ check() {
fi
}
-for f in write1 write2 write4 write8 read2 read4; do
+for f in write1 write2 write4 write8; do
check $f rsp 1
check $f push 1
- check $f pop 6
+ check $f pop 8
done
-for f in read1 read8; do
+for f in read1; do
check $f rsp 1
check $f push 2
- check $f pop 12
+ check $f pop 16
+done
+
+for f in read2 read4 read8; do
+ check $f rsp 1
+ check $f push 3
+ check $f pop 24
done
for f in func_entry func_exit; do
Modified: compiler-rt/trunk/lib/tsan/rtl/tsan_interface_atomic.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_interface_atomic.cc?rev=353401&r1=353400&r2=353401&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_interface_atomic.cc (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_interface_atomic.cc Thu Feb 7 04:43:09 2019
@@ -474,7 +474,7 @@ static morder convert_morder(morder mo)
#define SCOPED_ATOMIC(func, ...) \
ThreadState *const thr = cur_thread(); \
- if (thr->ignore_sync || thr->ignore_interceptors) { \
+ if (UNLIKELY(thr->ignore_sync || thr->ignore_interceptors)) { \
ProcessPendingSignals(thr); \
return NoTsanAtomic##func(__VA_ARGS__); \
} \
Modified: compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cc
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cc?rev=353401&r1=353400&r2=353401&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cc (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_rtl.cc Thu Feb 7 04:43:09 2019
@@ -641,6 +641,7 @@ void MemoryAccessImpl1(ThreadState *thr,
// __m128i _mm_move_epi64(__m128i*);
// _mm_storel_epi64(u64*, __m128i);
u64 store_word = cur.raw();
+ bool stored = false;
// scan all the shadow values and dispatch to 4 categories:
// same, replace, candidate and race (see comments below).
@@ -665,16 +666,28 @@ void MemoryAccessImpl1(ThreadState *thr,
int idx = 0;
#include "tsan_update_shadow_word_inl.h"
idx = 1;
+ if (stored) {
#include "tsan_update_shadow_word_inl.h"
+ } else {
+#include "tsan_update_shadow_word_inl.h"
+ }
idx = 2;
+ if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+ } else {
#include "tsan_update_shadow_word_inl.h"
+ }
idx = 3;
+ if (stored) {
+#include "tsan_update_shadow_word_inl.h"
+ } else {
#include "tsan_update_shadow_word_inl.h"
+ }
#endif
// we did not find any races and had already stored
// the current access info, so we are done
- if (LIKELY(store_word == 0))
+ if (LIKELY(stored))
return;
// choose a random candidate slot and replace it
StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word);
@@ -814,7 +827,7 @@ void MemoryAccess(ThreadState *thr, uptr
}
#endif
- if (!SANITIZER_GO && *shadow_mem == kShadowRodata) {
+ if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) {
// Access to .rodata section, no races here.
// Measurements show that it can be 10-20% of all memory accesses.
StatInc(thr, StatMop);
@@ -825,7 +838,7 @@ void MemoryAccess(ThreadState *thr, uptr
}
FastState fast_state = thr->fast_state;
- if (fast_state.GetIgnoreBit()) {
+ if (UNLIKELY(fast_state.GetIgnoreBit())) {
StatInc(thr, StatMop);
StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead);
StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog));
Modified: compiler-rt/trunk/lib/tsan/rtl/tsan_update_shadow_word_inl.h
URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/tsan/rtl/tsan_update_shadow_word_inl.h?rev=353401&r1=353400&r2=353401&view=diff
==============================================================================
--- compiler-rt/trunk/lib/tsan/rtl/tsan_update_shadow_word_inl.h (original)
+++ compiler-rt/trunk/lib/tsan/rtl/tsan_update_shadow_word_inl.h Thu Feb 7 04:43:09 2019
@@ -17,31 +17,35 @@ do {
const unsigned kAccessSize = 1 << kAccessSizeLog;
u64 *sp = &shadow_mem[idx];
old = LoadShadow(sp);
- if (old.IsZero()) {
+ if (LIKELY(old.IsZero())) {
StatInc(thr, StatShadowZero);
- if (store_word)
+ if (!stored) {
StoreIfNotYetStored(sp, &store_word);
- // The above StoreIfNotYetStored could be done unconditionally
- // and it even shows 4% gain on synthetic benchmarks (r4307).
+ stored = true;
+ }
break;
}
// is the memory access equal to the previous?
- if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
+ if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) {
StatInc(thr, StatShadowSameSize);
// same thread?
- if (Shadow::TidsAreEqual(old, cur)) {
+ if (LIKELY(Shadow::TidsAreEqual(old, cur))) {
StatInc(thr, StatShadowSameThread);
- if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+ if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) {
StoreIfNotYetStored(sp, &store_word);
+ stored = true;
+ }
break;
}
StatInc(thr, StatShadowAnotherThread);
if (HappensBefore(old, thr)) {
- if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
+ if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) {
StoreIfNotYetStored(sp, &store_word);
+ stored = true;
+ }
break;
}
- if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
+ if (LIKELY(old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)))
break;
goto RACE;
}
@@ -55,7 +59,7 @@ do {
StatInc(thr, StatShadowAnotherThread);
if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
break;
- if (HappensBefore(old, thr))
+ if (LIKELY(HappensBefore(old, thr)))
break;
goto RACE;
}
More information about the llvm-commits
mailing list