[compiler-rt] r270650 - Add working set base runtime library
Evgenii Stepanov via llvm-commits
llvm-commits at lists.llvm.org
Wed May 25 12:43:33 PDT 2016
Hi,
this test is failing on the sanitizer bot:
http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux/builds/24103/steps/test%20standalone%20compiler-rt/logs/stdio
You probably did not get the mail from the bot because there were
other build errors.
On Tue, May 24, 2016 at 7:04 PM, Derek Bruening via llvm-commits
<llvm-commits at lists.llvm.org> wrote:
> Author: bruening
> Date: Tue May 24 21:04:04 2016
> New Revision: 270650
>
> URL: http://llvm.org/viewvc/llvm-project?rev=270650&view=rev
> Log:
> Add working set base runtime library
>
> Summary:
> Adds the base runtime library for the working set tool.
> Adds slowpath code for updating the shadow memory.
>
> To be added in the future:
> + Scan memory and report the total size.
> + Take samples for intermediate values.
>
> Reviewers: aizatsky
>
> Subscribers: kubabrecka, vitalybuka, zhaoqin, kcc, eugenis, llvm-commits
>
> Differential Revision: http://reviews.llvm.org/D20485
>
> Added:
> compiler-rt/trunk/lib/esan/working_set.cpp (with props)
> compiler-rt/trunk/lib/esan/working_set.h (with props)
> compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp (with props)
> Modified:
> compiler-rt/trunk/lib/esan/CMakeLists.txt
> compiler-rt/trunk/lib/esan/esan.cpp
> compiler-rt/trunk/lib/esan/esan_interface_internal.h
> compiler-rt/trunk/test/esan/lit.cfg
>
> Modified: compiler-rt/trunk/lib/esan/CMakeLists.txt
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/esan/CMakeLists.txt?rev=270650&r1=270649&r2=270650&view=diff
> ==============================================================================
> --- compiler-rt/trunk/lib/esan/CMakeLists.txt (original)
> +++ compiler-rt/trunk/lib/esan/CMakeLists.txt Tue May 24 21:04:04 2016
> @@ -11,7 +11,8 @@ set(ESAN_SOURCES
> esan.cpp
> esan_flags.cpp
> esan_interface.cpp
> - esan_interceptors.cpp)
> + esan_interceptors.cpp
> + working_set.cpp)
>
> foreach (arch ${ESAN_SUPPORTED_ARCH})
> add_compiler_rt_runtime(clang_rt.esan
>
> Modified: compiler-rt/trunk/lib/esan/esan.cpp
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/esan/esan.cpp?rev=270650&r1=270649&r2=270650&view=diff
> ==============================================================================
> --- compiler-rt/trunk/lib/esan/esan.cpp (original)
> +++ compiler-rt/trunk/lib/esan/esan.cpp Tue May 24 21:04:04 2016
> @@ -19,6 +19,7 @@
> #include "sanitizer_common/sanitizer_common.h"
> #include "sanitizer_common/sanitizer_flag_parser.h"
> #include "sanitizer_common/sanitizer_flags.h"
> +#include "working_set.h"
>
> // See comment below.
> extern "C" {
> @@ -31,6 +32,15 @@ bool EsanIsInitialized;
> ToolType WhichTool;
> ShadowMapping Mapping;
>
> +// Different tools use different scales within the same shadow mapping scheme.
> +// The scale used here must match that used by the compiler instrumentation.
> +// This array is indexed by the ToolType enum.
> +static const uptr ShadowScale[] = {
> + 0, // ESAN_None.
> + 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2.
> + 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6.
> +};
> +
> // We are combining multiple performance tuning tools under the umbrella of
> // one EfficiencySanitizer super-tool. Most of our tools have very similar
> // memory access instrumentation, shadow memory mapping, libc interception,
> @@ -57,6 +67,8 @@ void processRangeAccess(uptr PC, uptr Ad
> if (WhichTool == ESAN_CacheFrag) {
> // TODO(bruening): add shadow mapping and update shadow bits here.
> // We'll move this to cache_frag.cpp once we have something.
> + } else if (WhichTool == ESAN_WorkingSet) {
> + processRangeAccessWorkingSet(PC, Addr, Size, IsWrite);
> }
> }
>
> @@ -113,10 +125,7 @@ static bool verifyShadowScheme() {
> static void initializeShadow() {
> DCHECK(verifyShadowScheme());
>
> - if (WhichTool == ESAN_CacheFrag)
> - Mapping.initialize(2); // 4B:1B, so 4 to 1 == >>2.
> - else
> - UNREACHABLE("unknown tool shadow mapping");
> + Mapping.initialize(ShadowScale[WhichTool]);
>
> VPrintf(1, "Shadow scale=%d offset=%p\n", Mapping.Scale, Mapping.Offset);
>
> @@ -157,7 +166,7 @@ void initializeLibrary(ToolType Tool) {
> ::__cxa_atexit((void (*)())finalizeLibrary);
>
> VPrintf(1, "in esan::%s\n", __FUNCTION__);
> - if (WhichTool != ESAN_CacheFrag) {
> + if (WhichTool <= ESAN_None || WhichTool >= ESAN_Max) {
> Printf("ERROR: unknown tool %d requested\n", WhichTool);
> Die();
> }
> @@ -165,6 +174,12 @@ void initializeLibrary(ToolType Tool) {
> initializeShadow();
> initializeInterceptors();
>
> + if (WhichTool == ESAN_CacheFrag) {
> + // FIXME: add runtime code for this tool
> + } else if (WhichTool == ESAN_WorkingSet) {
> + initializeWorkingSet();
> + }
> +
> EsanIsInitialized = true;
> }
>
> @@ -175,6 +190,8 @@ int finalizeLibrary() {
> // strategy for how to generate a final report.
> // We'll move this to cache_frag.cpp once we have something.
> Report("%s is not finished: nothing yet to report\n", SanitizerToolName);
> + } else if (WhichTool == ESAN_WorkingSet) {
> + return finalizeWorkingSet();
> }
> return 0;
> }
>
> Modified: compiler-rt/trunk/lib/esan/esan_interface_internal.h
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/esan/esan_interface_internal.h?rev=270650&r1=270649&r2=270650&view=diff
> ==============================================================================
> --- compiler-rt/trunk/lib/esan/esan_interface_internal.h (original)
> +++ compiler-rt/trunk/lib/esan/esan_interface_internal.h Tue May 24 21:04:04 2016
> @@ -28,6 +28,8 @@ extern "C" {
> typedef enum Type : u32 {
> ESAN_None = 0,
> ESAN_CacheFrag,
> + ESAN_WorkingSet,
> + ESAN_Max,
> } ToolType;
>
> // This function should be called at the very beginning of the process,
>
> Added: compiler-rt/trunk/lib/esan/working_set.cpp
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/esan/working_set.cpp?rev=270650&view=auto
> ==============================================================================
> --- compiler-rt/trunk/lib/esan/working_set.cpp (added)
> +++ compiler-rt/trunk/lib/esan/working_set.cpp Tue May 24 21:04:04 2016
> @@ -0,0 +1,90 @@
> +//===-- working_set.cpp ---------------------------------------------------===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +//
> +// This file is a part of EfficiencySanitizer, a family of performance tuners.
> +//
> +// This file contains working-set-specific code.
> +//===----------------------------------------------------------------------===//
> +
> +#include "working_set.h"
> +#include "esan.h"
> +#include "esan_flags.h"
> +#include "esan_shadow.h"
> +
> +// We shadow every cache line of app memory with one shadow byte.
> +// - The highest bit of each shadow byte indicates whether the corresponding
> +// cache line has ever been accessed.
> +// - The lowest bit of each shadow byte indicates whether the corresponding
> +// cache line was accessed since the last sample.
> +// - The other bits can be used either for a single working set snapshot
> +// between two consecutive samples, or an aggregate working set snapshot
> +// over multiple sample periods (future work).
> +// We live with races in accessing each shadow byte.
> +typedef unsigned char byte;
> +
> +namespace __esan {
> +
> +// See the shadow byte layout description above.
> +static const u32 TotalWorkingSetBitIdx = 7;
> +static const u32 CurWorkingSetBitIdx = 0;
> +static const byte ShadowAccessedVal =
> + (1 << TotalWorkingSetBitIdx) | (1 << CurWorkingSetBitIdx);
> +
> +void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size,
> + bool IsWrite) {
> + if (Size == 0)
> + return;
> + SIZE_T I = 0;
> + uptr LineSize = getFlags()->cache_line_size;
> + // As Addr+Size could overflow at the top of a 32-bit address space,
> + // we avoid the simpler formula that rounds the start and end.
> + SIZE_T NumLines = Size / LineSize +
> + // Add any extra at the start or end adding on an extra line:
> + (LineSize - 1 + Addr % LineSize + Size % LineSize) / LineSize;
> + byte *Shadow = (byte *)appToShadow(Addr);
> + // Write shadow bytes until we're word-aligned.
> + while (I < NumLines && (uptr)Shadow % 4 != 0) {
> + if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal)
> + *Shadow |= ShadowAccessedVal;
> + ++Shadow;
> + ++I;
> + }
> + // Write whole shadow words at a time.
> + // Using a word-stride loop improves the runtime of a microbenchmark of
> + // memset calls by 10%.
> + u32 WordValue = ShadowAccessedVal | ShadowAccessedVal << 8 |
> + ShadowAccessedVal << 16 | ShadowAccessedVal << 24;
> + while (I + 4 <= NumLines) {
> + if ((*(u32*)Shadow & WordValue) != WordValue)
> + *(u32*)Shadow |= WordValue;
> + Shadow += 4;
> + I += 4;
> + }
> + // Write any trailing shadow bytes.
> + while (I < NumLines) {
> + if ((*Shadow & ShadowAccessedVal) != ShadowAccessedVal)
> + *Shadow |= ShadowAccessedVal;
> + ++Shadow;
> + ++I;
> + }
> +}
> +
> +void initializeWorkingSet() {
> + // The shadow mapping assumes 64 so this cannot be changed.
> + CHECK(getFlags()->cache_line_size == 64);
> +}
> +
> +int finalizeWorkingSet() {
> + // FIXME NYI: we need to add memory scanning to report the total lines
> + // touched, and later add sampling to get intermediate values.
> + Report("%s is not finished: nothing yet to report\n", SanitizerToolName);
> + return 0;
> +}
> +
> +} // namespace __esan
>
> Propchange: compiler-rt/trunk/lib/esan/working_set.cpp
> ------------------------------------------------------------------------------
> svn:eol-style = LF
>
> Added: compiler-rt/trunk/lib/esan/working_set.h
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/lib/esan/working_set.h?rev=270650&view=auto
> ==============================================================================
> --- compiler-rt/trunk/lib/esan/working_set.h (added)
> +++ compiler-rt/trunk/lib/esan/working_set.h Tue May 24 21:04:04 2016
> @@ -0,0 +1,30 @@
> +//===-- working_set.h -------------------------------------------*- C++ -*-===//
> +//
> +// The LLVM Compiler Infrastructure
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +//
> +// This file is a part of EfficiencySanitizer, a family of performance tuners.
> +//
> +// Header for working-set-specific code.
> +//===----------------------------------------------------------------------===//
> +
> +#ifndef WORKING_SET_H
> +#define WORKING_SET_H
> +
> +#include "interception/interception.h"
> +#include "sanitizer_common/sanitizer_internal_defs.h"
> +
> +namespace __esan {
> +
> +void initializeWorkingSet();
> +int finalizeWorkingSet();
> +void processRangeAccessWorkingSet(uptr PC, uptr Addr, SIZE_T Size,
> + bool IsWrite);
> +
> +} // namespace __esan
> +
> +#endif // WORKING_SET_H
>
> Propchange: compiler-rt/trunk/lib/esan/working_set.h
> ------------------------------------------------------------------------------
> svn:eol-style = LF
>
> Added: compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp?rev=270650&view=auto
> ==============================================================================
> --- compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp (added)
> +++ compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp Tue May 24 21:04:04 2016
> @@ -0,0 +1,21 @@
> +// RUN: %clang_esan_wset -O0 %s -o %t 2>&1
> +// RUN: %run %t 2>&1 | FileCheck %s
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <sys/mman.h>
> +#include <assert.h>
> +#include <string.h>
> +
> +int main(int argc, char **argv) {
> + const int size = 128*1024*1024;
> + char *p = (char *)mmap(0, size, PROT_READ | PROT_WRITE,
> + MAP_ANON | MAP_PRIVATE, -1, 0);
> + // Test the slowpath at different cache line boundaries.
> + for (int i = 0; i < 630; i++)
> + memset((char *)p + 63*i, i, 63*i);
> + munmap(p, size);
> + return 0;
> + // FIXME: once the memory scan and size report is in place add it here.
> + // CHECK: {{.*}}EfficiencySanitizer is not finished: nothing yet to report
> +}
>
> Propchange: compiler-rt/trunk/test/esan/TestCases/workingset-memset.cpp
> ------------------------------------------------------------------------------
> svn:eol-style = LF
>
> Modified: compiler-rt/trunk/test/esan/lit.cfg
> URL: http://llvm.org/viewvc/llvm-project/compiler-rt/trunk/test/esan/lit.cfg?rev=270650&r1=270649&r2=270650&view=diff
> ==============================================================================
> --- compiler-rt/trunk/test/esan/lit.cfg (original)
> +++ compiler-rt/trunk/test/esan/lit.cfg Tue May 24 21:04:04 2016
> @@ -13,12 +13,15 @@ base_cflags = ([config.target_cflags] +
> base_cxxflags = config.cxx_mode_flags + base_cflags
>
> frag_cflags = (["-fsanitize=efficiency-cache-frag"] + base_cflags)
> +wset_cflags = (["-fsanitize=efficiency-working-set"] + base_cflags)
>
> def build_invocation(compile_flags):
> return " " + " ".join([config.clang] + compile_flags) + " "
>
> config.substitutions.append( ("%clang_esan_frag ",
> build_invocation(frag_cflags)) )
> +config.substitutions.append( ("%clang_esan_wset ",
> + build_invocation(wset_cflags)) )
>
> default_esan_opts = ''
> config.substitutions.append(('%env_esan_opts=',
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list