[test-suite] r261854 - [cmake] Add support for timeit.sh (--use-perf) and --benchmarking-only.

[James Molloy via llvm-commits]

Hi Matthias,

The TEST_SUITE_USE_PERF part of the patch is adding functionality equivalent to USE_PERF=1 in the Makefile system. In this mode, we use the “perf” linux tool to do the timing. This hooks into the kernel and is much more accurate on Linux systems than the gettimeofday() used in timeit.c - orders of magnitude more accurate in fact.

The “timeit.sh” script was designed to act like timeit.c, so yes, extra features to timeit should probably be added to timeit.sh too.

That said, perhaps it’s possible/worthwhile to teach timeit.c how to drive perf itself (should be simply prefixing the exec’d binary and a different output grep regex), so we don’t have to deal with two drivers?

Cheers,

James


> On 25 Feb 2016, at 21:16, Matthias Braun <mbraun at apple.com> wrote:
> 
> Hey James,
> 
> I'd be interested to learn a bit more about the use case for the TEST_SUITE_USE_PERF part of this patch. Why do we have a 2nd version of timeit and in which situations would you want to use it?
> In fact I already introduced a few small additions to timeit.c that are currently only used the External/* benchmarks, do I need to extend timeit.sh in the future as well or can we find a way to only use time measurement system?
> 
> - Matthias
> 
>> On Feb 25, 2016, at 2:46 AM, James Molloy via llvm-commits <llvm-commits at lists.llvm.org> wrote:
>> 
>> Author: jamesm
>> Date: Thu Feb 25 04:46:33 2016
>> New Revision: 261854
>> 
>> URL: http://llvm.org/viewvc/llvm-project?rev=261854&view=rev
>> Log:
>> [cmake] Add support for timeit.sh (--use-perf) and --benchmarking-only.
>> 
>> Modified:
>>   test-suite/trunk/CMakeLists.txt
>>   test-suite/trunk/cmake/modules/SingleMultiSource.cmake
>>   test-suite/trunk/tools/CMakeLists.txt
>> 
>> Modified: test-suite/trunk/CMakeLists.txt
>> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/CMakeLists.txt?rev=261854&r1=261853&r2=261854&view=diff
>> ==============================================================================
>> --- test-suite/trunk/CMakeLists.txt (original)
>> +++ test-suite/trunk/CMakeLists.txt Thu Feb 25 04:46:33 2016
>> @@ -128,6 +128,14 @@ set(FPCMP ${CMAKE_BINARY_DIR}/tools/fpcm
>> set(CMAKE_C_COMPILE_OBJECT "${CMAKE_BINARY_DIR}/tools/timeit --summary <OBJECT>.time ${CMAKE_C_COMPILE_OBJECT}")
>> set(CMAKE_CXX_COMPILE_OBJECT "${CMAKE_BINARY_DIR}/tools/timeit --summary <OBJECT>.time ${CMAKE_CXX_COMPILE_OBJECT}")
>> 
>> +set(TEST_SUITE_BENCHMARKING_ONLY "OFF" CACHE BOOL
>> +  "Only run the benchmarking only subset")
>> +if(TEST_SUITE_BENCHMARKING_ONLY)
>> +  set(BENCHMARKING_ONLY "ON")
>> +else()
>> +  set(BENCHMARKING_ONLY)
>> +endif()
>> +
>> add_subdirectory(SingleSource)
>> add_subdirectory(MultiSource)
>> add_subdirectory(External)
>> 
>> Modified: test-suite/trunk/cmake/modules/SingleMultiSource.cmake
>> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/cmake/modules/SingleMultiSource.cmake?rev=261854&r1=261853&r2=261854&view=diff
>> ==============================================================================
>> --- test-suite/trunk/cmake/modules/SingleMultiSource.cmake (original)
>> +++ test-suite/trunk/cmake/modules/SingleMultiSource.cmake Thu Feb 25 04:46:33 2016
>> @@ -152,7 +152,10 @@ macro(test_suite_add_executable name mai
>>    endif()
>> 
>>    llvm_add_test(${name} ${source_exename})
>> -    add_dependencies(${source_exename} timeit-host timeit-target fpcmp-host)
>> +    if (NOT TEST_SUITE_USE_PERF)
>> +      add_dependencies(${source_exename} timeit-target)
>> +    endif()
>> +    add_dependencies(${source_exename} timeit-host fpcmp-host)
>>  endif()
>> endmacro()
>> 
>> 
>> Modified: test-suite/trunk/tools/CMakeLists.txt
>> URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/tools/CMakeLists.txt?rev=261854&r1=261853&r2=261854&view=diff
>> ==============================================================================
>> --- test-suite/trunk/tools/CMakeLists.txt (original)
>> +++ test-suite/trunk/tools/CMakeLists.txt Thu Feb 25 04:46:33 2016
>> @@ -1,7 +1,3 @@
>> -add_executable(timeit-target ${CMAKE_CURRENT_SOURCE_DIR}/timeit.c)
>> -
>> -# FIXME: Replicate Makefile.tools's logic for determining whether to use fpcmp/fpcmp.sh
>> -
>> # Note that we have to compile fpcmp and timeit for the host machine even when
>> # cross compiling to a different target. We use custom rules doing "cc file.c"
>> # as an ad-hoc solution for now. A proper solution would probably be based on
>> @@ -19,4 +15,15 @@ macro(llvm_add_host_executable targetnam
>> endmacro()
>> 
>> llvm_add_host_executable(fpcmp-host fpcmp fpcmp.c)
>> -llvm_add_host_executable(timeit-host timeit timeit.c)
>> +
>> +set(TEST_SUITE_USE_PERF "OFF" CACHE BOOL "Use perf (timeit.sh) instead of timeit.c")
>> +if (${TEST_SUITE_USE_PERF})
>> +  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/timeit
>> +    COMMAND cp ${CMAKE_CURRENT_SOURCE_DIR}/timeit.sh ${CMAKE_CURRENT_BINARY_DIR}/timeit
>> +    COMMAND chmod +x ${CMAKE_CURRENT_BINARY_DIR}/timeit
>> +    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/timeit.sh)
>> +  add_custom_target(timeit-host DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/timeit)
>> +else()
>> +  add_executable(timeit-target ${CMAKE_CURRENT_SOURCE_DIR}/timeit.c)
>> +  llvm_add_host_executable(timeit-host timeit timeit.c)
>> +endif()
>> 
>> 
>> _______________________________________________
>> llvm-commits mailing list
>> llvm-commits at lists.llvm.org
>> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
>