[test-suite] r260357 - lit: Add module to produce and merge profile data for benchmarks compiled with -fprofile-instr-generate

[Matthias Braun via llvm-commits]

Author: matze
Date: Tue Feb  9 21:56:41 2016
New Revision: 260357

URL: http://llvm.org/viewvc/llvm-project?rev=260357&view=rev
Log:
lit: Add module to produce and merge profile data for benchmarks compiled with -fprofile-instr-generate

Added:
    test-suite/trunk/litsupport/profilegen.py
Modified:
    test-suite/trunk/CMakeLists.txt
    test-suite/trunk/RunSafely.sh
    test-suite/trunk/lit.site.cfg.in
    test-suite/trunk/litsupport/test.py

Modified: test-suite/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/CMakeLists.txt?rev=260357&r1=260356&r2=260357&view=diff
==============================================================================
--- test-suite/trunk/CMakeLists.txt (original)
+++ test-suite/trunk/CMakeLists.txt Tue Feb  9 21:56:41 2016
@@ -50,6 +50,20 @@ set(TEST_SUITE_RUN_UNDER "" CACHE STRING
 set(TEST_SUITE_RUN_TYPE "train" CACHE STRING
     "Type of benchmark inputs (may be test,train or ref)")
 
+# Enable profile generate mode in lit. Note that this does not automatically
+# add something like -fprofile-instr-generate to the compiler flags.
+set(TEST_SUITE_PROFILE_GENERATE "FALSE" CACHE BOOL
+    "Enable lit profile generate mode")
+set(TEST_SUITE_LLVM_PROFDATA "llvm-profdata" CACHE STRING
+    "llvm-profdata executable")
+mark_as_advanced(TEST_SUITE_LLVM_PROFDATA)
+# Set value to python style True/False
+if(TEST_SUITE_PROFILE_GENERATE)
+  set(TEST_SUITE_PROFILE_GENERATE "True")
+else()
+  set(TEST_SUITE_PROFILE_GENERATE "False")
+endif()
+
 include(MakefileFunctions)
 include(SingleMultiSource)
 find_package(YACC)

Modified: test-suite/trunk/RunSafely.sh
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/RunSafely.sh?rev=260357&r1=260356&r2=260357&view=diff
==============================================================================
--- test-suite/trunk/RunSafely.sh (original)
+++ test-suite/trunk/RunSafely.sh Tue Feb  9 21:56:41 2016
@@ -199,10 +199,16 @@ else
   rm -f "${OUTFILE}.time${REMOTE_SUFFIX}" "${OUTFILE}${REMOTE_SUFFIX}"
   rm -f "${STDOUT_FILE}${REMOTE_SUFFIX}" "${STDERR_FILE}${REMOTE_SUFFIX}"
 
+  # Pass LLVM_PROFILE_FILE environment variable along
+  PREFIX=""
+  if [ "${LLVM_PROFILE_FILE}" != "" ]; then
+     PREFIX="LLVM_PROFILE_FILE=\"$LLVM_PROFILE_FILE\" "
+  fi
+
   # Create .command script
   COMMANDFILE="${OUTFILE}.command"
   rm -f "${COMMANDFILE}"
-  echo "$TIMEIT $TIMEITFLAGS $COMMAND" > "${COMMANDFILE}"
+  echo "${PREFIX}${TIMEIT} $TIMEITFLAGS $COMMAND" > "${COMMANDFILE}"
   chmod +x "${COMMANDFILE}"
 
   ( $RCLIENT $RFLAGS $RHOST "ls ${COMMANDFILE}" ) > /dev/null 2>&1

Modified: test-suite/trunk/lit.site.cfg.in
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/lit.site.cfg.in?rev=260357&r1=260356&r2=260357&view=diff
==============================================================================
--- test-suite/trunk/lit.site.cfg.in (original)
+++ test-suite/trunk/lit.site.cfg.in Tue Feb  9 21:56:41 2016
@@ -7,5 +7,7 @@ config.remote_host = "@TEST_SUITE_REMOTE
 config.remote_user = "@TEST_SUITE_REMOTE_USER@"
 config.remote_port = "@TEST_SUITE_REMOTE_PORT@"
 config.run_under = "@TEST_SUITE_RUN_UNDER@"
+config.profile_generate = @TEST_SUITE_PROFILE_GENERATE@
+config.llvm_profdata = "@TEST_SUITE_LLVM_PROFDATA@"
 
 lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/lit.cfg")

Added: test-suite/trunk/litsupport/profilegen.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/profilegen.py?rev=260357&view=auto
==============================================================================
--- test-suite/trunk/litsupport/profilegen.py (added)
+++ test-suite/trunk/litsupport/profilegen.py Tue Feb  9 21:56:41 2016
@@ -0,0 +1,29 @@
+import shellcommand
+
+
+def wrapScript(context, script):
+    """Adjust runscript to set a different value to the LLVM_PROFILE_FILE
+    environment variable for each execution."""
+    i = 0
+    adjusted_script = []
+    context.profilefiles = []
+    for line in script:
+        number = ""
+        if len(script) > 1:
+            number = "-%s" % (i,)
+            i += 1
+        profilefile = "%s%s.profraw" % (context.tmpBase, number)
+        prefix = "LLVM_PROFILE_FILE=%s " % quote(profilefile)
+        context.profilefiles.append(profilefile)
+        adjusted_script.append(prefix + line)
+    return adjusted_script
+
+
+def getMergeProfilesScript(context):
+    executable = shellcommand.getMainExecutable(context)
+    if not executable:
+        return None
+    profdatafile = executable + ".profdata"
+    mergecmd = [config.llvm_profdata, 'merge', '-output=%s' % profdatafile]
+    mergecmd += context.profilefiles
+    return [mergecmd]

Modified: test-suite/trunk/litsupport/test.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/test.py?rev=260357&r1=260356&r2=260357&view=diff
==============================================================================
--- test-suite/trunk/litsupport/test.py (original)
+++ test-suite/trunk/litsupport/test.py Tue Feb  9 21:56:41 2016
@@ -7,10 +7,12 @@ from lit.TestRunner import executeScript
     applySubstitutions, getTempPaths
 from lit import Test
 from lit.util import to_bytes, to_string
-import testscript
-import runsafely
-import perf
+
 import compiletime
+import perf
+import profilegen
+import runsafely
+import testscript
 import timeit
 
 
@@ -68,6 +70,9 @@ class TestSuiteTest(FileBasedTest):
 
         runscript = runsafely.wrapScript(context, runscript, suffix=".out")
 
+        if config.profile_generate:
+            runscript = profilegen.wrapScript(context, runscript)
+
         # Create the output directory if it does not already exist.
         lit.util.mkdir_p(os.path.dirname(tmpBase))
 
@@ -101,6 +106,11 @@ class TestSuiteTest(FileBasedTest):
                                                  suffix=".perf.out")
             runScript(context, context.profilescript) # ignore result
 
+        # Merge llvm profile data
+        if config.profile_generate:
+            mergescript = profilegen.getMergeProfilesScript(context)
+            runScript(context, mergescript) # ignore result
+
         # Run verification script (the "VERIFY:" part)
         if len(verifyscript) > 0:
             res = runScript(context, verifyscript)