[test-suite] r283649 - litsupport: Introduce stats module

[Matthias Braun via llvm-commits]

Author: matze
Date: Fri Oct  7 22:28:18 2016
New Revision: 283649

URL: http://llvm.org/viewvc/llvm-project?rev=283649&view=rev
Log:
litsupport: Introduce stats module

This adds a new litsupport module and the -DTEST_SUITE_COLLECT_STATS=1
option to cmake. Uses the -save-stats=obj switch (available in new clang
builds) to collect llvm statistics and outputs the sum of them along
with the lit results.

Added:
    test-suite/trunk/litsupport/stats.py
Modified:
    test-suite/trunk/CMakeLists.txt

Modified: test-suite/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/CMakeLists.txt?rev=283649&r1=283648&r2=283649&view=diff
==============================================================================
--- test-suite/trunk/CMakeLists.txt (original)
+++ test-suite/trunk/CMakeLists.txt Fri Oct  7 22:28:18 2016
@@ -162,6 +162,12 @@ endif()
 set(TEST_SUITE_BENCHMARKING_ONLY "OFF" CACHE BOOL
   "Only run the benchmarking only subset")
 
+set(TEST_SUITE_COLLECT_STATS "FALSE" CACHE BOOL
+    "Collect LLVM statistics")
+if(TEST_SUITE_COLLECT_STATS)
+  list(APPEND CFLAGS -save-stats=obj)
+  list(APPEND CXXFLAGS -save-stats=obj)
+endif()
 
 # Detect and include subdirectories
 # This allows to: Place additional test-suites into the toplevel test-suite
@@ -209,6 +215,9 @@ endif()
 if(TEST_SUITE_REMOTE_HOST)
   list(APPEND LIT_MODULES remote)
 endif()
+if(TEST_SUITE_COLLECT_STATS)
+  list(APPEND LIT_MODULES stats)
+endif()
 
 # Produce lit.site.cfg
 configure_file("${PROJECT_SOURCE_DIR}/lit.site.cfg.in" "${CMAKE_BINARY_DIR}/lit.site.cfg")

Added: test-suite/trunk/litsupport/stats.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/stats.py?rev=283649&view=auto
==============================================================================
--- test-suite/trunk/litsupport/stats.py (added)
+++ test-suite/trunk/litsupport/stats.py Fri Oct  7 22:28:18 2016
@@ -0,0 +1,47 @@
+"""test-suite/lit plugin to collect internal llvm json statistics.
+
+This assumes the benchmarks were built with the -save-stats=obj flag."""
+import json
+import lit.Test
+import logging
+import os
+from collections import defaultdict
+
+
+def _mergeStats(global_stats, statsfilename):
+    try:
+        f = open(statsfilename, "rt")
+        stats = json.load(f)
+    except Exception as e:
+        logging.warning("Could not read '%s'", statsfilename, exc_info=e)
+        return
+    for name, value in stats.iteritems():
+        global_stats[name] += value
+
+
+def _getStats(context):
+    # We compile multiple benchmarks in the same directory in SingleSource
+    # mode. Only look at compiletime files starting with the name of our test.
+    prefix = ""
+    if context.config.single_source:
+        prefix = "%s." % os.path.basename(context.executable)
+
+    stats = defaultdict(lambda: 0.0)
+    dir = os.path.dirname(context.test.getFilePath())
+    for path, subdirs, files in os.walk(dir):
+        for file in files:
+            if file.endswith('.stats') and file.startswith(prefix):
+                fullpath = os.path.join(path, file)
+                _mergeStats(stats, fullpath)
+
+    if len(stats) == 0:
+        logging.warning("No stats for '%s'", context.test.getFullName())
+
+    result = dict()
+    for key, value in stats.iteritems():
+        result[key] = lit.Test.toMetricValue(value)
+    return result
+
+
+def mutatePlan(context, plan):
+    plan.metric_collectors.append(_getStats)