[vmkit-commits] [vmkit] r180459 - Testing and benchmarking framework now supports batch running of Dacapo multiple times in parallel, synthesis of tests results and plotting of benchmarks data.
Peter Senna Tschudin
peter.senna at gmail.com
Thu Apr 25 10:13:46 PDT 2013
Author: peter.senna
Date: Thu Apr 25 12:12:09 2013
New Revision: 180459
URL: http://llvm.org/viewvc/llvm-project?rev=180459&view=rev
Log:
Testing and benchmarking framework now supports batch running of Dacapo multiple times in parallel, synthesis of tests results and plotting of benchmarks data.
(cherry picked from commit 09bc7111a6aa6f305658523ff9a09347ccc94fe6)
Added:
vmkit/trunk/tests/dacapo/augment-dacapo-data.sh
vmkit/trunk/tests/dacapo/dacapo-incinerator-off-on.plot
vmkit/trunk/tests/dacapo/reduce-logs.sh
vmkit/trunk/tests/dacapo/stats-incinerator-dacapo-2006-10-MR2.csv
vmkit/trunk/tests/dacapo/stats-j3-dacapo-2006-10-MR2.csv
vmkit/trunk/tests/dacapo/synth-dacapo-data-min-max-avg.awk
vmkit/trunk/tests/dacapo/synth-dacapo-data-std-dev.awk
vmkit/trunk/tests/dacapo/synth-logs.sh
Removed:
vmkit/trunk/tests/dacapo/collect-log-data.sh
Modified:
vmkit/trunk/tests/dacapo/run-all.sh
vmkit/trunk/tests/dacapo/run-bench.sh
Added: vmkit/trunk/tests/dacapo/augment-dacapo-data.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/augment-dacapo-data.sh?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/augment-dacapo-data.sh (added)
+++ vmkit/trunk/tests/dacapo/augment-dacapo-data.sh Thu Apr 25 12:12:09 2013
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Augment input data with statistical data about them: min, max, average, standard deviation
+
+# Augment statistical data with min, max and average
+data=$(awk -f synth-dacapo-data-min-max-avg.awk)
+header=$(echo "$data" | head -1)
+
+# Integrate augmented information into given input
+data=$(echo "$data" | tail -n $(( $(echo "$data" | wc -l) - 1)) )
+added_data=$(echo "$data" | grep -v ',-,-,-,-')
+
+augmented_data=$header
+
+for i in $added_data ; do
+ key=$(echo "$i" | cut -d ',' -f 1-3)
+ added_value=$(echo "$i" | cut -d ',' -f 5-)
+ augmented_data+=$'\n'
+ augmented_data+=$(echo "$data" | grep "^$key,[^-]" | sed "s/-,-,-,-/$added_value/g")
+done
+
+# Augment statistical data with standard deviation
+data=$(echo "$augmented_data" | awk -f synth-dacapo-data-std-dev.awk)
+
+# Integrate augmented information into given input
+data=$(echo "$data" | tail -n $(( $(echo "$data" | wc -l) - 1)) )
+added_data=$(echo "$data" | grep -v ',-$')
+
+augmented_data=$header
+
+for i in $added_data ; do
+ key=$(echo "$i" | cut -d ',' -f 1-3)
+ added_value=$(echo "$i" | cut -d ',' -f 8)
+
+ augmented_data+=$'\n'
+ augmented_data+=$(echo "$data" | grep "^$key,[^-]" | sed "s/,-/,$added_value/g")
+done
+
+echo "$augmented_data"
Removed: vmkit/trunk/tests/dacapo/collect-log-data.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/collect-log-data.sh?rev=180458&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/collect-log-data.sh (original)
+++ vmkit/trunk/tests/dacapo/collect-log-data.sh (removed)
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-dacapo_dir=/home/koutheir/PhD/VMKit/vmkit2/tests/dacapo
-log_collect_file=$dacapo_dir/logs/collected_passed.log
-log_synth_file=$dacapo_dir/logs/synth.log
-
-synth_awk=$(cat <<EOF
-BEGIN {
- FS = ",";
- getline;
-
- bench_version = \$1;
- bench_name = \$2;
- vm_name = \$3;
- val = \$4;
- nval = 1;
-}
-/^\$/ {}
-{
- if (\$1 == bench_version && \$2 == bench_name && \$3 == vm_name) {
- val += \$4;
- nval++;
- } else {
- val /= nval;
- print bench_version FS bench_name FS vm_name FS int(val);
-
- bench_version = \$1;
- bench_name = \$2;
- vm_name = \$3;
- val = \$4;
- nval = 1;
- }
-}
-END {
- val /= nval;
- print bench_version FS bench_name FS vm_name FS val;
-}
-EOF
-)
-
-cd "$dacapo_dir/logs"
-grep '^===== DaCapo [A-Za-z]\+ PASSED' tmp.* | sed 's/tmp[^_]\+_//g' | sed 's/ msec =====//g' | sed 's/\.log:===== DaCapo [a-z]\+ PASSED in /_/g' | tr '_' ',' | sort > $log_collect_file
-awk -e "$synth_awk" "$log_collect_file" > $log_synth_file
Added: vmkit/trunk/tests/dacapo/dacapo-incinerator-off-on.plot
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/dacapo-incinerator-off-on.plot?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/dacapo-incinerator-off-on.plot (added)
+++ vmkit/trunk/tests/dacapo/dacapo-incinerator-off-on.plot Thu Apr 25 12:12:09 2013
@@ -0,0 +1,35 @@
+#!/usr/bin/gnuplot -p
+
+set datafile separator ","
+set datafile missing '-'
+set datafile commentschars "#"
+
+set ylabel "Duration (lower is better)"
+set ytics border nomirror
+set grid ytics
+set yrange [0:*]
+set ydata time
+set timefmt "%S"
+set format y "%M:%S"
+
+set xtics axis nomirror rotate by -20
+set border 3
+
+set style data histogram
+set style histogram errorbars gap 1 lw 1
+set bars fullwidth
+set style fill solid border 0
+set boxwidth 1
+
+set title "Incinerator overhead in Dacapo benchmarks"
+
+set term wxt 0
+plot \
+ 'stats-j3-dacapo-2006-10-MR2.csv' using (column(4)/1000.0):(column(7)/1000.0):xtic(2) linecolor rgb "blue" title "J3", \
+ 'stats-incinerator-dacapo-2006-10-MR2.csv' using (column(4)/1000.0):(column(7)/1000.0):xtic(2) linecolor rgb "red" title "Incinerator"
+
+#set term latex
+#set term postscript clip 12
+#plot \
+# 'stats-j3-dacapo-2006-10-MR2.csv' using (column(4)/1000.0):(column(7)/1000.0):xtic(2) linecolor rgb "blue" title "J3", \
+# 'stats-incinerator-dacapo-2006-10-MR2.csv' using (column(4)/1000.0):(column(7)/1000.0):xtic(2) linecolor rgb "red" title "Incinerator"
Added: vmkit/trunk/tests/dacapo/reduce-logs.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/reduce-logs.sh?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/reduce-logs.sh (added)
+++ vmkit/trunk/tests/dacapo/reduce-logs.sh Thu Apr 25 12:12:09 2013
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Removes elementary data and only leave satistical data (min, max, average, standard deviation)
+
+log_dir=$1
+
+./synth-logs.sh "$log_dir" | sed 's/^\([^,]*,[^,]*,[^,]*\),[^,]*/\1/g' | uniq
Modified: vmkit/trunk/tests/dacapo/run-all.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/run-all.sh?rev=180459&r1=180458&r2=180459&view=diff
==============================================================================
--- vmkit/trunk/tests/dacapo/run-all.sh (original)
+++ vmkit/trunk/tests/dacapo/run-all.sh Thu Apr 25 12:12:09 2013
@@ -1,5 +1,6 @@
#!/bin/bash
+# Run all Dacapo tests muliples times, logging their results
# ./run-all.sh 40 12 2006-10-MR2 'sleep $(($RANDOM / 7276))s ; echo Hello'
times=$1
Modified: vmkit/trunk/tests/dacapo/run-bench.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/run-bench.sh?rev=180459&r1=180458&r2=180459&view=diff
==============================================================================
--- vmkit/trunk/tests/dacapo/run-bench.sh (original)
+++ vmkit/trunk/tests/dacapo/run-bench.sh Thu Apr 25 12:12:09 2013
@@ -1,5 +1,7 @@
#!/bin/bash
+# Run a Dacapo benchmark, logging its results
+
enable=$1
if [ "$enable" -ne "1" ]; then exit 0; fi
Added: vmkit/trunk/tests/dacapo/stats-incinerator-dacapo-2006-10-MR2.csv
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/stats-incinerator-dacapo-2006-10-MR2.csv?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/stats-incinerator-dacapo-2006-10-MR2.csv (added)
+++ vmkit/trunk/tests/dacapo/stats-incinerator-dacapo-2006-10-MR2.csv Thu Apr 25 12:12:09 2013
@@ -0,0 +1,12 @@
+#bench_suite,benchmark,vm,avg_duration_ms,min_duration_ms,max_duration_ms,std_dev_duration_ms
+2006-10-MR2,antlr,j3,19652.10,17615.00,27332.00,2248.77
+2006-10-MR2,bloat,j3,43040.47,38777.00,46606.00,1503.30
+2006-10-MR2,chart,j3,121454.95,90114.00,137105.00,11513.67
+2006-10-MR2,eclipse,j3,158202.62,136726.00,182058.00,8597.38
+2006-10-MR2,fop,j3,103006.12,78002.00,114097.00,8931.45
+2006-10-MR2,hsqldb,j3,31539.78,26758.00,34946.00,1880.57
+2006-10-MR2,jython,j3,57339.60,52566.00,59797.00,1720.84
+2006-10-MR2,luindex,j3,37254.55,30240.00,49588.00,4037.17
+2006-10-MR2,lusearch,j3,18871.15,12641.00,24093.00,3315.06
+2006-10-MR2,pmd,j3,46988.35,37216.00,62281.00,5648.83
+2006-10-MR2,xalan,j3,17236.15,12539.00,19815.00,1893.30
Added: vmkit/trunk/tests/dacapo/stats-j3-dacapo-2006-10-MR2.csv
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/stats-j3-dacapo-2006-10-MR2.csv?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/stats-j3-dacapo-2006-10-MR2.csv (added)
+++ vmkit/trunk/tests/dacapo/stats-j3-dacapo-2006-10-MR2.csv Thu Apr 25 12:12:09 2013
@@ -0,0 +1,12 @@
+#bench_suite,benchmark,vm,avg_duration_ms,min_duration_ms,max_duration_ms,std_dev_duration_ms
+2006-10-MR2,antlr,j3,19735.97,17593.00,30859.00,2801.75
+2006-10-MR2,bloat,j3,42072.12,40035.00,47651.00,1491.42
+2006-10-MR2,chart,j3,119134.68,95089.00,140917.00,11348.92
+2006-10-MR2,eclipse,j3,160059.03,137516.00,207306.00,18388.06
+2006-10-MR2,fop,j3,99011.45,77498.00,119261.00,11201.14
+2006-10-MR2,hsqldb,j3,28953.33,26835.00,30806.00,1068.32
+2006-10-MR2,jython,j3,56398.65,52766.00,58694.00,1249.38
+2006-10-MR2,luindex,j3,36773.30,30243.00,43885.00,4531.07
+2006-10-MR2,lusearch,j3,17314.83,12121.00,27206.00,3245.44
+2006-10-MR2,pmd,j3,43720.15,37125.00,53931.00,5370.74
+2006-10-MR2,xalan,j3,15828.10,11996.00,18063.00,1656.58
Added: vmkit/trunk/tests/dacapo/synth-dacapo-data-min-max-avg.awk
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/synth-dacapo-data-min-max-avg.awk?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/synth-dacapo-data-min-max-avg.awk (added)
+++ vmkit/trunk/tests/dacapo/synth-dacapo-data-min-max-avg.awk Thu Apr 25 12:12:09 2013
@@ -0,0 +1,48 @@
+BEGIN {
+ FS = ",";
+
+ getline;
+ printf($1 FS $2 FS $3 FS $4 FS "avg_" $4 FS "min_" $4 FS "max_" $4 FS "std_dev_" $4 RS);
+
+ getline;
+
+ bench_suite = $1;
+ benchmark = $2;
+ vm = $3;
+ avg_duration_ms = $4;
+ min_duration_ms = $4;
+ max_duration_ms = $4;
+ count = 1;
+
+ printf("%s" FS "-" FS "-" FS "-" FS "-" RS, $0);
+}
+/^$/ { /* Ignore empty lines */ }
+{
+ if ($1 == bench_suite && $2 == benchmark && $3 == vm) {
+ avg_duration_ms += $4;
+ if ($4 < min_duration_ms) min_duration_ms = $4;
+ if ($4 > max_duration_ms) max_duration_ms = $4;
+ count++;
+ } else {
+ avg_duration_ms /= count;
+ printf("%s" FS "%s" FS "%s" FS "-" FS "%.2f" FS "%.2f" FS "%.2f" FS "-" RS,
+ bench_suite, benchmark, vm,
+ avg_duration_ms, min_duration_ms, max_duration_ms);
+
+ bench_suite = $1;
+ benchmark = $2;
+ vm = $3;
+ avg_duration_ms = $4;
+ min_duration_ms = $4;
+ max_duration_ms = $4;
+ count = 1;
+ }
+
+ printf("%s" FS "-" FS "-" FS "-" FS "-" RS, $0);
+}
+END {
+ avg_duration_ms /= count;
+ printf("%s" FS "%s" FS "%s" FS "-" FS "%.2f" FS "%.2f" FS "%.2f" FS "-" RS,
+ bench_suite, benchmark, vm,
+ avg_duration_ms, min_duration_ms, max_duration_ms);
+}
Added: vmkit/trunk/tests/dacapo/synth-dacapo-data-std-dev.awk
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/synth-dacapo-data-std-dev.awk?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/synth-dacapo-data-std-dev.awk (added)
+++ vmkit/trunk/tests/dacapo/synth-dacapo-data-std-dev.awk Thu Apr 25 12:12:09 2013
@@ -0,0 +1,50 @@
+BEGIN {
+ FS = ",";
+
+ getline;
+ print $0;
+
+ getline;
+
+ bench_suite = $1;
+ benchmark = $2;
+ vm = $3;
+ duration_ms = $4;
+ avg_duration_ms = $5;
+ min_duration_ms = $6;
+ max_duration_ms = $7;
+ std_dev_duration_ms = (duration_ms - avg_duration_ms)^2;
+ count = 1;
+
+ print $0;
+}
+/^$/ { /* Ignore empty lines */ }
+{
+ if ($1 == bench_suite && $2 == benchmark && $3 == vm) {
+ std_dev_duration_ms += ($4 - $5)^2;
+ count++;
+ } else {
+ std_dev_duration_ms = sqrt(std_dev_duration_ms / count);
+ printf("%s" FS "%s" FS "%s" FS "-" FS "%.2f" FS "%.2f" FS "%.2f" FS "%.2f" RS,
+ bench_suite, benchmark, vm,
+ avg_duration_ms, min_duration_ms, max_duration_ms, std_dev_duration_ms);
+
+ bench_suite = $1;
+ benchmark = $2;
+ vm = $3;
+ duration_ms = $4;
+ avg_duration_ms = $5;
+ min_duration_ms = $6;
+ max_duration_ms = $7;
+ std_dev_duration_ms = (duration_ms - avg_duration_ms)^2;
+ count = 1;
+ }
+
+ print $0;
+}
+END {
+ std_dev_duration_ms = sqrt(std_dev_duration_ms / count);
+ printf("%s" FS "%s" FS "%s" FS "-" FS "%.2f" FS "%.2f" FS "%.2f" FS "%.2f" RS,
+ bench_suite, benchmark, vm,
+ avg_duration_ms, min_duration_ms, max_duration_ms, std_dev_duration_ms);
+}
Added: vmkit/trunk/tests/dacapo/synth-logs.sh
URL: http://llvm.org/viewvc/llvm-project/vmkit/trunk/tests/dacapo/synth-logs.sh?rev=180459&view=auto
==============================================================================
--- vmkit/trunk/tests/dacapo/synth-logs.sh (added)
+++ vmkit/trunk/tests/dacapo/synth-logs.sh Thu Apr 25 12:12:09 2013
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Collect results from benchmarks logs, then synthesize them, producing statistical data about them.
+
+log_dir=$1
+cur_dir=$(pwd)
+
+cd "$log_dir"
+data=$(echo '#bench_suite,benchmark,vm,duration_ms')
+data+=$'\n'
+data+=$(grep '^===== DaCapo [A-Za-z]\+ PASSED' tmp.* | sed 's/tmp[^_]\+_//g' | sed 's/ msec =====//g' | sed 's/\.log:===== DaCapo [a-z]\+ PASSED in /_/g' | tr '_' ',' | sort)
+
+cd "$cur_dir"
+echo "$data" | ./augment-dacapo-data.sh
More information about the vmkit-commits
mailing list