[test-suite] r237777 - Disable more short-running programs in benchmark-only mode.

Kristof Beyls kristof.beyls at arm.com
Wed May 20 00:35:31 PDT 2015 

Author: kbeyls
Date: Wed May 20 02:35:22 2015
New Revision: 237777

URL: http://llvm.org/viewvc/llvm-project?rev=237777&view=rev
Log:
Disable more short-running programs in benchmark-only mode.

A substantial source of performance noisiness of the LNT test-suite
are programs running very shortly.

The following programs, are disabled in benchmark-only mode by this commit:
* SingleSource/UnitTests/Vector: constpool simple:
  both don't have any loops in the code.
* SingleSource/UnitTests/Vector/AArch64: aarch64_neon_intrinsics:
  doesn't have any loop in the code.
* SingleSource/UnitTests/Vector/NEON: simple:
  doesn't have any loop in the code.
* SingleSource/UnitTests: 2005-07-15-Bitfield-ABI 2006-01-23-UnionInit
  2007-04-10-BitfieldTest:
  doesn't have any loop in the code.
* MultiSource/Benchmarks/Prolangs-C: loader:
  This program exits immediately because no arguments are given on the command
  line. Unless  someone creates inputs for this program, this should not be
  considered a benchmark.

* MultiSource/Benchmarks/McCat:
  - 15-trie produces trie data structures, but the program has no loops
    so it's probably IO bound, and therefore shouldn't be considered as a
    benchmark.
* MultiSource/Benchmarks/Prolangs-C:
  - cdecl: The benchmark parses about 70 C declarations
* MultiSource/Benchmarks/MiBench:
  - office-stringsearch searches a few hundred substrings in a set of a few
    hundred strings.
  - telecom-adpcm seems to spend most of its time in IO - and should have a
    loop to do the main data transformation multiple times on the same
    buffer if run as a benchmark.
* SingleSource/Benchmarks/Stanford:
  - IntMM: does 10 matrix multiplications of size 40x40.
* SingleSource/Regression/C/Makefile:
  - matrixTranspose: transposes a 32x32 matrix 10 times.
  - sumarray2d: creates a 100x100 matrix and sums all the elements in it.
  - test_indvars: traverses a 20000-element matrix twice.
* SingleSource/UnitTests/SignlessTypes:
  - rem: does 100 gcd computations.

The following programs also run very shortly, but I do think they have value as
a benchmark - so they are not disabled by this commit:
* SingleSource/Benchmarks/Misc/lowercase
* SingleSource/Benchmarks/Shootout/objinst
* SingleSource/Benchmarks/Shootout-C++/objinst
For all three, it seems that the main computation in the benchmark is
completely optimized away. Keeping these running as benchmarks should allow us
to catch if llvm ever regresses in being able to optimize away the main
computation in these programs.



Modified:
    test-suite/trunk/MultiSource/Benchmarks/McCat/Makefile
    test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile
    test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile
    test-suite/trunk/SingleSource/Benchmarks/Stanford/Makefile
    test-suite/trunk/SingleSource/Regression/C/Makefile
    test-suite/trunk/SingleSource/UnitTests/Makefile
    test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile
    test-suite/trunk/SingleSource/UnitTests/Vector/AArch64/Makefile
    test-suite/trunk/SingleSource/UnitTests/Vector/Makefile
    test-suite/trunk/SingleSource/UnitTests/Vector/NEON/Makefile

Modified: test-suite/trunk/MultiSource/Benchmarks/McCat/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/McCat/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/McCat/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/McCat/Makefile Wed May 20 02:35:22 2015
@@ -3,5 +3,9 @@
 LEVEL = ../../..
 PARALLEL_DIRS  := 01-qbsort 04-bisect 08-main 12-IOtest 17-bintr 03-testtrie 05-eks 09-vor 15-trie 18-imp
 
+ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out 15-trie, $(PARALLEL_DIRS))
+endif
+
 
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile Wed May 20 02:35:22 2015
@@ -36,8 +36,10 @@ ifeq ($(ARCH),XCore)
 endif
 
 ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out office-stringsearch, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out office-ispell, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out security-blowfish, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out telecom-adpcm, $(PARALLEL_DIRS))
 endif
 
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile Wed May 20 02:35:22 2015
@@ -26,7 +26,9 @@ ifdef BENCHMARKING_ONLY
 PARALLEL_DIRS := $(filter-out allroots, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out archie-client, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out assembler, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out cdecl, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out compiler, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out loader, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out fixoutput, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out football, $(PARALLEL_DIRS))
 PARALLEL_DIRS := $(filter-out plot2fig, $(PARALLEL_DIRS))

Modified: test-suite/trunk/SingleSource/Benchmarks/Stanford/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Benchmarks/Stanford/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Benchmarks/Stanford/Makefile (original)
+++ test-suite/trunk/SingleSource/Benchmarks/Stanford/Makefile Wed May 20 02:35:22 2015
@@ -2,4 +2,8 @@ LEVEL = ../../..
 LDFLAGS += -lm
 FP_TOLERANCE = 0.001
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += IntMM
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Regression/C/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Regression/C/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Regression/C/Makefile (original)
+++ test-suite/trunk/SingleSource/Regression/C/Makefile Wed May 20 02:35:22 2015
@@ -30,13 +30,16 @@ PROGRAMS_TO_SKIP += \
 	ConstructorDestructorAttributes \
 	DuffsDevice \
 	globalrefs \
+	matrixTranspose \
 	pointer_arithmetic \
 	PR10189 \
 	PR1386 \
 	PR491 \
 	PR640 \
 	sumarray \
+	sumarray2d \
 	sumarraymalloc \
+	test_indvars \
 	testtrace
 endif
 

Modified: test-suite/trunk/SingleSource/UnitTests/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Makefile Wed May 20 02:35:22 2015
@@ -102,8 +102,10 @@ PROGRAMS_TO_SKIP += \
 	2005-05-11-Popcount-ffs-fls \
 	2005-05-12-Int64ToFP \
 	2005-05-13-SDivTwo \
+	2005-07-15-Bitfield-ABI \
 	2005-07-17-INT-To-FP \
 	2005-11-29-LongSwitch \
+	2006-01-23-UnionInit \
 	2006-01-29-SimpleIndirectCall \
 	2006-02-04-DivRem \
 	2006-12-01-float_varg \
@@ -112,6 +114,7 @@ PROGRAMS_TO_SKIP += \
 	2006-12-11-LoadConstants \
 	2007-01-04-KNR-Args \
 	2007-03-02-VaCopy \
+	2007-04-10-BitfieldTest \
 	2007-04-25-weak \
 	2008-04-18-LoopBug \
 	2008-04-20-LoopBug2 \

Modified: test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile Wed May 20 02:35:22 2015
@@ -12,7 +12,7 @@ SEED := $(shell date +%j)
 RUN_OPTIONS := 31415926
 
 ifdef BENCHMARKING_ONLY
-PROGRAMS_TO_SKIP += cast2 cast-bug ccc div factor shr
+PROGRAMS_TO_SKIP += cast2 cast-bug ccc div factor shr rem
 endif
 
 LDFLAGS += -lstdc++

Modified: test-suite/trunk/SingleSource/UnitTests/Vector/AArch64/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vector/AArch64/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Vector/AArch64/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Vector/AArch64/Makefile Wed May 20 02:35:22 2015
@@ -3,6 +3,10 @@
 DIRS = 
 LEVEL = ../../../..
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += aarch64_neon_intrinsics
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc
 
 CFLAGS += -std=c99

Modified: test-suite/trunk/SingleSource/UnitTests/Vector/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vector/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Vector/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Vector/Makefile Wed May 20 02:35:22 2015
@@ -38,7 +38,7 @@ endif
 endif
 
 ifdef BENCHMARKING_ONLY
-PROGRAMS_TO_SKIP += build divides sumarray sumarray-dbl
+PROGRAMS_TO_SKIP += build constpool divides simple sumarray sumarray-dbl
 endif
 
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/Vector/NEON/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vector/NEON/Makefile?rev=237777&r1=237776&r2=237777&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Vector/NEON/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Vector/NEON/Makefile Wed May 20 02:35:22 2015
@@ -3,6 +3,10 @@
 DIRS = 
 LEVEL = ../../../..
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += simple
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc
 
 CFLAGS += -std=c99

More information about the llvm-commits mailing list