[test-suite] r210251 - Add a benchmarking-only mode to the test suite

Hal Finkel hfinkel at anl.gov
Thu Jun 5 02:24:06 PDT 2014


Author: hfinkel
Date: Thu Jun  5 04:24:05 2014
New Revision: 210251

URL: http://llvm.org/viewvc/llvm-project?rev=210251&view=rev
Log:
Add a benchmarking-only mode to the test suite

This adds a benchmarking-only mode to the test suite, which excludes those
programs unlikely to use useful as benchmarks. When BENCHMARKING_ONLY is
defined, the following programs should be excluded:

MultiSource/Applications/Burg/burg
MultiSource/Applications/treecc/treecc
MultiSource/Benchmarks/MiBench/office-ispell/office-ispell
MultiSource/Benchmarks/MiBench/security-blowfish/security-blowfish
MultiSource/Benchmarks/Prolangs-C/allroots/allroots
MultiSource/Benchmarks/Prolangs-C/archie-client/archie
MultiSource/Benchmarks/Prolangs-C/assembler/assembler
MultiSource/Benchmarks/Prolangs-C/compiler/compiler
MultiSource/Benchmarks/Prolangs-C++/deriv1/deriv1
MultiSource/Benchmarks/Prolangs-C++/deriv2/deriv2
MultiSource/Benchmarks/Prolangs-C++/family/family
MultiSource/Benchmarks/Prolangs-C/fixoutput/fixoutput
MultiSource/Benchmarks/Prolangs-C/football/football
MultiSource/Benchmarks/Prolangs-C++/fsm/fsm
MultiSource/Benchmarks/Prolangs-C++/garage/garage
MultiSource/Benchmarks/Prolangs-C++/NP/np
MultiSource/Benchmarks/Prolangs-C++/objects/objects
MultiSource/Benchmarks/Prolangs-C++/office/office
MultiSource/Benchmarks/Prolangs-C/plot2fig/plot2fig
MultiSource/Benchmarks/Prolangs-C++/shapes/shapes
MultiSource/Benchmarks/Prolangs-C/simulator/simulator
MultiSource/Benchmarks/Prolangs-C/TimberWolfMC/timberwolfmc
MultiSource/Benchmarks/Prolangs-C++/trees/trees
MultiSource/Benchmarks/Prolangs-C/unix-smail/unix-smail
MultiSource/Benchmarks/Prolangs-C/unix-tbl/unix-tbl
MultiSource/Benchmarks/Prolangs-C++/vcirc/vcirc
SingleSource/Benchmarks/McGill/exptree
SingleSource/Benchmarks/Shootout-C++/hello
SingleSource/Benchmarks/Shootout-C++/reversefile
SingleSource/Benchmarks/Shootout-C++/spellcheck
SingleSource/Benchmarks/Shootout-C++/sumcol
SingleSource/Benchmarks/Shootout-C++/wc
SingleSource/Benchmarks/Shootout-C++/wordfreq
SingleSource/Benchmarks/Shootout/hello
SingleSource/Regression/C++/2003-05-14-array-init
SingleSource/Regression/C++/2003-05-14-expr_stmt
SingleSource/Regression/C/2003-05-14-initialize-string
SingleSource/Regression/C/2003-05-21-BitfieldHandling
SingleSource/Regression/C/2003-05-21-UnionBitfields
SingleSource/Regression/C/2003-05-21-UnionTest
SingleSource/Regression/C/2003-05-22-LocalTypeTest
SingleSource/Regression/C/2003-05-22-VarSizeArray
SingleSource/Regression/C/2003-05-23-TransparentUnion
SingleSource/Regression/C++/2003-06-08-BaseType
SingleSource/Regression/C++/2003-06-08-VirtualFunctions
SingleSource/Regression/C++/2003-06-13-Crasher
SingleSource/Regression/C/2003-06-16-InvalidInitializer
SingleSource/Regression/C/2003-06-16-VolatileError
SingleSource/Regression/C++/2003-08-20-EnumSizeProblem
SingleSource/Regression/C++/2003-09-29-NonPODsByValue
SingleSource/Regression/C/2003-10-12-GlobalVarInitializers
SingleSource/Regression/C/2004-02-03-AggregateCopy
SingleSource/Regression/C/2004-03-15-IndirectGoto
SingleSource/Regression/C/2005-05-06-LongLongSignedShift
SingleSource/Regression/C/2008-01-07-LongDouble
SingleSource/Regression/C++/2008-01-29-ParamAliasesReturn
SingleSource/Regression/C++/2011-03-28-Bitfield
SingleSource/Regression/C/badidx
SingleSource/Regression/C/bigstack
SingleSource/Regression/C++/BuiltinTypeInfo
SingleSource/Regression/C/callargs
SingleSource/Regression/C/casts
SingleSource/Regression/C/compare
SingleSource/Regression/C/ConstructorDestructorAttributes
SingleSource/Regression/C/DuffsDevice
SingleSource/Regression/C++/EH/class_hierarchy
SingleSource/Regression/C++/EH/ConditionalExpr
SingleSource/Regression/C++/EH/ctor_dtor_count
SingleSource/Regression/C++/EH/ctor_dtor_count-2
SingleSource/Regression/C++/EH/dead_try_block
SingleSource/Regression/C++/EH/exception_spec_test
SingleSource/Regression/C++/EH/function_try_block
SingleSource/Regression/C++/EH/inlined_cleanup
SingleSource/Regression/C++/EH/recursive-throw
SingleSource/Regression/C++/EH/simple_rethrow
SingleSource/Regression/C++/EH/simple_throw
SingleSource/Regression/C++/EH/throw_rethrow_test
SingleSource/Regression/C++/fixups
SingleSource/Regression/C++/global_ctor
SingleSource/Regression/C/globalrefs
SingleSource/Regression/C++/global_type
SingleSource/Regression/C++/ofstream_ctor
SingleSource/Regression/C/pointer_arithmetic
SingleSource/Regression/C++/pointer_member
SingleSource/Regression/C++/pointer_method
SingleSource/Regression/C++/pointer_method2
SingleSource/Regression/C/PR10189
SingleSource/Regression/C/PR1386
SingleSource/Regression/C/PR491
SingleSource/Regression/C/PR640
SingleSource/Regression/C++/short_circuit_dtor
SingleSource/Regression/C/sumarray
SingleSource/Regression/C/sumarraymalloc
SingleSource/Regression/C/testtrace
SingleSource/UnitTests/2002-04-17-PrintfChar
SingleSource/UnitTests/2002-05-02-ArgumentTest
SingleSource/UnitTests/2002-05-02-CastTest
SingleSource/UnitTests/2002-05-02-CastTest1
SingleSource/UnitTests/2002-05-02-CastTest2
SingleSource/UnitTests/2002-05-02-CastTest3
SingleSource/UnitTests/2002-05-02-ManyArguments
SingleSource/UnitTests/2002-05-03-NotTest
SingleSource/UnitTests/2002-05-19-DivTest
SingleSource/UnitTests/2002-08-02-CastTest
SingleSource/UnitTests/2002-08-02-CastTest2
SingleSource/UnitTests/2002-08-19-CodegenBug
SingleSource/UnitTests/2002-10-09-ArrayResolution
SingleSource/UnitTests/2002-10-12-StructureArgs
SingleSource/UnitTests/2002-10-12-StructureArgsSimple
SingleSource/UnitTests/2002-10-13-BadLoad
SingleSource/UnitTests/2002-12-13-MishaTest
SingleSource/UnitTests/2003-04-22-Switch
SingleSource/UnitTests/2003-05-02-DependentPHI
SingleSource/UnitTests/2003-05-07-VarArgs
SingleSource/UnitTests/2003-05-12-MinIntProblem
SingleSource/UnitTests/2003-05-14-AtExit
SingleSource/UnitTests/2003-05-26-Shorts
SingleSource/UnitTests/2003-05-31-CastToBool
SingleSource/UnitTests/2003-05-31-LongShifts
SingleSource/UnitTests/2003-07-06-IntOverflow
SingleSource/UnitTests/2003-07-08-BitOpsTest
SingleSource/UnitTests/2003-07-09-LoadShorts
SingleSource/UnitTests/2003-07-09-SignedArgs
SingleSource/UnitTests/2003-07-10-SignConversions
SingleSource/UnitTests/2003-08-05-CastFPToUint
SingleSource/UnitTests/2003-08-11-VaListArg
SingleSource/UnitTests/2003-08-20-FoldBug
SingleSource/UnitTests/2003-09-18-BitFieldTest
SingleSource/UnitTests/2003-10-13-SwitchTest
SingleSource/UnitTests/2003-10-29-ScalarReplBug
SingleSource/UnitTests/2004-02-02-NegativeZero
SingleSource/UnitTests/2004-06-20-StaticBitfieldInit
SingleSource/UnitTests/2004-11-28-GlobalBoolLayout
SingleSource/UnitTests/2005-05-11-Popcount-ffs-fls
SingleSource/UnitTests/2005-05-12-Int64ToFP
SingleSource/UnitTests/2005-05-13-SDivTwo
SingleSource/UnitTests/2005-07-17-INT-To-FP
SingleSource/UnitTests/2005-11-29-LongSwitch
SingleSource/UnitTests/2006-01-29-SimpleIndirectCall
SingleSource/UnitTests/2006-02-04-DivRem
SingleSource/UnitTests/2006-12-01-float_varg
SingleSource/UnitTests/2006-12-04-DynAllocAndRestore
SingleSource/UnitTests/2006-12-07-Compare64BitConstant
SingleSource/UnitTests/2006-12-11-LoadConstants
SingleSource/UnitTests/2007-01-04-KNR-Args
SingleSource/UnitTests/2007-03-02-VaCopy
SingleSource/UnitTests/2007-04-25-weak
SingleSource/UnitTests/2008-04-18-LoopBug
SingleSource/UnitTests/2008-04-20-LoopBug2
SingleSource/UnitTests/2008-07-13-InlineSetjmp
SingleSource/UnitTests/2009-04-16-BitfieldInitialization
SingleSource/UnitTests/2009-12-07-StructReturn
SingleSource/UnitTests/2010-05-24-BitfieldTest
SingleSource/UnitTests/AtomicOps
SingleSource/UnitTests/block-byref-cxxobj-test
SingleSource/UnitTests/block-byref-test
SingleSource/UnitTests/block-call-r7674133
SingleSource/UnitTests/block-copied-in-cxxobj
SingleSource/UnitTests/block-copied-in-cxxobj-1
SingleSource/UnitTests/blockstret
SingleSource/UnitTests/byval-alignment
SingleSource/UnitTests/conditional-gnu-ext
SingleSource/UnitTests/conditional-gnu-ext-cxx
SingleSource/UnitTests/DefaultInitDynArrays
SingleSource/UnitTests/FloatPrecision
SingleSource/UnitTests/initp1
SingleSource/UnitTests/member-function-pointers
SingleSource/UnitTests/printargs
SingleSource/UnitTests/SignlessTypes/cast2
SingleSource/UnitTests/SignlessTypes/cast-bug
SingleSource/UnitTests/SignlessTypes/ccc
SingleSource/UnitTests/SignlessTypes/div
SingleSource/UnitTests/SignlessTypes/factor
SingleSource/UnitTests/SignlessTypes/Large/cast
SingleSource/UnitTests/SignlessTypes/shr
SingleSource/UnitTests/stmtexpr
SingleSource/UnitTests/StructModifyTest
SingleSource/UnitTests/TestLoop
SingleSource/UnitTests/Threads/2010-12-08-tls
SingleSource/UnitTests/Threads/tls
SingleSource/UnitTests/Vector/build
SingleSource/UnitTests/Vector/divides
SingleSource/UnitTests/Vector/sumarray
SingleSource/UnitTests/Vector/sumarray-dbl
SingleSource/UnitTests/vla

SingleSource/UnitTests/Vector/Altivec/2007-01-07-lvsl-lvsr-Regression
SingleSource/UnitTests/Vector/Altivec/alti.sdot
SingleSource/UnitTests/Vector/Altivec/casts
SingleSource/UnitTests/Vector/Altivec/test1

SingleSource/UnitTests/ms_struct-bitfield
SingleSource/UnitTests/ms_struct-bitfield-1
SingleSource/UnitTests/ms_struct-bitfield-init
SingleSource/UnitTests/ms_struct-bitfield-init-1
SingleSource/UnitTests/ms_struct_pack_layout
SingleSource/UnitTests/ms_struct_pack_layout-1

I constructed this exclusion list primarily based on the following experiment:
I ran the test suite 10 times in three configurations: 1) On an IBM POWER7 (P7)
with -O3 -mvsx, 2) On a P7 at -O0 and 3) On an Intel Xeon E5430 with -O3 all
using make -j6. I then used the ministat utility (which performs a T test) to
compare the timings of the two P7 configurations against each other and the
Xeon configuration, requiring a detectable difference at 99.5% confidence. I
looked for tests that showed no significant difference in all three
comparisons. The running configuration here is purposefully noisy, the idea is
to eliminate those tests that are significantly sensitive to startup time, file
I/O time, memory bandwidth, etc., or just too short, and by running many tests
in parallel (non-deterministically), my hope is to eliminate those tests can
cannot usefully serve as benchmarks in a "normal" environment.

Modified:
    test-suite/trunk/MultiSource/Applications/Makefile
    test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile
    test-suite/trunk/MultiSource/Benchmarks/Prolangs-C++/Makefile
    test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile
    test-suite/trunk/SingleSource/Benchmarks/McGill/Makefile
    test-suite/trunk/SingleSource/Benchmarks/Shootout-C++/Makefile
    test-suite/trunk/SingleSource/Benchmarks/Shootout/Makefile
    test-suite/trunk/SingleSource/Regression/C++/EH/Makefile
    test-suite/trunk/SingleSource/Regression/C++/Makefile
    test-suite/trunk/SingleSource/Regression/C/Makefile
    test-suite/trunk/SingleSource/UnitTests/Makefile
    test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Large/Makefile
    test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile
    test-suite/trunk/SingleSource/UnitTests/Threads/Makefile
    test-suite/trunk/SingleSource/UnitTests/Vector/Altivec/Makefile
    test-suite/trunk/SingleSource/UnitTests/Vector/Makefile

Modified: test-suite/trunk/MultiSource/Applications/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Applications/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Applications/Makefile (original)
+++ test-suite/trunk/MultiSource/Applications/Makefile Thu Jun  5 04:24:05 2014
@@ -42,4 +42,9 @@ PARALLEL_DIRS := $(filter-out obsequi, $
 PARALLEL_DIRS := $(filter-out lemon, $(PARALLEL_DIRS))
 endif
 
+ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out Burg, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out treecc, $(PARALLEL_DIRS))
+endif
+
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/MiBench/Makefile Thu Jun  5 04:24:05 2014
@@ -35,4 +35,9 @@ ifeq ($(ARCH),XCore)
   PARALLEL_DIRS := $(filter-out office-ispell, $(PARALLEL_DIRS))
 endif
 
+ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out office-ispell, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out security-blowfish, $(PARALLEL_DIRS))
+endif
+
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/MultiSource/Benchmarks/Prolangs-C++/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/Prolangs-C%2B%2B/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/Prolangs-C++/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/Prolangs-C++/Makefile Thu Jun  5 04:24:05 2014
@@ -1,4 +1,19 @@
 LEVEL = ../../..
 PARALLEL_DIRS  := NP city deriv1 deriv2 employ family fsm garage \
                   life objects ocean office primes shapes simul trees vcirc
+
+ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out deriv1, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out deriv2, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out family, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out fsm, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out garage, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out NP, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out objects, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out office, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out shapes, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out trees, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out vcirc, $(PARALLEL_DIRS))
+endif
+
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/Prolangs-C/Makefile Thu Jun  5 04:24:05 2014
@@ -22,4 +22,18 @@ PARALLEL_DIRS := $(filter-out archie-cli
 PARALLEL_DIRS := $(filter-out unix-smail, $(PARALLEL_DIRS))
 endif
 
+ifdef BENCHMARKING_ONLY
+PARALLEL_DIRS := $(filter-out allroots, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out archie-client, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out assembler, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out compiler, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out fixoutput, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out football, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out plot2fig, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out simulator, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out TimberWolfMC, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out unix-smail, $(PARALLEL_DIRS))
+PARALLEL_DIRS := $(filter-out unix-tbl, $(PARALLEL_DIRS))
+endif
+
 include $(LEVEL)/Makefile.programs

Modified: test-suite/trunk/SingleSource/Benchmarks/McGill/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Benchmarks/McGill/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Benchmarks/McGill/Makefile (original)
+++ test-suite/trunk/SingleSource/Benchmarks/McGill/Makefile Thu Jun  5 04:24:05 2014
@@ -2,4 +2,8 @@ LEVEL = ../../..
 LDFLAGS += -lm 
 FP_TOLERANCE := 0.001
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += exptree
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Benchmarks/Shootout-C++/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Benchmarks/Shootout-C%2B%2B/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Benchmarks/Shootout-C++/Makefile (original)
+++ test-suite/trunk/SingleSource/Benchmarks/Shootout-C++/Makefile Thu Jun  5 04:24:05 2014
@@ -13,5 +13,9 @@ RUNTIMELIMIT:=10800
 endif
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += hello reversefile spellcheck sumcol wc wordfreq
+endif
+
 LDFLAGS += -lstdc++
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Benchmarks/Shootout/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Benchmarks/Shootout/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Benchmarks/Shootout/Makefile (original)
+++ test-suite/trunk/SingleSource/Benchmarks/Shootout/Makefile Thu Jun  5 04:24:05 2014
@@ -5,4 +5,8 @@ ifeq ($(ARCH),XCore)
 XCORE_TARGET_NEEDS_MEMORY := 256
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += hello
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Regression/C++/EH/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Regression/C%2B%2B/EH/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Regression/C++/EH/Makefile (original)
+++ test-suite/trunk/SingleSource/Regression/C++/EH/Makefile Thu Jun  5 04:24:05 2014
@@ -3,5 +3,21 @@ REQUIRES_EH_SUPPORT = 1
 
 include $(LEVEL)/Makefile.config
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += \
+	class_hierarchy \
+	ConditionalExpr \
+	ctor_dtor_count \
+	ctor_dtor_count-2 \
+	dead_try_block \
+	exception_spec_test \
+	function_try_block \
+	inlined_cleanup \
+	recursive-throw \
+	simple_rethrow \
+	simple_throw \
+	throw_rethrow_test
+endif
+
 LDFLAGS += -lstdc++
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Regression/C++/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Regression/C%2B%2B/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Regression/C++/Makefile (original)
+++ test-suite/trunk/SingleSource/Regression/C++/Makefile Thu Jun  5 04:24:05 2014
@@ -8,5 +8,27 @@ LEVEL = ../../..
 
 DIRS=EH
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += \
+	2003-05-14-array-init \
+	2003-05-14-expr_stmt \
+	2003-06-08-BaseType \
+	2003-06-08-VirtualFunctions \
+	2003-06-13-Crasher \
+	2003-08-20-EnumSizeProblem \
+	2003-09-29-NonPODsByValue \
+	2008-01-29-ParamAliasesReturn \
+	2011-03-28-Bitfield \
+	BuiltinTypeInfo \
+	fixups \
+	global_ctor \
+	global_type \
+	ofstream_ctor \
+	pointer_member \
+	pointer_method \
+	pointer_method2 \
+	short_circuit_dtor
+endif
+
 LDFLAGS += -lstdc++
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/Regression/C/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Regression/C/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/Regression/C/Makefile (original)
+++ test-suite/trunk/SingleSource/Regression/C/Makefile Thu Jun  5 04:24:05 2014
@@ -6,5 +6,39 @@ EXEC_XFAILS = casts
 endif
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += \
+	2003-05-14-initialize-string \
+	2003-05-21-BitfieldHandling \
+	2003-05-21-UnionBitfields \
+	2003-05-21-UnionTest \
+	2003-05-22-LocalTypeTest \
+	2003-05-22-VarSizeArray \
+	2003-05-23-TransparentUnion \
+	2003-06-16-InvalidInitializer \
+	2003-06-16-VolatileError \
+	2003-10-12-GlobalVarInitializers \
+	2004-02-03-AggregateCopy \
+	2004-03-15-IndirectGoto \
+	2005-05-06-LongLongSignedShift \
+	2008-01-07-LongDouble \
+	badidx \
+	bigstack \
+	callargs \
+	casts \
+	compare \
+	ConstructorDestructorAttributes \
+	DuffsDevice \
+	globalrefs \
+	pointer_arithmetic \
+	PR10189 \
+	PR1386 \
+	PR491 \
+	PR640 \
+	sumarray \
+	sumarraymalloc \
+	testtrace
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc
 

Modified: test-suite/trunk/SingleSource/UnitTests/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Makefile Thu Jun  5 04:24:05 2014
@@ -48,5 +48,93 @@ PROGRAMS_TO_SKIP += initp1
 XCORE_TARGET_NEEDS_MEMORY := 1
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += \
+	2002-04-17-PrintfChar \
+	2002-05-02-ArgumentTest \
+	2002-05-02-CastTest \
+	2002-05-02-CastTest1 \
+	2002-05-02-CastTest2 \
+	2002-05-02-CastTest3 \
+	2002-05-02-ManyArguments \
+	2002-05-03-NotTest \
+	2002-05-19-DivTest \
+	2002-08-02-CastTest \
+	2002-08-02-CastTest2 \
+	2002-08-19-CodegenBug \
+	2002-10-09-ArrayResolution \
+	2002-10-12-StructureArgs \
+	2002-10-12-StructureArgsSimple \
+	2002-10-13-BadLoad \
+	2002-12-13-MishaTest \
+	2003-04-22-Switch \
+	2003-05-02-DependentPHI \
+	2003-05-07-VarArgs \
+	2003-05-12-MinIntProblem \
+	2003-05-14-AtExit \
+	2003-05-26-Shorts \
+	2003-05-31-CastToBool \
+	2003-05-31-LongShifts \
+	2003-07-06-IntOverflow \
+	2003-07-08-BitOpsTest \
+	2003-07-09-LoadShorts \
+	2003-07-09-SignedArgs \
+	2003-07-10-SignConversions \
+	2003-08-05-CastFPToUint \
+	2003-08-11-VaListArg \
+	2003-08-20-FoldBug \
+	2003-09-18-BitFieldTest \
+	2003-10-13-SwitchTest \
+	2003-10-29-ScalarReplBug \
+	2004-02-02-NegativeZero \
+	2004-06-20-StaticBitfieldInit \
+	2004-11-28-GlobalBoolLayout \
+	2005-05-11-Popcount-ffs-fls \
+	2005-05-12-Int64ToFP \
+	2005-05-13-SDivTwo \
+	2005-07-17-INT-To-FP \
+	2005-11-29-LongSwitch \
+	2006-01-29-SimpleIndirectCall \
+	2006-02-04-DivRem \
+	2006-12-01-float_varg \
+	2006-12-04-DynAllocAndRestore \
+	2006-12-07-Compare64BitConstant \
+	2006-12-11-LoadConstants \
+	2007-01-04-KNR-Args \
+	2007-03-02-VaCopy \
+	2007-04-25-weak \
+	2008-04-18-LoopBug \
+	2008-04-20-LoopBug2 \
+	2008-07-13-InlineSetjmp \
+	2009-04-16-BitfieldInitialization \
+	2009-12-07-StructReturn \
+	2010-05-24-BitfieldTest \
+	AtomicOps \
+	block-byref-cxxobj-test \
+	block-byref-test \
+	block-call-r7674133 \
+	block-copied-in-cxxobj \
+	block-copied-in-cxxobj-1 \
+	blockstret \
+	byval-alignment \
+	conditional-gnu-ext \
+	conditional-gnu-ext-cxx \
+	DefaultInitDynArrays \
+	FloatPrecision \
+	initp1 \
+	member-function-pointers \
+	printargs \
+	stmtexpr \
+	StructModifyTest \
+	TestLoop \
+	vla \
+	ms_struct-bitfield \
+	ms_struct-bitfield-1 \
+	ms_struct-bitfield-init \
+	ms_struct-bitfield-init-1 \
+	ms_struct_pack_layout \
+	ms_struct_pack_layout-1
+endif
+
 LDFLAGS += -lstdc++
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Large/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Large/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Large/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Large/Makefile Thu Jun  5 04:24:05 2014
@@ -9,4 +9,8 @@ EXEC_XFAILS = cast
 endif
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += cast
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/SignlessTypes/Makefile Thu Jun  5 04:24:05 2014
@@ -11,5 +11,9 @@ SEED := $(shell date +%j)
 # the same results regardless of whether you make clean or not.
 RUN_OPTIONS := 31415926
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += cast2 cast-bug ccc div factor shr
+endif
+
 LDFLAGS += -lstdc++
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/Threads/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Threads/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Threads/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Threads/Makefile Thu Jun  5 04:24:05 2014
@@ -15,4 +15,8 @@ ifeq ($(TARGET_OS),Darwin)
 PROGRAMS_TO_SKIP := tls 2010-12-08-tls
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += 2010-12-08-tls tls
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/Vector/Altivec/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vector/Altivec/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Vector/Altivec/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Vector/Altivec/Makefile Thu Jun  5 04:24:05 2014
@@ -13,4 +13,10 @@ TARGET_FLAGS += -maltivec
 TEST_TARGET_FLAGS += -maltivec
 LCCFLAGS += -maltivec
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += \
+	2007-01-07-lvsl-lvsr-Regression alti.sdot \
+	casts test1
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc

Modified: test-suite/trunk/SingleSource/UnitTests/Vector/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vector/Makefile?rev=210251&r1=210250&r2=210251&view=diff
==============================================================================
--- test-suite/trunk/SingleSource/UnitTests/Vector/Makefile (original)
+++ test-suite/trunk/SingleSource/UnitTests/Vector/Makefile Thu Jun  5 04:24:05 2014
@@ -37,6 +37,10 @@ RUNTIMELIMIT:=7200
 endif
 endif
 
+ifdef BENCHMARKING_ONLY
+PROGRAMS_TO_SKIP += build divides sumarray sumarray-dbl
+endif
+
 include $(LEVEL)/SingleSource/Makefile.singlesrc
 
 





More information about the llvm-commits mailing list