[test-suite] r248898 - [test-suite] Don't allow FP_TOLERANCE and HASH_PROGRAM_OUTPUT to occur toghether.

[Charlie Turner via llvm-commits]

Author: chatur01
Date: Wed Sep 30 08:25:10 2015
New Revision: 248898

URL: http://llvm.org/viewvc/llvm-project?rev=248898&view=rev
Log:
[test-suite] Don't allow FP_TOLERANCE and HASH_PROGRAM_OUTPUT to occur toghether.

Summary:
LNT's benchmarks produce their output into files which are then compared in
various ways.

The benchmarks output numerical scores, which can then be compared by a custom
program in the test-suite tree which does a floating-point comparison with a
given tolerance (fpcmp).

The test-suite system allows allows users to do a hash of the program output
for a subsequently speedier comparison. This is used for benchmarks that
generate a significant amount of output (currently defined as >100k), The
problem is that some benchmarks that chose this feature also chose a
floating-point comparison of these hashes, which at best is nonsense, and at
worse causes an infinite loop.

The fpcmp program is not resilient when it's input is weird like this. For a
particular case of an observed mafft failure, it caused fpcmp to go into an
infinite loop.

Reviewers: kristof.beyls, ddunbar, rengolin

Subscribers: rengolin, llvm-commits

Differential Revision: http://reviews.llvm.org/D13283

Modified:
    test-suite/trunk/Makefile.programs
    test-suite/trunk/MultiSource/Benchmarks/MiBench/automotive-basicmath/Makefile
    test-suite/trunk/MultiSource/Benchmarks/Olden/voronoi/Makefile
    test-suite/trunk/MultiSource/Benchmarks/VersaBench/beamformer/Makefile
    test-suite/trunk/MultiSource/Benchmarks/mafft/Makefile

Modified: test-suite/trunk/Makefile.programs
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/Makefile.programs?rev=248898&r1=248897&r2=248898&view=diff
==============================================================================
--- test-suite/trunk/Makefile.programs (original)
+++ test-suite/trunk/Makefile.programs Wed Sep 30 08:25:10 2015
@@ -500,6 +500,11 @@ endif
 # If the program wants its output hashed, add an output filter.
 ifdef ENABLE_HASHED_PROGRAM_OUTPUT
 ifdef HASH_PROGRAM_OUTPUT
+# If the program also wants a floating-point comparison, give
+# and error.
+ifdef FP_TOLERANCE
+$(error Cannot use both FP_TOLERNANCE and HASH_PROGRAM_OUTPUT)
+endif
 PROGRAM_OUTPUT_FILTER := $(PROGDIR)/HashProgramOutput.sh
 endif
 endif

Modified: test-suite/trunk/MultiSource/Benchmarks/MiBench/automotive-basicmath/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/MiBench/automotive-basicmath/Makefile?rev=248898&r1=248897&r2=248898&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/MiBench/automotive-basicmath/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/MiBench/automotive-basicmath/Makefile Wed Sep 30 08:25:10 2015
@@ -2,7 +2,6 @@ LEVEL = ../../../..
 
 PROG     = automotive-basicmath
 LDFLAGS  = -lm
-FP_TOLERANCE :=  0.00000000001
 HASH_PROGRAM_OUTPUT = 1
 
 include $(LEVEL)/MultiSource/Makefile.multisrc

Modified: test-suite/trunk/MultiSource/Benchmarks/Olden/voronoi/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/Olden/voronoi/Makefile?rev=248898&r1=248897&r2=248898&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/Olden/voronoi/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/Olden/voronoi/Makefile Wed Sep 30 08:25:10 2015
@@ -4,7 +4,6 @@ PROG     = voronoi
 INCLUDES = defines.h
 CPPFLAGS = -DTORONTO
 LDFLAGS  = -lm
-FP_TOLERANCE = 0.0001
 ifdef LARGE_PROBLEM_SIZE
 RUN_OPTIONS = 1000000 20 32 7
 else

Modified: test-suite/trunk/MultiSource/Benchmarks/VersaBench/beamformer/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/VersaBench/beamformer/Makefile?rev=248898&r1=248897&r2=248898&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/VersaBench/beamformer/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/VersaBench/beamformer/Makefile Wed Sep 30 08:25:10 2015
@@ -2,7 +2,6 @@ LEVEL = ../../../..
 LDFLAGS += -lm
 
 PROG     = beamformer
-FP_TOLERANCE = 0.000001
 ifdef LARGE_PROBLEM_SIZE
 RUN_OPTIONS = -i 400
 else 

Modified: test-suite/trunk/MultiSource/Benchmarks/mafft/Makefile
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/mafft/Makefile?rev=248898&r1=248897&r2=248898&view=diff
==============================================================================
--- test-suite/trunk/MultiSource/Benchmarks/mafft/Makefile (original)
+++ test-suite/trunk/MultiSource/Benchmarks/mafft/Makefile Wed Sep 30 08:25:10 2015
@@ -10,7 +10,6 @@ CPPFLAGS = -DLLVM
 RUN_OPTIONS = -b 62 -g 0.100 -f 2.00 -h 0.100 -L
 STDIN_FILENAME = $(PROJ_SRC_DIR)/pyruvate_decarboxylase.fasta
 LDFLAGS = -lm
-FP_TOLERANCE = 0.00001
 HASH_PROGRAM_OUTPUT = 1
 
 ifeq ($(ARCH),XCore)