[llvm-commits] CVS: llvm-test/Makefile.programs RunSafely.sh RunSafelyAndStable.sh
Reid Spencer
reid at x10sys.com
Sat Nov 25 00:56:24 PST 2006
Changes in directory llvm-test:
Makefile.programs updated: 1.242 -> 1.243
RunSafely.sh updated: 1.21 -> 1.22
RunSafelyAndStable.sh updated: 1.5 -> 1.6
---
Log message:
Fix the way exit code handling was being done so that it actually works.
Previously the exit code of "awk" was being recorded. Not good. This
version captures the program's exit code in the .time file and checks it,
if requested, on each program run. This allows us to write very simple
C/C++ tests that simply return a result code and the test will fail if
the result code varies from that produced by GCC.
---
Diffs of the changes: (+102 -134)
Makefile.programs | 36 +++++++++++---
RunSafely.sh | 74 ++++++++++++++++++-----------
RunSafelyAndStable.sh | 126 +++++++++++---------------------------------------
3 files changed, 102 insertions(+), 134 deletions(-)
Index: llvm-test/Makefile.programs
diff -u llvm-test/Makefile.programs:1.242 llvm-test/Makefile.programs:1.243
--- llvm-test/Makefile.programs:1.242 Tue Nov 21 11:58:38 2006
+++ llvm-test/Makefile.programs Sat Nov 25 02:56:10 2006
@@ -83,10 +83,15 @@
# correctly, this script has no effect, otherwise it will do things like print a
# stack trace of a core dump. It always returns "successful" so that tests will
# continue to be run.
+ifdef PROGRAM_REQUIRED_TO_EXIT_OK
+EXIT_OK := 1
+else
+EXIT_OK := 0
+endif
ifndef GET_STABLE_NUMBERS
-RUNSAFELY := $(PROGDIR)/RunSafely.sh $(RUNTIMELIMIT)
+RUNSAFELY := $(PROGDIR)/RunSafely.sh $(RUNTIMELIMIT) $(EXIT_OK)
else
-RUNSAFELY := $(PROGDIR)/RunSafelyAndStable.sh $(RUNTIMELIMIT)
+RUNSAFELY := $(PROGDIR)/RunSafelyAndStable.sh $(RUNTIMELIMIT) $(EXIT_OK)
endif
ifndef STDIN_FILENAME
@@ -351,37 +356,52 @@
$(PROGRAMS_TO_TEST:%=Output/%.out-nat): \
Output/%.out-nat: Output/%.native
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ @if test $$? -ne 0 ; then \
+ echo "TEST (nat): $* FAILED EXITOK!"; rm -f $@; \
+ fi
endif
$(PROGRAMS_TO_TEST:%=Output/%.out-lli): \
Output/%.out-lli: Output/%.llvm.bc $(LLI)
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) $(LLI_OPTS) $< $(RUN_OPTIONS)
+ @if test $$? -ne 0 ; then \
+ echo "TEST (lli): $* FAILED EXITOK!"; rm -f $@; \
+ fi
$(PROGRAMS_TO_TEST:%=Output/%.out-jit): \
Output/%.out-jit: Output/%.llvm.bc $(LLI)
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) $(JIT_OPTS) $< $(RUN_OPTIONS)
-ifdef PROGRAM_REQUIRED_TO_EXIT_OK
- @if test \! -f $@.exitok; then echo "TEST (jit): $* FAILED!"; rm -f $@; fi
-endif
+ @if test $$? -ne 0 ; then \
+ echo "TEST (jit): $* FAILED EXITOK!"; rm -f $@; \
+ fi
$(PROGRAMS_TO_TEST:%=Output/%.out-jit-beta): \
Output/%.out-jit-beta: Output/%.llvm.bc $(LLI)
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $(LLI) $(LLCBETAOPTION) $(JIT_OPTS) $< $(RUN_OPTIONS)
-ifdef PROGRAM_REQUIRED_TO_EXIT_OK
- @if test \! -f $@.exitok; then echo "TEST (jit-beta): $* FAILED!"; rm -f $@; fi
-endif
+ @if test $$? -ne 0 ; then \
+ echo "TEST (jit-beta): $* FAILED EXITOK!"; rm -f $@; \
+ fi
$(PROGRAMS_TO_TEST:%=Output/%.out-llc): \
Output/%.out-llc: Output/%.llc
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ @if test $$? -ne 0 ; then \
+ echo "TEST (llc): $* FAILED EXITOK!" ; rm -f $@; \
+ fi
$(PROGRAMS_TO_TEST:%=Output/%.out-llc-beta): \
Output/%.out-llc-beta: Output/%.llc-beta
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ @if test $$? -ne 0 ; then \
+ echo "TEST (llc-beta): $* FAILED EXITOK!" ; rm -f $@; \
+ fi
$(PROGRAMS_TO_TEST:%=Output/%.out-cbe): \
Output/%.out-cbe: Output/%.cbe
-$(RUNSAFELY) $(STDIN_FILENAME) $@ $< $(RUN_OPTIONS)
+ @if test $$? -ne 0 ; then \
+ echo "TEST (cbe): $* FAILED EXITOK!" ; rm -f $@; \
+ fi
# If a tolerance is set, pass it off to bugpoint
ifdef FP_TOLERANCE
Index: llvm-test/RunSafely.sh
diff -u llvm-test/RunSafely.sh:1.21 llvm-test/RunSafely.sh:1.22
--- llvm-test/RunSafely.sh:1.21 Wed Jun 7 15:11:31 2006
+++ llvm-test/RunSafely.sh Sat Nov 25 02:56:10 2006
@@ -8,22 +8,39 @@
# "successful" so that tests will continue to be run.
#
# This script funnels stdout and stderr from the program into the
-# first argument specified, and outputs a <outputfile>.time file which
-# contains a timing of the program.
+# fourth argument specified, and outputs a <outfile>.time file which
+# contains a timing of the program and the program's exit code.
+#
+# If the <exitok> (2nd) parameter is 0 then this script always
+# returns 0, regardless of the actual exit of the <program>.
+# If the <exitok> parameter is non-zero then this script returns
+# the exit code of the <program>. If there is an error in getting
+# the <program>'s exit code, this script returns 99.
#
-# Syntax: ./RunSafely.sh <ulimit> <stdinfile> <stdoutfile> <program> <args...>
+# Syntax:
+#
+# RunSafely.sh <timeout> <exitok> <infile> <outfile> <program> <args...>
+#
+# where:
+# <timeout> is the maximum number of seconds to let the <program> run
+# <exitok> is 1 if the program must exit with 0 return code
+# <infile> is a file from which standard input is directed
+# <outfile> is a file to which standard output and error are directed
+# <program> is the path to the program to run
+# <args...> are the arguments to pass to the program.
#
if [ $# -lt 4 ]; then
- echo "./RunSafely.sh <timeout> <stdinfile> <stdoutfile> <program> <args...>"
- exit 1
+ echo "./RunSafely.sh <timeout> <exitok> <infile> <outfile> <program> <args...>"
+ exit 1
fi
DIR=${0%%`basename $0`}
ULIMIT=$1
-INFILE=$2
-OUTFILE=$3
-PROGRAM=$4
-shift 4
+EXITOK=$2
+INFILE=$3
+OUTFILE=$4
+PROGRAM=$5
+shift 5
SYSTEM=`uname -s`
case $SYSTEM in
@@ -50,32 +67,34 @@
#
# Run the command, timing its execution.
# The standard output and standard error of $PROGRAM should go in $OUTFILE,
-# and the standard error of time should go in $OUTFILE.time.
+# and the standard error of time should go in $OUTFILE.time. Note that the
+# return code of the program is appended to the $OUTFILE on an "Exit Val ="
+# line.
#
-# Ah, the joys of shell programming!
# To get the time program and the specified program different output filenames,
# we tell time to launch a shell which in turn executes $PROGRAM with the
# necessary I/O redirection.
#
+COMMAND="$PROGRAM $*"
if [ $SYSTEM == Darwin ]; then
- ( time -p sh -c "${DIR}TimedExec.sh $ULIMIT $PROGRAM $* > $OUTFILE 2>&1 < $INFILE" ) 2>&1 | awk -- '\
-BEGIN { cpu = 0.0; }
-/^user/ { cpu += $2; print }
-/^sys/ { cpu += $2; print }
-!/^user/ && !/^sys/ { print }
-END { printf("program %f\n", cpu); }' > $OUTFILE.time
-else
- ( time -p sh -c "$PROGRAM $* > $OUTFILE 2>&1 < $INFILE" ) 2>&1 | awk -- '\
+ COMMAND="${DIR}TimedExec.sh $ULIMIT $COMMAND"
+fi
+
+( time -p sh -c "$COMMAND >$OUTFILE 2>&1 < $INFILE" ; echo exit $? ) 2>&1 \
+ | awk -- '\
BEGIN { cpu = 0.0; }
-/^user/ { cpu += $2; print }
-/^sys/ { cpu += $2; print }
-!/^user/ && !/^sys/ { print }
+/^user/ { cpu += $2; print; }
+/^sys/ { cpu += $2; print; }
+!/^user/ && !/^sys/ { print; }
END { printf("program %f\n", cpu); }' > $OUTFILE.time
-fi
-if test $? -eq 0
-then
- touch $OUTFILE.exitok
+if [ "$EXITOK" -ne 0 ] ; then
+ exitval=`grep '^exit ' $OUTFILE.time | sed -e 's/^exit //'`
+ if [ -z "$exitval" ] ; then
+ exitval=99
+ fi
+else
+ exitval=0
fi
if ls | egrep "^core" > /dev/null
@@ -95,5 +114,4 @@
$GDB -q -batch --command=StackTrace.$$ --core=$corefile $PROGRAM < /dev/null
rm -f StackTrace.$$ $corefile
fi
-
-exit 0
+exit "$exitval"
Index: llvm-test/RunSafelyAndStable.sh
diff -u llvm-test/RunSafelyAndStable.sh:1.5 llvm-test/RunSafelyAndStable.sh:1.6
--- llvm-test/RunSafelyAndStable.sh:1.5 Tue Aug 2 17:04:31 2005
+++ llvm-test/RunSafelyAndStable.sh Sat Nov 25 02:56:10 2006
@@ -2,112 +2,42 @@
#
# Program: RunSafelyAndStable.sh
#
-# Synopsis: This script runs another program three times. If the program works
-# correctly, this script has no effect, otherwise it will do things
-# like print a stack trace of a core dump. It always returns
-# "successful" so that tests will continue to be run.
+# Synopsis: This script runs another program three times by repeatedly
+# invoking the RunSafely.sh script. It collects the timings of
+# the program and reports the smallest one. The <outfile>.time
+# file will reflect the time and result code of the fastest
+# run.
+#
+# Syntax:
+# ./RunSafelyAndStable.sh <ulimit> <exitok> <infile> <outfile> \
+# <program> <args...>
#
-# This script funnels stdout and stderr from the program into the
-# first argument specified, and outputs a <outputfile>.time file which
-# contains timing information for the fastest of the three runs of the
-# program.
-#
-# Syntax:
-# ./RunSafelyAndStable.sh <ulimit> <stdinfile> <stdoutfile> <program> <args...>
+# See the RunSafely.sh script for more details.
#
+DIR=${0%%`basename $0`}
ULIMIT=$1
-INFILE=$2
-OUTFILE=$3
-PROGRAM=$4
-shift 4
-
-SYSTEM=`uname -s`
-
-case $SYSTEM in
- CYGWIN*)
- ;;
- Darwin*)
- # Disable core file emission, the script doesn't find it anyway because it is put
- # into /cores.
- ulimit -c 0
- ulimit -t $ULIMIT
- # To prevent infinite loops which fill up the disk, specify a limit on size of
- # files being output by the tests. 10 MB should be enough for anybody. ;)
- ulimit -f 10485760
- ;;
- *)
- ulimit -t $ULIMIT
- ulimit -c unlimited
- # To prevent infinite loops which fill up the disk, specify a limit on size of
- # files being output by the tests. 10 MB should be enough for anybody. ;)
- ulimit -f 10485760
-esac
-
-#
-# Run the command, timing its execution.
-# The standard output and standard error of $PROGRAM should go in $OUTFILE,
-# and the standard error of time should go in $OUTFILE.time.
-#
-# Ah, the joys of shell programming!
-# To get the time program and the specified program different output filenames,
-# we tell time to launch a shell which in turn executes $PROGRAM with the
-# necessary I/O redirection.
-#
-(time -p sh -c "$PROGRAM $* > $OUTFILE 2>&1 < $INFILE") 2>&1 | awk -- '\
-BEGIN { cpu = 0.0; }
-/^user/ { cpu += $2; print }
-/^sys/ { cpu += $2; print }
-!/^user/ && !/^sys/ { print }
-END { printf("program %f\n", cpu); }' > $OUTFILE.time1
-
-if test $? -eq 0
-then
- touch $OUTFILE.exitok
-fi
-
-if ls | egrep "^core" > /dev/null
-then
- # If we are on a sun4u machine (UltraSparc), then the code we're generating
- # is 64 bit code. In that case, use gdb-64 instead of gdb.
- myarch=`uname -m`
- if [ "$myarch" = "sun4u" ]
- then
- GDB="gdb-64"
- else
- GDB=gdb
- fi
-
- corefile=`ls core* | head -n 1`
- echo "where 100" > StackTrace.$$
- $GDB -q -batch --command=StackTrace.$$ --core=$corefile $PROGRAM < /dev/null
- rm -f StackTrace.$$ $corefile
- exit 0
-fi
-
+EXITOK=$2
+INFILE=$3
+OUTFILE=$4
+PROGRAM=$5
+shift 5
+
+# Run it the first time
+${DIR}/RunSafely.sh $ULIMIT $EXITOK $INFILE $OUTFILE $PROGRAM $*
+exitval=$?
+mv $OUTFILE.time $OUTFILE.time1
TIME1=`grep program $OUTFILE.time1 | sed 's/^program//'`
echo "Program $PROGRAM run #1 time: $TIME1"
-# Do the second and third runs now
-
-(time -p sh -c "$PROGRAM $* > $OUTFILE 2>&1 < $INFILE") 2>&1 | awk -- '\
-BEGIN { cpu = 0.0; }
-/^real/ { print }
-/^user/ { cpu += $2; print }
-/^sys/ { cpu += $2; print }
-!/^real/ && !/^user/ && !/^sys/ { print }
-END { printf("program %f\n", cpu); }' > $OUTFILE.time2
-
+# Run it the second time
+${DIR}/RunSafely.sh $ULIMIT $EXITOK $INFILE $OUTFILE $PROGRAM $*
+mv $OUTFILE.time $OUTFILE.time2
TIME2=`grep program $OUTFILE.time2 | sed 's/^program//'`
echo "Program $PROGRAM run #2 time: $TIME2"
-(time -p sh -c "$PROGRAM $* > $OUTFILE 2>&1 < $INFILE") 2>&1 | awk -- '\
-BEGIN { cpu = 0.0; }
-/^real/ { print }
-/^user/ { cpu += $2; print }
-/^sys/ { cpu += $2; print }
-!/^real/ && !/^user/ && !/^sys/ { print }
-END { printf("program %f\n", cpu); }' > $OUTFILE.time3
-
+# Run it the third time
+${DIR}/RunSafely.sh $ULIMIT $EXITOK $INFILE $OUTFILE $PROGRAM $*
+mv $OUTFILE.time $OUTFILE.time3
TIME3=`grep program $OUTFILE.time3 | sed 's/^program//'`
echo "Program $PROGRAM run #3 time: $TIME3"
@@ -118,4 +48,4 @@
echo "Program $PROGRAM run #$SHORTEST was fastest"
cp $OUTFILE.time$SHORTEST $OUTFILE.time
-exit 0
+exit $exitval
More information about the llvm-commits
mailing list