[llvm-commits] CVS: llvm/utils/NightlyTest.pl NightlyTestTemplate.html
Chris Lattner
lattner at cs.uiuc.edu
Fri May 28 15:34:02 PDT 2004
Changes in directory llvm/utils:
NightlyTest.pl updated: 1.45 -> 1.46
NightlyTestTemplate.html updated: 1.23 -> 1.24
---
Log message:
Changes to make the nightly tester run the regression tests at night, yaay!
Change contributed by Reid Spencer.
---
Diffs of the changes: (+156 -13)
Index: llvm/utils/NightlyTest.pl
diff -u llvm/utils/NightlyTest.pl:1.45 llvm/utils/NightlyTest.pl:1.46
--- llvm/utils/NightlyTest.pl:1.45 Mon Jan 12 10:55:30 2004
+++ llvm/utils/NightlyTest.pl Fri May 28 15:30:23 2004
@@ -13,12 +13,16 @@
# -nocheckout Do not create, checkout, update, or configure
# the source tree.
# -noremove Do not remove the BUILDDIR after it has been built.
+# -nofeaturetests Do not run the feature tests.
+# -noregressiontests Do not run the regression tests.
# -notest Do not even attempt to run the test programs. Implies
# -norunningtests.
# -norunningtests Do not run the Olden benchmark suite with
# LARGE_PROBLEM_SIZE enabled.
# -parallel Run two parallel jobs with GNU Make.
# -enable-linscan Enable linearscan tests
+# -verbose Turn on some debug output
+# -debug Print information useful only to maintainers of this script.
#
# CVSROOT is the CVS repository from which the tree will be checked out,
# specified either in the full :method:user at host:/dir syntax, or
@@ -36,9 +40,13 @@
my $HOME = $ENV{'HOME'};
my $CVSRootDir = $ENV{'CVSROOT'};
$CVSRootDir = "/home/vadve/shared/PublicCVS"
- unless $CVSRootDir;
-my $BuildDir = "$HOME/buildtest";
-my $WebDir = "$HOME/cvs/testresults-X86";
+ unless $CVSRootDir;
+my $BuildDir = $ENV{'BUILDDIR'};
+ $BuildDir = "$HOME/buildtest"
+ unless $BuildDir;
+my $WebDir = $ENV{'WEBDIR'};
+ $WebDir = "$HOME/cvs/testresults-X86"
+ unless $WebDir;
# Calculate the date prefix...
@TIME = localtime;
@@ -116,10 +124,14 @@
# Command line argument settings...
my $NOCHECKOUT = 0;
my $NOREMOVE = 0;
+my $NOFEATURES = 0;
+my $NOREGRESSIONS = 0;
my $NOTEST = 0;
my $NORUNNINGTESTS = 0;
my $MAKEOPTS = "";
my $ENABLELINEARSCAN = "";
+my $VERBOSE = 0;
+my $DEBUG = 0;
# Parse arguments...
while (scalar(@ARGV) and ($_ = $ARGV[0], /^[-+]/)) {
@@ -129,10 +141,14 @@
# List command line options here...
if (/^-nocheckout$/) { $NOCHECKOUT = 1; next; }
if (/^-noremove$/) { $NOREMOVE = 1; next; }
+ if (/^-nofeaturetests$/) { $NOFEATURES = 1; next; }
+ if (/^-noregressiontests$/){ $NOREGRESSIONS = 1; next; }
if (/^-notest$/) { $NOTEST = 1; $NORUNNINGTESTS = 1; next; }
if (/^-norunningtests$/) { $NORUNNINGTESTS = 1; next; }
if (/^-parallel$/) { $MAKEOPTS = "-j2 -l3.0"; next; }
if (/^-enable-linscan$/) { $ENABLELINEARSCAN = "ENABLE_LINEARSCAN=1"; next; }
+ if (/^-verbose$/) { $VERBOSE = 1; next; }
+ if (/^-debug$/) { $DEBUG = 1; next; }
print "Unknown option: $_ : ignoring!\n";
}
@@ -145,10 +161,12 @@
$WebDir = $ARGV[2];
}
+
my $Template = "$BuildDir/llvm/utils/NightlyTestTemplate.html";
my $Prefix = "$WebDir/$DATE";
-if (0) {
+if ($VERBOSE) {
+ print "INITIALIZED\n";
print "CVS Root = $CVSRootDir\n";
print "BuildDir = $BuildDir\n";
print "WebDir = $WebDir\n";
@@ -160,6 +178,13 @@
# Create the CVS repository directory
#
if (!$NOCHECKOUT) {
+ if (-d $BuildDir) {
+ if (!$NOREMOVE) {
+ rmdir $BuildDir or die "Could not remove CVS checkout directory $BuildDir!";
+ } else {
+ die "CVS checkout directory $BuildDir already exists!";
+ }
+ }
mkdir $BuildDir or die "Could not create CVS checkout directory $BuildDir!";
}
chdir $BuildDir or die "Could not change to CVS checkout directory $BuildDir!";
@@ -170,12 +195,17 @@
#
$CVSOPT = "";
$CVSOPT = "-z3" if $CVSRootDir =~ /^:ext:/; # Use compression if going over ssh.
-system "(time -p cvs $CVSOPT -d $CVSRootDir co llvm) > $Prefix-CVS-Log.txt 2>&1"
- if (!$NOCHECKOUT);
+if (!$NOCHECKOUT) {
+ if ( $VERBOSE ) { print "CHECKOUT STAGE\n"; }
+ system "(time -p cvs $CVSOPT -d $CVSRootDir co llvm) > $Prefix-CVS-Log.txt 2>&1";
+}
chdir "llvm" or die "Could not change into llvm directory!";
-system "cvs up -P -d > /dev/null 2>&1" if (!$NOCHECKOUT);
+if (!$NOCHECKOUT) {
+ if ( $VERBOSE ) { print "UPDATE STAGE\n"; }
+ system "cvs update -P -d > /dev/null 2>&1" ;
+}
# Read in the HTML template file...
my $TemplateContents = ReadFile $Template;
@@ -193,8 +223,10 @@
# Build the entire tree, saving build messages to the build log
#
if (!$NOCHECKOUT) {
+ if ( $VERBOSE ) { print "CONFIGURE STAGE\n"; }
system "(time -p ./configure --enable-jit --enable-spec --with-objroot=.) > $Prefix-Build-Log.txt 2>&1";
+ if ( $VERBOSE ) { print "BUILD STAGE\n"; }
# Build the entire tree, capturing the output into $Prefix-Build-Log.txt
system "(time -p gmake $MAKEOPTS) >> $Prefix-Build-Log.txt 2>&1";
}
@@ -232,6 +264,84 @@
print "BUILD ERROR\n";
}
+sub GetQMTestResults { # (filename)
+ my ($filename) = @_;
+ my @lines;
+ my $firstline;
+ $/ = "\n"; #Make sure we're going line at a time.
+ if (open SRCHFILE, $filename) {
+ while ( <SRCHFILE> ) {
+ if ( m/^--- TEST RESULTS/ ) {
+ push(@lines, $_); last;
+ }
+ }
+ while ( <SRCHFILE> ) {
+ if ( length($_) > 1 ) {
+ if ( ! m/^gmake:/ && ! m/^ qmtest.target:/ && !/^ local/ ) {
+ push(@lines,$_);
+ }
+ }
+ }
+ close SRCHFILE;
+ }
+ my $content = join("", at lines);
+ return "<pre>\n at lines</pre>\n";
+}
+
+
+# Get results of feature tests.
+my $FeatureTestResults; # String containing the results of the feature tests
+my $FeatureTime; # System+CPU Time for feature tests
+my $FeatureWallTime; # Wall Clock Time for feature tests
+if (!$NOFEATURES) {
+ if ( $VERBOSE ) { print "FEATURE TEST STAGE\n"; }
+ my $feature_output = "$Prefix-FeatureTests-Log.txt";
+
+ # Run the feature tests so we can summarize the results
+ system "(time -p gmake -C test Feature.t) > $feature_output 2>&1";
+
+ # Extract test results
+ $FeatureTestResults = GetQMTestResults("$feature_output");
+
+ # Extract time of feature tests
+ my $FeatureTimeU = GetRegexNum "^user", 0, "([0-9.]+)", "$feature_output";
+ my $FeatureTimeS = GetRegexNum "^sys", 0, "([0-9.]+)", "$feature_output";
+ $FeatureTime = $FeatureTimeU+$FeatureTimeS; # FeatureTime = User+System
+ $FeatureWallTime = GetRegexNum "^real", 0,"([0-9.]+)","$feature_output";
+ # Run the regression tests so we can summarize the results
+} else {
+ $FeatureTestResults = "Skipped by user choice.";
+ $FeatureTime = "0.0";
+ $FeatureWallTime = "0.0";
+}
+
+if (!$NOREGRESSIONS) {
+ if ( $VERBOSE ) { print "REGRESSION TEST STAGE\n"; }
+ my $regression_output = "$Prefix-RegressionTests-Log.txt";
+
+ # Run the regression tests so we can summarize the results
+ system "(time -p gmake -C test Regression.t) > $regression_output 2>&1";
+
+ # Extract test results
+ $RegressionTestResults = GetQMTestResults("$regression_output");
+
+ # Extract time of regressions tests
+ my $RegressionTimeU = GetRegexNum "^user", 0, "([0-9.]+)", "$regression_output";
+ my $RegressionTimeS = GetRegexNum "^sys", 0, "([0-9.]+)", "$regression_output";
+ $RegressionTime = $RegressionTimeU+$RegressionTimeS; # RegressionTime = User+System
+ $RegressionWallTime = GetRegexNum "^real", 0,"([0-9.]+)","$regression_output";
+} else {
+ $RegressionTestResults = "Skipped by user choice.";
+ $RegressionTime = "0.0";
+ $RegressionWallTime = "0.0";
+}
+
+if ($DEBUG) {
+ print $FeatureTestResults;
+ print $RegressionTestResults;
+}
+
+if ( $VERBOSE ) { print "BUILD INFORMATION COLLECTION STAGE\n"; }
#
# Get warnings from the build
#
@@ -267,6 +377,7 @@
#
# Get some statistics about CVS commits over the current day...
#
+if ($VERBOSE) { print "CVS HISTORY ANALYSIS STAGE\n"; }
@CVSHistory = split "\n", `cvs history -D '1 day ago' -a -xAMROCGUW`;
#print join "\n", @CVSHistory; print "\n";
@@ -361,13 +472,25 @@
# If we build the tree successfully, run the nightly programs tests...
if ($BuildError eq "") {
+ if ( $VERBOSE ) {
+ print "SingleSource TEST STAGE\n";
+ }
$SingleSourceProgramsTable = TestDirectory("SingleSource");
+ if ( $VERBOSE ) {
+ print "MultiSource TEST STAGE\n";
+ }
$MultiSourceProgramsTable = TestDirectory("MultiSource");
+ if ( $VERBOSE ) {
+ print "External TEST STAGE\n";
+ }
$ExternalProgramsTable = TestDirectory("External");
system "cat $Prefix-SingleSource-Tests.txt $Prefix-MultiSource-Tests.txt ".
" $Prefix-External-Tests.txt | sort > $Prefix-Tests.txt";
}
+if ( $VERBOSE ) {
+ print "TEST INFORMATION COLLECTION STAGE\n";
+}
my ($TestsAdded, $TestsRemoved, $TestsFixed, $TestsBroken) = ("","","","");
if ($TestError) {
@@ -493,6 +616,9 @@
AddRecord($LOC, "running_loc.txt");
AddRecord($BuildTime, "running_build_time.txt");
+if ( $VERBOSE ) {
+ print "GRAPH GENERATION STAGE\n";
+}
#
# Rebuild the graphs now...
#
@@ -509,12 +635,13 @@
#
# Print out information...
#
-if (0) {
+if ($VERBOSE) {
print "DateString: $DateString\n";
print "CVS Checkout: $CVSCheckoutTime seconds\n";
print "Files/Dirs/LOC in CVS: $NumFilesInCVS/$NumDirsInCVS/$LOC\n";
-
print "Build Time: $BuildTime seconds\n";
+ print "Feature Test Time: $FeatureTime seconds\n";
+ print "Regression Test Time: $RegressionTime seconds\n";
print "Libraries/Executables/Objects built: $NumLibraries/$NumExecutables/$NumObjects\n";
print "WARNINGS:\n $WarningsList\n";
@@ -532,6 +659,9 @@
# Output the files...
#
+if ( $VERBOSE ) {
+ print "OUTPUT STAGE\n";
+}
# Main HTML file...
my $Output;
eval "\$Output = <<ENDOFFILE;$TemplateContents\nENDOFFILE\n";
Index: llvm/utils/NightlyTestTemplate.html
diff -u llvm/utils/NightlyTestTemplate.html:1.23 llvm/utils/NightlyTestTemplate.html:1.24
--- llvm/utils/NightlyTestTemplate.html:1.23 Thu Dec 18 21:34:46 2003
+++ llvm/utils/NightlyTestTemplate.html Fri May 28 15:30:23 2004
@@ -17,6 +17,7 @@
<a href="#Changes">Changes</a><br>
<a href="#Trends">Trends</a><br>
<a href="#Programs">Programs</a><br>
+<a href="#Feature">Feature</a><br>
<a href="#Regression">Regression</a><br>
</td></tr></table></td></tr></table>
@@ -65,6 +66,10 @@
(<b>$ConfigWallTime</b> seconds wall time)<br>
Time to build CVS tree: <b>$BuildTime</b> seconds
(<b>$BuildWallTime</b> seconds wall time)<br>
+ Time to run feature tests: <b>$FeatureTime</b> seconds
+ (<b>$FeatureWallTime</b> seconds wall time)<br>
+ Time to run regression tests: <b>$RegressionTime</b> seconds
+ (<b>$RegressionWallTime</b> seconds wall time)<br>
Number of object files compiled: <b>$NumObjects</b><br>
Number of libraries linked: <b>$NumLibraries</b><br>
Number of executables linked:<b> $NumExecutables</b><br>
@@ -233,13 +238,21 @@
-<br><br><center>
+<br/><br/><center>
<table border="0" cellspacing="0" cellpadding="2"><tr><td bgcolor="#000000">
<table border="0" cellpadding="10" cellspacing="0"><tr><td bgcolor="#DDAA77"
-<font size=+2 face=Verdana><b><a name="Regression">Regression Tests</font></b>
-</td></tr></table></td></tr></table></center><p>
+<font size=+2 face=Verdana><b><a name="Feature">Feature Test Results</font></b>
+</td></tr></table></td></tr></table></center>
+<br/>
+$FeatureTestResults
-Not yet implemented<p>
+<br/><br/><center>
+<table border="0" cellspacing="0" cellpadding="2"><tr><td bgcolor="#000000">
+<table border="0" cellpadding="10" cellspacing="0"><tr><td bgcolor="#DDAA77"
+<font size=+2 face=Verdana><b><a name="Regression">Regression Tests</font></b>
+</td></tr></table></td></tr></table></center>
+<br/>
+$RegressionTestResults
</td></tr>
More information about the llvm-commits
mailing list