r241073 - [Static Analyzer] Minor improvements to SATest.
Gabor Horvath
xazax.hun at gmail.com
Tue Jun 30 08:31:18 PDT 2015
Author: xazax
Date: Tue Jun 30 10:31:17 2015
New Revision: 241073
URL: http://llvm.org/viewvc/llvm-project?rev=241073&view=rev
Log:
[Static Analyzer] Minor improvements to SATest.
Differential Revision: http://reviews.llvm.org/D10812
Modified:
cfe/trunk/utils/analyzer/CmpRuns.py
cfe/trunk/utils/analyzer/SATestBuild.py
Modified: cfe/trunk/utils/analyzer/CmpRuns.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/analyzer/CmpRuns.py?rev=241073&r1=241072&r2=241073&view=diff
==============================================================================
--- cfe/trunk/utils/analyzer/CmpRuns.py (original)
+++ cfe/trunk/utils/analyzer/CmpRuns.py Tue Jun 30 10:31:17 2015
@@ -300,7 +300,7 @@ def dumpScanBuildResultsDiff(dirA, dirB,
print >>auxLog, "('TOTAL NEW REPORTS', %r)" % TotalReports
print >>auxLog, "('TOTAL DIFFERENCES', %r)" % foundDiffs
- return foundDiffs
+ return foundDiffs, len(resultsA.diagnostics), len(resultsB.diagnostics)
def main():
from optparse import OptionParser
Modified: cfe/trunk/utils/analyzer/SATestBuild.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/analyzer/SATestBuild.py?rev=241073&r1=241072&r2=241073&view=diff
==============================================================================
--- cfe/trunk/utils/analyzer/SATestBuild.py (original)
+++ cfe/trunk/utils/analyzer/SATestBuild.py Tue Jun 30 10:31:17 2015
@@ -46,6 +46,7 @@ import math
import shutil
import time
import plistlib
+import argparse
from subprocess import check_call, CalledProcessError
#------------------------------------------------------------------------------
@@ -216,6 +217,8 @@ def runScanBuild(Dir, SBOutputDir, PBuil
SBPrefix = "scan-build " + SBOptions + " "
for Command in SBCommandFile:
Command = Command.strip()
+ if len(Command) == 0:
+ continue;
# If using 'make', auto imply a -jX argument
# to speed up analysis. xcodebuild will
# automatically use the maximum number of cores.
@@ -404,7 +407,11 @@ class Discarder(object):
pass # do nothing
# Compare the warnings produced by scan-build.
-def runCmpResults(Dir):
+# Strictness defines the success criteria for the test:
+# 0 - success if there are no crashes or analyzer failure.
+# 1 - success if there are no difference in the number of reported bugs.
+# 2 - success if all the bug reports are identical.
+def runCmpResults(Dir, Strictness = 0):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
@@ -448,11 +455,18 @@ def runCmpResults(Dir):
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
- NumDiffs = CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
+ NumDiffs, ReportsInRef, ReportsInNew = \
+ CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if (NumDiffs > 0) :
print "Warning: %r differences in diagnostics. See %s" % \
(NumDiffs, DiffsPath,)
+ if Strictness >= 2 and NumDiffs > 0:
+ print "Error: Diffs found in strict mode (2)."
+ sys.exit(-1)
+ elif Strictness >= 1 and ReportsInRef != ReportsInNew:
+ print "Error: The number of results are different in strict mode (1)."
+ sys.exit(-1)
print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
return (NumDiffs > 0)
@@ -486,7 +500,7 @@ def updateSVN(Mode, ProjectsMap):
print "Error: SVN update failed."
sys.exit(-1)
-def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None):
+def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Dir=None, Strictness = 0):
print " \n\n--- Building project %s" % (ID,)
TBegin = time.time()
@@ -505,12 +519,12 @@ def testProject(ID, ProjectBuildMode, Is
checkBuild(SBOutputDir)
if IsReferenceBuild == False:
- runCmpResults(Dir)
+ runCmpResults(Dir, Strictness)
print "Completed tests for project %s (time: %.2f)." % \
(ID, (time.time()-TBegin))
-def testAll(IsReferenceBuild = False, UpdateSVN = False):
+def testAll(IsReferenceBuild = False, UpdateSVN = False, Strictness = 0):
PMapFile = open(getProjectMapPath(), "rb")
try:
# Validate the input.
@@ -532,7 +546,7 @@ def testAll(IsReferenceBuild = False, Up
# Test the projects.
PMapFile.seek(0)
for I in csv.reader(PMapFile):
- testProject(I[0], int(I[1]), IsReferenceBuild)
+ testProject(I[0], int(I[1]), IsReferenceBuild, None, Strictness)
# Add reference results to SVN.
if UpdateSVN == True:
@@ -545,18 +559,25 @@ def testAll(IsReferenceBuild = False, Up
PMapFile.close()
if __name__ == '__main__':
+ # Parse command line arguments.
+ Parser = argparse.ArgumentParser(description='Test the Clang Static Analyzer.')
+ Parser.add_argument('--strictness', dest='strictness', type=int, default=0,
+ help='0 to fail on runtime errors, 1 to fail when the number\
+ of found bugs are different from the reference, 2 to \
+ fail on any difference from the reference. Default is 0.')
+ Parser.add_argument('-r', dest='regenerate', action='store_true', default=False,
+ help='Regenerate reference output.')
+ Parser.add_argument('-rs', dest='update_reference', action='store_true',
+ default=False, help='Regenerate reference output and update svn.')
+ Args = Parser.parse_args()
+
IsReference = False
UpdateSVN = False
- if len(sys.argv) >= 2:
- if sys.argv[1] == "-r":
- IsReference = True
- elif sys.argv[1] == "-rs":
- IsReference = True
- UpdateSVN = True
- else:
- print >> sys.stderr, 'Usage: ', sys.argv[0],\
- '[-r|-rs]' \
- 'Use -r to regenerate reference output' \
- 'Use -rs to regenerate reference output and update svn'
+ Strictness = Args.strictness
+ if Args.regenerate:
+ IsReference = True
+ elif Args.update_reference:
+ IsReference = True
+ UpdateSVN = True
- testAll(IsReference, UpdateSVN)
+ testAll(IsReference, UpdateSVN, Strictness)
More information about the cfe-commits
mailing list