[cfe-commits] r141337 - in /cfe/trunk/utils/analyzer: SATestAdd.py SATestBuild.py
Anna Zaks
ganna at apple.com
Thu Oct 6 16:26:27 PDT 2011
Author: zaks
Date: Thu Oct 6 18:26:27 2011
New Revision: 141337
URL: http://llvm.org/viewvc/llvm-project?rev=141337&view=rev
Log:
[analyzer] Static Analyzer Qualification Infrastructure: Scripts to support basic testing of the analyzer on external projects. This can be used as a basis for setting up a buildbot.
Added:
cfe/trunk/utils/analyzer/SATestAdd.py
cfe/trunk/utils/analyzer/SATestBuild.py
Added: cfe/trunk/utils/analyzer/SATestAdd.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/analyzer/SATestAdd.py?rev=141337&view=auto
==============================================================================
--- cfe/trunk/utils/analyzer/SATestAdd.py (added)
+++ cfe/trunk/utils/analyzer/SATestAdd.py Thu Oct 6 18:26:27 2011
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+"""
+Static Analyzer qualification infrastructure: adding a new project to
+the Repository Directory.
+
+ Add a new project for testing: build it and add to the Project Map file.
+ Assumes it's being run from the Repository Directory.
+ The project directory should be added inside the Repository Directory and
+ have the same name as the project ID
+
+ The project should use the following files for set up:
+ - pre_run_static_analyzer.sh - prepare the build environment.
+ Ex: make clean can be a part of it.
+ - run_static_analyzer.cmd - a list of commands to run through scan-build.
+ Each command should be on a separate line.
+ Choose from: configure, make, xcodebuild
+"""
+import SATestBuild
+
+import os
+import csv
+import sys
+
+# Add a new project for testing: build it and add to the Project Map file.
+# Params:
+# Dir is the directory where the sources are.
+# ID is a short string used to identify a project.
+def addNewProject(ID) :
+ CurDir = os.path.abspath(os.curdir)
+ Dir = SATestBuild.getProjectDir(ID)
+ if not os.path.exists(Dir):
+ print "Error: Project directory is missing: %s" % Dir
+ sys.exit(-1)
+
+ # Build the project.
+ SATestBuild.testProject(ID, True, Dir)
+
+ # Add the project ID to the project map.
+ ProjectMapPath = os.path.join(CurDir, SATestBuild.ProjectMapFile)
+ if os.path.exists(ProjectMapPath):
+ PMapFile = open(ProjectMapPath, "r+b")
+ else:
+ print "Warning: Creating the Project Map file!!"
+ PMapFile = open(ProjectMapPath, "w+b")
+ try:
+ PMapReader = csv.reader(PMapFile)
+ for I in PMapReader:
+ IID = I[0]
+ if ID == IID:
+ print >> sys.stderr, 'Warning: Project with ID \'', ID, \
+ '\' already exists.'
+ sys.exit(-1)
+
+ PMapWriter = csv.writer(PMapFile)
+ PMapWriter.writerow( (ID, Dir) );
+ finally:
+ PMapFile.close()
+
+ print "The project map is updated: ", ProjectMapPath
+
+
+# TODO: Add an option not to build.
+# TODO: Set the path to the Repository directory.
+if __name__ == '__main__':
+ if len(sys.argv) < 2:
+ print >> sys.stderr, 'Usage: ', sys.argv[0],\
+ '[project ID]'
+ sys.exit(-1)
+
+ addNewProject(sys.argv[1])
Added: cfe/trunk/utils/analyzer/SATestBuild.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/utils/analyzer/SATestBuild.py?rev=141337&view=auto
==============================================================================
--- cfe/trunk/utils/analyzer/SATestBuild.py (added)
+++ cfe/trunk/utils/analyzer/SATestBuild.py Thu Oct 6 18:26:27 2011
@@ -0,0 +1,307 @@
+#!/usr/bin/env python
+
+"""
+Static Analyzer qualification infrastructure.
+
+The goal is to test the analyzer against different projects, check for failures,
+compare results, and measure performance.
+
+Repository Directory will contain sources of the projects as well as the
+information on how to build them and the expected output.
+Repository Directory structure:
+ - ProjectMap file
+ - Historical Performance Data
+ - Project Dir1
+ - ReferenceOutput
+ - Project Dir2
+ - ReferenceOutput
+ ..
+
+To test the build of the analyzer one would:
+ - Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that
+ the build directory does not pollute the repository to min network traffic).
+ - Build all projects, until error. Produce logs to report errors.
+ - Compare results.
+
+The files which should be kept around for failure investigations:
+ RepositoryCopy/Project DirI/ScanBuildResults
+ RepositoryCopy/Project DirI/run_static_analyzer.log
+
+Assumptions (TODO: shouldn't need to assume these.):
+ The script is being run from the Repository Directory.
+ The compiler for scan-build is in the PATH.
+ export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH
+
+For more logging, set the env variables:
+ zaks:TI zaks$ export CCC_ANALYZER_LOG=1
+ zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1
+"""
+import CmpRuns
+
+import os
+import csv
+import sys
+import glob
+import shutil
+import time
+import plistlib
+from subprocess import check_call
+
+# Project map stores info about all the "registered" projects.
+ProjectMapFile = "projectMap.csv"
+
+# Names of the project specific scripts.
+# The script that needs to be executed before the build can start.
+PreprocessScript = "pre_run_static_analyzer.sh"
+# This is a file containing commands for scan-build.
+BuildScript = "run_static_analyzer.cmd"
+
+# The log file name.
+BuildLogName = "run_static_analyzer.log"
+# Summary file - contains the summary of the failures. Ex: This info can be be
+# displayed when buildbot detects a build failure.
+NumOfFailuresInSummary = 10
+FailuresSummaryFileName = "failures.txt"
+# Summary of the result diffs.
+DiffsSummaryFileName = "diffs.txt"
+
+# The scan-build result directory.
+SBOutputDirName = "ScanBuildResults"
+SBOutputDirReferencePrefix = "Ref"
+
+Verbose = 1
+
+def getProjectMapPath():
+ ProjectMapPath = os.path.join(os.path.abspath(os.curdir),
+ ProjectMapFile)
+ if not os.path.exists(ProjectMapPath):
+ print "Error: Cannot find the Project Map file " + ProjectMapPath +\
+ "\nRunning script for the wrong directory?"
+ sys.exit(-1)
+ return ProjectMapPath
+
+def getProjectDir(ID):
+ return os.path.join(os.path.abspath(os.curdir), ID)
+
+# Run pre-processing script if any.
+def runPreProcessingScript(Dir, PBuildLogFile):
+ ScriptPath = os.path.join(Dir, PreprocessScript)
+ if os.path.exists(ScriptPath):
+ try:
+ if Verbose == 1:
+ print " Executing: %s" % (ScriptPath,)
+ check_call("chmod +x %s" % ScriptPath, cwd = Dir,
+ stderr=PBuildLogFile,
+ stdout=PBuildLogFile,
+ shell=True)
+ check_call(ScriptPath, cwd = Dir, stderr=PBuildLogFile,
+ stdout=PBuildLogFile,
+ shell=True)
+ except:
+ print "Error: The pre-processing step failed. See ", \
+ PBuildLogFile.name, " for details."
+ sys.exit(-1)
+
+# Build the project with scan-build by reading in the commands and
+# prefixing them with the scan-build options.
+def runScanBuild(Dir, SBOutputDir, PBuildLogFile):
+ BuildScriptPath = os.path.join(Dir, BuildScript)
+ if not os.path.exists(BuildScriptPath):
+ print "Error: build script is not defined: %s" % BuildScriptPath
+ sys.exit(-1)
+ SBOptions = "-plist -o " + SBOutputDir + " "
+ SBOptions += "-enable-checker core,deadcode.DeadStores"
+ try:
+ SBCommandFile = open(BuildScriptPath, "r")
+ SBPrefix = "scan-build " + SBOptions + " "
+ for Command in SBCommandFile:
+ SBCommand = SBPrefix + Command
+ if Verbose == 1:
+ print " Executing: %s" % (SBCommand,)
+ check_call(SBCommand, cwd = Dir, stderr=PBuildLogFile,
+ stdout=PBuildLogFile,
+ shell=True)
+ except:
+ print "Error: scan-build failed. See ",PBuildLogFile.name,\
+ " for details."
+ sys.exit(-1)
+
+def buildProject(Dir, SBOutputDir):
+ TBegin = time.time()
+
+ BuildLogPath = os.path.join(Dir, BuildLogName)
+ print "Log file: %s" % (BuildLogPath,)
+
+ # Clean up the log file.
+ if (os.path.exists(BuildLogPath)) :
+ RmCommand = "rm " + BuildLogPath
+ if Verbose == 1:
+ print " Executing: %s." % (RmCommand,)
+ check_call(RmCommand, shell=True)
+
+ # Open the log file.
+ PBuildLogFile = open(BuildLogPath, "wb+")
+ try:
+ # Clean up scan build results.
+ if (os.path.exists(SBOutputDir)) :
+ RmCommand = "rm -r " + SBOutputDir
+ if Verbose == 1:
+ print " Executing: %s" % (RmCommand,)
+ check_call(RmCommand, stderr=PBuildLogFile,
+ stdout=PBuildLogFile, shell=True)
+
+ runPreProcessingScript(Dir, PBuildLogFile)
+ runScanBuild(Dir, SBOutputDir, PBuildLogFile)
+ finally:
+ PBuildLogFile.close()
+
+ print "Build complete (time: %.2f). See the log for more details: %s" % \
+ ((time.time()-TBegin), BuildLogPath)
+
+# A plist file is created for each call to the analyzer(each source file).
+# We are only interested on the once that have bug reports, so delete the rest.
+def CleanUpEmptyPlists(SBOutputDir):
+ for F in glob.glob(SBOutputDir + "/*/*.plist"):
+ P = os.path.join(SBOutputDir, F)
+
+ Data = plistlib.readPlist(P)
+ # Delete empty reports.
+ if not Data['files']:
+ os.remove(P)
+ continue
+
+# Given the scan-build output directory, checks if the build failed
+# (by searching for the failures directories). If there are failures, it
+# creates a summary file in the output directory.
+def checkBuild(SBOutputDir):
+ # Check if there are failures.
+ Failures = glob.glob(SBOutputDir + "/*/failures/*.stderr.txt")
+ TotalFailed = len(Failures);
+ if TotalFailed == 0:
+ CleanUpEmptyPlists(SBOutputDir)
+ Plists = glob.glob(SBOutputDir + "/*/*.plist")
+ print "Number of bug reports (non empty plist files) produced: %d" %\
+ len(Plists)
+ return;
+
+ # Create summary file to display when the build fails.
+ SummaryPath = os.path.join(SBOutputDir, FailuresSummaryFileName);
+ if (Verbose > 0):
+ print " Creating the failures summary file %s." % (SummaryPath,)
+
+ SummaryLog = open(SummaryPath, "w+")
+ try:
+ SummaryLog.write("Total of %d failures discovered.\n" % (TotalFailed,))
+ if TotalFailed > NumOfFailuresInSummary:
+ SummaryLog.write("See the first %d below.\n"
+ % (NumOfFailuresInSummary,))
+ # TODO: Add a line "See the results folder for more."
+
+ FailuresCopied = NumOfFailuresInSummary
+ Idx = 0
+ for FailLogPathI in glob.glob(SBOutputDir + "/*/failures/*.stderr.txt"):
+ if Idx >= NumOfFailuresInSummary:
+ break;
+ Idx += 1
+ SummaryLog.write("\n-- Error #%d -----------\n" % (Idx,));
+ FailLogI = open(FailLogPathI, "r");
+ try:
+ shutil.copyfileobj(FailLogI, SummaryLog);
+ finally:
+ FailLogI.close()
+ finally:
+ SummaryLog.close()
+
+ print "Error: Scan-build failed. See ", \
+ os.path.join(SBOutputDir, FailuresSummaryFileName)
+ sys.exit(-1)
+
+# Auxiliary object to discard stdout.
+class Discarder(object):
+ def write(self, text):
+ pass # do nothing
+
+# Compare the warnings produced by scan-build.
+def runCmpResults(Dir):
+ TBegin = time.time()
+
+ RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
+ NewDir = os.path.join(Dir, SBOutputDirName)
+
+ # We have to go one level down the directory tree.
+ RefList = glob.glob(RefDir + "/*")
+ NewList = glob.glob(NewDir + "/*")
+ if len(RefList) == 0 or len(NewList) == 0:
+ return False
+ assert(len(RefList) == len(NewList))
+
+ # There might be more then one folder underneath - one per each scan-build
+ # command (Ex: one for configure and one for make).
+ if (len(RefList) > 1):
+ # Assume that the corresponding folders have the same names.
+ RefList.sort()
+ NewList.sort()
+
+ # Iterate and find the differences.
+ HaveDiffs = False
+ PairList = zip(RefList, NewList)
+ for P in PairList:
+ RefDir = P[0]
+ NewDir = P[1]
+
+ assert(RefDir != NewDir)
+ if Verbose == 1:
+ print " Comparing Results: %s %s" % (RefDir, NewDir)
+
+ DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
+ Opts = CmpRuns.CmpOptions(DiffsPath)
+ # Discard everything coming out of stdout (CmpRun produces a lot of them).
+ OLD_STDOUT = sys.stdout
+ sys.stdout = Discarder()
+ # Scan the results, delete empty plist files.
+ HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
+ sys.stdout = OLD_STDOUT
+ if HaveDiffs:
+ print "Warning: difference in diagnostics. See %s" % (DiffsPath,)
+ HaveDiffs=True
+
+ print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
+ return HaveDiffs
+
+def testProject(ID, IsReferenceBuild, Dir=None):
+ TBegin = time.time()
+
+ if Dir is None :
+ Dir = getProjectDir(ID)
+ if Verbose == 1:
+ print " Build directory: %s." % (Dir,)
+
+ # Set the build results directory.
+ if IsReferenceBuild == True :
+ SBOutputDir = os.path.join(Dir, SBOutputDirReferencePrefix + \
+ SBOutputDirName)
+ else :
+ SBOutputDir = os.path.join(Dir, SBOutputDirName)
+
+ buildProject(Dir, SBOutputDir)
+
+ checkBuild(SBOutputDir)
+
+ if IsReferenceBuild == False:
+ runCmpResults(Dir)
+
+ print "Completed tests for project %s (time: %.2f)." % \
+ (ID, (time.time()-TBegin))
+
+def testAll(IsReferenceBuild=False):
+ PMapFile = open(getProjectMapPath(), "rb")
+ try:
+ PMapReader = csv.reader(PMapFile)
+ for I in PMapReader:
+ print " --- Building project %s" % (I[0],)
+ testProject(I[0], IsReferenceBuild)
+ finally:
+ PMapFile.close()
+
+if __name__ == '__main__':
+ testAll()
More information about the cfe-commits
mailing list