[Analyzer Tests] Run static analyzer integration tests until the end,

Do not stop at the first failure.

Differential Revision: https://reviews.llvm.org/D38589

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@314992 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 20 insertions, 8 deletions

diff --git a/utils/analyzer/SATestBuild.py b/utils/analyzer/SATestBuild.py
index 26c519543e..f2d2df6a67 100755
--- a/utils/analyzer/SATestBuild.py
+++ b/utils/analyzer/SATestBuild.py
@@ -570,7 +570,11 @@ def runCmpResults(Dir, Strictness=0):
       0 - success if there are no crashes or analyzer failure.
       1 - success if there are no difference in the number of reported bugs.
       2 - success if all the bug reports are identical.
+
+    :return success: Whether tests pass according to the Strictness
+    criteria.
     """
+    TestsPassed = True
     TBegin = time.time()
 
     RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
@@ -586,8 +590,6 @@ def runCmpResults(Dir, Strictness=0):
         RefList.remove(RefLogDir)
     NewList.remove(os.path.join(NewDir, LogFolderName))
 
-    if len(RefList) == 0 or len(NewList) == 0:
-        return False
     assert(len(RefList) == len(NewList))
 
     # There might be more then one folder underneath - one per each scan-build
@@ -624,15 +626,15 @@ def runCmpResults(Dir, Strictness=0):
                   (NumDiffs, DiffsPath,)
         if Strictness >= 2 and NumDiffs > 0:
             print "Error: Diffs found in strict mode (2)."
-            sys.exit(-1)
+            TestsPassed = False
         elif Strictness >= 1 and ReportsInRef != ReportsInNew:
             print "Error: The number of results are different in "\
                   "strict mode (1)."
-            sys.exit(-1)
+            TestsPassed = False
 
     print "Diagnostic comparison complete (time: %.2f)." % (
           time.time() - TBegin)
-    return (NumDiffs > 0)
+    return TestsPassed
 
 
 def cleanupReferenceResults(SBOutputDir):
@@ -686,6 +688,11 @@ def updateSVN(Mode, PMapFile):
 
 
 def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
+    """
+    Test a given project.
+    :return TestsPassed: Whether tests have passed according
+    to the :param Strictness: criteria.
+    """
     print " \n\n--- Building project %s" % (ID,)
 
     TBegin = time.time()
@@ -704,11 +711,13 @@ def testProject(ID, ProjectBuildMode, IsReferenceBuild=False, Strictness=0):
 
     if IsReferenceBuild:
         cleanupReferenceResults(SBOutputDir)
+        TestsPassed = True
     else:
-        runCmpResults(Dir, Strictness)
+        TestsPassed = runCmpResults(Dir, Strictness)
 
     print "Completed tests for project %s (time: %.2f)." % \
           (ID, (time.time() - TBegin))
+    return TestsPassed
 
 
 def isCommentCSVLine(Entries):
@@ -749,6 +758,7 @@ def validateProjectFile(PMapFile):
 
 
 def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
+    TestsPassed = True
     with projectFileHandler() as PMapFile:
         validateProjectFile(PMapFile)
 
@@ -760,7 +770,7 @@ def testAll(IsReferenceBuild=False, UpdateSVN=False, Strictness=0):
 
         # Test the projects.
         for (ProjName, ProjBuildMode) in iterateOverProjects(PMapFile):
-            testProject(
+            TestsPassed &= testProject(
                 ProjName, int(ProjBuildMode), IsReferenceBuild, Strictness)
 
         # Re-add reference results to SVN.
@@ -793,4 +803,6 @@ if __name__ == '__main__':
         IsReference = True
         UpdateSVN = True
 
-    testAll(IsReference, UpdateSVN, Strictness)
+    TestsPassed = testAll(IsReference, UpdateSVN, Strictness)
+    if not TestsPassed:
+        sys.exit(-1)