Give lit a --xunit-xml-output option for saving results in xunit format

  --xunit-xml-output saves test results to disk in JUnit's xml format. This will allow Jenkins to report the details of a lit run.

  Based on a patch by David Chisnall.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@223163 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/utils/lit/lit/Test.py b/utils/lit/lit/Test.py
index e51bf1297762dfb0dd0cfea3cc316064537b9b52..2e0f478337bc8036022207a95f219e643c294f78 100644 (file)
--- a/utils/lit/lit/Test.py
+++ b/utils/lit/lit/Test.py
@@ -1,4 +1,5 @@
 import os
+from xml.sax.saxutils import escape
 
 # Test result codes.
 
@@ -194,3 +195,17 @@ class Test:
                 return True
 
         return False
+
+
+    def getJUnitXML(self):
+        test_name = self.path_in_suite[-1]
+        test_path = self.path_in_suite[:-1]
+ 
+        xml = "<testcase classname='" + self.suite.name + "." + "/".join(test_path) + "'" + " name='" + test_name + "'"
+        xml += " time='%.2f'" % (self.result.elapsed,)
+        if self.result.code.isFailure:
+          xml += ">\n\t<failure >\n" + escape(self.result.output)
+          xml += "\n\t</failure>\n</testcase>"
+        else:
+          xml += "/>"
+        return xml
\ No newline at end of file

diff --git a/utils/lit/lit/main.py b/utils/lit/lit/main.py
index 7343d242cba6688c93246ac396f2a02ffbb43d64..90f724e913605665421a5a0bdd82384241be6dfd 100755 (executable)
--- a/utils/lit/lit/main.py
+++ b/utils/lit/lit/main.py
@@ -196,6 +196,9 @@ def main(builtinParameters = {}):
     group.add_option("", "--no-execute", dest="noExecute",
                      help="Don't execute any tests (assume PASS)",
                      action="store_true", default=False)
+    group.add_option("", "--xunit-xml-output", dest="xunit_output_file",
+                      help=("Write XUnit-compatible XML test reports to the"
+                            " specified file"), default=None)
     parser.add_option_group(group)
 
     group = OptionGroup(parser, "Test Selection")
@@ -287,10 +290,10 @@ def main(builtinParameters = {}):
     if opts.showSuites or opts.showTests:
         # Aggregate the tests by suite.
         suitesAndTests = {}
-        for t in run.tests:
-            if t.suite not in suitesAndTests:
-                suitesAndTests[t.suite] = []
-            suitesAndTests[t.suite].append(t)
+        for result_test in run.tests:
+            if result_test.suite not in suitesAndTests:
+                suitesAndTests[result_test.suite] = []
+            suitesAndTests[result_test.suite].append(result_test)
         suitesAndTests = list(suitesAndTests.items())
         suitesAndTests.sort(key = lambda item: item[0].name)
 
@@ -323,8 +326,8 @@ def main(builtinParameters = {}):
         except:
             parser.error("invalid regular expression for --filter: %r" % (
                     opts.filter))
-        run.tests = [t for t in run.tests
-                     if rex.search(t.getFullName())]
+        run.tests = [result_test for result_test in run.tests
+                     if rex.search(result_test.getFullName())]
 
     # Then select the order.
     if opts.shuffle:
@@ -332,7 +335,7 @@ def main(builtinParameters = {}):
     elif opts.incremental:
         sort_by_incremental_cache(run)
     else:
-        run.tests.sort(key = lambda t: t.getFullName())
+        run.tests.sort(key = lambda result_test: result_test.getFullName())
 
     # Finally limit the number of tests, if desired.
     if opts.maxTests is not None:
@@ -422,6 +425,36 @@ def main(builtinParameters = {}):
         if N:
             print('  %s: %d' % (name,N))
 
+    if opts.xunit_output_file:
+        # Collect the tests, indexed by test suite
+        by_suite = {}
+        for result_test in run.tests:
+            suite = result_test.suite.config.name
+            if suite not in by_suite:
+                by_suite[suite] = {
+                                   'passes'   : 0,
+                                   'failures' : 0,
+                                   'tests'    : [] }
+            by_suite[suite]['tests'].append(result_test)
+            if result_test.result.code.isFailure:
+                by_suite[suite]['failures'] += 1
+            else:
+                by_suite[suite]['passes'] += 1
+        xunit_output_file = open(opts.xunit_output_file, "w")
+        xunit_output_file.write("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n")
+        xunit_output_file.write("<testsuites>\n")
+        for suite_name, suite in by_suite.items():
+            xunit_output_file.write("<testsuite name='" + suite_name + "'")
+            xunit_output_file.write(" tests='" + str(suite['passes'] + 
+              suite['failures']) + "'")
+            xunit_output_file.write(" failures='" + str(suite['failures']) + 
+              "'>\n")
+            for result_test in suite['tests']:
+                xunit_output_file.write(result_test.getJUnitXML() + "\n")
+            xunit_output_file.write("</testsuite>\n")
+        xunit_output_file.write("</testsuites>")
+        xunit_output_file.close()
+
     # If we encountered any additional errors, exit abnormally.
     if litConfig.numErrors:
         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)