utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.run
  16 import lit.util
  17 import lit.discovery
  18
  19 class TestingProgressDisplay(object):
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.progressBar = progressBar
  25         self.completed = 0
  26
  27     def finish(self):
  28         if self.progressBar:
  29             self.progressBar.clear()
  30         elif self.opts.quiet:
  31             pass
  32         elif self.opts.succinct:
  33             sys.stdout.write('\n')
  34
  35     def update(self, test):
  36         self.completed += 1
  37         if self.progressBar:
  38             self.progressBar.update(float(self.completed)/self.numTests,
  39                                     test.getFullName())
  40
  41         if not test.result.code.isFailure and \
  42                 (self.opts.quiet or self.opts.succinct):
  43             return
  44
  45         if self.progressBar:
  46             self.progressBar.clear()
  47
  48         print('%s: %s (%d of %d)' % (test.result.code.name, test.getFullName(),
  49                                      self.completed, self.numTests))
  50
  51         if test.result.code.isFailure and self.opts.showOutput:
  52             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  53                                               '*'*20))
  54             print(test.result.output)
  55             print("*" * 20)
  56
  57         sys.stdout.flush()
  58
  59 def main(builtinParameters = {}):
  60     # Bump the GIL check interval, its more important to get any one thread to a
  61     # blocking operation (hopefully exec) than to try and unblock other threads.
  62     #
  63     # FIXME: This is a hack.
  64     sys.setcheckinterval(1000)
  65
  66     global options
  67     from optparse import OptionParser, OptionGroup
  68     parser = OptionParser("usage: %prog [options] {file-or-path}")
  69
  70     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
  71                       help="Number of testing threads",
  72                       type=int, action="store", default=None)
  73     parser.add_option("", "--config-prefix", dest="configPrefix",
  74                       metavar="NAME", help="Prefix for 'lit' config files",
  75                       action="store", default=None)
  76     parser.add_option("", "--param", dest="userParameters",
  77                       metavar="NAME=VAL",
  78                       help="Add 'NAME' = 'VAL' to the user defined parameters",
  79                       type=str, action="append", default=[])
  80
  81     group = OptionGroup(parser, "Output Format")
  82     # FIXME: I find these names very confusing, although I like the
  83     # functionality.
  84     group.add_option("-q", "--quiet", dest="quiet",
  85                      help="Suppress no error output",
  86                      action="store_true", default=False)
  87     group.add_option("-s", "--succinct", dest="succinct",
  88                      help="Reduce amount of output",
  89                      action="store_true", default=False)
  90     group.add_option("-v", "--verbose", dest="showOutput",
  91                      help="Show all test output",
  92                      action="store_true", default=False)
  93     group.add_option("", "--no-progress-bar", dest="useProgressBar",
  94                      help="Do not use curses based progress bar",
  95                      action="store_false", default=True)
  96     parser.add_option_group(group)
  97
  98     group = OptionGroup(parser, "Test Execution")
  99     group.add_option("", "--path", dest="path",
 100                      help="Additional paths to add to testing environment",
 101                      action="append", type=str, default=[])
 102     group.add_option("", "--vg", dest="useValgrind",
 103                      help="Run tests under valgrind",
 104                      action="store_true", default=False)
 105     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 106                      help="Check for memory leaks under valgrind",
 107                      action="store_true", default=False)
 108     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 109                      help="Specify an extra argument for valgrind",
 110                      type=str, action="append", default=[])
 111     group.add_option("", "--time-tests", dest="timeTests",
 112                      help="Track elapsed wall time for each test",
 113                      action="store_true", default=False)
 114     group.add_option("", "--no-execute", dest="noExecute",
 115                      help="Don't execute any tests (assume PASS)",
 116                      action="store_true", default=False)
 117     parser.add_option_group(group)
 118
 119     group = OptionGroup(parser, "Test Selection")
 120     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 121                      help="Maximum number of tests to run",
 122                      action="store", type=int, default=None)
 123     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 124                      help="Maximum time to spend testing (in seconds)",
 125                      action="store", type=float, default=None)
 126     group.add_option("", "--shuffle", dest="shuffle",
 127                      help="Run tests in random order",
 128                      action="store_true", default=False)
 129     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 130                      help=("Only run tests with paths matching the given "
 131                            "regular expression"),
 132                      action="store", default=None)
 133     parser.add_option_group(group)
 134
 135     group = OptionGroup(parser, "Debug and Experimental Options")
 136     group.add_option("", "--debug", dest="debug",
 137                       help="Enable debugging (for 'lit' development)",
 138                       action="store_true", default=False)
 139     group.add_option("", "--show-suites", dest="showSuites",
 140                       help="Show discovered test suites",
 141                       action="store_true", default=False)
 142     group.add_option("", "--show-tests", dest="showTests",
 143                       help="Show all discovered tests",
 144                       action="store_true", default=False)
 145     parser.add_option_group(group)
 146
 147     (opts, args) = parser.parse_args()
 148
 149     if not args:
 150         parser.error('No inputs specified')
 151
 152     if opts.numThreads is None:
 153 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 154 # http://bugs.python.org/issue1731717
 155 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 156 # threads by default there.
 157        if sys.hexversion >= 0x2050200:
 158                opts.numThreads = lit.util.detectCPUs()
 159        else:
 160                opts.numThreads = 1
 161
 162     inputs = args
 163
 164     # Create the user defined parameters.
 165     userParams = dict(builtinParameters)
 166     for entry in opts.userParameters:
 167         if '=' not in entry:
 168             name,val = entry,''
 169         else:
 170             name,val = entry.split('=', 1)
 171         userParams[name] = val
 172
 173     # Create the global config object.
 174     litConfig = lit.LitConfig.LitConfig(
 175         progname = os.path.basename(sys.argv[0]),
 176         path = opts.path,
 177         quiet = opts.quiet,
 178         useValgrind = opts.useValgrind,
 179         valgrindLeakCheck = opts.valgrindLeakCheck,
 180         valgrindArgs = opts.valgrindArgs,
 181         noExecute = opts.noExecute,
 182         debug = opts.debug,
 183         isWindows = (platform.system()=='Windows'),
 184         params = userParams,
 185         config_prefix = opts.configPrefix)
 186
 187     # Perform test discovery.
 188     run = lit.run.Run(litConfig,
 189                       lit.discovery.find_tests_for_inputs(litConfig, inputs))
 190
 191     if opts.showSuites or opts.showTests:
 192         # Aggregate the tests by suite.
 193         suitesAndTests = {}
 194         for t in run.tests:
 195             if t.suite not in suitesAndTests:
 196                 suitesAndTests[t.suite] = []
 197             suitesAndTests[t.suite].append(t)
 198         suitesAndTests = list(suitesAndTests.items())
 199         suitesAndTests.sort(key = lambda item: item[0].name)
 200
 201         # Show the suites, if requested.
 202         if opts.showSuites:
 203             print('-- Test Suites --')
 204             for ts,ts_tests in suitesAndTests:
 205                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 206                 print('    Source Root: %s' % ts.source_root)
 207                 print('    Exec Root  : %s' % ts.exec_root)
 208
 209         # Show the tests, if requested.
 210         if opts.showTests:
 211             print('-- Available Tests --')
 212             for ts,ts_tests in suitesAndTests:
 213                 ts_tests.sort(key = lambda test: test.path_in_suite)
 214                 for test in ts_tests:
 215                     print('  %s' % (test.getFullName(),))
 216
 217         # Exit.
 218         sys.exit(0)
 219
 220     # Select and order the tests.
 221     numTotalTests = len(run.tests)
 222
 223     # First, select based on the filter expression if given.
 224     if opts.filter:
 225         try:
 226             rex = re.compile(opts.filter)
 227         except:
 228             parser.error("invalid regular expression for --filter: %r" % (
 229                     opts.filter))
 230         run.tests = [t for t in run.tests
 231                      if rex.search(t.getFullName())]
 232
 233     # Then select the order.
 234     if opts.shuffle:
 235         random.shuffle(run.tests)
 236     else:
 237         run.tests.sort(key = lambda t: t.getFullName())
 238
 239     # Finally limit the number of tests, if desired.
 240     if opts.maxTests is not None:
 241         run.tests = run.tests[:opts.maxTests]
 242
 243     # Don't create more threads than tests.
 244     opts.numThreads = min(len(run.tests), opts.numThreads)
 245
 246     extra = ''
 247     if len(run.tests) != numTotalTests:
 248         extra = ' of %d' % numTotalTests
 249     header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra,
 250                                                       opts.numThreads)
 251
 252     progressBar = None
 253     if not opts.quiet:
 254         if opts.succinct and opts.useProgressBar:
 255             try:
 256                 tc = lit.ProgressBar.TerminalController()
 257                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 258             except ValueError:
 259                 print(header)
 260                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 261         else:
 262             print(header)
 263
 264     startTime = time.time()
 265     display = TestingProgressDisplay(opts, len(run.tests), progressBar)
 266     try:
 267         run.execute_tests(display, opts.numThreads, opts.maxTime)
 268     except KeyboardInterrupt:
 269         sys.exit(2)
 270     display.finish()
 271
 272     if not opts.quiet:
 273         print('Testing Time: %.2fs'%(time.time() - startTime))
 274
 275     # List test results organized by kind.
 276     hasFailures = False
 277     byCode = {}
 278     for test in run.tests:
 279         if test.result.code not in byCode:
 280             byCode[test.result.code] = []
 281         byCode[test.result.code].append(test)
 282         if test.result.code.isFailure:
 283             hasFailures = True
 284
 285     # Print each test in any of the failing groups.
 286     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 287                        ('Failing Tests', lit.Test.FAIL),
 288                        ('Unresolved Tests', lit.Test.UNRESOLVED)):
 289         elts = byCode.get(code)
 290         if not elts:
 291             continue
 292         print('*'*20)
 293         print('%s (%d):' % (title, len(elts)))
 294         for test in elts:
 295             print('    %s' % test.getFullName())
 296         sys.stdout.write('\n')
 297
 298     if opts.timeTests and run.tests:
 299         # Order by time.
 300         test_times = [(test.getFullName(), test.result.elapsed)
 301                       for test in run.tests]
 302         lit.util.printHistogram(test_times, title='Tests')
 303
 304     for name,code in (('Expected Passes    ', lit.Test.PASS),
 305                       ('Expected Failures  ', lit.Test.XFAIL),
 306                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 307                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 308                       ('Unexpected Passes  ', lit.Test.XPASS),
 309                       ('Unexpected Failures', lit.Test.FAIL),):
 310         if opts.quiet and not code.isFailure:
 311             continue
 312         N = len(byCode.get(code,[]))
 313         if N:
 314             print('  %s: %d' % (name,N))
 315
 316     # If we encountered any additional errors, exit abnormally.
 317     if litConfig.numErrors:
 318         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 319         sys.exit(2)
 320
 321     # Warn about warnings.
 322     if litConfig.numWarnings:
 323         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 324
 325     if hasFailures:
 326         sys.exit(1)
 327     sys.exit(0)
 328
 329 if __name__=='__main__':
 330     main()