utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time, threading, traceback
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.Util
  16
  17 import lit.discovery
  18
  19 class TestingProgressDisplay:
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.lock = threading.Lock()
  25         self.progressBar = progressBar
  26         self.completed = 0
  27
  28     def update(self, test):
  29         # Avoid locking overhead in quiet mode
  30         if self.opts.quiet and not test.result.isFailure:
  31             self.completed += 1
  32             return
  33
  34         # Output lock.
  35         self.lock.acquire()
  36         try:
  37             self.handleUpdate(test)
  38         finally:
  39             self.lock.release()
  40
  41     def finish(self):
  42         if self.progressBar:
  43             self.progressBar.clear()
  44         elif self.opts.quiet:
  45             pass
  46         elif self.opts.succinct:
  47             sys.stdout.write('\n')
  48
  49     def handleUpdate(self, test):
  50         self.completed += 1
  51         if self.progressBar:
  52             self.progressBar.update(float(self.completed)/self.numTests,
  53                                     test.getFullName())
  54
  55         if self.opts.succinct and not test.result.isFailure:
  56             return
  57
  58         if self.progressBar:
  59             self.progressBar.clear()
  60
  61         print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  62                                      self.completed, self.numTests))
  63
  64         if test.result.isFailure and self.opts.showOutput:
  65             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  66                                               '*'*20))
  67             print(test.output)
  68             print("*" * 20)
  69
  70         sys.stdout.flush()
  71
  72 class TestProvider:
  73     def __init__(self, tests, maxTime):
  74         self.maxTime = maxTime
  75         self.iter = iter(tests)
  76         self.lock = threading.Lock()
  77         self.startTime = time.time()
  78         self.canceled = False
  79
  80     def cancel(self):
  81         self.lock.acquire()
  82         self.canceled = True
  83         self.lock.release()
  84
  85     def get(self):
  86         # Check if we have run out of time.
  87         if self.maxTime is not None:
  88             if time.time() - self.startTime > self.maxTime:
  89                 return None
  90
  91         # Otherwise take the next test.
  92         self.lock.acquire()
  93         if self.canceled:
  94           self.lock.release()
  95           return None
  96         for item in self.iter:
  97             break
  98         else:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print('\nCtrl-C detected, goodbye.')
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = lit.Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def runTests(numThreads, litConfig, provider, display):
 141     # If only using one testing thread, don't use threads at all; this lets us
 142     # profile, among other things.
 143     if numThreads == 1:
 144         t = Tester(litConfig, provider, display)
 145         t.run()
 146         return
 147
 148     # Otherwise spin up the testing threads and wait for them to finish.
 149     testers = [Tester(litConfig, provider, display)
 150                for i in range(numThreads)]
 151     for t in testers:
 152         t.start()
 153     try:
 154         for t in testers:
 155             t.join()
 156     except KeyboardInterrupt:
 157         sys.exit(2)
 158
 159 def main(builtinParameters = {}):
 160     # Bump the GIL check interval, its more important to get any one thread to a
 161     # blocking operation (hopefully exec) than to try and unblock other threads.
 162     #
 163     # FIXME: This is a hack.
 164     import sys
 165     sys.setcheckinterval(1000)
 166
 167     global options
 168     from optparse import OptionParser, OptionGroup
 169     parser = OptionParser("usage: %prog [options] {file-or-path}")
 170
 171     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 172                       help="Number of testing threads",
 173                       type=int, action="store", default=None)
 174     parser.add_option("", "--config-prefix", dest="configPrefix",
 175                       metavar="NAME", help="Prefix for 'lit' config files",
 176                       action="store", default=None)
 177     parser.add_option("", "--param", dest="userParameters",
 178                       metavar="NAME=VAL",
 179                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 180                       type=str, action="append", default=[])
 181
 182     group = OptionGroup(parser, "Output Format")
 183     # FIXME: I find these names very confusing, although I like the
 184     # functionality.
 185     group.add_option("-q", "--quiet", dest="quiet",
 186                      help="Suppress no error output",
 187                      action="store_true", default=False)
 188     group.add_option("-s", "--succinct", dest="succinct",
 189                      help="Reduce amount of output",
 190                      action="store_true", default=False)
 191     group.add_option("-v", "--verbose", dest="showOutput",
 192                      help="Show all test output",
 193                      action="store_true", default=False)
 194     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 195                      help="Do not use curses based progress bar",
 196                      action="store_false", default=True)
 197     parser.add_option_group(group)
 198
 199     group = OptionGroup(parser, "Test Execution")
 200     group.add_option("", "--path", dest="path",
 201                      help="Additional paths to add to testing environment",
 202                      action="append", type=str, default=[])
 203     group.add_option("", "--vg", dest="useValgrind",
 204                      help="Run tests under valgrind",
 205                      action="store_true", default=False)
 206     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 207                      help="Check for memory leaks under valgrind",
 208                      action="store_true", default=False)
 209     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 210                      help="Specify an extra argument for valgrind",
 211                      type=str, action="append", default=[])
 212     group.add_option("", "--time-tests", dest="timeTests",
 213                      help="Track elapsed wall time for each test",
 214                      action="store_true", default=False)
 215     group.add_option("", "--no-execute", dest="noExecute",
 216                      help="Don't execute any tests (assume PASS)",
 217                      action="store_true", default=False)
 218     parser.add_option_group(group)
 219
 220     group = OptionGroup(parser, "Test Selection")
 221     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 222                      help="Maximum number of tests to run",
 223                      action="store", type=int, default=None)
 224     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 225                      help="Maximum time to spend testing (in seconds)",
 226                      action="store", type=float, default=None)
 227     group.add_option("", "--shuffle", dest="shuffle",
 228                      help="Run tests in random order",
 229                      action="store_true", default=False)
 230     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 231                      help=("Only run tests with paths matching the given "
 232                            "regular expression"),
 233                      action="store", default=None)
 234     parser.add_option_group(group)
 235
 236     group = OptionGroup(parser, "Debug and Experimental Options")
 237     group.add_option("", "--debug", dest="debug",
 238                       help="Enable debugging (for 'lit' development)",
 239                       action="store_true", default=False)
 240     group.add_option("", "--show-suites", dest="showSuites",
 241                       help="Show discovered test suites",
 242                       action="store_true", default=False)
 243     group.add_option("", "--show-tests", dest="showTests",
 244                       help="Show all discovered tests",
 245                       action="store_true", default=False)
 246     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 247                       help="Repeat tests N times (for timing)",
 248                       action="store", default=None, type=int)
 249     parser.add_option_group(group)
 250
 251     (opts, args) = parser.parse_args()
 252
 253     if not args:
 254         parser.error('No inputs specified')
 255
 256     if opts.numThreads is None:
 257 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 258 # http://bugs.python.org/issue1731717
 259 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 260 # threads by default there.
 261        if sys.hexversion >= 0x2050200:
 262                opts.numThreads = lit.Util.detectCPUs()
 263        else:
 264                opts.numThreads = 1
 265
 266     inputs = args
 267
 268     # Create the user defined parameters.
 269     userParams = dict(builtinParameters)
 270     for entry in opts.userParameters:
 271         if '=' not in entry:
 272             name,val = entry,''
 273         else:
 274             name,val = entry.split('=', 1)
 275         userParams[name] = val
 276
 277     # Create the global config object.
 278     litConfig = lit.LitConfig.LitConfig(
 279         progname = os.path.basename(sys.argv[0]),
 280         path = opts.path,
 281         quiet = opts.quiet,
 282         useValgrind = opts.useValgrind,
 283         valgrindLeakCheck = opts.valgrindLeakCheck,
 284         valgrindArgs = opts.valgrindArgs,
 285         noExecute = opts.noExecute,
 286         debug = opts.debug,
 287         isWindows = (platform.system()=='Windows'),
 288         params = userParams,
 289         config_prefix = opts.configPrefix)
 290
 291     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 292
 293     if opts.showSuites or opts.showTests:
 294         # Aggregate the tests by suite.
 295         suitesAndTests = {}
 296         for t in tests:
 297             if t.suite not in suitesAndTests:
 298                 suitesAndTests[t.suite] = []
 299             suitesAndTests[t.suite].append(t)
 300         suitesAndTests = suitesAndTests.items()
 301         suitesAndTests.sort(key = lambda item: item[0].name)
 302
 303         # Show the suites, if requested.
 304         if opts.showSuites:
 305             print('-- Test Suites --')
 306             for ts,ts_tests in suitesAndTests:
 307                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 308                 print('    Source Root: %s' % ts.source_root)
 309                 print('    Exec Root  : %s' % ts.exec_root)
 310
 311         # Show the tests, if requested.
 312         if opts.showTests:
 313             print('-- Available Tests --')
 314             for ts,ts_tests in suitesAndTests:
 315                 ts_tests.sort(key = lambda test: test.path_in_suite)
 316                 for test in ts_tests:
 317                     print('  %s' % (test.getFullName(),))
 318
 319     # Select and order the tests.
 320     numTotalTests = len(tests)
 321
 322     # First, select based on the filter expression if given.
 323     if opts.filter:
 324         try:
 325             rex = re.compile(opts.filter)
 326         except:
 327             parser.error("invalid regular expression for --filter: %r" % (
 328                     opts.filter))
 329         tests = [t for t in tests
 330                  if rex.search(t.getFullName())]
 331
 332     # Then select the order.
 333     if opts.shuffle:
 334         random.shuffle(tests)
 335     else:
 336         tests.sort(key = lambda t: t.getFullName())
 337
 338     # Finally limit the number of tests, if desired.
 339     if opts.maxTests is not None:
 340         tests = tests[:opts.maxTests]
 341
 342     # Don't create more threads than tests.
 343     opts.numThreads = min(len(tests), opts.numThreads)
 344
 345     extra = ''
 346     if len(tests) != numTotalTests:
 347         extra = ' of %d' % numTotalTests
 348     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 349                                                       opts.numThreads)
 350
 351     if opts.repeatTests:
 352         tests = [t.copyWithIndex(i)
 353                  for t in tests
 354                  for i in range(opts.repeatTests)]
 355
 356     progressBar = None
 357     if not opts.quiet:
 358         if opts.succinct and opts.useProgressBar:
 359             try:
 360                 tc = lit.ProgressBar.TerminalController()
 361                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 362             except ValueError:
 363                 print(header)
 364                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 365         else:
 366             print(header)
 367
 368     startTime = time.time()
 369     display = TestingProgressDisplay(opts, len(tests), progressBar)
 370     provider = TestProvider(tests, opts.maxTime)
 371
 372     try:
 373       import win32api
 374     except ImportError:
 375       pass
 376     else:
 377       def console_ctrl_handler(type):
 378         provider.cancel()
 379         return True
 380       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 381
 382     runTests(opts.numThreads, litConfig, provider, display)
 383     display.finish()
 384
 385     if not opts.quiet:
 386         print('Testing Time: %.2fs'%(time.time() - startTime))
 387
 388     # Update results for any tests which weren't run.
 389     for t in tests:
 390         if t.result is None:
 391             t.setResult(lit.Test.UNRESOLVED, '', 0.0)
 392
 393     # List test results organized by kind.
 394     hasFailures = False
 395     byCode = {}
 396     for t in tests:
 397         if t.result not in byCode:
 398             byCode[t.result] = []
 399         byCode[t.result].append(t)
 400         if t.result.isFailure:
 401             hasFailures = True
 402
 403     # FIXME: Show unresolved and (optionally) unsupported tests.
 404     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 405                        ('Failing Tests', lit.Test.FAIL)):
 406         elts = byCode.get(code)
 407         if not elts:
 408             continue
 409         print('*'*20)
 410         print('%s (%d):' % (title, len(elts)))
 411         for t in elts:
 412             print('    %s' % t.getFullName())
 413         sys.stdout.write('\n')
 414
 415     if opts.timeTests:
 416         # Collate, in case we repeated tests.
 417         times = {}
 418         for t in tests:
 419             key = t.getFullName()
 420             times[key] = times.get(key, 0.) + t.elapsed
 421
 422         byTime = list(times.items())
 423         byTime.sort(key = lambda item: item[1])
 424         if byTime:
 425             lit.Util.printHistogram(byTime, title='Tests')
 426
 427     for name,code in (('Expected Passes    ', lit.Test.PASS),
 428                       ('Expected Failures  ', lit.Test.XFAIL),
 429                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 430                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 431                       ('Unexpected Passes  ', lit.Test.XPASS),
 432                       ('Unexpected Failures', lit.Test.FAIL),):
 433         if opts.quiet and not code.isFailure:
 434             continue
 435         N = len(byCode.get(code,[]))
 436         if N:
 437             print('  %s: %d' % (name,N))
 438
 439     # If we encountered any additional errors, exit abnormally.
 440     if litConfig.numErrors:
 441         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 442         sys.exit(2)
 443
 444     # Warn about warnings.
 445     if litConfig.numWarnings:
 446         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 447
 448     if hasFailures:
 449         sys.exit(1)
 450     sys.exit(0)
 451
 452 if __name__=='__main__':
 453     main()