utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time, threading, traceback
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.Util
  16
  17 import lit.discovery
  18
  19 class TestingProgressDisplay:
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.lock = threading.Lock()
  25         self.progressBar = progressBar
  26         self.completed = 0
  27
  28     def update(self, test):
  29         # Avoid locking overhead in quiet mode
  30         if self.opts.quiet and not test.result.isFailure:
  31             self.completed += 1
  32             return
  33
  34         # Output lock.
  35         self.lock.acquire()
  36         try:
  37             self.handleUpdate(test)
  38         finally:
  39             self.lock.release()
  40
  41     def finish(self):
  42         if self.progressBar:
  43             self.progressBar.clear()
  44         elif self.opts.quiet:
  45             pass
  46         elif self.opts.succinct:
  47             sys.stdout.write('\n')
  48
  49     def handleUpdate(self, test):
  50         self.completed += 1
  51         if self.progressBar:
  52             self.progressBar.update(float(self.completed)/self.numTests,
  53                                     test.getFullName())
  54
  55         if self.opts.succinct and not test.result.isFailure:
  56             return
  57
  58         if self.progressBar:
  59             self.progressBar.clear()
  60
  61         print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  62                                      self.completed, self.numTests))
  63
  64         if test.result.isFailure and self.opts.showOutput:
  65             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  66                                               '*'*20))
  67             print(test.output)
  68             print("*" * 20)
  69
  70         sys.stdout.flush()
  71
  72 class TestProvider:
  73     def __init__(self, tests, maxTime):
  74         self.maxTime = maxTime
  75         self.iter = iter(tests)
  76         self.lock = threading.Lock()
  77         self.startTime = time.time()
  78         self.canceled = False
  79
  80     def cancel(self):
  81         self.lock.acquire()
  82         self.canceled = True
  83         self.lock.release()
  84
  85     def get(self):
  86         # Check if we have run out of time.
  87         if self.maxTime is not None:
  88             if time.time() - self.startTime > self.maxTime:
  89                 return None
  90
  91         # Otherwise take the next test.
  92         self.lock.acquire()
  93         if self.canceled:
  94           self.lock.release()
  95           return None
  96         for item in self.iter:
  97             break
  98         else:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print('\nCtrl-C detected, goodbye.')
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = lit.Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def runTests(numThreads, litConfig, provider, display):
 141     # If only using one testing thread, don't use threads at all; this lets us
 142     # profile, among other things.
 143     if numThreads == 1:
 144         t = Tester(litConfig, provider, display)
 145         t.run()
 146         return
 147
 148     # Otherwise spin up the testing threads and wait for them to finish.
 149     testers = [Tester(litConfig, provider, display)
 150                for i in range(numThreads)]
 151     for t in testers:
 152         t.start()
 153     try:
 154         for t in testers:
 155             t.join()
 156     except KeyboardInterrupt:
 157         sys.exit(2)
 158
 159 def main(builtinParameters = {}):
 160     # Bump the GIL check interval, its more important to get any one thread to a
 161     # blocking operation (hopefully exec) than to try and unblock other threads.
 162     #
 163     # FIXME: This is a hack.
 164     import sys
 165     sys.setcheckinterval(1000)
 166
 167     global options
 168     from optparse import OptionParser, OptionGroup
 169     parser = OptionParser("usage: %prog [options] {file-or-path}")
 170
 171     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 172                       help="Number of testing threads",
 173                       type=int, action="store", default=None)
 174     parser.add_option("", "--config-prefix", dest="configPrefix",
 175                       metavar="NAME", help="Prefix for 'lit' config files",
 176                       action="store", default=None)
 177     parser.add_option("", "--param", dest="userParameters",
 178                       metavar="NAME=VAL",
 179                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 180                       type=str, action="append", default=[])
 181
 182     group = OptionGroup(parser, "Output Format")
 183     # FIXME: I find these names very confusing, although I like the
 184     # functionality.
 185     group.add_option("-q", "--quiet", dest="quiet",
 186                      help="Suppress no error output",
 187                      action="store_true", default=False)
 188     group.add_option("-s", "--succinct", dest="succinct",
 189                      help="Reduce amount of output",
 190                      action="store_true", default=False)
 191     group.add_option("-v", "--verbose", dest="showOutput",
 192                      help="Show all test output",
 193                      action="store_true", default=False)
 194     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 195                      help="Do not use curses based progress bar",
 196                      action="store_false", default=True)
 197     parser.add_option_group(group)
 198
 199     group = OptionGroup(parser, "Test Execution")
 200     group.add_option("", "--path", dest="path",
 201                      help="Additional paths to add to testing environment",
 202                      action="append", type=str, default=[])
 203     group.add_option("", "--vg", dest="useValgrind",
 204                      help="Run tests under valgrind",
 205                      action="store_true", default=False)
 206     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 207                      help="Check for memory leaks under valgrind",
 208                      action="store_true", default=False)
 209     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 210                      help="Specify an extra argument for valgrind",
 211                      type=str, action="append", default=[])
 212     group.add_option("", "--time-tests", dest="timeTests",
 213                      help="Track elapsed wall time for each test",
 214                      action="store_true", default=False)
 215     parser.add_option_group(group)
 216
 217     group = OptionGroup(parser, "Test Selection")
 218     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 219                      help="Maximum number of tests to run",
 220                      action="store", type=int, default=None)
 221     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 222                      help="Maximum time to spend testing (in seconds)",
 223                      action="store", type=float, default=None)
 224     group.add_option("", "--shuffle", dest="shuffle",
 225                      help="Run tests in random order",
 226                      action="store_true", default=False)
 227     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 228                      help=("Only run tests with paths matching the given "
 229                            "regular expression"),
 230                      action="store", default=None)
 231     parser.add_option_group(group)
 232
 233     group = OptionGroup(parser, "Debug and Experimental Options")
 234     group.add_option("", "--debug", dest="debug",
 235                       help="Enable debugging (for 'lit' development)",
 236                       action="store_true", default=False)
 237     group.add_option("", "--show-suites", dest="showSuites",
 238                       help="Show discovered test suites",
 239                       action="store_true", default=False)
 240     group.add_option("", "--show-tests", dest="showTests",
 241                       help="Show all discovered tests",
 242                       action="store_true", default=False)
 243     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 244                       help="Repeat tests N times (for timing)",
 245                       action="store", default=None, type=int)
 246     parser.add_option_group(group)
 247
 248     (opts, args) = parser.parse_args()
 249
 250     if not args:
 251         parser.error('No inputs specified')
 252
 253     if opts.numThreads is None:
 254 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 255 # http://bugs.python.org/issue1731717
 256 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 257 # threads by default there.
 258        if sys.hexversion >= 0x2050200:
 259                opts.numThreads = lit.Util.detectCPUs()
 260        else:
 261                opts.numThreads = 1
 262
 263     inputs = args
 264
 265     # Create the user defined parameters.
 266     userParams = dict(builtinParameters)
 267     for entry in opts.userParameters:
 268         if '=' not in entry:
 269             name,val = entry,''
 270         else:
 271             name,val = entry.split('=', 1)
 272         userParams[name] = val
 273
 274     # Create the global config object.
 275     litConfig = lit.LitConfig.LitConfig(
 276         progname = os.path.basename(sys.argv[0]),
 277         path = opts.path,
 278         quiet = opts.quiet,
 279         useValgrind = opts.useValgrind,
 280         valgrindLeakCheck = opts.valgrindLeakCheck,
 281         valgrindArgs = opts.valgrindArgs,
 282         debug = opts.debug,
 283         isWindows = (platform.system()=='Windows'),
 284         params = userParams,
 285         config_prefix = opts.configPrefix)
 286
 287     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 288
 289     if opts.showSuites or opts.showTests:
 290         # Aggregate the tests by suite.
 291         suitesAndTests = {}
 292         for t in tests:
 293             if t.suite not in suitesAndTests:
 294                 suitesAndTests[t.suite] = []
 295             suitesAndTests[t.suite].append(t)
 296         suitesAndTests = suitesAndTests.items()
 297         suitesAndTests.sort(key = lambda item: item[0].name)
 298
 299         # Show the suites, if requested.
 300         if opts.showSuites:
 301             print('-- Test Suites --')
 302             for ts,ts_tests in suitesAndTests:
 303                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 304                 print('    Source Root: %s' % ts.source_root)
 305                 print('    Exec Root  : %s' % ts.exec_root)
 306
 307         # Show the tests, if requested.
 308         if opts.showTests:
 309             print('-- Available Tests --')
 310             for ts,ts_tests in suitesAndTests:
 311                 ts_tests.sort(key = lambda test: test.path_in_suite)
 312                 for test in ts_tests:
 313                     print('  %s' % (test.getFullName(),))
 314
 315     # Select and order the tests.
 316     numTotalTests = len(tests)
 317
 318     # First, select based on the filter expression if given.
 319     if opts.filter:
 320         try:
 321             rex = re.compile(opts.filter)
 322         except:
 323             parser.error("invalid regular expression for --filter: %r" % (
 324                     opts.filter))
 325         tests = [t for t in tests
 326                  if rex.search(t.getFullName())]
 327
 328     # Then select the order.
 329     if opts.shuffle:
 330         random.shuffle(tests)
 331     else:
 332         tests.sort(key = lambda t: t.getFullName())
 333
 334     # Finally limit the number of tests, if desired.
 335     if opts.maxTests is not None:
 336         tests = tests[:opts.maxTests]
 337
 338     # Don't create more threads than tests.
 339     opts.numThreads = min(len(tests), opts.numThreads)
 340
 341     extra = ''
 342     if len(tests) != numTotalTests:
 343         extra = ' of %d' % numTotalTests
 344     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 345                                                       opts.numThreads)
 346
 347     if opts.repeatTests:
 348         tests = [t.copyWithIndex(i)
 349                  for t in tests
 350                  for i in range(opts.repeatTests)]
 351
 352     progressBar = None
 353     if not opts.quiet:
 354         if opts.succinct and opts.useProgressBar:
 355             try:
 356                 tc = lit.ProgressBar.TerminalController()
 357                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 358             except ValueError:
 359                 print(header)
 360                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 361         else:
 362             print(header)
 363
 364     startTime = time.time()
 365     display = TestingProgressDisplay(opts, len(tests), progressBar)
 366     provider = TestProvider(tests, opts.maxTime)
 367
 368     try:
 369       import win32api
 370     except ImportError:
 371       pass
 372     else:
 373       def console_ctrl_handler(type):
 374         provider.cancel()
 375         return True
 376       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 377
 378     runTests(opts.numThreads, litConfig, provider, display)
 379     display.finish()
 380
 381     if not opts.quiet:
 382         print('Testing Time: %.2fs'%(time.time() - startTime))
 383
 384     # Update results for any tests which weren't run.
 385     for t in tests:
 386         if t.result is None:
 387             t.setResult(lit.Test.UNRESOLVED, '', 0.0)
 388
 389     # List test results organized by kind.
 390     hasFailures = False
 391     byCode = {}
 392     for t in tests:
 393         if t.result not in byCode:
 394             byCode[t.result] = []
 395         byCode[t.result].append(t)
 396         if t.result.isFailure:
 397             hasFailures = True
 398
 399     # FIXME: Show unresolved and (optionally) unsupported tests.
 400     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 401                        ('Failing Tests', lit.Test.FAIL)):
 402         elts = byCode.get(code)
 403         if not elts:
 404             continue
 405         print('*'*20)
 406         print('%s (%d):' % (title, len(elts)))
 407         for t in elts:
 408             print('    %s' % t.getFullName())
 409         sys.stdout.write('\n')
 410
 411     if opts.timeTests:
 412         # Collate, in case we repeated tests.
 413         times = {}
 414         for t in tests:
 415             key = t.getFullName()
 416             times[key] = times.get(key, 0.) + t.elapsed
 417
 418         byTime = list(times.items())
 419         byTime.sort(key = lambda item: item[1])
 420         if byTime:
 421             lit.Util.printHistogram(byTime, title='Tests')
 422
 423     for name,code in (('Expected Passes    ', lit.Test.PASS),
 424                       ('Expected Failures  ', lit.Test.XFAIL),
 425                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 426                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 427                       ('Unexpected Passes  ', lit.Test.XPASS),
 428                       ('Unexpected Failures', lit.Test.FAIL),):
 429         if opts.quiet and not code.isFailure:
 430             continue
 431         N = len(byCode.get(code,[]))
 432         if N:
 433             print('  %s: %d' % (name,N))
 434
 435     # If we encountered any additional errors, exit abnormally.
 436     if litConfig.numErrors:
 437         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 438         sys.exit(2)
 439
 440     # Warn about warnings.
 441     if litConfig.numWarnings:
 442         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 443
 444     if hasFailures:
 445         sys.exit(1)
 446     sys.exit(0)
 447
 448 if __name__=='__main__':
 449     main()