utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 import LitConfig
  16 import Test
  17
  18 import lit.discovery
  19
  20 class TestingProgressDisplay:
  21     def __init__(self, opts, numTests, progressBar=None):
  22         self.opts = opts
  23         self.numTests = numTests
  24         self.current = None
  25         self.lock = threading.Lock()
  26         self.progressBar = progressBar
  27         self.completed = 0
  28
  29     def update(self, test):
  30         # Avoid locking overhead in quiet mode
  31         if self.opts.quiet and not test.result.isFailure:
  32             self.completed += 1
  33             return
  34
  35         # Output lock.
  36         self.lock.acquire()
  37         try:
  38             self.handleUpdate(test)
  39         finally:
  40             self.lock.release()
  41
  42     def finish(self):
  43         if self.progressBar:
  44             self.progressBar.clear()
  45         elif self.opts.quiet:
  46             pass
  47         elif self.opts.succinct:
  48             sys.stdout.write('\n')
  49
  50     def handleUpdate(self, test):
  51         self.completed += 1
  52         if self.progressBar:
  53             self.progressBar.update(float(self.completed)/self.numTests,
  54                                     test.getFullName())
  55
  56         if self.opts.succinct and not test.result.isFailure:
  57             return
  58
  59         if self.progressBar:
  60             self.progressBar.clear()
  61
  62         print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  63                                      self.completed, self.numTests))
  64
  65         if test.result.isFailure and self.opts.showOutput:
  66             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  67                                               '*'*20))
  68             print(test.output)
  69             print("*" * 20)
  70
  71         sys.stdout.flush()
  72
  73 class TestProvider:
  74     def __init__(self, tests, maxTime):
  75         self.maxTime = maxTime
  76         self.iter = iter(tests)
  77         self.lock = threading.Lock()
  78         self.startTime = time.time()
  79         self.canceled = False
  80
  81     def cancel(self):
  82         self.lock.acquire()
  83         self.canceled = True
  84         self.lock.release()
  85
  86     def get(self):
  87         # Check if we have run out of time.
  88         if self.maxTime is not None:
  89             if time.time() - self.startTime > self.maxTime:
  90                 return None
  91
  92         # Otherwise take the next test.
  93         self.lock.acquire()
  94         if self.canceled:
  95           self.lock.release()
  96           return None
  97         for item in self.iter:
  98             break
  99         else:
 100             item = None
 101         self.lock.release()
 102         return item
 103
 104 class Tester(threading.Thread):
 105     def __init__(self, litConfig, provider, display):
 106         threading.Thread.__init__(self)
 107         self.litConfig = litConfig
 108         self.provider = provider
 109         self.display = display
 110
 111     def run(self):
 112         while 1:
 113             item = self.provider.get()
 114             if item is None:
 115                 break
 116             self.runTest(item)
 117
 118     def runTest(self, test):
 119         result = None
 120         startTime = time.time()
 121         try:
 122             result, output = test.config.test_format.execute(test,
 123                                                              self.litConfig)
 124         except KeyboardInterrupt:
 125             # This is a sad hack. Unfortunately subprocess goes
 126             # bonkers with ctrl-c and we start forking merrily.
 127             print('\nCtrl-C detected, goodbye.')
 128             os.kill(0,9)
 129         except:
 130             if self.litConfig.debug:
 131                 raise
 132             result = Test.UNRESOLVED
 133             output = 'Exception during script execution:\n'
 134             output += traceback.format_exc()
 135             output += '\n'
 136         elapsed = time.time() - startTime
 137
 138         test.setResult(result, output, elapsed)
 139         self.display.update(test)
 140
 141 def runTests(numThreads, litConfig, provider, display):
 142     # If only using one testing thread, don't use threads at all; this lets us
 143     # profile, among other things.
 144     if numThreads == 1:
 145         t = Tester(litConfig, provider, display)
 146         t.run()
 147         return
 148
 149     # Otherwise spin up the testing threads and wait for them to finish.
 150     testers = [Tester(litConfig, provider, display)
 151                for i in range(numThreads)]
 152     for t in testers:
 153         t.start()
 154     try:
 155         for t in testers:
 156             t.join()
 157     except KeyboardInterrupt:
 158         sys.exit(2)
 159
 160 def main(builtinParameters = {}):
 161     # Bump the GIL check interval, its more important to get any one thread to a
 162     # blocking operation (hopefully exec) than to try and unblock other threads.
 163     #
 164     # FIXME: This is a hack.
 165     import sys
 166     sys.setcheckinterval(1000)
 167
 168     global options
 169     from optparse import OptionParser, OptionGroup
 170     parser = OptionParser("usage: %prog [options] {file-or-path}")
 171
 172     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 173                       help="Number of testing threads",
 174                       type=int, action="store", default=None)
 175     parser.add_option("", "--config-prefix", dest="configPrefix",
 176                       metavar="NAME", help="Prefix for 'lit' config files",
 177                       action="store", default=None)
 178     parser.add_option("", "--param", dest="userParameters",
 179                       metavar="NAME=VAL",
 180                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 181                       type=str, action="append", default=[])
 182
 183     group = OptionGroup(parser, "Output Format")
 184     # FIXME: I find these names very confusing, although I like the
 185     # functionality.
 186     group.add_option("-q", "--quiet", dest="quiet",
 187                      help="Suppress no error output",
 188                      action="store_true", default=False)
 189     group.add_option("-s", "--succinct", dest="succinct",
 190                      help="Reduce amount of output",
 191                      action="store_true", default=False)
 192     group.add_option("-v", "--verbose", dest="showOutput",
 193                      help="Show all test output",
 194                      action="store_true", default=False)
 195     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 196                      help="Do not use curses based progress bar",
 197                      action="store_false", default=True)
 198     parser.add_option_group(group)
 199
 200     group = OptionGroup(parser, "Test Execution")
 201     group.add_option("", "--path", dest="path",
 202                      help="Additional paths to add to testing environment",
 203                      action="append", type=str, default=[])
 204     group.add_option("", "--vg", dest="useValgrind",
 205                      help="Run tests under valgrind",
 206                      action="store_true", default=False)
 207     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 208                      help="Check for memory leaks under valgrind",
 209                      action="store_true", default=False)
 210     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 211                      help="Specify an extra argument for valgrind",
 212                      type=str, action="append", default=[])
 213     group.add_option("", "--time-tests", dest="timeTests",
 214                      help="Track elapsed wall time for each test",
 215                      action="store_true", default=False)
 216     parser.add_option_group(group)
 217
 218     group = OptionGroup(parser, "Test Selection")
 219     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 220                      help="Maximum number of tests to run",
 221                      action="store", type=int, default=None)
 222     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 223                      help="Maximum time to spend testing (in seconds)",
 224                      action="store", type=float, default=None)
 225     group.add_option("", "--shuffle", dest="shuffle",
 226                      help="Run tests in random order",
 227                      action="store_true", default=False)
 228     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 229                      help=("Only run tests with paths matching the given "
 230                            "regular expression"),
 231                      action="store", default=None)
 232     parser.add_option_group(group)
 233
 234     group = OptionGroup(parser, "Debug and Experimental Options")
 235     group.add_option("", "--debug", dest="debug",
 236                       help="Enable debugging (for 'lit' development)",
 237                       action="store_true", default=False)
 238     group.add_option("", "--show-suites", dest="showSuites",
 239                       help="Show discovered test suites",
 240                       action="store_true", default=False)
 241     group.add_option("", "--show-tests", dest="showTests",
 242                       help="Show all discovered tests",
 243                       action="store_true", default=False)
 244     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 245                       help="Repeat tests N times (for timing)",
 246                       action="store", default=None, type=int)
 247     parser.add_option_group(group)
 248
 249     (opts, args) = parser.parse_args()
 250
 251     if not args:
 252         parser.error('No inputs specified')
 253
 254     if opts.numThreads is None:
 255 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 256 # http://bugs.python.org/issue1731717
 257 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 258 # threads by default there.
 259        if sys.hexversion >= 0x2050200:
 260                opts.numThreads = Util.detectCPUs()
 261        else:
 262                opts.numThreads = 1
 263
 264     inputs = args
 265
 266     # Create the user defined parameters.
 267     userParams = dict(builtinParameters)
 268     for entry in opts.userParameters:
 269         if '=' not in entry:
 270             name,val = entry,''
 271         else:
 272             name,val = entry.split('=', 1)
 273         userParams[name] = val
 274
 275     # Create the global config object.
 276     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 277                                     path = opts.path,
 278                                     quiet = opts.quiet,
 279                                     useValgrind = opts.useValgrind,
 280                                     valgrindLeakCheck = opts.valgrindLeakCheck,
 281                                     valgrindArgs = opts.valgrindArgs,
 282                                     debug = opts.debug,
 283                                     isWindows = (platform.system()=='Windows'),
 284                                     params = userParams,
 285                                     config_prefix = opts.configPrefix)
 286
 287     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 288
 289     if opts.showSuites or opts.showTests:
 290         # Aggregate the tests by suite.
 291         suitesAndTests = {}
 292         for t in tests:
 293             if t.suite not in suitesAndTests:
 294                 suitesAndTests[t.suite] = []
 295             suitesAndTests[t.suite].append(t)
 296         suitesAndTests = suitesAndTests.items()
 297         suitesAndTests.sort(key = lambda item: item[0].name)
 298
 299         # Show the suites, if requested.
 300         if opts.showSuites:
 301             print('-- Test Suites --')
 302             for ts,ts_tests in suitesAndTests:
 303                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 304                 print('    Source Root: %s' % ts.source_root)
 305                 print('    Exec Root  : %s' % ts.exec_root)
 306
 307         # Show the tests, if requested.
 308         if opts.showTests:
 309             print('-- Available Tests --')
 310             for ts,ts_tests in suitesAndTests:
 311                 ts_tests.sort(key = lambda test: test.path_in_suite)
 312                 for test in ts_tests:
 313                     print('  %s' % (test.getFullName(),))
 314
 315     # Select and order the tests.
 316     numTotalTests = len(tests)
 317
 318     # First, select based on the filter expression if given.
 319     if opts.filter:
 320         try:
 321             rex = re.compile(opts.filter)
 322         except:
 323             parser.error("invalid regular expression for --filter: %r" % (
 324                     opts.filter))
 325         tests = [t for t in tests
 326                  if rex.search(t.getFullName())]
 327
 328     # Then select the order.
 329     if opts.shuffle:
 330         random.shuffle(tests)
 331     else:
 332         tests.sort(key = lambda t: t.getFullName())
 333
 334     # Finally limit the number of tests, if desired.
 335     if opts.maxTests is not None:
 336         tests = tests[:opts.maxTests]
 337
 338     # Don't create more threads than tests.
 339     opts.numThreads = min(len(tests), opts.numThreads)
 340
 341     extra = ''
 342     if len(tests) != numTotalTests:
 343         extra = ' of %d' % numTotalTests
 344     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 345                                                       opts.numThreads)
 346
 347     if opts.repeatTests:
 348         tests = [t.copyWithIndex(i)
 349                  for t in tests
 350                  for i in range(opts.repeatTests)]
 351
 352     progressBar = None
 353     if not opts.quiet:
 354         if opts.succinct and opts.useProgressBar:
 355             try:
 356                 tc = ProgressBar.TerminalController()
 357                 progressBar = ProgressBar.ProgressBar(tc, header)
 358             except ValueError:
 359                 print(header)
 360                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 361         else:
 362             print(header)
 363
 364     startTime = time.time()
 365     display = TestingProgressDisplay(opts, len(tests), progressBar)
 366     provider = TestProvider(tests, opts.maxTime)
 367
 368     try:
 369       import win32api
 370     except ImportError:
 371       pass
 372     else:
 373       def console_ctrl_handler(type):
 374         provider.cancel()
 375         return True
 376       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 377
 378     runTests(opts.numThreads, litConfig, provider, display)
 379     display.finish()
 380
 381     if not opts.quiet:
 382         print('Testing Time: %.2fs'%(time.time() - startTime))
 383
 384     # Update results for any tests which weren't run.
 385     for t in tests:
 386         if t.result is None:
 387             t.setResult(Test.UNRESOLVED, '', 0.0)
 388
 389     # List test results organized by kind.
 390     hasFailures = False
 391     byCode = {}
 392     for t in tests:
 393         if t.result not in byCode:
 394             byCode[t.result] = []
 395         byCode[t.result].append(t)
 396         if t.result.isFailure:
 397             hasFailures = True
 398
 399     # FIXME: Show unresolved and (optionally) unsupported tests.
 400     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 401                        ('Failing Tests', Test.FAIL)):
 402         elts = byCode.get(code)
 403         if not elts:
 404             continue
 405         print('*'*20)
 406         print('%s (%d):' % (title, len(elts)))
 407         for t in elts:
 408             print('    %s' % t.getFullName())
 409         sys.stdout.write('\n')
 410
 411     if opts.timeTests:
 412         # Collate, in case we repeated tests.
 413         times = {}
 414         for t in tests:
 415             key = t.getFullName()
 416             times[key] = times.get(key, 0.) + t.elapsed
 417
 418         byTime = list(times.items())
 419         byTime.sort(key = lambda item: item[1])
 420         if byTime:
 421             Util.printHistogram(byTime, title='Tests')
 422
 423     for name,code in (('Expected Passes    ', Test.PASS),
 424                       ('Expected Failures  ', Test.XFAIL),
 425                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 426                       ('Unresolved Tests   ', Test.UNRESOLVED),
 427                       ('Unexpected Passes  ', Test.XPASS),
 428                       ('Unexpected Failures', Test.FAIL),):
 429         if opts.quiet and not code.isFailure:
 430             continue
 431         N = len(byCode.get(code,[]))
 432         if N:
 433             print('  %s: %d' % (name,N))
 434
 435     # If we encountered any additional errors, exit abnormally.
 436     if litConfig.numErrors:
 437         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 438         sys.exit(2)
 439
 440     # Warn about warnings.
 441     if litConfig.numWarnings:
 442         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 443
 444     if hasFailures:
 445         sys.exit(1)
 446     sys.exit(0)
 447
 448 if __name__=='__main__':
 449     main()