utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time, threading, traceback
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.Util
  16
  17 import lit.discovery
  18
  19 class TestingProgressDisplay:
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.lock = threading.Lock()
  25         self.progressBar = progressBar
  26         self.completed = 0
  27
  28     def update(self, test):
  29         # Avoid locking overhead in quiet mode
  30         if self.opts.quiet and not test.result.isFailure:
  31             self.completed += 1
  32             return
  33
  34         # Output lock.
  35         self.lock.acquire()
  36         try:
  37             self.handleUpdate(test)
  38         finally:
  39             self.lock.release()
  40
  41     def finish(self):
  42         if self.progressBar:
  43             self.progressBar.clear()
  44         elif self.opts.quiet:
  45             pass
  46         elif self.opts.succinct:
  47             sys.stdout.write('\n')
  48
  49     def handleUpdate(self, test):
  50         self.completed += 1
  51         if self.progressBar:
  52             self.progressBar.update(float(self.completed)/self.numTests,
  53                                     test.getFullName())
  54
  55         if self.opts.succinct and not test.result.isFailure:
  56             return
  57
  58         if self.progressBar:
  59             self.progressBar.clear()
  60
  61         print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  62                                      self.completed, self.numTests))
  63
  64         if test.result.isFailure and self.opts.showOutput:
  65             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  66                                               '*'*20))
  67             print(test.output)
  68             print("*" * 20)
  69
  70         sys.stdout.flush()
  71
  72 class TestProvider:
  73     def __init__(self, tests, maxTime):
  74         self.maxTime = maxTime
  75         self.iter = iter(tests)
  76         self.lock = threading.Lock()
  77         self.startTime = time.time()
  78         self.canceled = False
  79
  80     def cancel(self):
  81         self.lock.acquire()
  82         self.canceled = True
  83         self.lock.release()
  84
  85     def get(self):
  86         # Check if we have run out of time.
  87         if self.maxTime is not None:
  88             if time.time() - self.startTime > self.maxTime:
  89                 return None
  90
  91         # Otherwise take the next test.
  92         self.lock.acquire()
  93         if self.canceled:
  94           self.lock.release()
  95           return None
  96         for item in self.iter:
  97             break
  98         else:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print('\nCtrl-C detected, goodbye.')
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = lit.Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def runTests(numThreads, litConfig, provider, display):
 141     # If only using one testing thread, don't use threads at all; this lets us
 142     # profile, among other things.
 143     if numThreads == 1:
 144         t = Tester(litConfig, provider, display)
 145         t.run()
 146         return
 147
 148     # Otherwise spin up the testing threads and wait for them to finish.
 149     testers = [Tester(litConfig, provider, display)
 150                for i in range(numThreads)]
 151     for t in testers:
 152         t.start()
 153     try:
 154         for t in testers:
 155             t.join()
 156     except KeyboardInterrupt:
 157         sys.exit(2)
 158
 159 def main(builtinParameters = {}):
 160     # Bump the GIL check interval, its more important to get any one thread to a
 161     # blocking operation (hopefully exec) than to try and unblock other threads.
 162     #
 163     # FIXME: This is a hack.
 164     sys.setcheckinterval(1000)
 165
 166     global options
 167     from optparse import OptionParser, OptionGroup
 168     parser = OptionParser("usage: %prog [options] {file-or-path}")
 169
 170     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 171                       help="Number of testing threads",
 172                       type=int, action="store", default=None)
 173     parser.add_option("", "--config-prefix", dest="configPrefix",
 174                       metavar="NAME", help="Prefix for 'lit' config files",
 175                       action="store", default=None)
 176     parser.add_option("", "--param", dest="userParameters",
 177                       metavar="NAME=VAL",
 178                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 179                       type=str, action="append", default=[])
 180
 181     group = OptionGroup(parser, "Output Format")
 182     # FIXME: I find these names very confusing, although I like the
 183     # functionality.
 184     group.add_option("-q", "--quiet", dest="quiet",
 185                      help="Suppress no error output",
 186                      action="store_true", default=False)
 187     group.add_option("-s", "--succinct", dest="succinct",
 188                      help="Reduce amount of output",
 189                      action="store_true", default=False)
 190     group.add_option("-v", "--verbose", dest="showOutput",
 191                      help="Show all test output",
 192                      action="store_true", default=False)
 193     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 194                      help="Do not use curses based progress bar",
 195                      action="store_false", default=True)
 196     parser.add_option_group(group)
 197
 198     group = OptionGroup(parser, "Test Execution")
 199     group.add_option("", "--path", dest="path",
 200                      help="Additional paths to add to testing environment",
 201                      action="append", type=str, default=[])
 202     group.add_option("", "--vg", dest="useValgrind",
 203                      help="Run tests under valgrind",
 204                      action="store_true", default=False)
 205     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 206                      help="Check for memory leaks under valgrind",
 207                      action="store_true", default=False)
 208     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 209                      help="Specify an extra argument for valgrind",
 210                      type=str, action="append", default=[])
 211     group.add_option("", "--time-tests", dest="timeTests",
 212                      help="Track elapsed wall time for each test",
 213                      action="store_true", default=False)
 214     group.add_option("", "--no-execute", dest="noExecute",
 215                      help="Don't execute any tests (assume PASS)",
 216                      action="store_true", default=False)
 217     parser.add_option_group(group)
 218
 219     group = OptionGroup(parser, "Test Selection")
 220     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 221                      help="Maximum number of tests to run",
 222                      action="store", type=int, default=None)
 223     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 224                      help="Maximum time to spend testing (in seconds)",
 225                      action="store", type=float, default=None)
 226     group.add_option("", "--shuffle", dest="shuffle",
 227                      help="Run tests in random order",
 228                      action="store_true", default=False)
 229     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 230                      help=("Only run tests with paths matching the given "
 231                            "regular expression"),
 232                      action="store", default=None)
 233     parser.add_option_group(group)
 234
 235     group = OptionGroup(parser, "Debug and Experimental Options")
 236     group.add_option("", "--debug", dest="debug",
 237                       help="Enable debugging (for 'lit' development)",
 238                       action="store_true", default=False)
 239     group.add_option("", "--show-suites", dest="showSuites",
 240                       help="Show discovered test suites",
 241                       action="store_true", default=False)
 242     group.add_option("", "--show-tests", dest="showTests",
 243                       help="Show all discovered tests",
 244                       action="store_true", default=False)
 245     parser.add_option_group(group)
 246
 247     (opts, args) = parser.parse_args()
 248
 249     if not args:
 250         parser.error('No inputs specified')
 251
 252     if opts.numThreads is None:
 253 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 254 # http://bugs.python.org/issue1731717
 255 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 256 # threads by default there.
 257        if sys.hexversion >= 0x2050200:
 258                opts.numThreads = lit.Util.detectCPUs()
 259        else:
 260                opts.numThreads = 1
 261
 262     inputs = args
 263
 264     # Create the user defined parameters.
 265     userParams = dict(builtinParameters)
 266     for entry in opts.userParameters:
 267         if '=' not in entry:
 268             name,val = entry,''
 269         else:
 270             name,val = entry.split('=', 1)
 271         userParams[name] = val
 272
 273     # Create the global config object.
 274     litConfig = lit.LitConfig.LitConfig(
 275         progname = os.path.basename(sys.argv[0]),
 276         path = opts.path,
 277         quiet = opts.quiet,
 278         useValgrind = opts.useValgrind,
 279         valgrindLeakCheck = opts.valgrindLeakCheck,
 280         valgrindArgs = opts.valgrindArgs,
 281         noExecute = opts.noExecute,
 282         debug = opts.debug,
 283         isWindows = (platform.system()=='Windows'),
 284         params = userParams,
 285         config_prefix = opts.configPrefix)
 286
 287     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 288
 289     if opts.showSuites or opts.showTests:
 290         # Aggregate the tests by suite.
 291         suitesAndTests = {}
 292         for t in tests:
 293             if t.suite not in suitesAndTests:
 294                 suitesAndTests[t.suite] = []
 295             suitesAndTests[t.suite].append(t)
 296         suitesAndTests = list(suitesAndTests.items())
 297         suitesAndTests.sort(key = lambda item: item[0].name)
 298
 299         # Show the suites, if requested.
 300         if opts.showSuites:
 301             print('-- Test Suites --')
 302             for ts,ts_tests in suitesAndTests:
 303                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 304                 print('    Source Root: %s' % ts.source_root)
 305                 print('    Exec Root  : %s' % ts.exec_root)
 306
 307         # Show the tests, if requested.
 308         if opts.showTests:
 309             print('-- Available Tests --')
 310             for ts,ts_tests in suitesAndTests:
 311                 ts_tests.sort(key = lambda test: test.path_in_suite)
 312                 for test in ts_tests:
 313                     print('  %s' % (test.getFullName(),))
 314
 315     # Select and order the tests.
 316     numTotalTests = len(tests)
 317
 318     # First, select based on the filter expression if given.
 319     if opts.filter:
 320         try:
 321             rex = re.compile(opts.filter)
 322         except:
 323             parser.error("invalid regular expression for --filter: %r" % (
 324                     opts.filter))
 325         tests = [t for t in tests
 326                  if rex.search(t.getFullName())]
 327
 328     # Then select the order.
 329     if opts.shuffle:
 330         random.shuffle(tests)
 331     else:
 332         tests.sort(key = lambda t: t.getFullName())
 333
 334     # Finally limit the number of tests, if desired.
 335     if opts.maxTests is not None:
 336         tests = tests[:opts.maxTests]
 337
 338     # Don't create more threads than tests.
 339     opts.numThreads = min(len(tests), opts.numThreads)
 340
 341     extra = ''
 342     if len(tests) != numTotalTests:
 343         extra = ' of %d' % numTotalTests
 344     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 345                                                       opts.numThreads)
 346
 347     progressBar = None
 348     if not opts.quiet:
 349         if opts.succinct and opts.useProgressBar:
 350             try:
 351                 tc = lit.ProgressBar.TerminalController()
 352                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 353             except ValueError:
 354                 print(header)
 355                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 356         else:
 357             print(header)
 358
 359     startTime = time.time()
 360     display = TestingProgressDisplay(opts, len(tests), progressBar)
 361     provider = TestProvider(tests, opts.maxTime)
 362
 363     try:
 364       import win32api
 365     except ImportError:
 366       pass
 367     else:
 368       def console_ctrl_handler(type):
 369         provider.cancel()
 370         return True
 371       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 372
 373     runTests(opts.numThreads, litConfig, provider, display)
 374     display.finish()
 375
 376     if not opts.quiet:
 377         print('Testing Time: %.2fs'%(time.time() - startTime))
 378
 379     # Update results for any tests which weren't run.
 380     for t in tests:
 381         if t.result is None:
 382             t.setResult(lit.Test.UNRESOLVED, '', 0.0)
 383
 384     # List test results organized by kind.
 385     hasFailures = False
 386     byCode = {}
 387     for t in tests:
 388         if t.result not in byCode:
 389             byCode[t.result] = []
 390         byCode[t.result].append(t)
 391         if t.result.isFailure:
 392             hasFailures = True
 393
 394     # Print each test in any of the failing groups.
 395     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 396                        ('Failing Tests', lit.Test.FAIL),
 397                        ('Unresolved Tests', lit.Test.UNRESOLVED)):
 398         elts = byCode.get(code)
 399         if not elts:
 400             continue
 401         print('*'*20)
 402         print('%s (%d):' % (title, len(elts)))
 403         for t in elts:
 404             print('    %s' % t.getFullName())
 405         sys.stdout.write('\n')
 406
 407     if opts.timeTests:
 408         # Collate, in case we repeated tests.
 409         times = {}
 410         for t in tests:
 411             key = t.getFullName()
 412             times[key] = times.get(key, 0.) + t.elapsed
 413
 414         byTime = list(times.items())
 415         byTime.sort(key = lambda item: item[1])
 416         if byTime:
 417             lit.Util.printHistogram(byTime, title='Tests')
 418
 419     for name,code in (('Expected Passes    ', lit.Test.PASS),
 420                       ('Expected Failures  ', lit.Test.XFAIL),
 421                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 422                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 423                       ('Unexpected Passes  ', lit.Test.XPASS),
 424                       ('Unexpected Failures', lit.Test.FAIL),):
 425         if opts.quiet and not code.isFailure:
 426             continue
 427         N = len(byCode.get(code,[]))
 428         if N:
 429             print('  %s: %d' % (name,N))
 430
 431     # If we encountered any additional errors, exit abnormally.
 432     if litConfig.numErrors:
 433         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 434         sys.exit(2)
 435
 436     # Warn about warnings.
 437     if litConfig.numWarnings:
 438         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 439
 440     if hasFailures:
 441         sys.exit(1)
 442     sys.exit(0)
 443
 444 if __name__=='__main__':
 445     main()