utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time, threading, traceback
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.Util
  16
  17 import lit.discovery
  18
  19 class TestingProgressDisplay:
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.lock = threading.Lock()
  25         self.progressBar = progressBar
  26         self.completed = 0
  27
  28     def update(self, test):
  29         # Avoid locking overhead in quiet mode
  30         if self.opts.quiet and not test.result.isFailure:
  31             self.completed += 1
  32             return
  33
  34         # Output lock.
  35         self.lock.acquire()
  36         try:
  37             self.handleUpdate(test)
  38         finally:
  39             self.lock.release()
  40
  41     def finish(self):
  42         if self.progressBar:
  43             self.progressBar.clear()
  44         elif self.opts.quiet:
  45             pass
  46         elif self.opts.succinct:
  47             sys.stdout.write('\n')
  48
  49     def handleUpdate(self, test):
  50         self.completed += 1
  51         if self.progressBar:
  52             self.progressBar.update(float(self.completed)/self.numTests,
  53                                     test.getFullName())
  54
  55         if self.opts.succinct and not test.result.isFailure:
  56             return
  57
  58         if self.progressBar:
  59             self.progressBar.clear()
  60
  61         print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  62                                      self.completed, self.numTests))
  63
  64         if test.result.isFailure and self.opts.showOutput:
  65             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  66                                               '*'*20))
  67             print(test.output)
  68             print("*" * 20)
  69
  70         sys.stdout.flush()
  71
  72 class TestProvider:
  73     def __init__(self, tests, maxTime):
  74         self.maxTime = maxTime
  75         self.iter = iter(tests)
  76         self.lock = threading.Lock()
  77         self.startTime = time.time()
  78         self.canceled = False
  79
  80     def cancel(self):
  81         self.lock.acquire()
  82         self.canceled = True
  83         self.lock.release()
  84
  85     def get(self):
  86         # Check if we have run out of time.
  87         if self.maxTime is not None:
  88             if time.time() - self.startTime > self.maxTime:
  89                 return None
  90
  91         # Otherwise take the next test.
  92         self.lock.acquire()
  93         if self.canceled:
  94           self.lock.release()
  95           return None
  96         for item in self.iter:
  97             break
  98         else:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print('\nCtrl-C detected, goodbye.')
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = lit.Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def runTests(numThreads, litConfig, provider, display):
 141     # If only using one testing thread, don't use threads at all; this lets us
 142     # profile, among other things.
 143     if numThreads == 1:
 144         t = Tester(litConfig, provider, display)
 145         t.run()
 146         return
 147
 148     # Otherwise spin up the testing threads and wait for them to finish.
 149     testers = [Tester(litConfig, provider, display)
 150                for i in range(numThreads)]
 151     for t in testers:
 152         t.start()
 153     try:
 154         for t in testers:
 155             t.join()
 156     except KeyboardInterrupt:
 157         sys.exit(2)
 158
 159 def main(builtinParameters = {}):
 160     # Bump the GIL check interval, its more important to get any one thread to a
 161     # blocking operation (hopefully exec) than to try and unblock other threads.
 162     #
 163     # FIXME: This is a hack.
 164     sys.setcheckinterval(1000)
 165
 166     global options
 167     from optparse import OptionParser, OptionGroup
 168     parser = OptionParser("usage: %prog [options] {file-or-path}")
 169
 170     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 171                       help="Number of testing threads",
 172                       type=int, action="store", default=None)
 173     parser.add_option("", "--config-prefix", dest="configPrefix",
 174                       metavar="NAME", help="Prefix for 'lit' config files",
 175                       action="store", default=None)
 176     parser.add_option("", "--param", dest="userParameters",
 177                       metavar="NAME=VAL",
 178                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 179                       type=str, action="append", default=[])
 180
 181     group = OptionGroup(parser, "Output Format")
 182     # FIXME: I find these names very confusing, although I like the
 183     # functionality.
 184     group.add_option("-q", "--quiet", dest="quiet",
 185                      help="Suppress no error output",
 186                      action="store_true", default=False)
 187     group.add_option("-s", "--succinct", dest="succinct",
 188                      help="Reduce amount of output",
 189                      action="store_true", default=False)
 190     group.add_option("-v", "--verbose", dest="showOutput",
 191                      help="Show all test output",
 192                      action="store_true", default=False)
 193     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 194                      help="Do not use curses based progress bar",
 195                      action="store_false", default=True)
 196     parser.add_option_group(group)
 197
 198     group = OptionGroup(parser, "Test Execution")
 199     group.add_option("", "--path", dest="path",
 200                      help="Additional paths to add to testing environment",
 201                      action="append", type=str, default=[])
 202     group.add_option("", "--vg", dest="useValgrind",
 203                      help="Run tests under valgrind",
 204                      action="store_true", default=False)
 205     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 206                      help="Check for memory leaks under valgrind",
 207                      action="store_true", default=False)
 208     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 209                      help="Specify an extra argument for valgrind",
 210                      type=str, action="append", default=[])
 211     group.add_option("", "--time-tests", dest="timeTests",
 212                      help="Track elapsed wall time for each test",
 213                      action="store_true", default=False)
 214     group.add_option("", "--no-execute", dest="noExecute",
 215                      help="Don't execute any tests (assume PASS)",
 216                      action="store_true", default=False)
 217     parser.add_option_group(group)
 218
 219     group = OptionGroup(parser, "Test Selection")
 220     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 221                      help="Maximum number of tests to run",
 222                      action="store", type=int, default=None)
 223     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 224                      help="Maximum time to spend testing (in seconds)",
 225                      action="store", type=float, default=None)
 226     group.add_option("", "--shuffle", dest="shuffle",
 227                      help="Run tests in random order",
 228                      action="store_true", default=False)
 229     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 230                      help=("Only run tests with paths matching the given "
 231                            "regular expression"),
 232                      action="store", default=None)
 233     parser.add_option_group(group)
 234
 235     group = OptionGroup(parser, "Debug and Experimental Options")
 236     group.add_option("", "--debug", dest="debug",
 237                       help="Enable debugging (for 'lit' development)",
 238                       action="store_true", default=False)
 239     group.add_option("", "--show-suites", dest="showSuites",
 240                       help="Show discovered test suites",
 241                       action="store_true", default=False)
 242     group.add_option("", "--show-tests", dest="showTests",
 243                       help="Show all discovered tests",
 244                       action="store_true", default=False)
 245     parser.add_option_group(group)
 246
 247     (opts, args) = parser.parse_args()
 248
 249     if not args:
 250         parser.error('No inputs specified')
 251
 252     if opts.numThreads is None:
 253 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 254 # http://bugs.python.org/issue1731717
 255 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 256 # threads by default there.
 257        if sys.hexversion >= 0x2050200:
 258                opts.numThreads = lit.Util.detectCPUs()
 259        else:
 260                opts.numThreads = 1
 261
 262     inputs = args
 263
 264     # Create the user defined parameters.
 265     userParams = dict(builtinParameters)
 266     for entry in opts.userParameters:
 267         if '=' not in entry:
 268             name,val = entry,''
 269         else:
 270             name,val = entry.split('=', 1)
 271         userParams[name] = val
 272
 273     # Create the global config object.
 274     litConfig = lit.LitConfig.LitConfig(
 275         progname = os.path.basename(sys.argv[0]),
 276         path = opts.path,
 277         quiet = opts.quiet,
 278         useValgrind = opts.useValgrind,
 279         valgrindLeakCheck = opts.valgrindLeakCheck,
 280         valgrindArgs = opts.valgrindArgs,
 281         noExecute = opts.noExecute,
 282         debug = opts.debug,
 283         isWindows = (platform.system()=='Windows'),
 284         params = userParams,
 285         config_prefix = opts.configPrefix)
 286
 287     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 288
 289     if opts.showSuites or opts.showTests:
 290         # Aggregate the tests by suite.
 291         suitesAndTests = {}
 292         for t in tests:
 293             if t.suite not in suitesAndTests:
 294                 suitesAndTests[t.suite] = []
 295             suitesAndTests[t.suite].append(t)
 296         suitesAndTests = list(suitesAndTests.items())
 297         suitesAndTests.sort(key = lambda item: item[0].name)
 298
 299         # Show the suites, if requested.
 300         if opts.showSuites:
 301             print('-- Test Suites --')
 302             for ts,ts_tests in suitesAndTests:
 303                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 304                 print('    Source Root: %s' % ts.source_root)
 305                 print('    Exec Root  : %s' % ts.exec_root)
 306
 307         # Show the tests, if requested.
 308         if opts.showTests:
 309             print('-- Available Tests --')
 310             for ts,ts_tests in suitesAndTests:
 311                 ts_tests.sort(key = lambda test: test.path_in_suite)
 312                 for test in ts_tests:
 313                     print('  %s' % (test.getFullName(),))
 314
 315         # Exit.
 316         sys.exit(0)
 317
 318     # Select and order the tests.
 319     numTotalTests = len(tests)
 320
 321     # First, select based on the filter expression if given.
 322     if opts.filter:
 323         try:
 324             rex = re.compile(opts.filter)
 325         except:
 326             parser.error("invalid regular expression for --filter: %r" % (
 327                     opts.filter))
 328         tests = [t for t in tests
 329                  if rex.search(t.getFullName())]
 330
 331     # Then select the order.
 332     if opts.shuffle:
 333         random.shuffle(tests)
 334     else:
 335         tests.sort(key = lambda t: t.getFullName())
 336
 337     # Finally limit the number of tests, if desired.
 338     if opts.maxTests is not None:
 339         tests = tests[:opts.maxTests]
 340
 341     # Don't create more threads than tests.
 342     opts.numThreads = min(len(tests), opts.numThreads)
 343
 344     extra = ''
 345     if len(tests) != numTotalTests:
 346         extra = ' of %d' % numTotalTests
 347     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 348                                                       opts.numThreads)
 349
 350     progressBar = None
 351     if not opts.quiet:
 352         if opts.succinct and opts.useProgressBar:
 353             try:
 354                 tc = lit.ProgressBar.TerminalController()
 355                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 356             except ValueError:
 357                 print(header)
 358                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 359         else:
 360             print(header)
 361
 362     startTime = time.time()
 363     display = TestingProgressDisplay(opts, len(tests), progressBar)
 364     provider = TestProvider(tests, opts.maxTime)
 365
 366     try:
 367       import win32api
 368     except ImportError:
 369       pass
 370     else:
 371       def console_ctrl_handler(type):
 372         provider.cancel()
 373         return True
 374       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 375
 376     runTests(opts.numThreads, litConfig, provider, display)
 377     display.finish()
 378
 379     if not opts.quiet:
 380         print('Testing Time: %.2fs'%(time.time() - startTime))
 381
 382     # Update results for any tests which weren't run.
 383     for t in tests:
 384         if t.result is None:
 385             t.setResult(lit.Test.UNRESOLVED, '', 0.0)
 386
 387     # List test results organized by kind.
 388     hasFailures = False
 389     byCode = {}
 390     for t in tests:
 391         if t.result not in byCode:
 392             byCode[t.result] = []
 393         byCode[t.result].append(t)
 394         if t.result.isFailure:
 395             hasFailures = True
 396
 397     # Print each test in any of the failing groups.
 398     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 399                        ('Failing Tests', lit.Test.FAIL),
 400                        ('Unresolved Tests', lit.Test.UNRESOLVED)):
 401         elts = byCode.get(code)
 402         if not elts:
 403             continue
 404         print('*'*20)
 405         print('%s (%d):' % (title, len(elts)))
 406         for t in elts:
 407             print('    %s' % t.getFullName())
 408         sys.stdout.write('\n')
 409
 410     if opts.timeTests:
 411         # Collate, in case we repeated tests.
 412         times = {}
 413         for t in tests:
 414             key = t.getFullName()
 415             times[key] = times.get(key, 0.) + t.elapsed
 416
 417         byTime = list(times.items())
 418         byTime.sort(key = lambda item: item[1])
 419         if byTime:
 420             lit.Util.printHistogram(byTime, title='Tests')
 421
 422     for name,code in (('Expected Passes    ', lit.Test.PASS),
 423                       ('Expected Failures  ', lit.Test.XFAIL),
 424                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 425                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 426                       ('Unexpected Passes  ', lit.Test.XPASS),
 427                       ('Unexpected Failures', lit.Test.FAIL),):
 428         if opts.quiet and not code.isFailure:
 429             continue
 430         N = len(byCode.get(code,[]))
 431         if N:
 432             print('  %s: %d' % (name,N))
 433
 434     # If we encountered any additional errors, exit abnormally.
 435     if litConfig.numErrors:
 436         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 437         sys.exit(2)
 438
 439     # Warn about warnings.
 440     if litConfig.numWarnings:
 441         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 442
 443     if hasFailures:
 444         sys.exit(1)
 445     sys.exit(0)
 446
 447 if __name__=='__main__':
 448     main()