utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 from __future__ import absolute_import
  10 import math, os, platform, random, re, sys, time, threading, traceback
  11
  12 import lit.ProgressBar
  13 import lit.LitConfig
  14 import lit.Test
  15 import lit.util
  16
  17 import lit.discovery
  18
  19 class TestingProgressDisplay:
  20     def __init__(self, opts, numTests, progressBar=None):
  21         self.opts = opts
  22         self.numTests = numTests
  23         self.current = None
  24         self.lock = threading.Lock()
  25         self.progressBar = progressBar
  26         self.completed = 0
  27
  28     def update(self, test):
  29         # Avoid locking overhead in quiet mode
  30         if self.opts.quiet and not test.result.code.isFailure:
  31             self.completed += 1
  32             return
  33
  34         # Output lock.
  35         self.lock.acquire()
  36         try:
  37             self.handleUpdate(test)
  38         finally:
  39             self.lock.release()
  40
  41     def finish(self):
  42         if self.progressBar:
  43             self.progressBar.clear()
  44         elif self.opts.quiet:
  45             pass
  46         elif self.opts.succinct:
  47             sys.stdout.write('\n')
  48
  49     def handleUpdate(self, test):
  50         self.completed += 1
  51         if self.progressBar:
  52             self.progressBar.update(float(self.completed)/self.numTests,
  53                                     test.getFullName())
  54
  55         if self.opts.succinct and not test.result.code.isFailure:
  56             return
  57
  58         if self.progressBar:
  59             self.progressBar.clear()
  60
  61         print('%s: %s (%d of %d)' % (test.result.code.name, test.getFullName(),
  62                                      self.completed, self.numTests))
  63
  64         if test.result.code.isFailure and self.opts.showOutput:
  65             print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  66                                               '*'*20))
  67             print(test.result.output)
  68             print("*" * 20)
  69
  70         sys.stdout.flush()
  71
  72 class TestProvider:
  73     def __init__(self, tests, maxTime):
  74         self.maxTime = maxTime
  75         self.iter = iter(tests)
  76         self.lock = threading.Lock()
  77         self.startTime = time.time()
  78         self.canceled = False
  79
  80     def cancel(self):
  81         self.lock.acquire()
  82         self.canceled = True
  83         self.lock.release()
  84
  85     def get(self):
  86         # Check if we have run out of time.
  87         if self.maxTime is not None:
  88             if time.time() - self.startTime > self.maxTime:
  89                 return None
  90
  91         # Otherwise take the next test.
  92         self.lock.acquire()
  93         if self.canceled:
  94           self.lock.release()
  95           return None
  96         for item in self.iter:
  97             break
  98         else:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result = test.config.test_format.execute(test, self.litConfig)
 122
 123             # Support deprecated result from execute() which returned the result
 124             # code and additional output as a tuple.
 125             if isinstance(result, tuple):
 126                 code, output = result
 127                 result = lit.Test.Result(code, output)
 128             elif not isinstance(result, lit.Test.Result):
 129                 raise ValueError("unexpected result from test execution")
 130         except KeyboardInterrupt:
 131             # This is a sad hack. Unfortunately subprocess goes
 132             # bonkers with ctrl-c and we start forking merrily.
 133             print('\nCtrl-C detected, goodbye.')
 134             os.kill(0,9)
 135         except:
 136             if self.litConfig.debug:
 137                 raise
 138             output = 'Exception during script execution:\n'
 139             output += traceback.format_exc()
 140             output += '\n'
 141             result = lit.Test.Result(lit.Test.UNRESOLVED, output)
 142         result.elapsed = time.time() - startTime
 143
 144         test.setResult(result)
 145         self.display.update(test)
 146
 147 def runTests(numThreads, litConfig, provider, display):
 148     # If only using one testing thread, don't use threads at all; this lets us
 149     # profile, among other things.
 150     if numThreads == 1:
 151         t = Tester(litConfig, provider, display)
 152         t.run()
 153         return
 154
 155     # Otherwise spin up the testing threads and wait for them to finish.
 156     testers = [Tester(litConfig, provider, display)
 157                for i in range(numThreads)]
 158     for t in testers:
 159         t.start()
 160     try:
 161         for t in testers:
 162             t.join()
 163     except KeyboardInterrupt:
 164         sys.exit(2)
 165
 166 def main(builtinParameters = {}):
 167     # Bump the GIL check interval, its more important to get any one thread to a
 168     # blocking operation (hopefully exec) than to try and unblock other threads.
 169     #
 170     # FIXME: This is a hack.
 171     sys.setcheckinterval(1000)
 172
 173     global options
 174     from optparse import OptionParser, OptionGroup
 175     parser = OptionParser("usage: %prog [options] {file-or-path}")
 176
 177     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 178                       help="Number of testing threads",
 179                       type=int, action="store", default=None)
 180     parser.add_option("", "--config-prefix", dest="configPrefix",
 181                       metavar="NAME", help="Prefix for 'lit' config files",
 182                       action="store", default=None)
 183     parser.add_option("", "--param", dest="userParameters",
 184                       metavar="NAME=VAL",
 185                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 186                       type=str, action="append", default=[])
 187
 188     group = OptionGroup(parser, "Output Format")
 189     # FIXME: I find these names very confusing, although I like the
 190     # functionality.
 191     group.add_option("-q", "--quiet", dest="quiet",
 192                      help="Suppress no error output",
 193                      action="store_true", default=False)
 194     group.add_option("-s", "--succinct", dest="succinct",
 195                      help="Reduce amount of output",
 196                      action="store_true", default=False)
 197     group.add_option("-v", "--verbose", dest="showOutput",
 198                      help="Show all test output",
 199                      action="store_true", default=False)
 200     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 201                      help="Do not use curses based progress bar",
 202                      action="store_false", default=True)
 203     parser.add_option_group(group)
 204
 205     group = OptionGroup(parser, "Test Execution")
 206     group.add_option("", "--path", dest="path",
 207                      help="Additional paths to add to testing environment",
 208                      action="append", type=str, default=[])
 209     group.add_option("", "--vg", dest="useValgrind",
 210                      help="Run tests under valgrind",
 211                      action="store_true", default=False)
 212     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 213                      help="Check for memory leaks under valgrind",
 214                      action="store_true", default=False)
 215     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 216                      help="Specify an extra argument for valgrind",
 217                      type=str, action="append", default=[])
 218     group.add_option("", "--time-tests", dest="timeTests",
 219                      help="Track elapsed wall time for each test",
 220                      action="store_true", default=False)
 221     group.add_option("", "--no-execute", dest="noExecute",
 222                      help="Don't execute any tests (assume PASS)",
 223                      action="store_true", default=False)
 224     parser.add_option_group(group)
 225
 226     group = OptionGroup(parser, "Test Selection")
 227     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 228                      help="Maximum number of tests to run",
 229                      action="store", type=int, default=None)
 230     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 231                      help="Maximum time to spend testing (in seconds)",
 232                      action="store", type=float, default=None)
 233     group.add_option("", "--shuffle", dest="shuffle",
 234                      help="Run tests in random order",
 235                      action="store_true", default=False)
 236     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 237                      help=("Only run tests with paths matching the given "
 238                            "regular expression"),
 239                      action="store", default=None)
 240     parser.add_option_group(group)
 241
 242     group = OptionGroup(parser, "Debug and Experimental Options")
 243     group.add_option("", "--debug", dest="debug",
 244                       help="Enable debugging (for 'lit' development)",
 245                       action="store_true", default=False)
 246     group.add_option("", "--show-suites", dest="showSuites",
 247                       help="Show discovered test suites",
 248                       action="store_true", default=False)
 249     group.add_option("", "--show-tests", dest="showTests",
 250                       help="Show all discovered tests",
 251                       action="store_true", default=False)
 252     parser.add_option_group(group)
 253
 254     (opts, args) = parser.parse_args()
 255
 256     if not args:
 257         parser.error('No inputs specified')
 258
 259     if opts.numThreads is None:
 260 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 261 # http://bugs.python.org/issue1731717
 262 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 263 # threads by default there.
 264        if sys.hexversion >= 0x2050200:
 265                opts.numThreads = lit.util.detectCPUs()
 266        else:
 267                opts.numThreads = 1
 268
 269     inputs = args
 270
 271     # Create the user defined parameters.
 272     userParams = dict(builtinParameters)
 273     for entry in opts.userParameters:
 274         if '=' not in entry:
 275             name,val = entry,''
 276         else:
 277             name,val = entry.split('=', 1)
 278         userParams[name] = val
 279
 280     # Create the global config object.
 281     litConfig = lit.LitConfig.LitConfig(
 282         progname = os.path.basename(sys.argv[0]),
 283         path = opts.path,
 284         quiet = opts.quiet,
 285         useValgrind = opts.useValgrind,
 286         valgrindLeakCheck = opts.valgrindLeakCheck,
 287         valgrindArgs = opts.valgrindArgs,
 288         noExecute = opts.noExecute,
 289         debug = opts.debug,
 290         isWindows = (platform.system()=='Windows'),
 291         params = userParams,
 292         config_prefix = opts.configPrefix)
 293
 294     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 295
 296     if opts.showSuites or opts.showTests:
 297         # Aggregate the tests by suite.
 298         suitesAndTests = {}
 299         for t in tests:
 300             if t.suite not in suitesAndTests:
 301                 suitesAndTests[t.suite] = []
 302             suitesAndTests[t.suite].append(t)
 303         suitesAndTests = list(suitesAndTests.items())
 304         suitesAndTests.sort(key = lambda item: item[0].name)
 305
 306         # Show the suites, if requested.
 307         if opts.showSuites:
 308             print('-- Test Suites --')
 309             for ts,ts_tests in suitesAndTests:
 310                 print('  %s - %d tests' %(ts.name, len(ts_tests)))
 311                 print('    Source Root: %s' % ts.source_root)
 312                 print('    Exec Root  : %s' % ts.exec_root)
 313
 314         # Show the tests, if requested.
 315         if opts.showTests:
 316             print('-- Available Tests --')
 317             for ts,ts_tests in suitesAndTests:
 318                 ts_tests.sort(key = lambda test: test.path_in_suite)
 319                 for test in ts_tests:
 320                     print('  %s' % (test.getFullName(),))
 321
 322         # Exit.
 323         sys.exit(0)
 324
 325     # Select and order the tests.
 326     numTotalTests = len(tests)
 327
 328     # First, select based on the filter expression if given.
 329     if opts.filter:
 330         try:
 331             rex = re.compile(opts.filter)
 332         except:
 333             parser.error("invalid regular expression for --filter: %r" % (
 334                     opts.filter))
 335         tests = [t for t in tests
 336                  if rex.search(t.getFullName())]
 337
 338     # Then select the order.
 339     if opts.shuffle:
 340         random.shuffle(tests)
 341     else:
 342         tests.sort(key = lambda t: t.getFullName())
 343
 344     # Finally limit the number of tests, if desired.
 345     if opts.maxTests is not None:
 346         tests = tests[:opts.maxTests]
 347
 348     # Don't create more threads than tests.
 349     opts.numThreads = min(len(tests), opts.numThreads)
 350
 351     extra = ''
 352     if len(tests) != numTotalTests:
 353         extra = ' of %d' % numTotalTests
 354     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 355                                                       opts.numThreads)
 356
 357     progressBar = None
 358     if not opts.quiet:
 359         if opts.succinct and opts.useProgressBar:
 360             try:
 361                 tc = lit.ProgressBar.TerminalController()
 362                 progressBar = lit.ProgressBar.ProgressBar(tc, header)
 363             except ValueError:
 364                 print(header)
 365                 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
 366         else:
 367             print(header)
 368
 369     startTime = time.time()
 370     display = TestingProgressDisplay(opts, len(tests), progressBar)
 371     provider = TestProvider(tests, opts.maxTime)
 372
 373     try:
 374       import win32api
 375     except ImportError:
 376       pass
 377     else:
 378       def console_ctrl_handler(type):
 379         provider.cancel()
 380         return True
 381       win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
 382
 383     runTests(opts.numThreads, litConfig, provider, display)
 384     display.finish()
 385
 386     if not opts.quiet:
 387         print('Testing Time: %.2fs'%(time.time() - startTime))
 388
 389     # Update results for any tests which weren't run.
 390     for test in tests:
 391         if test.result is None:
 392             test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
 393
 394     # List test results organized by kind.
 395     hasFailures = False
 396     byCode = {}
 397     for test in tests:
 398         if test.result.code not in byCode:
 399             byCode[test.result.code] = []
 400         byCode[test.result.code].append(test)
 401         if test.result.code.isFailure:
 402             hasFailures = True
 403
 404     # Print each test in any of the failing groups.
 405     for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
 406                        ('Failing Tests', lit.Test.FAIL),
 407                        ('Unresolved Tests', lit.Test.UNRESOLVED)):
 408         elts = byCode.get(code)
 409         if not elts:
 410             continue
 411         print('*'*20)
 412         print('%s (%d):' % (title, len(elts)))
 413         for test in elts:
 414             print('    %s' % test.getFullName())
 415         sys.stdout.write('\n')
 416
 417     if opts.timeTests and tests:
 418         # Order by time.
 419         test_times = [(test.getFullName(), test.result.elapsed)
 420                       for test in tests]
 421         lit.util.printHistogram(test_times, title='Tests')
 422
 423     for name,code in (('Expected Passes    ', lit.Test.PASS),
 424                       ('Expected Failures  ', lit.Test.XFAIL),
 425                       ('Unsupported Tests  ', lit.Test.UNSUPPORTED),
 426                       ('Unresolved Tests   ', lit.Test.UNRESOLVED),
 427                       ('Unexpected Passes  ', lit.Test.XPASS),
 428                       ('Unexpected Failures', lit.Test.FAIL),):
 429         if opts.quiet and not code.isFailure:
 430             continue
 431         N = len(byCode.get(code,[]))
 432         if N:
 433             print('  %s: %d' % (name,N))
 434
 435     # If we encountered any additional errors, exit abnormally.
 436     if litConfig.numErrors:
 437         sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
 438         sys.exit(2)
 439
 440     # Warn about warnings.
 441     if litConfig.numWarnings:
 442         sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
 443
 444     if hasFailures:
 445         sys.exit(1)
 446     sys.exit(0)
 447
 448 if __name__=='__main__':
 449     main()