utils/lit/lit/main.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 import LitConfig
  16 import Test
  17
  18 import lit.discovery
  19
  20 class TestingProgressDisplay:
  21     def __init__(self, opts, numTests, progressBar=None):
  22         self.opts = opts
  23         self.numTests = numTests
  24         self.current = None
  25         self.lock = threading.Lock()
  26         self.progressBar = progressBar
  27         self.completed = 0
  28
  29     def update(self, test):
  30         # Avoid locking overhead in quiet mode
  31         if self.opts.quiet and not test.result.isFailure:
  32             self.completed += 1
  33             return
  34
  35         # Output lock.
  36         self.lock.acquire()
  37         try:
  38             self.handleUpdate(test)
  39         finally:
  40             self.lock.release()
  41
  42     def finish(self):
  43         if self.progressBar:
  44             self.progressBar.clear()
  45         elif self.opts.quiet:
  46             pass
  47         elif self.opts.succinct:
  48             sys.stdout.write('\n')
  49
  50     def handleUpdate(self, test):
  51         self.completed += 1
  52         if self.progressBar:
  53             self.progressBar.update(float(self.completed)/self.numTests,
  54                                     test.getFullName())
  55
  56         if self.opts.succinct and not test.result.isFailure:
  57             return
  58
  59         if self.progressBar:
  60             self.progressBar.clear()
  61
  62         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  63                                      self.completed, self.numTests)
  64
  65         if test.result.isFailure and self.opts.showOutput:
  66             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  67                                               '*'*20)
  68             print test.output
  69             print "*" * 20
  70
  71         sys.stdout.flush()
  72
  73 class TestProvider:
  74     def __init__(self, tests, maxTime):
  75         self.maxTime = maxTime
  76         self.iter = iter(tests)
  77         self.lock = threading.Lock()
  78         self.startTime = time.time()
  79
  80     def get(self):
  81         # Check if we have run out of time.
  82         if self.maxTime is not None:
  83             if time.time() - self.startTime > self.maxTime:
  84                 return None
  85
  86         # Otherwise take the next test.
  87         self.lock.acquire()
  88         try:
  89             item = self.iter.next()
  90         except StopIteration:
  91             item = None
  92         self.lock.release()
  93         return item
  94
  95 class Tester(threading.Thread):
  96     def __init__(self, litConfig, provider, display):
  97         threading.Thread.__init__(self)
  98         self.litConfig = litConfig
  99         self.provider = provider
 100         self.display = display
 101
 102     def run(self):
 103         while 1:
 104             item = self.provider.get()
 105             if item is None:
 106                 break
 107             self.runTest(item)
 108
 109     def runTest(self, test):
 110         result = None
 111         startTime = time.time()
 112         try:
 113             result, output = test.config.test_format.execute(test,
 114                                                              self.litConfig)
 115         except KeyboardInterrupt:
 116             # This is a sad hack. Unfortunately subprocess goes
 117             # bonkers with ctrl-c and we start forking merrily.
 118             print '\nCtrl-C detected, goodbye.'
 119             os.kill(0,9)
 120         except:
 121             if self.litConfig.debug:
 122                 raise
 123             result = Test.UNRESOLVED
 124             output = 'Exception during script execution:\n'
 125             output += traceback.format_exc()
 126             output += '\n'
 127         elapsed = time.time() - startTime
 128
 129         test.setResult(result, output, elapsed)
 130         self.display.update(test)
 131
 132 def runTests(numThreads, litConfig, provider, display):
 133     # If only using one testing thread, don't use threads at all; this lets us
 134     # profile, among other things.
 135     if numThreads == 1:
 136         t = Tester(litConfig, provider, display)
 137         t.run()
 138         return
 139
 140     # Otherwise spin up the testing threads and wait for them to finish.
 141     testers = [Tester(litConfig, provider, display)
 142                for i in range(numThreads)]
 143     for t in testers:
 144         t.start()
 145     try:
 146         for t in testers:
 147             t.join()
 148     except KeyboardInterrupt:
 149         sys.exit(2)
 150
 151 def main(builtinParameters = {}):
 152     # Bump the GIL check interval, its more important to get any one thread to a
 153     # blocking operation (hopefully exec) than to try and unblock other threads.
 154     #
 155     # FIXME: This is a hack.
 156     import sys
 157     sys.setcheckinterval(1000)
 158
 159     global options
 160     from optparse import OptionParser, OptionGroup
 161     parser = OptionParser("usage: %prog [options] {file-or-path}")
 162
 163     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 164                       help="Number of testing threads",
 165                       type=int, action="store", default=None)
 166     parser.add_option("", "--config-prefix", dest="configPrefix",
 167                       metavar="NAME", help="Prefix for 'lit' config files",
 168                       action="store", default=None)
 169     parser.add_option("", "--param", dest="userParameters",
 170                       metavar="NAME=VAL",
 171                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 172                       type=str, action="append", default=[])
 173
 174     group = OptionGroup(parser, "Output Format")
 175     # FIXME: I find these names very confusing, although I like the
 176     # functionality.
 177     group.add_option("-q", "--quiet", dest="quiet",
 178                      help="Suppress no error output",
 179                      action="store_true", default=False)
 180     group.add_option("-s", "--succinct", dest="succinct",
 181                      help="Reduce amount of output",
 182                      action="store_true", default=False)
 183     group.add_option("-v", "--verbose", dest="showOutput",
 184                      help="Show all test output",
 185                      action="store_true", default=False)
 186     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 187                      help="Do not use curses based progress bar",
 188                      action="store_false", default=True)
 189     parser.add_option_group(group)
 190
 191     group = OptionGroup(parser, "Test Execution")
 192     group.add_option("", "--path", dest="path",
 193                      help="Additional paths to add to testing environment",
 194                      action="append", type=str, default=[])
 195     group.add_option("", "--vg", dest="useValgrind",
 196                      help="Run tests under valgrind",
 197                      action="store_true", default=False)
 198     group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
 199                      help="Check for memory leaks under valgrind",
 200                      action="store_true", default=False)
 201     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 202                      help="Specify an extra argument for valgrind",
 203                      type=str, action="append", default=[])
 204     group.add_option("", "--time-tests", dest="timeTests",
 205                      help="Track elapsed wall time for each test",
 206                      action="store_true", default=False)
 207     group.add_option("", "--no-execute", dest="noExecute",
 208                      help="Don't execute any tests (assume PASS)",
 209                      action="store_true", default=False)
 210     parser.add_option_group(group)
 211
 212     group = OptionGroup(parser, "Test Selection")
 213     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 214                      help="Maximum number of tests to run",
 215                      action="store", type=int, default=None)
 216     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 217                      help="Maximum time to spend testing (in seconds)",
 218                      action="store", type=float, default=None)
 219     group.add_option("", "--shuffle", dest="shuffle",
 220                      help="Run tests in random order",
 221                      action="store_true", default=False)
 222     group.add_option("", "--filter", dest="filter", metavar="REGEX",
 223                      help=("Only run tests with paths matching the given "
 224                            "regular expression"),
 225                      action="store", default=None)
 226     parser.add_option_group(group)
 227
 228     group = OptionGroup(parser, "Debug and Experimental Options")
 229     group.add_option("", "--debug", dest="debug",
 230                       help="Enable debugging (for 'lit' development)",
 231                       action="store_true", default=False)
 232     group.add_option("", "--show-suites", dest="showSuites",
 233                       help="Show discovered test suites",
 234                       action="store_true", default=False)
 235     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 236                       help="Repeat tests N times (for timing)",
 237                       action="store", default=None, type=int)
 238     parser.add_option_group(group)
 239
 240     (opts, args) = parser.parse_args()
 241
 242     if not args:
 243         parser.error('No inputs specified')
 244
 245     if opts.numThreads is None:
 246 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
 247 # http://bugs.python.org/issue1731717
 248 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
 249 # threads by default there.
 250        if sys.hexversion >= 0x2050200:
 251                opts.numThreads = Util.detectCPUs()
 252        else:
 253                opts.numThreads = 1
 254
 255     inputs = args
 256
 257     # Create the user defined parameters.
 258     userParams = dict(builtinParameters)
 259     for entry in opts.userParameters:
 260         if '=' not in entry:
 261             name,val = entry,''
 262         else:
 263             name,val = entry.split('=', 1)
 264         userParams[name] = val
 265
 266     # Create the global config object.
 267     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 268                                     path = opts.path,
 269                                     quiet = opts.quiet,
 270                                     useValgrind = opts.useValgrind,
 271                                     valgrindLeakCheck = opts.valgrindLeakCheck,
 272                                     valgrindArgs = opts.valgrindArgs,
 273                                     noExecute = opts.noExecute,
 274                                     ignoreStdErr = False,
 275                                     debug = opts.debug,
 276                                     isWindows = (platform.system()=='Windows'),
 277                                     params = userParams,
 278                                     config_prefix = opts.configPrefix)
 279
 280     tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
 281
 282     if opts.showSuites:
 283         suitesAndTests = {}
 284         for t in tests:
 285             if t.suite not in suitesAndTests:
 286                 suitesAndTests[t.suite] = []
 287             suitesAndTests[t.suite].append(t)
 288
 289         print '-- Test Suites --'
 290         suitesAndTests = suitesAndTests.items()
 291         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 292         for ts,ts_tests in suitesAndTests:
 293             print '  %s - %d tests' %(ts.name, len(ts_tests))
 294             print '    Source Root: %s' % ts.source_root
 295             print '    Exec Root  : %s' % ts.exec_root
 296
 297     # Select and order the tests.
 298     numTotalTests = len(tests)
 299
 300     # First, select based on the filter expression if given.
 301     if opts.filter:
 302         try:
 303             rex = re.compile(opts.filter)
 304         except:
 305             parser.error("invalid regular expression for --filter: %r" % (
 306                     opts.filter))
 307         tests = [t for t in tests
 308                  if rex.search(t.getFullName())]
 309
 310     # Then select the order.
 311     if opts.shuffle:
 312         random.shuffle(tests)
 313     else:
 314         tests.sort(key = lambda t: t.getFullName())
 315
 316     # Finally limit the number of tests, if desired.
 317     if opts.maxTests is not None:
 318         tests = tests[:opts.maxTests]
 319
 320     # Don't create more threads than tests.
 321     opts.numThreads = min(len(tests), opts.numThreads)
 322
 323     extra = ''
 324     if len(tests) != numTotalTests:
 325         extra = ' of %d' % numTotalTests
 326     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 327                                                       opts.numThreads)
 328
 329     if opts.repeatTests:
 330         tests = [t.copyWithIndex(i)
 331                  for t in tests
 332                  for i in range(opts.repeatTests)]
 333
 334     progressBar = None
 335     if not opts.quiet:
 336         if opts.succinct and opts.useProgressBar:
 337             try:
 338                 tc = ProgressBar.TerminalController()
 339                 progressBar = ProgressBar.ProgressBar(tc, header)
 340             except ValueError:
 341                 print header
 342                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 343         else:
 344             print header
 345
 346     startTime = time.time()
 347     display = TestingProgressDisplay(opts, len(tests), progressBar)
 348     provider = TestProvider(tests, opts.maxTime)
 349     runTests(opts.numThreads, litConfig, provider, display)
 350     display.finish()
 351
 352     if not opts.quiet:
 353         print 'Testing Time: %.2fs'%(time.time() - startTime)
 354
 355     # Update results for any tests which weren't run.
 356     for t in tests:
 357         if t.result is None:
 358             t.setResult(Test.UNRESOLVED, '', 0.0)
 359
 360     # List test results organized by kind.
 361     hasFailures = False
 362     byCode = {}
 363     for t in tests:
 364         if t.result not in byCode:
 365             byCode[t.result] = []
 366         byCode[t.result].append(t)
 367         if t.result.isFailure:
 368             hasFailures = True
 369
 370     # FIXME: Show unresolved and (optionally) unsupported tests.
 371     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 372                        ('Failing Tests', Test.FAIL)):
 373         elts = byCode.get(code)
 374         if not elts:
 375             continue
 376         print '*'*20
 377         print '%s (%d):' % (title, len(elts))
 378         for t in elts:
 379             print '    %s' % t.getFullName()
 380         print
 381
 382     if opts.timeTests:
 383         # Collate, in case we repeated tests.
 384         times = {}
 385         for t in tests:
 386             key = t.getFullName()
 387             times[key] = times.get(key, 0.) + t.elapsed
 388
 389         byTime = list(times.items())
 390         byTime.sort(key = lambda (name,elapsed): elapsed)
 391         if byTime:
 392             Util.printHistogram(byTime, title='Tests')
 393
 394     for name,code in (('Expected Passes    ', Test.PASS),
 395                       ('Expected Failures  ', Test.XFAIL),
 396                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 397                       ('Unresolved Tests   ', Test.UNRESOLVED),
 398                       ('Unexpected Passes  ', Test.XPASS),
 399                       ('Unexpected Failures', Test.FAIL),):
 400         if opts.quiet and not code.isFailure:
 401             continue
 402         N = len(byCode.get(code,[]))
 403         if N:
 404             print '  %s: %d' % (name,N)
 405
 406     # If we encountered any additional errors, exit abnormally.
 407     if litConfig.numErrors:
 408         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 409         sys.exit(2)
 410
 411     # Warn about warnings.
 412     if litConfig.numWarnings:
 413         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 414
 415     if hasFailures:
 416         sys.exit(1)
 417     sys.exit(0)
 418
 419 if __name__=='__main__':
 420     main()