utils/lit/lit/lit.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 from TestingConfig import TestingConfig
  16 import LitConfig
  17 import Test
  18
  19 # Configuration files to look for when discovering test suites. These can be
  20 # overridden with --config-prefix.
  21 #
  22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
  23 gConfigName = 'lit.cfg'
  24 gSiteConfigName = 'lit.site.cfg'
  25
  26 kLocalConfigName = 'lit.local.cfg'
  27
  28 class TestingProgressDisplay:
  29     def __init__(self, opts, numTests, progressBar=None):
  30         self.opts = opts
  31         self.numTests = numTests
  32         self.current = None
  33         self.lock = threading.Lock()
  34         self.progressBar = progressBar
  35         self.completed = 0
  36
  37     def update(self, test):
  38         # Avoid locking overhead in quiet mode
  39         if self.opts.quiet and not test.result.isFailure:
  40             self.completed += 1
  41             return
  42
  43         # Output lock.
  44         self.lock.acquire()
  45         try:
  46             self.handleUpdate(test)
  47         finally:
  48             self.lock.release()
  49
  50     def finish(self):
  51         if self.progressBar:
  52             self.progressBar.clear()
  53         elif self.opts.quiet:
  54             pass
  55         elif self.opts.succinct:
  56             sys.stdout.write('\n')
  57
  58     def handleUpdate(self, test):
  59         self.completed += 1
  60         if self.progressBar:
  61             self.progressBar.update(float(self.completed)/self.numTests,
  62                                     test.getFullName())
  63
  64         if self.opts.succinct and not test.result.isFailure:
  65             return
  66
  67         if self.progressBar:
  68             self.progressBar.clear()
  69
  70         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  71                                      self.completed, self.numTests)
  72
  73         if test.result.isFailure and self.opts.showOutput:
  74             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  75                                               '*'*20)
  76             print test.output
  77             print "*" * 20
  78
  79         sys.stdout.flush()
  80
  81 class TestProvider:
  82     def __init__(self, tests, maxTime):
  83         self.maxTime = maxTime
  84         self.iter = iter(tests)
  85         self.lock = threading.Lock()
  86         self.startTime = time.time()
  87
  88     def get(self):
  89         # Check if we have run out of time.
  90         if self.maxTime is not None:
  91             if time.time() - self.startTime > self.maxTime:
  92                 return None
  93
  94         # Otherwise take the next test.
  95         self.lock.acquire()
  96         try:
  97             item = self.iter.next()
  98         except StopIteration:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print '\nCtrl-C detected, goodbye.'
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def dirContainsTestSuite(path):
 141     cfgpath = os.path.join(path, gSiteConfigName)
 142     if os.path.exists(cfgpath):
 143         return cfgpath
 144     cfgpath = os.path.join(path, gConfigName)
 145     if os.path.exists(cfgpath):
 146         return cfgpath
 147
 148 def getTestSuite(item, litConfig, cache):
 149     """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
 150
 151     Find the test suite containing @arg item.
 152
 153     @retval (None, ...) - Indicates no test suite contains @arg item.
 154     @retval (suite, relative_path) - The suite that @arg item is in, and its
 155     relative path inside that suite.
 156     """
 157     def search1(path):
 158         # Check for a site config or a lit config.
 159         cfgpath = dirContainsTestSuite(path)
 160
 161         # If we didn't find a config file, keep looking.
 162         if not cfgpath:
 163             parent,base = os.path.split(path)
 164             if parent == path:
 165                 return (None, ())
 166
 167             ts, relative = search(parent)
 168             return (ts, relative + (base,))
 169
 170         # We found a config file, load it.
 171         if litConfig.debug:
 172             litConfig.note('loading suite config %r' % cfgpath)
 173
 174         cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
 175         source_root = os.path.realpath(cfg.test_source_root or path)
 176         exec_root = os.path.realpath(cfg.test_exec_root or path)
 177         return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
 178
 179     def search(path):
 180         # Check for an already instantiated test suite.
 181         res = cache.get(path)
 182         if res is None:
 183             cache[path] = res = search1(path)
 184         return res
 185
 186     # Canonicalize the path.
 187     item = os.path.realpath(item)
 188
 189     # Skip files and virtual components.
 190     components = []
 191     while not os.path.isdir(item):
 192         parent,base = os.path.split(item)
 193         if parent == item:
 194             return (None, ())
 195         components.append(base)
 196         item = parent
 197     components.reverse()
 198
 199     ts, relative = search(item)
 200     return ts, tuple(relative + tuple(components))
 201
 202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
 203     def search1(path_in_suite):
 204         # Get the parent config.
 205         if not path_in_suite:
 206             parent = ts.config
 207         else:
 208             parent = search(path_in_suite[:-1])
 209
 210         # Load the local configuration.
 211         source_path = ts.getSourcePath(path_in_suite)
 212         cfgpath = os.path.join(source_path, kLocalConfigName)
 213         if litConfig.debug:
 214             litConfig.note('loading local config %r' % cfgpath)
 215         return TestingConfig.frompath(cfgpath, parent, litConfig,
 216                                     mustExist = False,
 217                                     config = parent.clone(cfgpath))
 218
 219     def search(path_in_suite):
 220         key = (ts, path_in_suite)
 221         res = cache.get(key)
 222         if res is None:
 223             cache[key] = res = search1(path_in_suite)
 224         return res
 225
 226     return search(path_in_suite)
 227
 228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
 229     # Find the test suite for this input and its relative path.
 230     ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
 231     if ts is None:
 232         litConfig.warning('unable to find test suite for %r' % path)
 233         return (),()
 234
 235     if litConfig.debug:
 236         litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
 237                                                         path_in_suite))
 238
 239     return ts, getTestsInSuite(ts, path_in_suite, litConfig,
 240                                testSuiteCache, localConfigCache)
 241
 242 def getTestsInSuite(ts, path_in_suite, litConfig,
 243                     testSuiteCache, localConfigCache):
 244     # Check that the source path exists (errors here are reported by the
 245     # caller).
 246     source_path = ts.getSourcePath(path_in_suite)
 247     if not os.path.exists(source_path):
 248         return
 249
 250     # Check if the user named a test directly.
 251     if not os.path.isdir(source_path):
 252         lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
 253         yield Test.Test(ts, path_in_suite, lc)
 254         return
 255
 256     # Otherwise we have a directory to search for tests, start by getting the
 257     # local configuration.
 258     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 259
 260     # Search for tests.
 261     for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
 262                                                   litConfig, lc):
 263         yield res
 264
 265     # Search subdirectories.
 266     for filename in os.listdir(source_path):
 267         # FIXME: This doesn't belong here?
 268         if filename in ('Output', '.svn') or filename in lc.excludes:
 269             continue
 270
 271         # Ignore non-directories.
 272         file_sourcepath = os.path.join(source_path, filename)
 273         if not os.path.isdir(file_sourcepath):
 274             continue
 275
 276         # Check for nested test suites, first in the execpath in case there is a
 277         # site configuration and then in the source path.
 278         file_execpath = ts.getExecPath(path_in_suite + (filename,))
 279         if dirContainsTestSuite(file_execpath):
 280             sub_ts, subiter = getTests(file_execpath, litConfig,
 281                                        testSuiteCache, localConfigCache)
 282         elif dirContainsTestSuite(file_sourcepath):
 283             sub_ts, subiter = getTests(file_sourcepath, litConfig,
 284                                        testSuiteCache, localConfigCache)
 285         else:
 286             # Otherwise, continue loading from inside this test suite.
 287             subiter = getTestsInSuite(ts, path_in_suite + (filename,),
 288                                       litConfig, testSuiteCache,
 289                                       localConfigCache)
 290             sub_ts = None
 291
 292         N = 0
 293         for res in subiter:
 294             N += 1
 295             yield res
 296         if sub_ts and not N:
 297             litConfig.warning('test suite %r contained no tests' % sub_ts.name)
 298
 299 def runTests(numThreads, litConfig, provider, display):
 300     # If only using one testing thread, don't use threads at all; this lets us
 301     # profile, among other things.
 302     if numThreads == 1:
 303         t = Tester(litConfig, provider, display)
 304         t.run()
 305         return
 306
 307     # Otherwise spin up the testing threads and wait for them to finish.
 308     testers = [Tester(litConfig, provider, display)
 309                for i in range(numThreads)]
 310     for t in testers:
 311         t.start()
 312     try:
 313         for t in testers:
 314             t.join()
 315     except KeyboardInterrupt:
 316         sys.exit(2)
 317
 318 def main():
 319     # Bump the GIL check interval, its more important to get any one thread to a
 320     # blocking operation (hopefully exec) than to try and unblock other threads.
 321     #
 322     # FIXME: This is a hack.
 323     import sys
 324     sys.setcheckinterval(1000)
 325
 326     global options
 327     from optparse import OptionParser, OptionGroup
 328     parser = OptionParser("usage: %prog [options] {file-or-path}")
 329
 330     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 331                       help="Number of testing threads",
 332                       type=int, action="store", default=None)
 333     parser.add_option("", "--config-prefix", dest="configPrefix",
 334                       metavar="NAME", help="Prefix for 'lit' config files",
 335                       action="store", default=None)
 336     parser.add_option("", "--param", dest="userParameters",
 337                       metavar="NAME=VAL",
 338                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 339                       type=str, action="append", default=[])
 340
 341     group = OptionGroup(parser, "Output Format")
 342     # FIXME: I find these names very confusing, although I like the
 343     # functionality.
 344     group.add_option("-q", "--quiet", dest="quiet",
 345                      help="Suppress no error output",
 346                      action="store_true", default=False)
 347     group.add_option("-s", "--succinct", dest="succinct",
 348                      help="Reduce amount of output",
 349                      action="store_true", default=False)
 350     group.add_option("-v", "--verbose", dest="showOutput",
 351                      help="Show all test output",
 352                      action="store_true", default=False)
 353     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 354                      help="Do not use curses based progress bar",
 355                      action="store_false", default=True)
 356     parser.add_option_group(group)
 357
 358     group = OptionGroup(parser, "Test Execution")
 359     group.add_option("", "--path", dest="path",
 360                      help="Additional paths to add to testing environment",
 361                      action="append", type=str, default=[])
 362     group.add_option("", "--vg", dest="useValgrind",
 363                      help="Run tests under valgrind",
 364                      action="store_true", default=False)
 365     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 366                      help="Specify an extra argument for valgrind",
 367                      type=str, action="append", default=[])
 368     group.add_option("", "--time-tests", dest="timeTests",
 369                      help="Track elapsed wall time for each test",
 370                      action="store_true", default=False)
 371     group.add_option("", "--no-execute", dest="noExecute",
 372                      help="Don't execute any tests (assume PASS)",
 373                      action="store_true", default=False)
 374     parser.add_option_group(group)
 375
 376     group = OptionGroup(parser, "Test Selection")
 377     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 378                      help="Maximum number of tests to run",
 379                      action="store", type=int, default=None)
 380     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 381                      help="Maximum time to spend testing (in seconds)",
 382                      action="store", type=float, default=None)
 383     group.add_option("", "--shuffle", dest="shuffle",
 384                      help="Run tests in random order",
 385                      action="store_true", default=False)
 386     parser.add_option_group(group)
 387
 388     group = OptionGroup(parser, "Debug and Experimental Options")
 389     group.add_option("", "--debug", dest="debug",
 390                       help="Enable debugging (for 'lit' development)",
 391                       action="store_true", default=False)
 392     group.add_option("", "--show-suites", dest="showSuites",
 393                       help="Show discovered test suites",
 394                       action="store_true", default=False)
 395     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
 396                       help="Don't run Tcl scripts using 'sh'",
 397                       action="store_false", default=True)
 398     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 399                       help="Repeat tests N times (for timing)",
 400                       action="store", default=None, type=int)
 401     parser.add_option_group(group)
 402
 403     (opts, args) = parser.parse_args()
 404
 405     if not args:
 406         parser.error('No inputs specified')
 407
 408     if opts.configPrefix is not None:
 409         global gConfigName, gSiteConfigName
 410         gConfigName = '%s.cfg' % opts.configPrefix
 411         gSiteConfigName = '%s.site.cfg' % opts.configPrefix
 412
 413     if opts.numThreads is None:
 414         opts.numThreads = Util.detectCPUs()
 415
 416     inputs = args
 417
 418     # Create the user defined parameters.
 419     userParams = {}
 420     for entry in opts.userParameters:
 421         if '=' not in entry:
 422             name,val = entry,''
 423         else:
 424             name,val = entry.split('=', 1)
 425         userParams[name] = val
 426
 427     # Create the global config object.
 428     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 429                                     path = opts.path,
 430                                     quiet = opts.quiet,
 431                                     useValgrind = opts.useValgrind,
 432                                     valgrindArgs = opts.valgrindArgs,
 433                                     useTclAsSh = opts.useTclAsSh,
 434                                     noExecute = opts.noExecute,
 435                                     debug = opts.debug,
 436                                     isWindows = (platform.system()=='Windows'),
 437                                     params = userParams)
 438
 439     # Load the tests from the inputs.
 440     tests = []
 441     testSuiteCache = {}
 442     localConfigCache = {}
 443     for input in inputs:
 444         prev = len(tests)
 445         tests.extend(getTests(input, litConfig,
 446                               testSuiteCache, localConfigCache)[1])
 447         if prev == len(tests):
 448             litConfig.warning('input %r contained no tests' % input)
 449
 450     # If there were any errors during test discovery, exit now.
 451     if litConfig.numErrors:
 452         print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
 453         sys.exit(2)
 454
 455     if opts.showSuites:
 456         suitesAndTests = dict([(ts,[])
 457                                for ts,_ in testSuiteCache.values()
 458                                if ts])
 459         for t in tests:
 460             suitesAndTests[t.suite].append(t)
 461
 462         print '-- Test Suites --'
 463         suitesAndTests = suitesAndTests.items()
 464         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 465         for ts,ts_tests in suitesAndTests:
 466             print '  %s - %d tests' %(ts.name, len(ts_tests))
 467             print '    Source Root: %s' % ts.source_root
 468             print '    Exec Root  : %s' % ts.exec_root
 469
 470     # Select and order the tests.
 471     numTotalTests = len(tests)
 472     if opts.shuffle:
 473         random.shuffle(tests)
 474     else:
 475         tests.sort(key = lambda t: t.getFullName())
 476     if opts.maxTests is not None:
 477         tests = tests[:opts.maxTests]
 478
 479     extra = ''
 480     if len(tests) != numTotalTests:
 481         extra = ' of %d' % numTotalTests
 482     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 483                                                       opts.numThreads)
 484
 485     if opts.repeatTests:
 486         tests = [t.copyWithIndex(i)
 487                  for t in tests
 488                  for i in range(opts.repeatTests)]
 489
 490     progressBar = None
 491     if not opts.quiet:
 492         if opts.succinct and opts.useProgressBar:
 493             try:
 494                 tc = ProgressBar.TerminalController()
 495                 progressBar = ProgressBar.ProgressBar(tc, header)
 496             except ValueError:
 497                 print header
 498                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 499         else:
 500             print header
 501
 502     # Don't create more threads than tests.
 503     opts.numThreads = min(len(tests), opts.numThreads)
 504
 505     startTime = time.time()
 506     display = TestingProgressDisplay(opts, len(tests), progressBar)
 507     provider = TestProvider(tests, opts.maxTime)
 508     runTests(opts.numThreads, litConfig, provider, display)
 509     display.finish()
 510
 511     if not opts.quiet:
 512         print 'Testing Time: %.2fs'%(time.time() - startTime)
 513
 514     # Update results for any tests which weren't run.
 515     for t in tests:
 516         if t.result is None:
 517             t.setResult(Test.UNRESOLVED, '', 0.0)
 518
 519     # List test results organized by kind.
 520     hasFailures = False
 521     byCode = {}
 522     for t in tests:
 523         if t.result not in byCode:
 524             byCode[t.result] = []
 525         byCode[t.result].append(t)
 526         if t.result.isFailure:
 527             hasFailures = True
 528
 529     # FIXME: Show unresolved and (optionally) unsupported tests.
 530     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 531                        ('Failing Tests', Test.FAIL)):
 532         elts = byCode.get(code)
 533         if not elts:
 534             continue
 535         print '*'*20
 536         print '%s (%d):' % (title, len(elts))
 537         for t in elts:
 538             print '    %s' % t.getFullName()
 539         print
 540
 541     if opts.timeTests:
 542         # Collate, in case we repeated tests.
 543         times = {}
 544         for t in tests:
 545             key = t.getFullName()
 546             times[key] = times.get(key, 0.) + t.elapsed
 547
 548         byTime = list(times.items())
 549         byTime.sort(key = lambda (name,elapsed): elapsed)
 550         if byTime:
 551             Util.printHistogram(byTime, title='Tests')
 552
 553     for name,code in (('Expected Passes    ', Test.PASS),
 554                       ('Expected Failures  ', Test.XFAIL),
 555                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 556                       ('Unresolved Tests   ', Test.UNRESOLVED),
 557                       ('Unexpected Passes  ', Test.XPASS),
 558                       ('Unexpected Failures', Test.FAIL),):
 559         if opts.quiet and not code.isFailure:
 560             continue
 561         N = len(byCode.get(code,[]))
 562         if N:
 563             print '  %s: %d' % (name,N)
 564
 565     # If we encountered any additional errors, exit abnormally.
 566     if litConfig.numErrors:
 567         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 568         sys.exit(2)
 569
 570     # Warn about warnings.
 571     if litConfig.numWarnings:
 572         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 573
 574     if hasFailures:
 575         sys.exit(1)
 576     sys.exit(0)
 577
 578 if __name__=='__main__':
 579     main()