utils/lit/lit.py

   1 #!/usr/bin/env python
   2
   3 """
   4 lit - LLVM Integrated Tester.
   5
   6 See lit.pod for more information.
   7 """
   8
   9 import math, os, platform, random, re, sys, time, threading, traceback
  10
  11 import ProgressBar
  12 import TestRunner
  13 import Util
  14
  15 from TestingConfig import TestingConfig
  16 import LitConfig
  17 import Test
  18
  19 # Configuration files to look for when discovering test suites. These can be
  20 # overridden with --config-prefix.
  21 #
  22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
  23 gConfigName = 'lit.cfg'
  24 gSiteConfigName = 'lit.site.cfg'
  25
  26 kLocalConfigName = 'lit.local.cfg'
  27
  28 class TestingProgressDisplay:
  29     def __init__(self, opts, numTests, progressBar=None):
  30         self.opts = opts
  31         self.numTests = numTests
  32         self.current = None
  33         self.lock = threading.Lock()
  34         self.progressBar = progressBar
  35         self.completed = 0
  36
  37     def update(self, test):
  38         # Avoid locking overhead in quiet mode
  39         if self.opts.quiet and not test.result.isFailure:
  40             self.completed += 1
  41             return
  42
  43         # Output lock.
  44         self.lock.acquire()
  45         try:
  46             self.handleUpdate(test)
  47         finally:
  48             self.lock.release()
  49
  50     def finish(self):
  51         if self.progressBar:
  52             self.progressBar.clear()
  53         elif self.opts.quiet:
  54             pass
  55         elif self.opts.succinct:
  56             sys.stdout.write('\n')
  57
  58     def handleUpdate(self, test):
  59         self.completed += 1
  60         if self.progressBar:
  61             self.progressBar.update(float(self.completed)/self.numTests,
  62                                     test.getFullName())
  63
  64         if self.opts.succinct and not test.result.isFailure:
  65             return
  66
  67         if self.progressBar:
  68             self.progressBar.clear()
  69
  70         print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
  71                                      self.completed, self.numTests)
  72
  73         if test.result.isFailure and self.opts.showOutput:
  74             print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
  75                                               '*'*20)
  76             print test.output
  77             print "*" * 20
  78
  79         sys.stdout.flush()
  80
  81 class TestProvider:
  82     def __init__(self, tests, maxTime):
  83         self.maxTime = maxTime
  84         self.iter = iter(tests)
  85         self.lock = threading.Lock()
  86         self.startTime = time.time()
  87
  88     def get(self):
  89         # Check if we have run out of time.
  90         if self.maxTime is not None:
  91             if time.time() - self.startTime > self.maxTime:
  92                 return None
  93
  94         # Otherwise take the next test.
  95         self.lock.acquire()
  96         try:
  97             item = self.iter.next()
  98         except StopIteration:
  99             item = None
 100         self.lock.release()
 101         return item
 102
 103 class Tester(threading.Thread):
 104     def __init__(self, litConfig, provider, display):
 105         threading.Thread.__init__(self)
 106         self.litConfig = litConfig
 107         self.provider = provider
 108         self.display = display
 109
 110     def run(self):
 111         while 1:
 112             item = self.provider.get()
 113             if item is None:
 114                 break
 115             self.runTest(item)
 116
 117     def runTest(self, test):
 118         result = None
 119         startTime = time.time()
 120         try:
 121             result, output = test.config.test_format.execute(test,
 122                                                              self.litConfig)
 123         except KeyboardInterrupt:
 124             # This is a sad hack. Unfortunately subprocess goes
 125             # bonkers with ctrl-c and we start forking merrily.
 126             print '\nCtrl-C detected, goodbye.'
 127             os.kill(0,9)
 128         except:
 129             if self.litConfig.debug:
 130                 raise
 131             result = Test.UNRESOLVED
 132             output = 'Exception during script execution:\n'
 133             output += traceback.format_exc()
 134             output += '\n'
 135         elapsed = time.time() - startTime
 136
 137         test.setResult(result, output, elapsed)
 138         self.display.update(test)
 139
 140 def dirContainsTestSuite(path):
 141     cfgpath = os.path.join(path, gSiteConfigName)
 142     if os.path.exists(cfgpath):
 143         return cfgpath
 144     cfgpath = os.path.join(path, gConfigName)
 145     if os.path.exists(cfgpath):
 146         return cfgpath
 147
 148 def getTestSuite(item, litConfig, cache):
 149     """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
 150
 151     Find the test suite containing @arg item.
 152
 153     @retval (None, ...) - Indicates no test suite contains @arg item.
 154     @retval (suite, relative_path) - The suite that @arg item is in, and its
 155     relative path inside that suite.
 156     """
 157     def search1(path):
 158         # Check for a site config or a lit config.
 159         cfgpath = dirContainsTestSuite(path)
 160
 161         # If we didn't find a config file, keep looking.
 162         if not cfgpath:
 163             parent,base = os.path.split(path)
 164             if parent == path:
 165                 return (None, ())
 166
 167             ts, relative = search(parent)
 168             return (ts, relative + (base,))
 169
 170         # We found a config file, load it.
 171         if litConfig.debug:
 172             litConfig.note('loading suite config %r' % cfgpath)
 173
 174         cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
 175         source_root = os.path.realpath(cfg.test_source_root or path)
 176         exec_root = os.path.realpath(cfg.test_exec_root or path)
 177         return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
 178
 179     def search(path):
 180         # Check for an already instantiated test suite.
 181         res = cache.get(path)
 182         if res is None:
 183             cache[path] = res = search1(path)
 184         return res
 185
 186     # Canonicalize the path.
 187     item = os.path.realpath(item)
 188
 189     # Skip files and virtual components.
 190     components = []
 191     while not os.path.isdir(item):
 192         parent,base = os.path.split(item)
 193         if parent == item:
 194             return (None, ())
 195         components.append(base)
 196         item = parent
 197     components.reverse()
 198
 199     ts, relative = search(item)
 200     return ts, tuple(relative + tuple(components))
 201
 202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
 203     def search1(path_in_suite):
 204         # Get the parent config.
 205         if not path_in_suite:
 206             parent = ts.config
 207         else:
 208             parent = search(path_in_suite[:-1])
 209
 210         # Load the local configuration.
 211         source_path = ts.getSourcePath(path_in_suite)
 212         cfgpath = os.path.join(source_path, kLocalConfigName)
 213         if litConfig.debug:
 214             litConfig.note('loading local config %r' % cfgpath)
 215         return TestingConfig.frompath(cfgpath, parent, litConfig,
 216                                     mustExist = False,
 217                                     config = parent.clone(cfgpath))
 218
 219     def search(path_in_suite):
 220         key = (ts, path_in_suite)
 221         res = cache.get(key)
 222         if res is None:
 223             cache[key] = res = search1(path_in_suite)
 224         return res
 225
 226     return search(path_in_suite)
 227
 228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
 229     # Find the test suite for this input and its relative path.
 230     ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
 231     if ts is None:
 232         litConfig.warning('unable to find test suite for %r' % path)
 233         return (),()
 234
 235     if litConfig.debug:
 236         litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
 237                                                         path_in_suite))
 238
 239     return ts, getTestsInSuite(ts, path_in_suite, litConfig,
 240                                testSuiteCache, localConfigCache)
 241
 242 def getTestsInSuite(ts, path_in_suite, litConfig,
 243                     testSuiteCache, localConfigCache):
 244     # Check that the source path exists (errors here are reported by the
 245     # caller).
 246     source_path = ts.getSourcePath(path_in_suite)
 247     if not os.path.exists(source_path):
 248         return
 249
 250     # Check if the user named a test directly.
 251     if not os.path.isdir(source_path):
 252         lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
 253         yield Test.Test(ts, path_in_suite, lc)
 254         return
 255
 256     # Otherwise we have a directory to search for tests, start by getting the
 257     # local configuration.
 258     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 259
 260     # Search for tests.
 261     for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
 262                                                   litConfig, lc):
 263         yield res
 264
 265     # Search subdirectories.
 266     for filename in os.listdir(source_path):
 267         # FIXME: This doesn't belong here?
 268         if filename in ('Output', '.svn') or filename in lc.excludes:
 269             continue
 270
 271         # Ignore non-directories.
 272         file_sourcepath = os.path.join(source_path, filename)
 273         if not os.path.isdir(file_sourcepath):
 274             continue
 275
 276         # Check for nested test suites, first in the execpath in case there is a
 277         # site configuration and then in the source path.
 278         file_execpath = ts.getExecPath(path_in_suite + (filename,))
 279         if dirContainsTestSuite(file_execpath):
 280             sub_ts, subiter = getTests(file_execpath, litConfig,
 281                                        testSuiteCache, localConfigCache)
 282         elif dirContainsTestSuite(file_sourcepath):
 283             sub_ts, subiter = getTests(file_sourcepath, litConfig,
 284                                        testSuiteCache, localConfigCache)
 285         else:
 286             # Otherwise, continue loading from inside this test suite.
 287             subiter = getTestsInSuite(ts, path_in_suite + (filename,),
 288                                       litConfig, testSuiteCache,
 289                                       localConfigCache)
 290             sub_ts = None
 291
 292         N = 0
 293         for res in subiter:
 294             N += 1
 295             yield res
 296         if sub_ts and not N:
 297             litConfig.warning('test suite %r contained no tests' % sub_ts.name)
 298
 299 def runTests(numThreads, litConfig, provider, display):
 300     # If only using one testing thread, don't use threads at all; this lets us
 301     # profile, among other things.
 302     if numThreads == 1:
 303         t = Tester(litConfig, provider, display)
 304         t.run()
 305         return
 306
 307     # Otherwise spin up the testing threads and wait for them to finish.
 308     testers = [Tester(litConfig, provider, display)
 309                for i in range(numThreads)]
 310     for t in testers:
 311         t.start()
 312     try:
 313         for t in testers:
 314             t.join()
 315     except KeyboardInterrupt:
 316         sys.exit(2)
 317
 318 def main():
 319     global options
 320     from optparse import OptionParser, OptionGroup
 321     parser = OptionParser("usage: %prog [options] {file-or-path}")
 322
 323     parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
 324                       help="Number of testing threads",
 325                       type=int, action="store", default=None)
 326     parser.add_option("", "--config-prefix", dest="configPrefix",
 327                       metavar="NAME", help="Prefix for 'lit' config files",
 328                       action="store", default=None)
 329     parser.add_option("", "--param", dest="userParameters",
 330                       metavar="NAME=VAL",
 331                       help="Add 'NAME' = 'VAL' to the user defined parameters",
 332                       type=str, action="append", default=[])
 333
 334     group = OptionGroup(parser, "Output Format")
 335     # FIXME: I find these names very confusing, although I like the
 336     # functionality.
 337     group.add_option("-q", "--quiet", dest="quiet",
 338                      help="Suppress no error output",
 339                      action="store_true", default=False)
 340     group.add_option("-s", "--succinct", dest="succinct",
 341                      help="Reduce amount of output",
 342                      action="store_true", default=False)
 343     group.add_option("-v", "--verbose", dest="showOutput",
 344                      help="Show all test output",
 345                      action="store_true", default=False)
 346     group.add_option("", "--no-progress-bar", dest="useProgressBar",
 347                      help="Do not use curses based progress bar",
 348                      action="store_false", default=True)
 349     parser.add_option_group(group)
 350
 351     group = OptionGroup(parser, "Test Execution")
 352     group.add_option("", "--path", dest="path",
 353                      help="Additional paths to add to testing environment",
 354                      action="append", type=str, default=[])
 355     group.add_option("", "--vg", dest="useValgrind",
 356                      help="Run tests under valgrind",
 357                      action="store_true", default=False)
 358     group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
 359                      help="Specify an extra argument for valgrind",
 360                      type=str, action="append", default=[])
 361     group.add_option("", "--time-tests", dest="timeTests",
 362                      help="Track elapsed wall time for each test",
 363                      action="store_true", default=False)
 364     group.add_option("", "--no-execute", dest="noExecute",
 365                      help="Don't execute any tests (assume PASS)",
 366                      action="store_true", default=False)
 367     parser.add_option_group(group)
 368
 369     group = OptionGroup(parser, "Test Selection")
 370     group.add_option("", "--max-tests", dest="maxTests", metavar="N",
 371                      help="Maximum number of tests to run",
 372                      action="store", type=int, default=None)
 373     group.add_option("", "--max-time", dest="maxTime", metavar="N",
 374                      help="Maximum time to spend testing (in seconds)",
 375                      action="store", type=float, default=None)
 376     group.add_option("", "--shuffle", dest="shuffle",
 377                      help="Run tests in random order",
 378                      action="store_true", default=False)
 379     parser.add_option_group(group)
 380
 381     group = OptionGroup(parser, "Debug and Experimental Options")
 382     group.add_option("", "--debug", dest="debug",
 383                       help="Enable debugging (for 'lit' development)",
 384                       action="store_true", default=False)
 385     group.add_option("", "--show-suites", dest="showSuites",
 386                       help="Show discovered test suites",
 387                       action="store_true", default=False)
 388     group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
 389                       help="Don't run Tcl scripts using 'sh'",
 390                       action="store_false", default=True)
 391     group.add_option("", "--repeat", dest="repeatTests", metavar="N",
 392                       help="Repeat tests N times (for timing)",
 393                       action="store", default=None, type=int)
 394     parser.add_option_group(group)
 395
 396     (opts, args) = parser.parse_args()
 397
 398     if not args:
 399         parser.error('No inputs specified')
 400
 401     if opts.configPrefix is not None:
 402         global gConfigName, gSiteConfigName
 403         gConfigName = '%s.cfg' % opts.configPrefix
 404         gSiteConfigName = '%s.site.cfg' % opts.configPrefix
 405
 406     if opts.numThreads is None:
 407         opts.numThreads = Util.detectCPUs()
 408
 409     inputs = args
 410
 411     # Create the user defined parameters.
 412     userParams = {}
 413     for entry in opts.userParameters:
 414         if '=' not in entry:
 415             name,val = entry,''
 416         else:
 417             name,val = entry.split('=', 1)
 418         userParams[name] = val
 419
 420     # Create the global config object.
 421     litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
 422                                     path = opts.path,
 423                                     quiet = opts.quiet,
 424                                     useValgrind = opts.useValgrind,
 425                                     valgrindArgs = opts.valgrindArgs,
 426                                     useTclAsSh = opts.useTclAsSh,
 427                                     noExecute = opts.noExecute,
 428                                     debug = opts.debug,
 429                                     isWindows = (platform.system()=='Windows'),
 430                                     params = userParams)
 431
 432     # Load the tests from the inputs.
 433     tests = []
 434     testSuiteCache = {}
 435     localConfigCache = {}
 436     for input in inputs:
 437         prev = len(tests)
 438         tests.extend(getTests(input, litConfig,
 439                               testSuiteCache, localConfigCache)[1])
 440         if prev == len(tests):
 441             litConfig.warning('input %r contained no tests' % input)
 442
 443     # If there were any errors during test discovery, exit now.
 444     if litConfig.numErrors:
 445         print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
 446         sys.exit(2)
 447
 448     if opts.showSuites:
 449         suitesAndTests = dict([(ts,[])
 450                                for ts,_ in testSuiteCache.values()
 451                                if ts])
 452         for t in tests:
 453             suitesAndTests[t.suite].append(t)
 454
 455         print '-- Test Suites --'
 456         suitesAndTests = suitesAndTests.items()
 457         suitesAndTests.sort(key = lambda (ts,_): ts.name)
 458         for ts,ts_tests in suitesAndTests:
 459             print '  %s - %d tests' %(ts.name, len(ts_tests))
 460             print '    Source Root: %s' % ts.source_root
 461             print '    Exec Root  : %s' % ts.exec_root
 462
 463     # Select and order the tests.
 464     numTotalTests = len(tests)
 465     if opts.shuffle:
 466         random.shuffle(tests)
 467     else:
 468         tests.sort(key = lambda t: t.getFullName())
 469     if opts.maxTests is not None:
 470         tests = tests[:opts.maxTests]
 471
 472     extra = ''
 473     if len(tests) != numTotalTests:
 474         extra = ' of %d' % numTotalTests
 475     header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
 476                                                       opts.numThreads)
 477
 478     if opts.repeatTests:
 479         tests = [t.copyWithIndex(i)
 480                  for t in tests
 481                  for i in range(opts.repeatTests)]
 482
 483     progressBar = None
 484     if not opts.quiet:
 485         if opts.succinct and opts.useProgressBar:
 486             try:
 487                 tc = ProgressBar.TerminalController()
 488                 progressBar = ProgressBar.ProgressBar(tc, header)
 489             except ValueError:
 490                 print header
 491                 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
 492         else:
 493             print header
 494
 495     # Don't create more threads than tests.
 496     opts.numThreads = min(len(tests), opts.numThreads)
 497
 498     startTime = time.time()
 499     display = TestingProgressDisplay(opts, len(tests), progressBar)
 500     provider = TestProvider(tests, opts.maxTime)
 501     runTests(opts.numThreads, litConfig, provider, display)
 502     display.finish()
 503
 504     if not opts.quiet:
 505         print 'Testing Time: %.2fs'%(time.time() - startTime)
 506
 507     # Update results for any tests which weren't run.
 508     for t in tests:
 509         if t.result is None:
 510             t.setResult(Test.UNRESOLVED, '', 0.0)
 511
 512     # List test results organized by kind.
 513     hasFailures = False
 514     byCode = {}
 515     for t in tests:
 516         if t.result not in byCode:
 517             byCode[t.result] = []
 518         byCode[t.result].append(t)
 519         if t.result.isFailure:
 520             hasFailures = True
 521
 522     # FIXME: Show unresolved and (optionally) unsupported tests.
 523     for title,code in (('Unexpected Passing Tests', Test.XPASS),
 524                        ('Failing Tests', Test.FAIL)):
 525         elts = byCode.get(code)
 526         if not elts:
 527             continue
 528         print '*'*20
 529         print '%s (%d):' % (title, len(elts))
 530         for t in elts:
 531             print '    %s' % t.getFullName()
 532         print
 533
 534     if opts.timeTests:
 535         # Collate, in case we repeated tests.
 536         times = {}
 537         for t in tests:
 538             key = t.getFullName()
 539             times[key] = times.get(key, 0.) + t.elapsed
 540
 541         byTime = list(times.items())
 542         byTime.sort(key = lambda (name,elapsed): elapsed)
 543         if byTime:
 544             Util.printHistogram(byTime, title='Tests')
 545
 546     for name,code in (('Expected Passes    ', Test.PASS),
 547                       ('Expected Failures  ', Test.XFAIL),
 548                       ('Unsupported Tests  ', Test.UNSUPPORTED),
 549                       ('Unresolved Tests   ', Test.UNRESOLVED),
 550                       ('Unexpected Passes  ', Test.XPASS),
 551                       ('Unexpected Failures', Test.FAIL),):
 552         if opts.quiet and not code.isFailure:
 553             continue
 554         N = len(byCode.get(code,[]))
 555         if N:
 556             print '  %s: %d' % (name,N)
 557
 558     # If we encountered any additional errors, exit abnormally.
 559     if litConfig.numErrors:
 560         print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
 561         sys.exit(2)
 562
 563     # Warn about warnings.
 564     if litConfig.numWarnings:
 565         print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
 566
 567     if hasFailures:
 568         sys.exit(1)
 569     sys.exit(0)
 570
 571 if __name__=='__main__':
 572     # Bump the GIL check interval, its more important to get any one thread to a
 573     # blocking operation (hopefully exec) than to try and unblock other threads.
 574     import sys
 575     sys.setcheckinterval(1000)
 576     main()