4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
15 from TestingConfig import TestingConfig
19 # Configuration files to look for when discovering test suites. These can be
20 # overridden with --config-prefix.
22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
23 gConfigName = 'lit.cfg'
24 gSiteConfigName = 'lit.site.cfg'
26 kLocalConfigName = 'lit.local.cfg'
28 class TestingProgressDisplay:
29 def __init__(self, opts, numTests, progressBar=None):
31 self.numTests = numTests
33 self.lock = threading.Lock()
34 self.progressBar = progressBar
37 def update(self, test):
38 # Avoid locking overhead in quiet mode
39 if self.opts.quiet and not test.result.isFailure:
46 self.handleUpdate(test)
52 self.progressBar.clear()
55 elif self.opts.succinct:
56 sys.stdout.write('\n')
58 def handleUpdate(self, test):
61 self.progressBar.update(float(self.completed)/self.numTests,
64 if self.opts.succinct and not test.result.isFailure:
68 self.progressBar.clear()
70 print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
71 self.completed, self.numTests)
73 if test.result.isFailure and self.opts.showOutput:
74 print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
82 def __init__(self, tests, maxTime):
83 self.maxTime = maxTime
84 self.iter = iter(tests)
85 self.lock = threading.Lock()
86 self.startTime = time.time()
89 # Check if we have run out of time.
90 if self.maxTime is not None:
91 if time.time() - self.startTime > self.maxTime:
94 # Otherwise take the next test.
97 item = self.iter.next()
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print '\nCtrl-C detected, goodbye.'
129 if self.litConfig.debug:
131 result = Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def dirContainsTestSuite(path):
141 cfgpath = os.path.join(path, gSiteConfigName)
142 if os.path.exists(cfgpath):
144 cfgpath = os.path.join(path, gConfigName)
145 if os.path.exists(cfgpath):
148 def getTestSuite(item, litConfig, cache):
149 """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
151 Find the test suite containing @arg item.
153 @retval (None, ...) - Indicates no test suite contains @arg item.
154 @retval (suite, relative_path) - The suite that @arg item is in, and its
155 relative path inside that suite.
158 # Check for a site config or a lit config.
159 cfgpath = dirContainsTestSuite(path)
161 # If we didn't find a config file, keep looking.
163 parent,base = os.path.split(path)
167 ts, relative = search(parent)
168 return (ts, relative + (base,))
170 # We found a config file, load it.
172 litConfig.note('loading suite config %r' % cfgpath)
174 cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
175 source_root = os.path.realpath(cfg.test_source_root or path)
176 exec_root = os.path.realpath(cfg.test_exec_root or path)
177 return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
180 # Check for an already instantiated test suite.
181 res = cache.get(path)
183 cache[path] = res = search1(path)
186 # Canonicalize the path.
187 item = os.path.realpath(item)
189 # Skip files and virtual components.
191 while not os.path.isdir(item):
192 parent,base = os.path.split(item)
195 components.append(base)
199 ts, relative = search(item)
200 return ts, tuple(relative + tuple(components))
202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
203 def search1(path_in_suite):
204 # Get the parent config.
205 if not path_in_suite:
208 parent = search(path_in_suite[:-1])
210 # Load the local configuration.
211 source_path = ts.getSourcePath(path_in_suite)
212 cfgpath = os.path.join(source_path, kLocalConfigName)
214 litConfig.note('loading local config %r' % cfgpath)
215 return TestingConfig.frompath(cfgpath, parent, litConfig,
217 config = parent.clone(cfgpath))
219 def search(path_in_suite):
220 key = (ts, path_in_suite)
223 cache[key] = res = search1(path_in_suite)
226 return search(path_in_suite)
228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
229 # Find the test suite for this input and its relative path.
230 ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
232 litConfig.warning('unable to find test suite for %r' % path)
236 litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
239 return ts, getTestsInSuite(ts, path_in_suite, litConfig,
240 testSuiteCache, localConfigCache)
242 def getTestsInSuite(ts, path_in_suite, litConfig,
243 testSuiteCache, localConfigCache):
244 # Check that the source path exists (errors here are reported by the
246 source_path = ts.getSourcePath(path_in_suite)
247 if not os.path.exists(source_path):
250 # Check if the user named a test directly.
251 if not os.path.isdir(source_path):
252 lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
253 yield Test.Test(ts, path_in_suite, lc)
256 # Otherwise we have a directory to search for tests, start by getting the
257 # local configuration.
258 lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
261 if lc.test_format is not None:
262 for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
266 # Search subdirectories.
267 for filename in os.listdir(source_path):
268 # FIXME: This doesn't belong here?
269 if filename in ('Output', '.svn') or filename in lc.excludes:
272 # Ignore non-directories.
273 file_sourcepath = os.path.join(source_path, filename)
274 if not os.path.isdir(file_sourcepath):
277 # Check for nested test suites, first in the execpath in case there is a
278 # site configuration and then in the source path.
279 file_execpath = ts.getExecPath(path_in_suite + (filename,))
280 if dirContainsTestSuite(file_execpath):
281 sub_ts, subiter = getTests(file_execpath, litConfig,
282 testSuiteCache, localConfigCache)
283 elif dirContainsTestSuite(file_sourcepath):
284 sub_ts, subiter = getTests(file_sourcepath, litConfig,
285 testSuiteCache, localConfigCache)
287 # Otherwise, continue loading from inside this test suite.
288 subiter = getTestsInSuite(ts, path_in_suite + (filename,),
289 litConfig, testSuiteCache,
298 litConfig.warning('test suite %r contained no tests' % sub_ts.name)
300 def runTests(numThreads, litConfig, provider, display):
301 # If only using one testing thread, don't use threads at all; this lets us
302 # profile, among other things.
304 t = Tester(litConfig, provider, display)
308 # Otherwise spin up the testing threads and wait for them to finish.
309 testers = [Tester(litConfig, provider, display)
310 for i in range(numThreads)]
316 except KeyboardInterrupt:
319 def load_test_suite(inputs):
322 # Create the global config object.
323 litConfig = LitConfig.LitConfig(progname = 'lit',
327 valgrindLeakCheck = False,
331 ignoreStdErr = False,
333 isWindows = (platform.system()=='Windows'),
336 # Load the tests from the inputs.
339 localConfigCache = {}
342 tests.extend(getTests(input, litConfig,
343 testSuiteCache, localConfigCache)[1])
344 if prev == len(tests):
345 litConfig.warning('input %r contained no tests' % input)
347 # If there were any errors during test discovery, exit now.
348 if litConfig.numErrors:
349 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
352 # Return a unittest test suite which just runs the tests in order.
353 def get_test_fn(test):
354 return unittest.FunctionTestCase(
355 lambda: test.config.test_format.execute(
357 description = test.getFullName())
359 from LitTestCase import LitTestCase
360 return unittest.TestSuite([LitTestCase(test, litConfig) for test in tests])
362 def main(builtinParameters = {}): # Bump the GIL check interval, its more important to get any one thread to a
363 # blocking operation (hopefully exec) than to try and unblock other threads.
365 # FIXME: This is a hack.
367 sys.setcheckinterval(1000)
370 from optparse import OptionParser, OptionGroup
371 parser = OptionParser("usage: %prog [options] {file-or-path}")
373 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
374 help="Number of testing threads",
375 type=int, action="store", default=None)
376 parser.add_option("", "--config-prefix", dest="configPrefix",
377 metavar="NAME", help="Prefix for 'lit' config files",
378 action="store", default=None)
379 parser.add_option("", "--param", dest="userParameters",
381 help="Add 'NAME' = 'VAL' to the user defined parameters",
382 type=str, action="append", default=[])
384 group = OptionGroup(parser, "Output Format")
385 # FIXME: I find these names very confusing, although I like the
387 group.add_option("-q", "--quiet", dest="quiet",
388 help="Suppress no error output",
389 action="store_true", default=False)
390 group.add_option("-s", "--succinct", dest="succinct",
391 help="Reduce amount of output",
392 action="store_true", default=False)
393 group.add_option("-v", "--verbose", dest="showOutput",
394 help="Show all test output",
395 action="store_true", default=False)
396 group.add_option("", "--no-progress-bar", dest="useProgressBar",
397 help="Do not use curses based progress bar",
398 action="store_false", default=True)
399 parser.add_option_group(group)
401 group = OptionGroup(parser, "Test Execution")
402 group.add_option("", "--path", dest="path",
403 help="Additional paths to add to testing environment",
404 action="append", type=str, default=[])
405 group.add_option("", "--vg", dest="useValgrind",
406 help="Run tests under valgrind",
407 action="store_true", default=False)
408 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
409 help="Check for memory leaks under valgrind",
410 action="store_true", default=False)
411 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
412 help="Specify an extra argument for valgrind",
413 type=str, action="append", default=[])
414 group.add_option("", "--time-tests", dest="timeTests",
415 help="Track elapsed wall time for each test",
416 action="store_true", default=False)
417 group.add_option("", "--no-execute", dest="noExecute",
418 help="Don't execute any tests (assume PASS)",
419 action="store_true", default=False)
420 parser.add_option_group(group)
422 group = OptionGroup(parser, "Test Selection")
423 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
424 help="Maximum number of tests to run",
425 action="store", type=int, default=None)
426 group.add_option("", "--max-time", dest="maxTime", metavar="N",
427 help="Maximum time to spend testing (in seconds)",
428 action="store", type=float, default=None)
429 group.add_option("", "--shuffle", dest="shuffle",
430 help="Run tests in random order",
431 action="store_true", default=False)
432 group.add_option("", "--filter", dest="filter", metavar="EXPRESSION",
433 help=("Only run tests with paths matching the given "
434 "regular expression"),
435 action="store", default=None)
436 parser.add_option_group(group)
438 group = OptionGroup(parser, "Debug and Experimental Options")
439 group.add_option("", "--debug", dest="debug",
440 help="Enable debugging (for 'lit' development)",
441 action="store_true", default=False)
442 group.add_option("", "--show-suites", dest="showSuites",
443 help="Show discovered test suites",
444 action="store_true", default=False)
445 group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
446 help="Don't run Tcl scripts using 'sh'",
447 action="store_false", default=True)
448 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
449 help="Repeat tests N times (for timing)",
450 action="store", default=None, type=int)
451 parser.add_option_group(group)
453 (opts, args) = parser.parse_args()
456 parser.error('No inputs specified')
458 if opts.configPrefix is not None:
459 global gConfigName, gSiteConfigName, kLocalConfigName
460 gConfigName = '%s.cfg' % opts.configPrefix
461 gSiteConfigName = '%s.site.cfg' % opts.configPrefix
462 kLocalConfigName = '%s.local.cfg' % opts.configPrefix
464 if opts.numThreads is None:
465 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
466 # http://bugs.python.org/issue1731717
467 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
468 # threads by default there.
469 if sys.hexversion >= 0x2050200:
470 opts.numThreads = Util.detectCPUs()
476 # Create the user defined parameters.
477 userParams = dict(builtinParameters)
478 for entry in opts.userParameters:
482 name,val = entry.split('=', 1)
483 userParams[name] = val
485 # Create the global config object.
486 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
489 useValgrind = opts.useValgrind,
490 valgrindLeakCheck = opts.valgrindLeakCheck,
491 valgrindArgs = opts.valgrindArgs,
492 useTclAsSh = opts.useTclAsSh,
493 noExecute = opts.noExecute,
494 ignoreStdErr = False,
496 isWindows = (platform.system()=='Windows'),
499 # Expand '@...' form in inputs.
502 if os.path.exists(input) or not input.startswith('@'):
503 actual_inputs.append(input)
510 actual_inputs.append(ln)
515 # Load the tests from the inputs.
518 localConfigCache = {}
519 for input in actual_inputs:
521 tests.extend(getTests(input, litConfig,
522 testSuiteCache, localConfigCache)[1])
523 if prev == len(tests):
524 litConfig.warning('input %r contained no tests' % input)
526 # If there were any errors during test discovery, exit now.
527 if litConfig.numErrors:
528 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
532 suitesAndTests = dict([(ts,[])
533 for ts,_ in testSuiteCache.values()
536 suitesAndTests[t.suite].append(t)
538 print '-- Test Suites --'
539 suitesAndTests = suitesAndTests.items()
540 suitesAndTests.sort(key = lambda (ts,_): ts.name)
541 for ts,ts_tests in suitesAndTests:
542 print ' %s - %d tests' %(ts.name, len(ts_tests))
543 print ' Source Root: %s' % ts.source_root
544 print ' Exec Root : %s' % ts.exec_root
546 # Select and order the tests.
547 numTotalTests = len(tests)
549 # First, select based on the filter expression if given.
552 rex = re.compile(opts.filter)
554 parser.error("invalid regular expression for --filter: %r" % (
556 tests = [t for t in tests
557 if rex.search(t.getFullName())]
559 # Then select the order.
561 random.shuffle(tests)
563 tests.sort(key = lambda t: t.getFullName())
565 # Finally limit the number of tests, if desired.
566 if opts.maxTests is not None:
567 tests = tests[:opts.maxTests]
569 # Don't create more threads than tests.
570 opts.numThreads = min(len(tests), opts.numThreads)
573 if len(tests) != numTotalTests:
574 extra = ' of %d' % numTotalTests
575 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
579 tests = [t.copyWithIndex(i)
581 for i in range(opts.repeatTests)]
585 if opts.succinct and opts.useProgressBar:
587 tc = ProgressBar.TerminalController()
588 progressBar = ProgressBar.ProgressBar(tc, header)
591 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
595 startTime = time.time()
596 display = TestingProgressDisplay(opts, len(tests), progressBar)
597 provider = TestProvider(tests, opts.maxTime)
598 runTests(opts.numThreads, litConfig, provider, display)
602 print 'Testing Time: %.2fs'%(time.time() - startTime)
604 # Update results for any tests which weren't run.
607 t.setResult(Test.UNRESOLVED, '', 0.0)
609 # List test results organized by kind.
613 if t.result not in byCode:
614 byCode[t.result] = []
615 byCode[t.result].append(t)
616 if t.result.isFailure:
619 # FIXME: Show unresolved and (optionally) unsupported tests.
620 for title,code in (('Unexpected Passing Tests', Test.XPASS),
621 ('Failing Tests', Test.FAIL)):
622 elts = byCode.get(code)
626 print '%s (%d):' % (title, len(elts))
628 print ' %s' % t.getFullName()
632 # Collate, in case we repeated tests.
635 key = t.getFullName()
636 times[key] = times.get(key, 0.) + t.elapsed
638 byTime = list(times.items())
639 byTime.sort(key = lambda (name,elapsed): elapsed)
641 Util.printHistogram(byTime, title='Tests')
643 for name,code in (('Expected Passes ', Test.PASS),
644 ('Expected Failures ', Test.XFAIL),
645 ('Unsupported Tests ', Test.UNSUPPORTED),
646 ('Unresolved Tests ', Test.UNRESOLVED),
647 ('Unexpected Passes ', Test.XPASS),
648 ('Unexpected Failures', Test.FAIL),):
649 if opts.quiet and not code.isFailure:
651 N = len(byCode.get(code,[]))
653 print ' %s: %d' % (name,N)
655 # If we encountered any additional errors, exit abnormally.
656 if litConfig.numErrors:
657 print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
660 # Warn about warnings.
661 if litConfig.numWarnings:
662 print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
668 if __name__=='__main__':