4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
15 from TestingConfig import TestingConfig
19 # Configuration files to look for when discovering test suites. These can be
20 # overridden with --config-prefix.
22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
23 gConfigName = 'lit.cfg'
24 gSiteConfigName = 'lit.site.cfg'
26 kLocalConfigName = 'lit.local.cfg'
28 class TestingProgressDisplay:
29 def __init__(self, opts, numTests, progressBar=None):
31 self.numTests = numTests
33 self.lock = threading.Lock()
34 self.progressBar = progressBar
37 def update(self, test):
38 # Avoid locking overhead in quiet mode
39 if self.opts.quiet and not test.result.isFailure:
46 self.handleUpdate(test)
52 self.progressBar.clear()
55 elif self.opts.succinct:
56 sys.stdout.write('\n')
58 def handleUpdate(self, test):
61 self.progressBar.update(float(self.completed)/self.numTests,
64 if self.opts.succinct and not test.result.isFailure:
68 self.progressBar.clear()
70 print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
71 self.completed, self.numTests)
73 if test.result.isFailure and self.opts.showOutput:
74 print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
82 def __init__(self, tests, maxTime):
83 self.maxTime = maxTime
84 self.iter = iter(tests)
85 self.lock = threading.Lock()
86 self.startTime = time.time()
89 # Check if we have run out of time.
90 if self.maxTime is not None:
91 if time.time() - self.startTime > self.maxTime:
94 # Otherwise take the next test.
97 item = self.iter.next()
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print '\nCtrl-C detected, goodbye.'
129 if self.litConfig.debug:
131 result = Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def dirContainsTestSuite(path):
141 cfgpath = os.path.join(path, gSiteConfigName)
142 if os.path.exists(cfgpath):
144 cfgpath = os.path.join(path, gConfigName)
145 if os.path.exists(cfgpath):
148 def getTestSuite(item, litConfig, cache):
149 """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
151 Find the test suite containing @arg item.
153 @retval (None, ...) - Indicates no test suite contains @arg item.
154 @retval (suite, relative_path) - The suite that @arg item is in, and its
155 relative path inside that suite.
158 # Check for a site config or a lit config.
159 cfgpath = dirContainsTestSuite(path)
161 # If we didn't find a config file, keep looking.
163 parent,base = os.path.split(path)
167 ts, relative = search(parent)
168 return (ts, relative + (base,))
170 # We found a config file, load it.
172 litConfig.note('loading suite config %r' % cfgpath)
174 cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
175 source_root = os.path.realpath(cfg.test_source_root or path)
176 exec_root = os.path.realpath(cfg.test_exec_root or path)
177 return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
180 # Check for an already instantiated test suite.
181 res = cache.get(path)
183 cache[path] = res = search1(path)
186 # Canonicalize the path.
187 item = os.path.realpath(item)
189 # Skip files and virtual components.
191 while not os.path.isdir(item):
192 parent,base = os.path.split(item)
195 components.append(base)
199 ts, relative = search(item)
200 return ts, tuple(relative + tuple(components))
202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
203 def search1(path_in_suite):
204 # Get the parent config.
205 if not path_in_suite:
208 parent = search(path_in_suite[:-1])
210 # Load the local configuration.
211 source_path = ts.getSourcePath(path_in_suite)
212 cfgpath = os.path.join(source_path, kLocalConfigName)
214 litConfig.note('loading local config %r' % cfgpath)
215 return TestingConfig.frompath(cfgpath, parent, litConfig,
217 config = parent.clone(cfgpath))
219 def search(path_in_suite):
220 key = (ts, path_in_suite)
223 cache[key] = res = search1(path_in_suite)
226 return search(path_in_suite)
228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
229 # Find the test suite for this input and its relative path.
230 ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
232 litConfig.warning('unable to find test suite for %r' % path)
236 litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
239 return ts, getTestsInSuite(ts, path_in_suite, litConfig,
240 testSuiteCache, localConfigCache)
242 def getTestsInSuite(ts, path_in_suite, litConfig,
243 testSuiteCache, localConfigCache):
244 # Check that the source path exists (errors here are reported by the
246 source_path = ts.getSourcePath(path_in_suite)
247 if not os.path.exists(source_path):
250 # Check if the user named a test directly.
251 if not os.path.isdir(source_path):
252 lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
253 yield Test.Test(ts, path_in_suite, lc)
256 # Otherwise we have a directory to search for tests, start by getting the
257 # local configuration.
258 lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
261 for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
265 # Search subdirectories.
266 for filename in os.listdir(source_path):
267 # FIXME: This doesn't belong here?
268 if filename in ('Output', '.svn') or filename in lc.excludes:
271 # Ignore non-directories.
272 file_sourcepath = os.path.join(source_path, filename)
273 if not os.path.isdir(file_sourcepath):
276 # Check for nested test suites, first in the execpath in case there is a
277 # site configuration and then in the source path.
278 file_execpath = ts.getExecPath(path_in_suite + (filename,))
279 if dirContainsTestSuite(file_execpath):
280 sub_ts, subiter = getTests(file_execpath, litConfig,
281 testSuiteCache, localConfigCache)
282 elif dirContainsTestSuite(file_sourcepath):
283 sub_ts, subiter = getTests(file_sourcepath, litConfig,
284 testSuiteCache, localConfigCache)
286 # Otherwise, continue loading from inside this test suite.
287 subiter = getTestsInSuite(ts, path_in_suite + (filename,),
288 litConfig, testSuiteCache,
297 litConfig.warning('test suite %r contained no tests' % sub_ts.name)
299 def runTests(numThreads, litConfig, provider, display):
300 # If only using one testing thread, don't use threads at all; this lets us
301 # profile, among other things.
303 t = Tester(litConfig, provider, display)
307 # Otherwise spin up the testing threads and wait for them to finish.
308 testers = [Tester(litConfig, provider, display)
309 for i in range(numThreads)]
315 except KeyboardInterrupt:
318 def load_test_suite(inputs):
321 # Create the global config object.
322 litConfig = LitConfig.LitConfig(progname = 'lit',
326 valgrindLeakCheck = False,
331 isWindows = (platform.system()=='Windows'),
334 # Load the tests from the inputs.
337 localConfigCache = {}
340 tests.extend(getTests(input, litConfig,
341 testSuiteCache, localConfigCache)[1])
342 if prev == len(tests):
343 litConfig.warning('input %r contained no tests' % input)
345 # If there were any errors during test discovery, exit now.
346 if litConfig.numErrors:
347 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
350 # Return a unittest test suite which just runs the tests in order.
351 def get_test_fn(test):
352 return unittest.FunctionTestCase(
353 lambda: test.config.test_format.execute(
355 description = test.getFullName())
357 from LitTestCase import LitTestCase
358 return unittest.TestSuite([LitTestCase(test, litConfig) for test in tests])
361 # Bump the GIL check interval, its more important to get any one thread to a
362 # blocking operation (hopefully exec) than to try and unblock other threads.
364 # FIXME: This is a hack.
366 sys.setcheckinterval(1000)
369 from optparse import OptionParser, OptionGroup
370 parser = OptionParser("usage: %prog [options] {file-or-path}")
372 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
373 help="Number of testing threads",
374 type=int, action="store", default=None)
375 parser.add_option("", "--config-prefix", dest="configPrefix",
376 metavar="NAME", help="Prefix for 'lit' config files",
377 action="store", default=None)
378 parser.add_option("", "--param", dest="userParameters",
380 help="Add 'NAME' = 'VAL' to the user defined parameters",
381 type=str, action="append", default=[])
383 group = OptionGroup(parser, "Output Format")
384 # FIXME: I find these names very confusing, although I like the
386 group.add_option("-q", "--quiet", dest="quiet",
387 help="Suppress no error output",
388 action="store_true", default=False)
389 group.add_option("-s", "--succinct", dest="succinct",
390 help="Reduce amount of output",
391 action="store_true", default=False)
392 group.add_option("-v", "--verbose", dest="showOutput",
393 help="Show all test output",
394 action="store_true", default=False)
395 group.add_option("", "--no-progress-bar", dest="useProgressBar",
396 help="Do not use curses based progress bar",
397 action="store_false", default=True)
398 parser.add_option_group(group)
400 group = OptionGroup(parser, "Test Execution")
401 group.add_option("", "--path", dest="path",
402 help="Additional paths to add to testing environment",
403 action="append", type=str, default=[])
404 group.add_option("", "--vg", dest="useValgrind",
405 help="Run tests under valgrind",
406 action="store_true", default=False)
407 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
408 help="Check for memory leaks under valgrind",
409 action="store_true", default=False)
410 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
411 help="Specify an extra argument for valgrind",
412 type=str, action="append", default=[])
413 group.add_option("", "--time-tests", dest="timeTests",
414 help="Track elapsed wall time for each test",
415 action="store_true", default=False)
416 group.add_option("", "--no-execute", dest="noExecute",
417 help="Don't execute any tests (assume PASS)",
418 action="store_true", default=False)
419 parser.add_option_group(group)
421 group = OptionGroup(parser, "Test Selection")
422 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
423 help="Maximum number of tests to run",
424 action="store", type=int, default=None)
425 group.add_option("", "--max-time", dest="maxTime", metavar="N",
426 help="Maximum time to spend testing (in seconds)",
427 action="store", type=float, default=None)
428 group.add_option("", "--shuffle", dest="shuffle",
429 help="Run tests in random order",
430 action="store_true", default=False)
431 parser.add_option_group(group)
433 group = OptionGroup(parser, "Debug and Experimental Options")
434 group.add_option("", "--debug", dest="debug",
435 help="Enable debugging (for 'lit' development)",
436 action="store_true", default=False)
437 group.add_option("", "--show-suites", dest="showSuites",
438 help="Show discovered test suites",
439 action="store_true", default=False)
440 group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
441 help="Don't run Tcl scripts using 'sh'",
442 action="store_false", default=True)
443 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
444 help="Repeat tests N times (for timing)",
445 action="store", default=None, type=int)
446 parser.add_option_group(group)
448 (opts, args) = parser.parse_args()
451 parser.error('No inputs specified')
453 if opts.configPrefix is not None:
454 global gConfigName, gSiteConfigName
455 gConfigName = '%s.cfg' % opts.configPrefix
456 gSiteConfigName = '%s.site.cfg' % opts.configPrefix
458 if opts.numThreads is None:
459 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
460 # http://bugs.python.org/issue1731717
461 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
462 # threads by default there.
463 if sys.hexversion >= 0x2050200:
464 opts.numThreads = Util.detectCPUs()
470 # Create the user defined parameters.
472 for entry in opts.userParameters:
476 name,val = entry.split('=', 1)
477 userParams[name] = val
479 # Create the global config object.
480 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
483 useValgrind = opts.useValgrind,
484 valgrindLeakCheck = opts.valgrindLeakCheck,
485 valgrindArgs = opts.valgrindArgs,
486 useTclAsSh = opts.useTclAsSh,
487 noExecute = opts.noExecute,
489 isWindows = (platform.system()=='Windows'),
492 # Load the tests from the inputs.
495 localConfigCache = {}
498 tests.extend(getTests(input, litConfig,
499 testSuiteCache, localConfigCache)[1])
500 if prev == len(tests):
501 litConfig.warning('input %r contained no tests' % input)
503 # If there were any errors during test discovery, exit now.
504 if litConfig.numErrors:
505 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
509 suitesAndTests = dict([(ts,[])
510 for ts,_ in testSuiteCache.values()
513 suitesAndTests[t.suite].append(t)
515 print '-- Test Suites --'
516 suitesAndTests = suitesAndTests.items()
517 suitesAndTests.sort(key = lambda (ts,_): ts.name)
518 for ts,ts_tests in suitesAndTests:
519 print ' %s - %d tests' %(ts.name, len(ts_tests))
520 print ' Source Root: %s' % ts.source_root
521 print ' Exec Root : %s' % ts.exec_root
523 # Select and order the tests.
524 numTotalTests = len(tests)
526 random.shuffle(tests)
528 tests.sort(key = lambda t: t.getFullName())
529 if opts.maxTests is not None:
530 tests = tests[:opts.maxTests]
533 if len(tests) != numTotalTests:
534 extra = ' of %d' % numTotalTests
535 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
539 tests = [t.copyWithIndex(i)
541 for i in range(opts.repeatTests)]
545 if opts.succinct and opts.useProgressBar:
547 tc = ProgressBar.TerminalController()
548 progressBar = ProgressBar.ProgressBar(tc, header)
551 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
555 # Don't create more threads than tests.
556 opts.numThreads = min(len(tests), opts.numThreads)
558 startTime = time.time()
559 display = TestingProgressDisplay(opts, len(tests), progressBar)
560 provider = TestProvider(tests, opts.maxTime)
561 runTests(opts.numThreads, litConfig, provider, display)
565 print 'Testing Time: %.2fs'%(time.time() - startTime)
567 # Update results for any tests which weren't run.
570 t.setResult(Test.UNRESOLVED, '', 0.0)
572 # List test results organized by kind.
576 if t.result not in byCode:
577 byCode[t.result] = []
578 byCode[t.result].append(t)
579 if t.result.isFailure:
582 # FIXME: Show unresolved and (optionally) unsupported tests.
583 for title,code in (('Unexpected Passing Tests', Test.XPASS),
584 ('Failing Tests', Test.FAIL)):
585 elts = byCode.get(code)
589 print '%s (%d):' % (title, len(elts))
591 print ' %s' % t.getFullName()
595 # Collate, in case we repeated tests.
598 key = t.getFullName()
599 times[key] = times.get(key, 0.) + t.elapsed
601 byTime = list(times.items())
602 byTime.sort(key = lambda (name,elapsed): elapsed)
604 Util.printHistogram(byTime, title='Tests')
606 for name,code in (('Expected Passes ', Test.PASS),
607 ('Expected Failures ', Test.XFAIL),
608 ('Unsupported Tests ', Test.UNSUPPORTED),
609 ('Unresolved Tests ', Test.UNRESOLVED),
610 ('Unexpected Passes ', Test.XPASS),
611 ('Unexpected Failures', Test.FAIL),):
612 if opts.quiet and not code.isFailure:
614 N = len(byCode.get(code,[]))
616 print ' %s: %d' % (name,N)
618 # If we encountered any additional errors, exit abnormally.
619 if litConfig.numErrors:
620 print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
623 # Warn about warnings.
624 if litConfig.numWarnings:
625 print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
631 if __name__=='__main__':