4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
15 from TestingConfig import TestingConfig
19 # Configuration files to look for when discovering test suites. These can be
20 # overridden with --config-prefix.
22 # FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ?
23 gConfigName = 'lit.cfg'
24 gSiteConfigName = 'lit.site.cfg'
26 kLocalConfigName = 'lit.local.cfg'
28 class TestingProgressDisplay:
29 def __init__(self, opts, numTests, progressBar=None):
31 self.numTests = numTests
33 self.lock = threading.Lock()
34 self.progressBar = progressBar
37 def update(self, test):
38 # Avoid locking overhead in quiet mode
39 if self.opts.quiet and not test.result.isFailure:
46 self.handleUpdate(test)
52 self.progressBar.clear()
55 elif self.opts.succinct:
56 sys.stdout.write('\n')
58 def handleUpdate(self, test):
61 self.progressBar.update(float(self.completed)/self.numTests,
64 if self.opts.succinct and not test.result.isFailure:
68 self.progressBar.clear()
70 print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
71 self.completed, self.numTests)
73 if test.result.isFailure and self.opts.showOutput:
74 print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
82 def __init__(self, tests, maxTime):
83 self.maxTime = maxTime
84 self.iter = iter(tests)
85 self.lock = threading.Lock()
86 self.startTime = time.time()
89 # Check if we have run out of time.
90 if self.maxTime is not None:
91 if time.time() - self.startTime > self.maxTime:
94 # Otherwise take the next test.
97 item = self.iter.next()
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print '\nCtrl-C detected, goodbye.'
129 if self.litConfig.debug:
131 result = Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def dirContainsTestSuite(path):
141 cfgpath = os.path.join(path, gSiteConfigName)
142 if os.path.exists(cfgpath):
144 cfgpath = os.path.join(path, gConfigName)
145 if os.path.exists(cfgpath):
148 def getTestSuite(item, litConfig, cache):
149 """getTestSuite(item, litConfig, cache) -> (suite, relative_path)
151 Find the test suite containing @arg item.
153 @retval (None, ...) - Indicates no test suite contains @arg item.
154 @retval (suite, relative_path) - The suite that @arg item is in, and its
155 relative path inside that suite.
158 # Check for a site config or a lit config.
159 cfgpath = dirContainsTestSuite(path)
161 # If we didn't find a config file, keep looking.
163 parent,base = os.path.split(path)
167 ts, relative = search(parent)
168 return (ts, relative + (base,))
170 # We found a config file, load it.
172 litConfig.note('loading suite config %r' % cfgpath)
174 cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True)
175 source_root = os.path.realpath(cfg.test_source_root or path)
176 exec_root = os.path.realpath(cfg.test_exec_root or path)
177 return Test.TestSuite(cfg.name, source_root, exec_root, cfg), ()
180 # Check for an already instantiated test suite.
181 res = cache.get(path)
183 cache[path] = res = search1(path)
186 # Canonicalize the path.
187 item = os.path.realpath(item)
189 # Skip files and virtual components.
191 while not os.path.isdir(item):
192 parent,base = os.path.split(item)
195 components.append(base)
199 ts, relative = search(item)
200 return ts, tuple(relative + tuple(components))
202 def getLocalConfig(ts, path_in_suite, litConfig, cache):
203 def search1(path_in_suite):
204 # Get the parent config.
205 if not path_in_suite:
208 parent = search(path_in_suite[:-1])
210 # Load the local configuration.
211 source_path = ts.getSourcePath(path_in_suite)
212 cfgpath = os.path.join(source_path, kLocalConfigName)
214 litConfig.note('loading local config %r' % cfgpath)
215 return TestingConfig.frompath(cfgpath, parent, litConfig,
217 config = parent.clone(cfgpath))
219 def search(path_in_suite):
220 key = (ts, path_in_suite)
223 cache[key] = res = search1(path_in_suite)
226 return search(path_in_suite)
228 def getTests(path, litConfig, testSuiteCache, localConfigCache):
229 # Find the test suite for this input and its relative path.
230 ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache)
232 litConfig.warning('unable to find test suite for %r' % path)
236 litConfig.note('resolved input %r to %r::%r' % (path, ts.name,
239 return getTestsInSuite(ts, path_in_suite, litConfig,
240 testSuiteCache, localConfigCache)
242 def getTestsInSuite(ts, path_in_suite, litConfig,
243 testSuiteCache, localConfigCache):
244 # Check that the source path exists (errors here are reported by the
246 source_path = ts.getSourcePath(path_in_suite)
247 if not os.path.exists(source_path):
250 # Check if the user named a test directly.
251 if not os.path.isdir(source_path):
252 lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache)
253 yield Test.Test(ts, path_in_suite, lc)
256 # Otherwise we have a directory to search for tests, start by getting the
257 # local configuration.
258 lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
261 for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,
265 # Search subdirectories.
266 for filename in os.listdir(source_path):
267 # FIXME: This doesn't belong here?
268 if filename in ('Output', '.svn') or filename in lc.excludes:
271 # Ignore non-directories.
272 file_sourcepath = os.path.join(source_path, filename)
273 if not os.path.isdir(file_sourcepath):
276 # Check for nested test suites, first in the execpath in case there is a
277 # site configuration and then in the source path.
278 file_execpath = ts.getExecPath(path_in_suite + (filename,))
279 if dirContainsTestSuite(file_execpath):
280 subiter = getTests(file_execpath, litConfig,
281 testSuiteCache, localConfigCache)
282 elif dirContainsTestSuite(file_sourcepath):
283 subiter = getTests(file_sourcepath, litConfig,
284 testSuiteCache, localConfigCache)
286 # Otherwise, continue loading from inside this test suite.
287 subiter = getTestsInSuite(ts, path_in_suite + (filename,),
288 litConfig, testSuiteCache,
294 def runTests(numThreads, litConfig, provider, display):
295 # If only using one testing thread, don't use threads at all; this lets us
296 # profile, among other things.
298 t = Tester(litConfig, provider, display)
302 # Otherwise spin up the testing threads and wait for them to finish.
303 testers = [Tester(litConfig, provider, display)
304 for i in range(numThreads)]
310 except KeyboardInterrupt:
315 from optparse import OptionParser, OptionGroup
316 parser = OptionParser("usage: %prog [options] {file-or-path}")
318 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
319 help="Number of testing threads",
320 type=int, action="store", default=None)
321 parser.add_option("", "--config-prefix", dest="configPrefix",
322 metavar="NAME", help="Prefix for 'lit' config files",
323 action="store", default=None)
325 group = OptionGroup(parser, "Output Format")
326 # FIXME: I find these names very confusing, although I like the
328 group.add_option("-q", "--quiet", dest="quiet",
329 help="Suppress no error output",
330 action="store_true", default=False)
331 group.add_option("-s", "--succinct", dest="succinct",
332 help="Reduce amount of output",
333 action="store_true", default=False)
334 group.add_option("-v", "--verbose", dest="showOutput",
335 help="Show all test output",
336 action="store_true", default=False)
337 group.add_option("", "--no-progress-bar", dest="useProgressBar",
338 help="Do not use curses based progress bar",
339 action="store_false", default=True)
340 parser.add_option_group(group)
342 group = OptionGroup(parser, "Test Execution")
343 group.add_option("", "--path", dest="path",
344 help="Additional paths to add to testing environment",
345 action="append", type=str, default=[])
346 group.add_option("", "--vg", dest="useValgrind",
347 help="Run tests under valgrind",
348 action="store_true", default=False)
349 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
350 help="Specify an extra argument for valgrind",
351 type=str, action="append", default=[])
352 group.add_option("", "--time-tests", dest="timeTests",
353 help="Track elapsed wall time for each test",
354 action="store_true", default=False)
355 group.add_option("", "--no-execute", dest="noExecute",
356 help="Don't execute any tests (assume PASS)",
357 action="store_true", default=False)
358 parser.add_option_group(group)
360 group = OptionGroup(parser, "Test Selection")
361 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
362 help="Maximum number of tests to run",
363 action="store", type=int, default=None)
364 group.add_option("", "--max-time", dest="maxTime", metavar="N",
365 help="Maximum time to spend testing (in seconds)",
366 action="store", type=float, default=None)
367 group.add_option("", "--shuffle", dest="shuffle",
368 help="Run tests in random order",
369 action="store_true", default=False)
370 parser.add_option_group(group)
372 group = OptionGroup(parser, "Debug and Experimental Options")
373 group.add_option("", "--debug", dest="debug",
374 help="Enable debugging (for 'lit' development)",
375 action="store_true", default=False)
376 group.add_option("", "--show-suites", dest="showSuites",
377 help="Show discovered test suites",
378 action="store_true", default=False)
379 group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh",
380 help="Don't run Tcl scripts using 'sh'",
381 action="store_false", default=True)
382 parser.add_option_group(group)
384 (opts, args) = parser.parse_args()
387 parser.error('No inputs specified')
389 if opts.configPrefix is not None:
390 global gConfigName, gSiteConfigName
391 gConfigName = '%s.cfg' % opts.configPrefix
392 gSiteConfigName = '%s.site.cfg' % opts.configPrefix
394 if opts.numThreads is None:
395 opts.numThreads = Util.detectCPUs()
399 # Create the global config object.
400 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
403 useValgrind = opts.useValgrind,
404 valgrindArgs = opts.valgrindArgs,
405 useTclAsSh = opts.useTclAsSh,
406 noExecute = opts.noExecute,
408 isWindows = (platform.system()=='Windows'))
410 # Load the tests from the inputs.
413 localConfigCache = {}
416 tests.extend(getTests(input, litConfig,
417 testSuiteCache, localConfigCache))
418 if prev == len(tests):
419 litConfig.warning('input %r contained no tests' % input)
421 # If there were any errors during test discovery, exit now.
422 if litConfig.numErrors:
423 print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors
427 suitesAndTests = dict([(ts,[])
428 for ts,_ in testSuiteCache.values()
431 suitesAndTests[t.suite].append(t)
433 print '-- Test Suites --'
434 suitesAndTests = suitesAndTests.items()
435 suitesAndTests.sort(key = lambda (ts,_): ts.name)
436 for ts,tests in suitesAndTests:
437 print ' %s - %d tests' %(ts.name, len(tests))
438 print ' Source Root: %s' % ts.source_root
439 print ' Exec Root : %s' % ts.exec_root
441 # Select and order the tests.
442 numTotalTests = len(tests)
444 random.shuffle(tests)
446 tests.sort(key = lambda t: t.getFullName())
447 if opts.maxTests is not None:
448 tests = tests[:opts.maxTests]
451 if len(tests) != numTotalTests:
452 extra = ' of %d' % numTotalTests
453 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
458 if opts.succinct and opts.useProgressBar:
460 tc = ProgressBar.TerminalController()
461 progressBar = ProgressBar.ProgressBar(tc, header)
464 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
468 # Don't create more threads than tests.
469 opts.numThreads = min(len(tests), opts.numThreads)
471 startTime = time.time()
472 display = TestingProgressDisplay(opts, len(tests), progressBar)
473 provider = TestProvider(tests, opts.maxTime)
474 runTests(opts.numThreads, litConfig, provider, display)
478 print 'Testing Time: %.2fs'%(time.time() - startTime)
480 # Update results for any tests which weren't run.
483 t.setResult(Test.UNRESOLVED, '', 0.0)
485 # List test results organized by kind.
489 if t.result not in byCode:
490 byCode[t.result] = []
491 byCode[t.result].append(t)
492 if t.result.isFailure:
495 # FIXME: Show unresolved and (optionally) unsupported tests.
496 for title,code in (('Unexpected Passing Tests', Test.XPASS),
497 ('Failing Tests', Test.FAIL)):
498 elts = byCode.get(code)
502 print '%s (%d):' % (title, len(elts))
504 print ' %s' % t.getFullName()
509 byTime.sort(key = lambda t: t.elapsed)
511 Util.printHistogram([(t.getFullName(), t.elapsed) for t in byTime],
514 for name,code in (('Expected Passes ', Test.PASS),
515 ('Expected Failures ', Test.XFAIL),
516 ('Unsupported Tests ', Test.UNSUPPORTED),
517 ('Unresolved Tests ', Test.UNRESOLVED),
518 ('Unexpected Passes ', Test.XPASS),
519 ('Unexpected Failures', Test.FAIL),):
520 if opts.quiet and not code.isFailure:
522 N = len(byCode.get(code,[]))
524 print ' %s: %d' % (name,N)
526 # If we encountered any additional errors, exit abnormally.
527 if litConfig.numErrors:
528 print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
531 # Warn about warnings.
532 if litConfig.numWarnings:
533 print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
539 if __name__=='__main__':
540 # Bump the GIL check interval, its more important to get any one thread to a
541 # blocking operation (hopefully exec) than to try and unblock other threads.
543 sys.setcheckinterval(1000)