4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 from __future__ import absolute_import
10 import math, os, platform, random, re, sys, time, threading, traceback
12 import lit.ProgressBar
19 class TestingProgressDisplay:
20 def __init__(self, opts, numTests, progressBar=None):
22 self.numTests = numTests
24 self.lock = threading.Lock()
25 self.progressBar = progressBar
28 def update(self, test):
29 # Avoid locking overhead in quiet mode
30 if self.opts.quiet and not test.result.isFailure:
37 self.handleUpdate(test)
43 self.progressBar.clear()
46 elif self.opts.succinct:
47 sys.stdout.write('\n')
49 def handleUpdate(self, test):
52 self.progressBar.update(float(self.completed)/self.numTests,
55 if self.opts.succinct and not test.result.isFailure:
59 self.progressBar.clear()
61 print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
62 self.completed, self.numTests))
64 if test.result.isFailure and self.opts.showOutput:
65 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
73 def __init__(self, tests, maxTime):
74 self.maxTime = maxTime
75 self.iter = iter(tests)
76 self.lock = threading.Lock()
77 self.startTime = time.time()
86 # Check if we have run out of time.
87 if self.maxTime is not None:
88 if time.time() - self.startTime > self.maxTime:
91 # Otherwise take the next test.
96 for item in self.iter:
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print('\nCtrl-C detected, goodbye.')
129 if self.litConfig.debug:
131 result = lit.Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def runTests(numThreads, litConfig, provider, display):
141 # If only using one testing thread, don't use threads at all; this lets us
142 # profile, among other things.
144 t = Tester(litConfig, provider, display)
148 # Otherwise spin up the testing threads and wait for them to finish.
149 testers = [Tester(litConfig, provider, display)
150 for i in range(numThreads)]
156 except KeyboardInterrupt:
159 def main(builtinParameters = {}):
160 # Bump the GIL check interval, its more important to get any one thread to a
161 # blocking operation (hopefully exec) than to try and unblock other threads.
163 # FIXME: This is a hack.
165 sys.setcheckinterval(1000)
168 from optparse import OptionParser, OptionGroup
169 parser = OptionParser("usage: %prog [options] {file-or-path}")
171 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
172 help="Number of testing threads",
173 type=int, action="store", default=None)
174 parser.add_option("", "--config-prefix", dest="configPrefix",
175 metavar="NAME", help="Prefix for 'lit' config files",
176 action="store", default=None)
177 parser.add_option("", "--param", dest="userParameters",
179 help="Add 'NAME' = 'VAL' to the user defined parameters",
180 type=str, action="append", default=[])
182 group = OptionGroup(parser, "Output Format")
183 # FIXME: I find these names very confusing, although I like the
185 group.add_option("-q", "--quiet", dest="quiet",
186 help="Suppress no error output",
187 action="store_true", default=False)
188 group.add_option("-s", "--succinct", dest="succinct",
189 help="Reduce amount of output",
190 action="store_true", default=False)
191 group.add_option("-v", "--verbose", dest="showOutput",
192 help="Show all test output",
193 action="store_true", default=False)
194 group.add_option("", "--no-progress-bar", dest="useProgressBar",
195 help="Do not use curses based progress bar",
196 action="store_false", default=True)
197 parser.add_option_group(group)
199 group = OptionGroup(parser, "Test Execution")
200 group.add_option("", "--path", dest="path",
201 help="Additional paths to add to testing environment",
202 action="append", type=str, default=[])
203 group.add_option("", "--vg", dest="useValgrind",
204 help="Run tests under valgrind",
205 action="store_true", default=False)
206 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
207 help="Check for memory leaks under valgrind",
208 action="store_true", default=False)
209 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
210 help="Specify an extra argument for valgrind",
211 type=str, action="append", default=[])
212 group.add_option("", "--time-tests", dest="timeTests",
213 help="Track elapsed wall time for each test",
214 action="store_true", default=False)
215 parser.add_option_group(group)
217 group = OptionGroup(parser, "Test Selection")
218 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
219 help="Maximum number of tests to run",
220 action="store", type=int, default=None)
221 group.add_option("", "--max-time", dest="maxTime", metavar="N",
222 help="Maximum time to spend testing (in seconds)",
223 action="store", type=float, default=None)
224 group.add_option("", "--shuffle", dest="shuffle",
225 help="Run tests in random order",
226 action="store_true", default=False)
227 group.add_option("", "--filter", dest="filter", metavar="REGEX",
228 help=("Only run tests with paths matching the given "
229 "regular expression"),
230 action="store", default=None)
231 parser.add_option_group(group)
233 group = OptionGroup(parser, "Debug and Experimental Options")
234 group.add_option("", "--debug", dest="debug",
235 help="Enable debugging (for 'lit' development)",
236 action="store_true", default=False)
237 group.add_option("", "--show-suites", dest="showSuites",
238 help="Show discovered test suites",
239 action="store_true", default=False)
240 group.add_option("", "--show-tests", dest="showTests",
241 help="Show all discovered tests",
242 action="store_true", default=False)
243 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
244 help="Repeat tests N times (for timing)",
245 action="store", default=None, type=int)
246 parser.add_option_group(group)
248 (opts, args) = parser.parse_args()
251 parser.error('No inputs specified')
253 if opts.numThreads is None:
254 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
255 # http://bugs.python.org/issue1731717
256 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
257 # threads by default there.
258 if sys.hexversion >= 0x2050200:
259 opts.numThreads = lit.Util.detectCPUs()
265 # Create the user defined parameters.
266 userParams = dict(builtinParameters)
267 for entry in opts.userParameters:
271 name,val = entry.split('=', 1)
272 userParams[name] = val
274 # Create the global config object.
275 litConfig = lit.LitConfig.LitConfig(
276 progname = os.path.basename(sys.argv[0]),
279 useValgrind = opts.useValgrind,
280 valgrindLeakCheck = opts.valgrindLeakCheck,
281 valgrindArgs = opts.valgrindArgs,
283 isWindows = (platform.system()=='Windows'),
285 config_prefix = opts.configPrefix)
287 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
289 if opts.showSuites or opts.showTests:
290 # Aggregate the tests by suite.
293 if t.suite not in suitesAndTests:
294 suitesAndTests[t.suite] = []
295 suitesAndTests[t.suite].append(t)
296 suitesAndTests = suitesAndTests.items()
297 suitesAndTests.sort(key = lambda item: item[0].name)
299 # Show the suites, if requested.
301 print('-- Test Suites --')
302 for ts,ts_tests in suitesAndTests:
303 print(' %s - %d tests' %(ts.name, len(ts_tests)))
304 print(' Source Root: %s' % ts.source_root)
305 print(' Exec Root : %s' % ts.exec_root)
307 # Show the tests, if requested.
309 print('-- Available Tests --')
310 for ts,ts_tests in suitesAndTests:
311 ts_tests.sort(key = lambda test: test.path_in_suite)
312 for test in ts_tests:
313 print(' %s' % (test.getFullName(),))
315 # Select and order the tests.
316 numTotalTests = len(tests)
318 # First, select based on the filter expression if given.
321 rex = re.compile(opts.filter)
323 parser.error("invalid regular expression for --filter: %r" % (
325 tests = [t for t in tests
326 if rex.search(t.getFullName())]
328 # Then select the order.
330 random.shuffle(tests)
332 tests.sort(key = lambda t: t.getFullName())
334 # Finally limit the number of tests, if desired.
335 if opts.maxTests is not None:
336 tests = tests[:opts.maxTests]
338 # Don't create more threads than tests.
339 opts.numThreads = min(len(tests), opts.numThreads)
342 if len(tests) != numTotalTests:
343 extra = ' of %d' % numTotalTests
344 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
348 tests = [t.copyWithIndex(i)
350 for i in range(opts.repeatTests)]
354 if opts.succinct and opts.useProgressBar:
356 tc = lit.ProgressBar.TerminalController()
357 progressBar = lit.ProgressBar.ProgressBar(tc, header)
360 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
364 startTime = time.time()
365 display = TestingProgressDisplay(opts, len(tests), progressBar)
366 provider = TestProvider(tests, opts.maxTime)
373 def console_ctrl_handler(type):
376 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
378 runTests(opts.numThreads, litConfig, provider, display)
382 print('Testing Time: %.2fs'%(time.time() - startTime))
384 # Update results for any tests which weren't run.
387 t.setResult(lit.Test.UNRESOLVED, '', 0.0)
389 # List test results organized by kind.
393 if t.result not in byCode:
394 byCode[t.result] = []
395 byCode[t.result].append(t)
396 if t.result.isFailure:
399 # FIXME: Show unresolved and (optionally) unsupported tests.
400 for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
401 ('Failing Tests', lit.Test.FAIL)):
402 elts = byCode.get(code)
406 print('%s (%d):' % (title, len(elts)))
408 print(' %s' % t.getFullName())
409 sys.stdout.write('\n')
412 # Collate, in case we repeated tests.
415 key = t.getFullName()
416 times[key] = times.get(key, 0.) + t.elapsed
418 byTime = list(times.items())
419 byTime.sort(key = lambda item: item[1])
421 lit.Util.printHistogram(byTime, title='Tests')
423 for name,code in (('Expected Passes ', lit.Test.PASS),
424 ('Expected Failures ', lit.Test.XFAIL),
425 ('Unsupported Tests ', lit.Test.UNSUPPORTED),
426 ('Unresolved Tests ', lit.Test.UNRESOLVED),
427 ('Unexpected Passes ', lit.Test.XPASS),
428 ('Unexpected Failures', lit.Test.FAIL),):
429 if opts.quiet and not code.isFailure:
431 N = len(byCode.get(code,[]))
433 print(' %s: %d' % (name,N))
435 # If we encountered any additional errors, exit abnormally.
436 if litConfig.numErrors:
437 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
440 # Warn about warnings.
441 if litConfig.numWarnings:
442 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
448 if __name__=='__main__':