4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
20 class TestingProgressDisplay:
21 def __init__(self, opts, numTests, progressBar=None):
23 self.numTests = numTests
25 self.lock = threading.Lock()
26 self.progressBar = progressBar
29 def update(self, test):
30 # Avoid locking overhead in quiet mode
31 if self.opts.quiet and not test.result.isFailure:
38 self.handleUpdate(test)
44 self.progressBar.clear()
47 elif self.opts.succinct:
48 sys.stdout.write('\n')
50 def handleUpdate(self, test):
53 self.progressBar.update(float(self.completed)/self.numTests,
56 if self.opts.succinct and not test.result.isFailure:
60 self.progressBar.clear()
62 print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
63 self.completed, self.numTests))
65 if test.result.isFailure and self.opts.showOutput:
66 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
74 def __init__(self, tests, maxTime):
75 self.maxTime = maxTime
76 self.iter = iter(tests)
77 self.lock = threading.Lock()
78 self.startTime = time.time()
87 # Check if we have run out of time.
88 if self.maxTime is not None:
89 if time.time() - self.startTime > self.maxTime:
92 # Otherwise take the next test.
97 for item in self.iter:
104 class Tester(threading.Thread):
105 def __init__(self, litConfig, provider, display):
106 threading.Thread.__init__(self)
107 self.litConfig = litConfig
108 self.provider = provider
109 self.display = display
113 item = self.provider.get()
118 def runTest(self, test):
120 startTime = time.time()
122 result, output = test.config.test_format.execute(test,
124 except KeyboardInterrupt:
125 # This is a sad hack. Unfortunately subprocess goes
126 # bonkers with ctrl-c and we start forking merrily.
127 print('\nCtrl-C detected, goodbye.')
130 if self.litConfig.debug:
132 result = Test.UNRESOLVED
133 output = 'Exception during script execution:\n'
134 output += traceback.format_exc()
136 elapsed = time.time() - startTime
138 test.setResult(result, output, elapsed)
139 self.display.update(test)
141 def runTests(numThreads, litConfig, provider, display):
142 # If only using one testing thread, don't use threads at all; this lets us
143 # profile, among other things.
145 t = Tester(litConfig, provider, display)
149 # Otherwise spin up the testing threads and wait for them to finish.
150 testers = [Tester(litConfig, provider, display)
151 for i in range(numThreads)]
157 except KeyboardInterrupt:
160 def main(builtinParameters = {}):
161 # Bump the GIL check interval, its more important to get any one thread to a
162 # blocking operation (hopefully exec) than to try and unblock other threads.
164 # FIXME: This is a hack.
166 sys.setcheckinterval(1000)
169 from optparse import OptionParser, OptionGroup
170 parser = OptionParser("usage: %prog [options] {file-or-path}")
172 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
173 help="Number of testing threads",
174 type=int, action="store", default=None)
175 parser.add_option("", "--config-prefix", dest="configPrefix",
176 metavar="NAME", help="Prefix for 'lit' config files",
177 action="store", default=None)
178 parser.add_option("", "--param", dest="userParameters",
180 help="Add 'NAME' = 'VAL' to the user defined parameters",
181 type=str, action="append", default=[])
183 group = OptionGroup(parser, "Output Format")
184 # FIXME: I find these names very confusing, although I like the
186 group.add_option("-q", "--quiet", dest="quiet",
187 help="Suppress no error output",
188 action="store_true", default=False)
189 group.add_option("-s", "--succinct", dest="succinct",
190 help="Reduce amount of output",
191 action="store_true", default=False)
192 group.add_option("-v", "--verbose", dest="showOutput",
193 help="Show all test output",
194 action="store_true", default=False)
195 group.add_option("", "--no-progress-bar", dest="useProgressBar",
196 help="Do not use curses based progress bar",
197 action="store_false", default=True)
198 parser.add_option_group(group)
200 group = OptionGroup(parser, "Test Execution")
201 group.add_option("", "--path", dest="path",
202 help="Additional paths to add to testing environment",
203 action="append", type=str, default=[])
204 group.add_option("", "--vg", dest="useValgrind",
205 help="Run tests under valgrind",
206 action="store_true", default=False)
207 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
208 help="Check for memory leaks under valgrind",
209 action="store_true", default=False)
210 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
211 help="Specify an extra argument for valgrind",
212 type=str, action="append", default=[])
213 group.add_option("", "--time-tests", dest="timeTests",
214 help="Track elapsed wall time for each test",
215 action="store_true", default=False)
216 parser.add_option_group(group)
218 group = OptionGroup(parser, "Test Selection")
219 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
220 help="Maximum number of tests to run",
221 action="store", type=int, default=None)
222 group.add_option("", "--max-time", dest="maxTime", metavar="N",
223 help="Maximum time to spend testing (in seconds)",
224 action="store", type=float, default=None)
225 group.add_option("", "--shuffle", dest="shuffle",
226 help="Run tests in random order",
227 action="store_true", default=False)
228 group.add_option("", "--filter", dest="filter", metavar="REGEX",
229 help=("Only run tests with paths matching the given "
230 "regular expression"),
231 action="store", default=None)
232 parser.add_option_group(group)
234 group = OptionGroup(parser, "Debug and Experimental Options")
235 group.add_option("", "--debug", dest="debug",
236 help="Enable debugging (for 'lit' development)",
237 action="store_true", default=False)
238 group.add_option("", "--show-suites", dest="showSuites",
239 help="Show discovered test suites",
240 action="store_true", default=False)
241 group.add_option("", "--show-tests", dest="showTests",
242 help="Show all discovered tests",
243 action="store_true", default=False)
244 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
245 help="Repeat tests N times (for timing)",
246 action="store", default=None, type=int)
247 parser.add_option_group(group)
249 (opts, args) = parser.parse_args()
252 parser.error('No inputs specified')
254 if opts.numThreads is None:
255 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
256 # http://bugs.python.org/issue1731717
257 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
258 # threads by default there.
259 if sys.hexversion >= 0x2050200:
260 opts.numThreads = Util.detectCPUs()
266 # Create the user defined parameters.
267 userParams = dict(builtinParameters)
268 for entry in opts.userParameters:
272 name,val = entry.split('=', 1)
273 userParams[name] = val
275 # Create the global config object.
276 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
279 useValgrind = opts.useValgrind,
280 valgrindLeakCheck = opts.valgrindLeakCheck,
281 valgrindArgs = opts.valgrindArgs,
283 isWindows = (platform.system()=='Windows'),
285 config_prefix = opts.configPrefix)
287 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
289 if opts.showSuites or opts.showTests:
290 # Aggregate the tests by suite.
293 if t.suite not in suitesAndTests:
294 suitesAndTests[t.suite] = []
295 suitesAndTests[t.suite].append(t)
296 suitesAndTests = suitesAndTests.items()
297 suitesAndTests.sort(key = lambda item: item[0].name)
299 # Show the suites, if requested.
301 print('-- Test Suites --')
302 for ts,ts_tests in suitesAndTests:
303 print(' %s - %d tests' %(ts.name, len(ts_tests)))
304 print(' Source Root: %s' % ts.source_root)
305 print(' Exec Root : %s' % ts.exec_root)
307 # Show the tests, if requested.
309 print('-- Available Tests --')
310 for ts,ts_tests in suitesAndTests:
311 ts_tests.sort(key = lambda test: test.path_in_suite)
312 for test in ts_tests:
313 print(' %s' % (test.getFullName(),))
315 # Select and order the tests.
316 numTotalTests = len(tests)
318 # First, select based on the filter expression if given.
321 rex = re.compile(opts.filter)
323 parser.error("invalid regular expression for --filter: %r" % (
325 tests = [t for t in tests
326 if rex.search(t.getFullName())]
328 # Then select the order.
330 random.shuffle(tests)
332 tests.sort(key = lambda t: t.getFullName())
334 # Finally limit the number of tests, if desired.
335 if opts.maxTests is not None:
336 tests = tests[:opts.maxTests]
338 # Don't create more threads than tests.
339 opts.numThreads = min(len(tests), opts.numThreads)
342 if len(tests) != numTotalTests:
343 extra = ' of %d' % numTotalTests
344 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
348 tests = [t.copyWithIndex(i)
350 for i in range(opts.repeatTests)]
354 if opts.succinct and opts.useProgressBar:
356 tc = ProgressBar.TerminalController()
357 progressBar = ProgressBar.ProgressBar(tc, header)
360 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
364 startTime = time.time()
365 display = TestingProgressDisplay(opts, len(tests), progressBar)
366 provider = TestProvider(tests, opts.maxTime)
373 def console_ctrl_handler(type):
376 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
378 runTests(opts.numThreads, litConfig, provider, display)
382 print('Testing Time: %.2fs'%(time.time() - startTime))
384 # Update results for any tests which weren't run.
387 t.setResult(Test.UNRESOLVED, '', 0.0)
389 # List test results organized by kind.
393 if t.result not in byCode:
394 byCode[t.result] = []
395 byCode[t.result].append(t)
396 if t.result.isFailure:
399 # FIXME: Show unresolved and (optionally) unsupported tests.
400 for title,code in (('Unexpected Passing Tests', Test.XPASS),
401 ('Failing Tests', Test.FAIL)):
402 elts = byCode.get(code)
406 print('%s (%d):' % (title, len(elts)))
408 print(' %s' % t.getFullName())
409 sys.stdout.write('\n')
412 # Collate, in case we repeated tests.
415 key = t.getFullName()
416 times[key] = times.get(key, 0.) + t.elapsed
418 byTime = list(times.items())
419 byTime.sort(key = lambda item: item[1])
421 Util.printHistogram(byTime, title='Tests')
423 for name,code in (('Expected Passes ', Test.PASS),
424 ('Expected Failures ', Test.XFAIL),
425 ('Unsupported Tests ', Test.UNSUPPORTED),
426 ('Unresolved Tests ', Test.UNRESOLVED),
427 ('Unexpected Passes ', Test.XPASS),
428 ('Unexpected Failures', Test.FAIL),):
429 if opts.quiet and not code.isFailure:
431 N = len(byCode.get(code,[]))
433 print(' %s: %d' % (name,N))
435 # If we encountered any additional errors, exit abnormally.
436 if litConfig.numErrors:
437 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
440 # Warn about warnings.
441 if litConfig.numWarnings:
442 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
448 if __name__=='__main__':