4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
20 class TestingProgressDisplay:
21 def __init__(self, opts, numTests, progressBar=None):
23 self.numTests = numTests
25 self.lock = threading.Lock()
26 self.progressBar = progressBar
29 def update(self, test):
30 # Avoid locking overhead in quiet mode
31 if self.opts.quiet and not test.result.isFailure:
38 self.handleUpdate(test)
44 self.progressBar.clear()
47 elif self.opts.succinct:
48 sys.stdout.write('\n')
50 def handleUpdate(self, test):
53 self.progressBar.update(float(self.completed)/self.numTests,
56 if self.opts.succinct and not test.result.isFailure:
60 self.progressBar.clear()
62 print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
63 self.completed, self.numTests))
65 if test.result.isFailure and self.opts.showOutput:
66 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
74 def __init__(self, tests, maxTime):
75 self.maxTime = maxTime
76 self.iter = iter(tests)
77 self.lock = threading.Lock()
78 self.startTime = time.time()
87 # Check if we have run out of time.
88 if self.maxTime is not None:
89 if time.time() - self.startTime > self.maxTime:
92 # Otherwise take the next test.
99 item = self.iter.next()
100 except StopIteration:
105 class Tester(threading.Thread):
106 def __init__(self, litConfig, provider, display):
107 threading.Thread.__init__(self)
108 self.litConfig = litConfig
109 self.provider = provider
110 self.display = display
114 item = self.provider.get()
119 def runTest(self, test):
121 startTime = time.time()
123 result, output = test.config.test_format.execute(test,
125 except KeyboardInterrupt:
126 # This is a sad hack. Unfortunately subprocess goes
127 # bonkers with ctrl-c and we start forking merrily.
128 print('\nCtrl-C detected, goodbye.')
131 if self.litConfig.debug:
133 result = Test.UNRESOLVED
134 output = 'Exception during script execution:\n'
135 output += traceback.format_exc()
137 elapsed = time.time() - startTime
139 test.setResult(result, output, elapsed)
140 self.display.update(test)
142 def runTests(numThreads, litConfig, provider, display):
143 # If only using one testing thread, don't use threads at all; this lets us
144 # profile, among other things.
146 t = Tester(litConfig, provider, display)
150 # Otherwise spin up the testing threads and wait for them to finish.
151 testers = [Tester(litConfig, provider, display)
152 for i in range(numThreads)]
158 except KeyboardInterrupt:
161 def main(builtinParameters = {}):
162 # Bump the GIL check interval, its more important to get any one thread to a
163 # blocking operation (hopefully exec) than to try and unblock other threads.
165 # FIXME: This is a hack.
167 sys.setcheckinterval(1000)
170 from optparse import OptionParser, OptionGroup
171 parser = OptionParser("usage: %prog [options] {file-or-path}")
173 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
174 help="Number of testing threads",
175 type=int, action="store", default=None)
176 parser.add_option("", "--config-prefix", dest="configPrefix",
177 metavar="NAME", help="Prefix for 'lit' config files",
178 action="store", default=None)
179 parser.add_option("", "--param", dest="userParameters",
181 help="Add 'NAME' = 'VAL' to the user defined parameters",
182 type=str, action="append", default=[])
184 group = OptionGroup(parser, "Output Format")
185 # FIXME: I find these names very confusing, although I like the
187 group.add_option("-q", "--quiet", dest="quiet",
188 help="Suppress no error output",
189 action="store_true", default=False)
190 group.add_option("-s", "--succinct", dest="succinct",
191 help="Reduce amount of output",
192 action="store_true", default=False)
193 group.add_option("-v", "--verbose", dest="showOutput",
194 help="Show all test output",
195 action="store_true", default=False)
196 group.add_option("", "--no-progress-bar", dest="useProgressBar",
197 help="Do not use curses based progress bar",
198 action="store_false", default=True)
199 parser.add_option_group(group)
201 group = OptionGroup(parser, "Test Execution")
202 group.add_option("", "--path", dest="path",
203 help="Additional paths to add to testing environment",
204 action="append", type=str, default=[])
205 group.add_option("", "--vg", dest="useValgrind",
206 help="Run tests under valgrind",
207 action="store_true", default=False)
208 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
209 help="Check for memory leaks under valgrind",
210 action="store_true", default=False)
211 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
212 help="Specify an extra argument for valgrind",
213 type=str, action="append", default=[])
214 group.add_option("", "--time-tests", dest="timeTests",
215 help="Track elapsed wall time for each test",
216 action="store_true", default=False)
217 parser.add_option_group(group)
219 group = OptionGroup(parser, "Test Selection")
220 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
221 help="Maximum number of tests to run",
222 action="store", type=int, default=None)
223 group.add_option("", "--max-time", dest="maxTime", metavar="N",
224 help="Maximum time to spend testing (in seconds)",
225 action="store", type=float, default=None)
226 group.add_option("", "--shuffle", dest="shuffle",
227 help="Run tests in random order",
228 action="store_true", default=False)
229 group.add_option("", "--filter", dest="filter", metavar="REGEX",
230 help=("Only run tests with paths matching the given "
231 "regular expression"),
232 action="store", default=None)
233 parser.add_option_group(group)
235 group = OptionGroup(parser, "Debug and Experimental Options")
236 group.add_option("", "--debug", dest="debug",
237 help="Enable debugging (for 'lit' development)",
238 action="store_true", default=False)
239 group.add_option("", "--show-suites", dest="showSuites",
240 help="Show discovered test suites",
241 action="store_true", default=False)
242 group.add_option("", "--show-tests", dest="showTests",
243 help="Show all discovered tests",
244 action="store_true", default=False)
245 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
246 help="Repeat tests N times (for timing)",
247 action="store", default=None, type=int)
248 parser.add_option_group(group)
250 (opts, args) = parser.parse_args()
253 parser.error('No inputs specified')
255 if opts.numThreads is None:
256 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
257 # http://bugs.python.org/issue1731717
258 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
259 # threads by default there.
260 if sys.hexversion >= 0x2050200:
261 opts.numThreads = Util.detectCPUs()
267 # Create the user defined parameters.
268 userParams = dict(builtinParameters)
269 for entry in opts.userParameters:
273 name,val = entry.split('=', 1)
274 userParams[name] = val
276 # Create the global config object.
277 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
280 useValgrind = opts.useValgrind,
281 valgrindLeakCheck = opts.valgrindLeakCheck,
282 valgrindArgs = opts.valgrindArgs,
284 isWindows = (platform.system()=='Windows'),
286 config_prefix = opts.configPrefix)
288 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
290 if opts.showSuites or opts.showTests:
291 # Aggregate the tests by suite.
294 if t.suite not in suitesAndTests:
295 suitesAndTests[t.suite] = []
296 suitesAndTests[t.suite].append(t)
297 suitesAndTests = suitesAndTests.items()
298 suitesAndTests.sort(key = lambda item: item[0].name)
300 # Show the suites, if requested.
302 print('-- Test Suites --')
303 for ts,ts_tests in suitesAndTests:
304 print(' %s - %d tests' %(ts.name, len(ts_tests)))
305 print(' Source Root: %s' % ts.source_root)
306 print(' Exec Root : %s' % ts.exec_root)
308 # Show the tests, if requested.
310 print('-- Available Tests --')
311 for ts,ts_tests in suitesAndTests:
312 ts_tests.sort(key = lambda test: test.path_in_suite)
313 for test in ts_tests:
314 print(' %s' % (test.getFullName(),))
316 # Select and order the tests.
317 numTotalTests = len(tests)
319 # First, select based on the filter expression if given.
322 rex = re.compile(opts.filter)
324 parser.error("invalid regular expression for --filter: %r" % (
326 tests = [t for t in tests
327 if rex.search(t.getFullName())]
329 # Then select the order.
331 random.shuffle(tests)
333 tests.sort(key = lambda t: t.getFullName())
335 # Finally limit the number of tests, if desired.
336 if opts.maxTests is not None:
337 tests = tests[:opts.maxTests]
339 # Don't create more threads than tests.
340 opts.numThreads = min(len(tests), opts.numThreads)
343 if len(tests) != numTotalTests:
344 extra = ' of %d' % numTotalTests
345 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
349 tests = [t.copyWithIndex(i)
351 for i in range(opts.repeatTests)]
355 if opts.succinct and opts.useProgressBar:
357 tc = ProgressBar.TerminalController()
358 progressBar = ProgressBar.ProgressBar(tc, header)
361 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
365 startTime = time.time()
366 display = TestingProgressDisplay(opts, len(tests), progressBar)
367 provider = TestProvider(tests, opts.maxTime)
374 def console_ctrl_handler(type):
377 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
379 runTests(opts.numThreads, litConfig, provider, display)
383 print('Testing Time: %.2fs'%(time.time() - startTime))
385 # Update results for any tests which weren't run.
388 t.setResult(Test.UNRESOLVED, '', 0.0)
390 # List test results organized by kind.
394 if t.result not in byCode:
395 byCode[t.result] = []
396 byCode[t.result].append(t)
397 if t.result.isFailure:
400 # FIXME: Show unresolved and (optionally) unsupported tests.
401 for title,code in (('Unexpected Passing Tests', Test.XPASS),
402 ('Failing Tests', Test.FAIL)):
403 elts = byCode.get(code)
407 print('%s (%d):' % (title, len(elts)))
409 print(' %s' % t.getFullName())
410 sys.stdout.write('\n')
413 # Collate, in case we repeated tests.
416 key = t.getFullName()
417 times[key] = times.get(key, 0.) + t.elapsed
419 byTime = list(times.items())
420 byTime.sort(key = lambda item: item[1])
422 Util.printHistogram(byTime, title='Tests')
424 for name,code in (('Expected Passes ', Test.PASS),
425 ('Expected Failures ', Test.XFAIL),
426 ('Unsupported Tests ', Test.UNSUPPORTED),
427 ('Unresolved Tests ', Test.UNRESOLVED),
428 ('Unexpected Passes ', Test.XPASS),
429 ('Unexpected Failures', Test.FAIL),):
430 if opts.quiet and not code.isFailure:
432 N = len(byCode.get(code,[]))
434 print(' %s: %d' % (name,N))
436 # If we encountered any additional errors, exit abnormally.
437 if litConfig.numErrors:
438 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
441 # Warn about warnings.
442 if litConfig.numWarnings:
443 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
449 if __name__=='__main__':