4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 from __future__ import absolute_import
10 import math, os, platform, random, re, sys, time, threading, traceback
12 import lit.ProgressBar
19 class TestingProgressDisplay:
20 def __init__(self, opts, numTests, progressBar=None):
22 self.numTests = numTests
24 self.lock = threading.Lock()
25 self.progressBar = progressBar
28 def update(self, test):
29 # Avoid locking overhead in quiet mode
30 if self.opts.quiet and not test.result.isFailure:
37 self.handleUpdate(test)
43 self.progressBar.clear()
46 elif self.opts.succinct:
47 sys.stdout.write('\n')
49 def handleUpdate(self, test):
52 self.progressBar.update(float(self.completed)/self.numTests,
55 if self.opts.succinct and not test.result.isFailure:
59 self.progressBar.clear()
61 print('%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
62 self.completed, self.numTests))
64 if test.result.isFailure and self.opts.showOutput:
65 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
73 def __init__(self, tests, maxTime):
74 self.maxTime = maxTime
75 self.iter = iter(tests)
76 self.lock = threading.Lock()
77 self.startTime = time.time()
86 # Check if we have run out of time.
87 if self.maxTime is not None:
88 if time.time() - self.startTime > self.maxTime:
91 # Otherwise take the next test.
96 for item in self.iter:
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print('\nCtrl-C detected, goodbye.')
129 if self.litConfig.debug:
131 result = lit.Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def runTests(numThreads, litConfig, provider, display):
141 # If only using one testing thread, don't use threads at all; this lets us
142 # profile, among other things.
144 t = Tester(litConfig, provider, display)
148 # Otherwise spin up the testing threads and wait for them to finish.
149 testers = [Tester(litConfig, provider, display)
150 for i in range(numThreads)]
156 except KeyboardInterrupt:
159 def main(builtinParameters = {}):
160 # Bump the GIL check interval, its more important to get any one thread to a
161 # blocking operation (hopefully exec) than to try and unblock other threads.
163 # FIXME: This is a hack.
165 sys.setcheckinterval(1000)
168 from optparse import OptionParser, OptionGroup
169 parser = OptionParser("usage: %prog [options] {file-or-path}")
171 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
172 help="Number of testing threads",
173 type=int, action="store", default=None)
174 parser.add_option("", "--config-prefix", dest="configPrefix",
175 metavar="NAME", help="Prefix for 'lit' config files",
176 action="store", default=None)
177 parser.add_option("", "--param", dest="userParameters",
179 help="Add 'NAME' = 'VAL' to the user defined parameters",
180 type=str, action="append", default=[])
182 group = OptionGroup(parser, "Output Format")
183 # FIXME: I find these names very confusing, although I like the
185 group.add_option("-q", "--quiet", dest="quiet",
186 help="Suppress no error output",
187 action="store_true", default=False)
188 group.add_option("-s", "--succinct", dest="succinct",
189 help="Reduce amount of output",
190 action="store_true", default=False)
191 group.add_option("-v", "--verbose", dest="showOutput",
192 help="Show all test output",
193 action="store_true", default=False)
194 group.add_option("", "--no-progress-bar", dest="useProgressBar",
195 help="Do not use curses based progress bar",
196 action="store_false", default=True)
197 parser.add_option_group(group)
199 group = OptionGroup(parser, "Test Execution")
200 group.add_option("", "--path", dest="path",
201 help="Additional paths to add to testing environment",
202 action="append", type=str, default=[])
203 group.add_option("", "--vg", dest="useValgrind",
204 help="Run tests under valgrind",
205 action="store_true", default=False)
206 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
207 help="Check for memory leaks under valgrind",
208 action="store_true", default=False)
209 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
210 help="Specify an extra argument for valgrind",
211 type=str, action="append", default=[])
212 group.add_option("", "--time-tests", dest="timeTests",
213 help="Track elapsed wall time for each test",
214 action="store_true", default=False)
215 group.add_option("", "--no-execute", dest="noExecute",
216 help="Don't execute any tests (assume PASS)",
217 action="store_true", default=False)
218 parser.add_option_group(group)
220 group = OptionGroup(parser, "Test Selection")
221 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
222 help="Maximum number of tests to run",
223 action="store", type=int, default=None)
224 group.add_option("", "--max-time", dest="maxTime", metavar="N",
225 help="Maximum time to spend testing (in seconds)",
226 action="store", type=float, default=None)
227 group.add_option("", "--shuffle", dest="shuffle",
228 help="Run tests in random order",
229 action="store_true", default=False)
230 group.add_option("", "--filter", dest="filter", metavar="REGEX",
231 help=("Only run tests with paths matching the given "
232 "regular expression"),
233 action="store", default=None)
234 parser.add_option_group(group)
236 group = OptionGroup(parser, "Debug and Experimental Options")
237 group.add_option("", "--debug", dest="debug",
238 help="Enable debugging (for 'lit' development)",
239 action="store_true", default=False)
240 group.add_option("", "--show-suites", dest="showSuites",
241 help="Show discovered test suites",
242 action="store_true", default=False)
243 group.add_option("", "--show-tests", dest="showTests",
244 help="Show all discovered tests",
245 action="store_true", default=False)
246 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
247 help="Repeat tests N times (for timing)",
248 action="store", default=None, type=int)
249 parser.add_option_group(group)
251 (opts, args) = parser.parse_args()
254 parser.error('No inputs specified')
256 if opts.numThreads is None:
257 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
258 # http://bugs.python.org/issue1731717
259 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
260 # threads by default there.
261 if sys.hexversion >= 0x2050200:
262 opts.numThreads = lit.Util.detectCPUs()
268 # Create the user defined parameters.
269 userParams = dict(builtinParameters)
270 for entry in opts.userParameters:
274 name,val = entry.split('=', 1)
275 userParams[name] = val
277 # Create the global config object.
278 litConfig = lit.LitConfig.LitConfig(
279 progname = os.path.basename(sys.argv[0]),
282 useValgrind = opts.useValgrind,
283 valgrindLeakCheck = opts.valgrindLeakCheck,
284 valgrindArgs = opts.valgrindArgs,
285 noExecute = opts.noExecute,
287 isWindows = (platform.system()=='Windows'),
289 config_prefix = opts.configPrefix)
291 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
293 if opts.showSuites or opts.showTests:
294 # Aggregate the tests by suite.
297 if t.suite not in suitesAndTests:
298 suitesAndTests[t.suite] = []
299 suitesAndTests[t.suite].append(t)
300 suitesAndTests = list(suitesAndTests.items())
301 suitesAndTests.sort(key = lambda item: item[0].name)
303 # Show the suites, if requested.
305 print('-- Test Suites --')
306 for ts,ts_tests in suitesAndTests:
307 print(' %s - %d tests' %(ts.name, len(ts_tests)))
308 print(' Source Root: %s' % ts.source_root)
309 print(' Exec Root : %s' % ts.exec_root)
311 # Show the tests, if requested.
313 print('-- Available Tests --')
314 for ts,ts_tests in suitesAndTests:
315 ts_tests.sort(key = lambda test: test.path_in_suite)
316 for test in ts_tests:
317 print(' %s' % (test.getFullName(),))
319 # Select and order the tests.
320 numTotalTests = len(tests)
322 # First, select based on the filter expression if given.
325 rex = re.compile(opts.filter)
327 parser.error("invalid regular expression for --filter: %r" % (
329 tests = [t for t in tests
330 if rex.search(t.getFullName())]
332 # Then select the order.
334 random.shuffle(tests)
336 tests.sort(key = lambda t: t.getFullName())
338 # Finally limit the number of tests, if desired.
339 if opts.maxTests is not None:
340 tests = tests[:opts.maxTests]
342 # Don't create more threads than tests.
343 opts.numThreads = min(len(tests), opts.numThreads)
346 if len(tests) != numTotalTests:
347 extra = ' of %d' % numTotalTests
348 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
352 tests = [t.copyWithIndex(i)
354 for i in range(opts.repeatTests)]
358 if opts.succinct and opts.useProgressBar:
360 tc = lit.ProgressBar.TerminalController()
361 progressBar = lit.ProgressBar.ProgressBar(tc, header)
364 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
368 startTime = time.time()
369 display = TestingProgressDisplay(opts, len(tests), progressBar)
370 provider = TestProvider(tests, opts.maxTime)
377 def console_ctrl_handler(type):
380 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
382 runTests(opts.numThreads, litConfig, provider, display)
386 print('Testing Time: %.2fs'%(time.time() - startTime))
388 # Update results for any tests which weren't run.
391 t.setResult(lit.Test.UNRESOLVED, '', 0.0)
393 # List test results organized by kind.
397 if t.result not in byCode:
398 byCode[t.result] = []
399 byCode[t.result].append(t)
400 if t.result.isFailure:
403 # FIXME: Show unresolved and (optionally) unsupported tests.
404 for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
405 ('Failing Tests', lit.Test.FAIL)):
406 elts = byCode.get(code)
410 print('%s (%d):' % (title, len(elts)))
412 print(' %s' % t.getFullName())
413 sys.stdout.write('\n')
416 # Collate, in case we repeated tests.
419 key = t.getFullName()
420 times[key] = times.get(key, 0.) + t.elapsed
422 byTime = list(times.items())
423 byTime.sort(key = lambda item: item[1])
425 lit.Util.printHistogram(byTime, title='Tests')
427 for name,code in (('Expected Passes ', lit.Test.PASS),
428 ('Expected Failures ', lit.Test.XFAIL),
429 ('Unsupported Tests ', lit.Test.UNSUPPORTED),
430 ('Unresolved Tests ', lit.Test.UNRESOLVED),
431 ('Unexpected Passes ', lit.Test.XPASS),
432 ('Unexpected Failures', lit.Test.FAIL),):
433 if opts.quiet and not code.isFailure:
435 N = len(byCode.get(code,[]))
437 print(' %s: %d' % (name,N))
439 # If we encountered any additional errors, exit abnormally.
440 if litConfig.numErrors:
441 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
444 # Warn about warnings.
445 if litConfig.numWarnings:
446 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
452 if __name__=='__main__':