4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 from __future__ import absolute_import
10 import math, os, platform, random, re, sys, time, threading, traceback
12 import lit.ProgressBar
20 class TestingProgressDisplay:
21 def __init__(self, opts, numTests, progressBar=None):
23 self.numTests = numTests
25 self.lock = threading.Lock()
26 self.progressBar = progressBar
29 def update(self, test):
30 # Avoid locking overhead in quiet mode
31 if self.opts.quiet and not test.result.code.isFailure:
38 self.handleUpdate(test)
44 self.progressBar.clear()
47 elif self.opts.succinct:
48 sys.stdout.write('\n')
50 def handleUpdate(self, test):
53 self.progressBar.update(float(self.completed)/self.numTests,
56 if self.opts.succinct and not test.result.code.isFailure:
60 self.progressBar.clear()
62 print('%s: %s (%d of %d)' % (test.result.code.name, test.getFullName(),
63 self.completed, self.numTests))
65 if test.result.code.isFailure and self.opts.showOutput:
66 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
68 print(test.result.output)
74 def __init__(self, tests, maxTime):
75 self.maxTime = maxTime
76 self.iter = iter(tests)
77 self.lock = threading.Lock()
78 self.startTime = time.time()
87 # Check if we have run out of time.
88 if self.maxTime is not None:
89 if time.time() - self.startTime > self.maxTime:
92 # Otherwise take the next test.
97 for item in self.iter:
104 class Tester(threading.Thread):
105 def __init__(self, run_instance, provider, display):
106 threading.Thread.__init__(self)
107 self.run_instance = run_instance
108 self.provider = provider
109 self.display = display
113 item = self.provider.get()
118 def runTest(self, test):
120 self.run_instance.execute_test(test)
121 except KeyboardInterrupt:
122 # This is a sad hack. Unfortunately subprocess goes
123 # bonkers with ctrl-c and we start forking merrily.
124 print('\nCtrl-C detected, goodbye.')
126 self.display.update(test)
128 def runTests(numThreads, run, provider, display):
129 # If only using one testing thread, don't use threads at all; this lets us
130 # profile, among other things.
132 t = Tester(run, provider, display)
136 # Otherwise spin up the testing threads and wait for them to finish.
137 testers = [Tester(run, provider, display)
138 for i in range(numThreads)]
144 except KeyboardInterrupt:
147 def main(builtinParameters = {}):
148 # Bump the GIL check interval, its more important to get any one thread to a
149 # blocking operation (hopefully exec) than to try and unblock other threads.
151 # FIXME: This is a hack.
152 sys.setcheckinterval(1000)
155 from optparse import OptionParser, OptionGroup
156 parser = OptionParser("usage: %prog [options] {file-or-path}")
158 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
159 help="Number of testing threads",
160 type=int, action="store", default=None)
161 parser.add_option("", "--config-prefix", dest="configPrefix",
162 metavar="NAME", help="Prefix for 'lit' config files",
163 action="store", default=None)
164 parser.add_option("", "--param", dest="userParameters",
166 help="Add 'NAME' = 'VAL' to the user defined parameters",
167 type=str, action="append", default=[])
169 group = OptionGroup(parser, "Output Format")
170 # FIXME: I find these names very confusing, although I like the
172 group.add_option("-q", "--quiet", dest="quiet",
173 help="Suppress no error output",
174 action="store_true", default=False)
175 group.add_option("-s", "--succinct", dest="succinct",
176 help="Reduce amount of output",
177 action="store_true", default=False)
178 group.add_option("-v", "--verbose", dest="showOutput",
179 help="Show all test output",
180 action="store_true", default=False)
181 group.add_option("", "--no-progress-bar", dest="useProgressBar",
182 help="Do not use curses based progress bar",
183 action="store_false", default=True)
184 parser.add_option_group(group)
186 group = OptionGroup(parser, "Test Execution")
187 group.add_option("", "--path", dest="path",
188 help="Additional paths to add to testing environment",
189 action="append", type=str, default=[])
190 group.add_option("", "--vg", dest="useValgrind",
191 help="Run tests under valgrind",
192 action="store_true", default=False)
193 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
194 help="Check for memory leaks under valgrind",
195 action="store_true", default=False)
196 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
197 help="Specify an extra argument for valgrind",
198 type=str, action="append", default=[])
199 group.add_option("", "--time-tests", dest="timeTests",
200 help="Track elapsed wall time for each test",
201 action="store_true", default=False)
202 group.add_option("", "--no-execute", dest="noExecute",
203 help="Don't execute any tests (assume PASS)",
204 action="store_true", default=False)
205 parser.add_option_group(group)
207 group = OptionGroup(parser, "Test Selection")
208 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
209 help="Maximum number of tests to run",
210 action="store", type=int, default=None)
211 group.add_option("", "--max-time", dest="maxTime", metavar="N",
212 help="Maximum time to spend testing (in seconds)",
213 action="store", type=float, default=None)
214 group.add_option("", "--shuffle", dest="shuffle",
215 help="Run tests in random order",
216 action="store_true", default=False)
217 group.add_option("", "--filter", dest="filter", metavar="REGEX",
218 help=("Only run tests with paths matching the given "
219 "regular expression"),
220 action="store", default=None)
221 parser.add_option_group(group)
223 group = OptionGroup(parser, "Debug and Experimental Options")
224 group.add_option("", "--debug", dest="debug",
225 help="Enable debugging (for 'lit' development)",
226 action="store_true", default=False)
227 group.add_option("", "--show-suites", dest="showSuites",
228 help="Show discovered test suites",
229 action="store_true", default=False)
230 group.add_option("", "--show-tests", dest="showTests",
231 help="Show all discovered tests",
232 action="store_true", default=False)
233 parser.add_option_group(group)
235 (opts, args) = parser.parse_args()
238 parser.error('No inputs specified')
240 if opts.numThreads is None:
241 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
242 # http://bugs.python.org/issue1731717
243 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
244 # threads by default there.
245 if sys.hexversion >= 0x2050200:
246 opts.numThreads = lit.util.detectCPUs()
252 # Create the user defined parameters.
253 userParams = dict(builtinParameters)
254 for entry in opts.userParameters:
258 name,val = entry.split('=', 1)
259 userParams[name] = val
261 # Create the global config object.
262 litConfig = lit.LitConfig.LitConfig(
263 progname = os.path.basename(sys.argv[0]),
266 useValgrind = opts.useValgrind,
267 valgrindLeakCheck = opts.valgrindLeakCheck,
268 valgrindArgs = opts.valgrindArgs,
269 noExecute = opts.noExecute,
271 isWindows = (platform.system()=='Windows'),
273 config_prefix = opts.configPrefix)
275 # Perform test discovery.
276 run = lit.run.Run(litConfig,
277 lit.discovery.find_tests_for_inputs(litConfig, inputs))
279 if opts.showSuites or opts.showTests:
280 # Aggregate the tests by suite.
283 if t.suite not in suitesAndTests:
284 suitesAndTests[t.suite] = []
285 suitesAndTests[t.suite].append(t)
286 suitesAndTests = list(suitesAndTests.items())
287 suitesAndTests.sort(key = lambda item: item[0].name)
289 # Show the suites, if requested.
291 print('-- Test Suites --')
292 for ts,ts_tests in suitesAndTests:
293 print(' %s - %d tests' %(ts.name, len(ts_tests)))
294 print(' Source Root: %s' % ts.source_root)
295 print(' Exec Root : %s' % ts.exec_root)
297 # Show the tests, if requested.
299 print('-- Available Tests --')
300 for ts,ts_tests in suitesAndTests:
301 ts_tests.sort(key = lambda test: test.path_in_suite)
302 for test in ts_tests:
303 print(' %s' % (test.getFullName(),))
308 # Select and order the tests.
309 numTotalTests = len(run.tests)
311 # First, select based on the filter expression if given.
314 rex = re.compile(opts.filter)
316 parser.error("invalid regular expression for --filter: %r" % (
318 run.tests = [t for t in run.tests
319 if rex.search(t.getFullName())]
321 # Then select the order.
323 random.shuffle(run.tests)
325 run.tests.sort(key = lambda t: t.getFullName())
327 # Finally limit the number of tests, if desired.
328 if opts.maxTests is not None:
329 run.tests = run.tests[:opts.maxTests]
331 # Don't create more threads than tests.
332 opts.numThreads = min(len(run.tests), opts.numThreads)
335 if len(run.tests) != numTotalTests:
336 extra = ' of %d' % numTotalTests
337 header = '-- Testing: %d%s tests, %d threads --'%(len(run.tests), extra,
342 if opts.succinct and opts.useProgressBar:
344 tc = lit.ProgressBar.TerminalController()
345 progressBar = lit.ProgressBar.ProgressBar(tc, header)
348 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
352 startTime = time.time()
353 display = TestingProgressDisplay(opts, len(run.tests), progressBar)
354 provider = TestProvider(run.tests, opts.maxTime)
361 def console_ctrl_handler(type):
364 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
366 runTests(opts.numThreads, run, provider, display)
370 print('Testing Time: %.2fs'%(time.time() - startTime))
372 # Update results for any tests which weren't run.
373 for test in run.tests:
374 if test.result is None:
375 test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
377 # List test results organized by kind.
380 for test in run.tests:
381 if test.result.code not in byCode:
382 byCode[test.result.code] = []
383 byCode[test.result.code].append(test)
384 if test.result.code.isFailure:
387 # Print each test in any of the failing groups.
388 for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
389 ('Failing Tests', lit.Test.FAIL),
390 ('Unresolved Tests', lit.Test.UNRESOLVED)):
391 elts = byCode.get(code)
395 print('%s (%d):' % (title, len(elts)))
397 print(' %s' % test.getFullName())
398 sys.stdout.write('\n')
400 if opts.timeTests and run.tests:
402 test_times = [(test.getFullName(), test.result.elapsed)
403 for test in run.tests]
404 lit.util.printHistogram(test_times, title='Tests')
406 for name,code in (('Expected Passes ', lit.Test.PASS),
407 ('Expected Failures ', lit.Test.XFAIL),
408 ('Unsupported Tests ', lit.Test.UNSUPPORTED),
409 ('Unresolved Tests ', lit.Test.UNRESOLVED),
410 ('Unexpected Passes ', lit.Test.XPASS),
411 ('Unexpected Failures', lit.Test.FAIL),):
412 if opts.quiet and not code.isFailure:
414 N = len(byCode.get(code,[]))
416 print(' %s: %d' % (name,N))
418 # If we encountered any additional errors, exit abnormally.
419 if litConfig.numErrors:
420 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
423 # Warn about warnings.
424 if litConfig.numWarnings:
425 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
431 if __name__=='__main__':