4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 from __future__ import absolute_import
10 import math, os, platform, random, re, sys, time, threading, traceback
12 import lit.ProgressBar
19 class TestingProgressDisplay:
20 def __init__(self, opts, numTests, progressBar=None):
22 self.numTests = numTests
24 self.lock = threading.Lock()
25 self.progressBar = progressBar
28 def update(self, test):
29 # Avoid locking overhead in quiet mode
30 if self.opts.quiet and not test.result.code.isFailure:
37 self.handleUpdate(test)
43 self.progressBar.clear()
46 elif self.opts.succinct:
47 sys.stdout.write('\n')
49 def handleUpdate(self, test):
52 self.progressBar.update(float(self.completed)/self.numTests,
55 if self.opts.succinct and not test.result.code.isFailure:
59 self.progressBar.clear()
61 print('%s: %s (%d of %d)' % (test.result.code.name, test.getFullName(),
62 self.completed, self.numTests))
64 if test.result.code.isFailure and self.opts.showOutput:
65 print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
67 print(test.result.output)
73 def __init__(self, tests, maxTime):
74 self.maxTime = maxTime
75 self.iter = iter(tests)
76 self.lock = threading.Lock()
77 self.startTime = time.time()
86 # Check if we have run out of time.
87 if self.maxTime is not None:
88 if time.time() - self.startTime > self.maxTime:
91 # Otherwise take the next test.
96 for item in self.iter:
103 class Tester(threading.Thread):
104 def __init__(self, litConfig, provider, display):
105 threading.Thread.__init__(self)
106 self.litConfig = litConfig
107 self.provider = provider
108 self.display = display
112 item = self.provider.get()
117 def runTest(self, test):
119 startTime = time.time()
121 result, output = test.config.test_format.execute(test,
123 except KeyboardInterrupt:
124 # This is a sad hack. Unfortunately subprocess goes
125 # bonkers with ctrl-c and we start forking merrily.
126 print('\nCtrl-C detected, goodbye.')
129 if self.litConfig.debug:
131 result = lit.Test.UNRESOLVED
132 output = 'Exception during script execution:\n'
133 output += traceback.format_exc()
135 elapsed = time.time() - startTime
137 test.setResult(result, output, elapsed)
138 self.display.update(test)
140 def runTests(numThreads, litConfig, provider, display):
141 # If only using one testing thread, don't use threads at all; this lets us
142 # profile, among other things.
144 t = Tester(litConfig, provider, display)
148 # Otherwise spin up the testing threads and wait for them to finish.
149 testers = [Tester(litConfig, provider, display)
150 for i in range(numThreads)]
156 except KeyboardInterrupt:
159 def main(builtinParameters = {}):
160 # Bump the GIL check interval, its more important to get any one thread to a
161 # blocking operation (hopefully exec) than to try and unblock other threads.
163 # FIXME: This is a hack.
164 sys.setcheckinterval(1000)
167 from optparse import OptionParser, OptionGroup
168 parser = OptionParser("usage: %prog [options] {file-or-path}")
170 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
171 help="Number of testing threads",
172 type=int, action="store", default=None)
173 parser.add_option("", "--config-prefix", dest="configPrefix",
174 metavar="NAME", help="Prefix for 'lit' config files",
175 action="store", default=None)
176 parser.add_option("", "--param", dest="userParameters",
178 help="Add 'NAME' = 'VAL' to the user defined parameters",
179 type=str, action="append", default=[])
181 group = OptionGroup(parser, "Output Format")
182 # FIXME: I find these names very confusing, although I like the
184 group.add_option("-q", "--quiet", dest="quiet",
185 help="Suppress no error output",
186 action="store_true", default=False)
187 group.add_option("-s", "--succinct", dest="succinct",
188 help="Reduce amount of output",
189 action="store_true", default=False)
190 group.add_option("-v", "--verbose", dest="showOutput",
191 help="Show all test output",
192 action="store_true", default=False)
193 group.add_option("", "--no-progress-bar", dest="useProgressBar",
194 help="Do not use curses based progress bar",
195 action="store_false", default=True)
196 parser.add_option_group(group)
198 group = OptionGroup(parser, "Test Execution")
199 group.add_option("", "--path", dest="path",
200 help="Additional paths to add to testing environment",
201 action="append", type=str, default=[])
202 group.add_option("", "--vg", dest="useValgrind",
203 help="Run tests under valgrind",
204 action="store_true", default=False)
205 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
206 help="Check for memory leaks under valgrind",
207 action="store_true", default=False)
208 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
209 help="Specify an extra argument for valgrind",
210 type=str, action="append", default=[])
211 group.add_option("", "--time-tests", dest="timeTests",
212 help="Track elapsed wall time for each test",
213 action="store_true", default=False)
214 group.add_option("", "--no-execute", dest="noExecute",
215 help="Don't execute any tests (assume PASS)",
216 action="store_true", default=False)
217 parser.add_option_group(group)
219 group = OptionGroup(parser, "Test Selection")
220 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
221 help="Maximum number of tests to run",
222 action="store", type=int, default=None)
223 group.add_option("", "--max-time", dest="maxTime", metavar="N",
224 help="Maximum time to spend testing (in seconds)",
225 action="store", type=float, default=None)
226 group.add_option("", "--shuffle", dest="shuffle",
227 help="Run tests in random order",
228 action="store_true", default=False)
229 group.add_option("", "--filter", dest="filter", metavar="REGEX",
230 help=("Only run tests with paths matching the given "
231 "regular expression"),
232 action="store", default=None)
233 parser.add_option_group(group)
235 group = OptionGroup(parser, "Debug and Experimental Options")
236 group.add_option("", "--debug", dest="debug",
237 help="Enable debugging (for 'lit' development)",
238 action="store_true", default=False)
239 group.add_option("", "--show-suites", dest="showSuites",
240 help="Show discovered test suites",
241 action="store_true", default=False)
242 group.add_option("", "--show-tests", dest="showTests",
243 help="Show all discovered tests",
244 action="store_true", default=False)
245 parser.add_option_group(group)
247 (opts, args) = parser.parse_args()
250 parser.error('No inputs specified')
252 if opts.numThreads is None:
253 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
254 # http://bugs.python.org/issue1731717
255 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
256 # threads by default there.
257 if sys.hexversion >= 0x2050200:
258 opts.numThreads = lit.util.detectCPUs()
264 # Create the user defined parameters.
265 userParams = dict(builtinParameters)
266 for entry in opts.userParameters:
270 name,val = entry.split('=', 1)
271 userParams[name] = val
273 # Create the global config object.
274 litConfig = lit.LitConfig.LitConfig(
275 progname = os.path.basename(sys.argv[0]),
278 useValgrind = opts.useValgrind,
279 valgrindLeakCheck = opts.valgrindLeakCheck,
280 valgrindArgs = opts.valgrindArgs,
281 noExecute = opts.noExecute,
283 isWindows = (platform.system()=='Windows'),
285 config_prefix = opts.configPrefix)
287 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
289 if opts.showSuites or opts.showTests:
290 # Aggregate the tests by suite.
293 if t.suite not in suitesAndTests:
294 suitesAndTests[t.suite] = []
295 suitesAndTests[t.suite].append(t)
296 suitesAndTests = list(suitesAndTests.items())
297 suitesAndTests.sort(key = lambda item: item[0].name)
299 # Show the suites, if requested.
301 print('-- Test Suites --')
302 for ts,ts_tests in suitesAndTests:
303 print(' %s - %d tests' %(ts.name, len(ts_tests)))
304 print(' Source Root: %s' % ts.source_root)
305 print(' Exec Root : %s' % ts.exec_root)
307 # Show the tests, if requested.
309 print('-- Available Tests --')
310 for ts,ts_tests in suitesAndTests:
311 ts_tests.sort(key = lambda test: test.path_in_suite)
312 for test in ts_tests:
313 print(' %s' % (test.getFullName(),))
318 # Select and order the tests.
319 numTotalTests = len(tests)
321 # First, select based on the filter expression if given.
324 rex = re.compile(opts.filter)
326 parser.error("invalid regular expression for --filter: %r" % (
328 tests = [t for t in tests
329 if rex.search(t.getFullName())]
331 # Then select the order.
333 random.shuffle(tests)
335 tests.sort(key = lambda t: t.getFullName())
337 # Finally limit the number of tests, if desired.
338 if opts.maxTests is not None:
339 tests = tests[:opts.maxTests]
341 # Don't create more threads than tests.
342 opts.numThreads = min(len(tests), opts.numThreads)
345 if len(tests) != numTotalTests:
346 extra = ' of %d' % numTotalTests
347 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
352 if opts.succinct and opts.useProgressBar:
354 tc = lit.ProgressBar.TerminalController()
355 progressBar = lit.ProgressBar.ProgressBar(tc, header)
358 progressBar = lit.ProgressBar.SimpleProgressBar('Testing: ')
362 startTime = time.time()
363 display = TestingProgressDisplay(opts, len(tests), progressBar)
364 provider = TestProvider(tests, opts.maxTime)
371 def console_ctrl_handler(type):
374 win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
376 runTests(opts.numThreads, litConfig, provider, display)
380 print('Testing Time: %.2fs'%(time.time() - startTime))
382 # Update results for any tests which weren't run.
384 if test.result is None:
385 test.setResult(lit.Test.UNRESOLVED, '', 0.0)
387 # List test results organized by kind.
391 if test.result.code not in byCode:
392 byCode[test.result.code] = []
393 byCode[test.result.code].append(test)
394 if test.result.code.isFailure:
397 # Print each test in any of the failing groups.
398 for title,code in (('Unexpected Passing Tests', lit.Test.XPASS),
399 ('Failing Tests', lit.Test.FAIL),
400 ('Unresolved Tests', lit.Test.UNRESOLVED)):
401 elts = byCode.get(code)
405 print('%s (%d):' % (title, len(elts)))
407 print(' %s' % test.getFullName())
408 sys.stdout.write('\n')
410 if opts.timeTests and tests:
412 test_times = [(test.getFullName(), test.result.elapsed)
414 lit.util.printHistogram(test_times, title='Tests')
416 for name,code in (('Expected Passes ', lit.Test.PASS),
417 ('Expected Failures ', lit.Test.XFAIL),
418 ('Unsupported Tests ', lit.Test.UNSUPPORTED),
419 ('Unresolved Tests ', lit.Test.UNRESOLVED),
420 ('Unexpected Passes ', lit.Test.XPASS),
421 ('Unexpected Failures', lit.Test.FAIL),):
422 if opts.quiet and not code.isFailure:
424 N = len(byCode.get(code,[]))
426 print(' %s: %d' % (name,N))
428 # If we encountered any additional errors, exit abnormally.
429 if litConfig.numErrors:
430 sys.stderr.write('\n%d error(s), exiting.\n' % litConfig.numErrors)
433 # Warn about warnings.
434 if litConfig.numWarnings:
435 sys.stderr.write('\n%d warning(s) in tests.\n' % litConfig.numWarnings)
441 if __name__=='__main__':