4 lit - LLVM Integrated Tester.
6 See lit.pod for more information.
9 import math, os, platform, random, re, sys, time, threading, traceback
20 class TestingProgressDisplay:
21 def __init__(self, opts, numTests, progressBar=None):
23 self.numTests = numTests
25 self.lock = threading.Lock()
26 self.progressBar = progressBar
29 def update(self, test):
30 # Avoid locking overhead in quiet mode
31 if self.opts.quiet and not test.result.isFailure:
38 self.handleUpdate(test)
44 self.progressBar.clear()
47 elif self.opts.succinct:
48 sys.stdout.write('\n')
50 def handleUpdate(self, test):
53 self.progressBar.update(float(self.completed)/self.numTests,
56 if self.opts.succinct and not test.result.isFailure:
60 self.progressBar.clear()
62 print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(),
63 self.completed, self.numTests)
65 if test.result.isFailure and self.opts.showOutput:
66 print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
74 def __init__(self, tests, maxTime):
75 self.maxTime = maxTime
76 self.iter = iter(tests)
77 self.lock = threading.Lock()
78 self.startTime = time.time()
81 # Check if we have run out of time.
82 if self.maxTime is not None:
83 if time.time() - self.startTime > self.maxTime:
86 # Otherwise take the next test.
89 item = self.iter.next()
95 class Tester(threading.Thread):
96 def __init__(self, litConfig, provider, display):
97 threading.Thread.__init__(self)
98 self.litConfig = litConfig
99 self.provider = provider
100 self.display = display
104 item = self.provider.get()
109 def runTest(self, test):
111 startTime = time.time()
113 result, output = test.config.test_format.execute(test,
115 except KeyboardInterrupt:
116 # This is a sad hack. Unfortunately subprocess goes
117 # bonkers with ctrl-c and we start forking merrily.
118 print '\nCtrl-C detected, goodbye.'
121 if self.litConfig.debug:
123 result = Test.UNRESOLVED
124 output = 'Exception during script execution:\n'
125 output += traceback.format_exc()
127 elapsed = time.time() - startTime
129 test.setResult(result, output, elapsed)
130 self.display.update(test)
132 def runTests(numThreads, litConfig, provider, display):
133 # If only using one testing thread, don't use threads at all; this lets us
134 # profile, among other things.
136 t = Tester(litConfig, provider, display)
140 # Otherwise spin up the testing threads and wait for them to finish.
141 testers = [Tester(litConfig, provider, display)
142 for i in range(numThreads)]
148 except KeyboardInterrupt:
151 def main(builtinParameters = {}):
152 # Bump the GIL check interval, its more important to get any one thread to a
153 # blocking operation (hopefully exec) than to try and unblock other threads.
155 # FIXME: This is a hack.
157 sys.setcheckinterval(1000)
160 from optparse import OptionParser, OptionGroup
161 parser = OptionParser("usage: %prog [options] {file-or-path}")
163 parser.add_option("-j", "--threads", dest="numThreads", metavar="N",
164 help="Number of testing threads",
165 type=int, action="store", default=None)
166 parser.add_option("", "--config-prefix", dest="configPrefix",
167 metavar="NAME", help="Prefix for 'lit' config files",
168 action="store", default=None)
169 parser.add_option("", "--param", dest="userParameters",
171 help="Add 'NAME' = 'VAL' to the user defined parameters",
172 type=str, action="append", default=[])
174 group = OptionGroup(parser, "Output Format")
175 # FIXME: I find these names very confusing, although I like the
177 group.add_option("-q", "--quiet", dest="quiet",
178 help="Suppress no error output",
179 action="store_true", default=False)
180 group.add_option("-s", "--succinct", dest="succinct",
181 help="Reduce amount of output",
182 action="store_true", default=False)
183 group.add_option("-v", "--verbose", dest="showOutput",
184 help="Show all test output",
185 action="store_true", default=False)
186 group.add_option("", "--no-progress-bar", dest="useProgressBar",
187 help="Do not use curses based progress bar",
188 action="store_false", default=True)
189 parser.add_option_group(group)
191 group = OptionGroup(parser, "Test Execution")
192 group.add_option("", "--path", dest="path",
193 help="Additional paths to add to testing environment",
194 action="append", type=str, default=[])
195 group.add_option("", "--vg", dest="useValgrind",
196 help="Run tests under valgrind",
197 action="store_true", default=False)
198 group.add_option("", "--vg-leak", dest="valgrindLeakCheck",
199 help="Check for memory leaks under valgrind",
200 action="store_true", default=False)
201 group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG",
202 help="Specify an extra argument for valgrind",
203 type=str, action="append", default=[])
204 group.add_option("", "--time-tests", dest="timeTests",
205 help="Track elapsed wall time for each test",
206 action="store_true", default=False)
207 group.add_option("", "--no-execute", dest="noExecute",
208 help="Don't execute any tests (assume PASS)",
209 action="store_true", default=False)
210 parser.add_option_group(group)
212 group = OptionGroup(parser, "Test Selection")
213 group.add_option("", "--max-tests", dest="maxTests", metavar="N",
214 help="Maximum number of tests to run",
215 action="store", type=int, default=None)
216 group.add_option("", "--max-time", dest="maxTime", metavar="N",
217 help="Maximum time to spend testing (in seconds)",
218 action="store", type=float, default=None)
219 group.add_option("", "--shuffle", dest="shuffle",
220 help="Run tests in random order",
221 action="store_true", default=False)
222 group.add_option("", "--filter", dest="filter", metavar="REGEX",
223 help=("Only run tests with paths matching the given "
224 "regular expression"),
225 action="store", default=None)
226 parser.add_option_group(group)
228 group = OptionGroup(parser, "Debug and Experimental Options")
229 group.add_option("", "--debug", dest="debug",
230 help="Enable debugging (for 'lit' development)",
231 action="store_true", default=False)
232 group.add_option("", "--show-suites", dest="showSuites",
233 help="Show discovered test suites",
234 action="store_true", default=False)
235 group.add_option("", "--repeat", dest="repeatTests", metavar="N",
236 help="Repeat tests N times (for timing)",
237 action="store", default=None, type=int)
238 parser.add_option_group(group)
240 (opts, args) = parser.parse_args()
243 parser.error('No inputs specified')
245 if opts.numThreads is None:
246 # Python <2.5 has a race condition causing lit to always fail with numThreads>1
247 # http://bugs.python.org/issue1731717
248 # I haven't seen this bug occur with 2.5.2 and later, so only enable multiple
249 # threads by default there.
250 if sys.hexversion >= 0x2050200:
251 opts.numThreads = Util.detectCPUs()
257 # Create the user defined parameters.
258 userParams = dict(builtinParameters)
259 for entry in opts.userParameters:
263 name,val = entry.split('=', 1)
264 userParams[name] = val
266 # Create the global config object.
267 litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]),
270 useValgrind = opts.useValgrind,
271 valgrindLeakCheck = opts.valgrindLeakCheck,
272 valgrindArgs = opts.valgrindArgs,
273 noExecute = opts.noExecute,
274 ignoreStdErr = False,
276 isWindows = (platform.system()=='Windows'),
278 config_prefix = opts.configPrefix)
280 tests = lit.discovery.find_tests_for_inputs(litConfig, inputs)
285 if t.suite not in suitesAndTests:
286 suitesAndTests[t.suite] = []
287 suitesAndTests[t.suite].append(t)
289 print '-- Test Suites --'
290 suitesAndTests = suitesAndTests.items()
291 suitesAndTests.sort(key = lambda (ts,_): ts.name)
292 for ts,ts_tests in suitesAndTests:
293 print ' %s - %d tests' %(ts.name, len(ts_tests))
294 print ' Source Root: %s' % ts.source_root
295 print ' Exec Root : %s' % ts.exec_root
297 # Select and order the tests.
298 numTotalTests = len(tests)
300 # First, select based on the filter expression if given.
303 rex = re.compile(opts.filter)
305 parser.error("invalid regular expression for --filter: %r" % (
307 tests = [t for t in tests
308 if rex.search(t.getFullName())]
310 # Then select the order.
312 random.shuffle(tests)
314 tests.sort(key = lambda t: t.getFullName())
316 # Finally limit the number of tests, if desired.
317 if opts.maxTests is not None:
318 tests = tests[:opts.maxTests]
320 # Don't create more threads than tests.
321 opts.numThreads = min(len(tests), opts.numThreads)
324 if len(tests) != numTotalTests:
325 extra = ' of %d' % numTotalTests
326 header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra,
330 tests = [t.copyWithIndex(i)
332 for i in range(opts.repeatTests)]
336 if opts.succinct and opts.useProgressBar:
338 tc = ProgressBar.TerminalController()
339 progressBar = ProgressBar.ProgressBar(tc, header)
342 progressBar = ProgressBar.SimpleProgressBar('Testing: ')
346 startTime = time.time()
347 display = TestingProgressDisplay(opts, len(tests), progressBar)
348 provider = TestProvider(tests, opts.maxTime)
349 runTests(opts.numThreads, litConfig, provider, display)
353 print 'Testing Time: %.2fs'%(time.time() - startTime)
355 # Update results for any tests which weren't run.
358 t.setResult(Test.UNRESOLVED, '', 0.0)
360 # List test results organized by kind.
364 if t.result not in byCode:
365 byCode[t.result] = []
366 byCode[t.result].append(t)
367 if t.result.isFailure:
370 # FIXME: Show unresolved and (optionally) unsupported tests.
371 for title,code in (('Unexpected Passing Tests', Test.XPASS),
372 ('Failing Tests', Test.FAIL)):
373 elts = byCode.get(code)
377 print '%s (%d):' % (title, len(elts))
379 print ' %s' % t.getFullName()
383 # Collate, in case we repeated tests.
386 key = t.getFullName()
387 times[key] = times.get(key, 0.) + t.elapsed
389 byTime = list(times.items())
390 byTime.sort(key = lambda (name,elapsed): elapsed)
392 Util.printHistogram(byTime, title='Tests')
394 for name,code in (('Expected Passes ', Test.PASS),
395 ('Expected Failures ', Test.XFAIL),
396 ('Unsupported Tests ', Test.UNSUPPORTED),
397 ('Unresolved Tests ', Test.UNRESOLVED),
398 ('Unexpected Passes ', Test.XPASS),
399 ('Unexpected Failures', Test.FAIL),):
400 if opts.quiet and not code.isFailure:
402 N = len(byCode.get(code,[]))
404 print ' %s: %d' % (name,N)
406 # If we encountered any additional errors, exit abnormally.
407 if litConfig.numErrors:
408 print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors
411 # Warn about warnings.
412 if litConfig.numWarnings:
413 print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings
419 if __name__=='__main__':