1 from __future__ import absolute_import
2 import os, signal, subprocess, sys
7 import lit.ShUtil as ShUtil
8 import lit.Test as Test
10 from lit.util import to_bytes, to_string
12 class InternalShellError(Exception):
13 def __init__(self, command, message):
14 self.command = command
15 self.message = message
17 kIsWindows = platform.system() == 'Windows'
19 # Don't use close_fds on Windows.
20 kUseCloseFDs = not kIsWindows
22 # Use temporary files to replace /dev/null on Windows.
23 kAvoidDevNull = kIsWindows
25 class ShellEnvironment(object):
27 """Mutable shell environment containing things like CWD and env vars.
29 Environment variables are not implemented, but cwd tracking is.
32 def __init__(self, cwd, env):
36 def executeShCmd(cmd, shenv, results):
37 if isinstance(cmd, ShUtil.Seq):
39 res = executeShCmd(cmd.lhs, shenv, results)
40 return executeShCmd(cmd.rhs, shenv, results)
43 raise InternalShellError(cmd,"unsupported shell operator: '&'")
46 res = executeShCmd(cmd.lhs, shenv, results)
48 res = executeShCmd(cmd.rhs, shenv, results)
52 res = executeShCmd(cmd.lhs, shenv, results)
57 res = executeShCmd(cmd.rhs, shenv, results)
60 raise ValueError('Unknown shell command: %r' % cmd.op)
61 assert isinstance(cmd, ShUtil.Pipeline)
63 # Handle shell builtins first.
64 if cmd.commands[0].args[0] == 'cd':
65 if len(cmd.commands) != 1:
66 raise ValueError("'cd' cannot be part of a pipeline")
67 if len(cmd.commands[0].args) != 2:
68 raise ValueError("'cd' supports only one argument")
69 newdir = cmd.commands[0].args[1]
70 # Update the cwd in the parent environment.
71 if os.path.isabs(newdir):
74 shenv.cwd = os.path.join(shenv.cwd, newdir)
75 # The cd builtin always succeeds. If the directory does not exist, the
76 # following Popen calls will fail instead.
80 input = subprocess.PIPE
84 # To avoid deadlock, we use a single stderr stream for piped
85 # output. This is null until we have seen some output using
87 for i,j in enumerate(cmd.commands):
88 # Reference the global environment by default.
90 if j.args[0] == 'env':
91 # Create a copy of the global environment and modify it for this one
92 # command. There might be multiple envs in a pipeline:
93 # env FOO=1 llc < %s | env BAR=2 llvm-mc | FileCheck %s
94 cmd_shenv = ShellEnvironment(shenv.cwd, shenv.env)
96 for arg_idx, arg in enumerate(j.args[1:]):
97 # Partition the string into KEY=VALUE.
98 key, eq, val = arg.partition('=')
99 # Stop if there was no equals.
102 cmd_shenv.env[key] = val
103 j.args = j.args[arg_idx+1:]
105 # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
106 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
107 # from a file are represented with a list [file, mode, file-object]
108 # where file-object is initially None.
109 redirects = [(0,), (1,), (2,)]
110 for r in j.redirects:
112 redirects[2] = [r[1], 'w', None]
113 elif r[0] == ('>>',2):
114 redirects[2] = [r[1], 'a', None]
115 elif r[0] == ('>&',2) and r[1] in '012':
116 redirects[2] = redirects[int(r[1])]
117 elif r[0] == ('>&',) or r[0] == ('&>',):
118 redirects[1] = redirects[2] = [r[1], 'w', None]
120 redirects[1] = [r[1], 'w', None]
121 elif r[0] == ('>>',):
122 redirects[1] = [r[1], 'a', None]
124 redirects[0] = [r[1], 'r', None]
126 raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
128 # Map from the final redirections to something subprocess can handle.
130 for index,r in enumerate(redirects):
135 raise InternalShellError(j,"Unsupported redirect for stdin")
137 result = subprocess.PIPE
139 result = subprocess.STDOUT
142 raise InternalShellError(j,"Unsupported redirect on stdout")
143 result = subprocess.PIPE
146 if kAvoidDevNull and r[0] == '/dev/null':
147 r[2] = tempfile.TemporaryFile(mode=r[1])
149 # Make sure relative paths are relative to the cwd.
150 redir_filename = os.path.join(cmd_shenv.cwd, r[0])
151 r[2] = open(redir_filename, r[1])
152 # Workaround a Win32 and/or subprocess bug when appending.
154 # FIXME: Actually, this is probably an instance of PR6753.
157 opened_files.append(r[2])
159 final_redirects.append(result)
161 stdin, stdout, stderr = final_redirects
163 # If stderr wants to come from stdout, but stdout isn't a pipe, then put
164 # stderr on a pipe and treat it as stdout.
165 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
166 stderr = subprocess.PIPE
167 stderrIsStdout = True
169 stderrIsStdout = False
171 # Don't allow stderr on a PIPE except for the last
172 # process, this could deadlock.
174 # FIXME: This is slow, but so is deadlock.
175 if stderr == subprocess.PIPE and j != cmd.commands[-1]:
176 stderr = tempfile.TemporaryFile(mode='w+b')
177 stderrTempFiles.append((i, stderr))
179 # Resolve the executable path ourselves.
182 # For paths relative to cwd, use the cwd of the shell environment.
183 if args[0].startswith('.'):
184 exe_in_cwd = os.path.join(cmd_shenv.cwd, args[0])
185 if os.path.isfile(exe_in_cwd):
186 executable = exe_in_cwd
188 executable = lit.util.which(args[0], cmd_shenv.env['PATH'])
190 raise InternalShellError(j, '%r: command not found' % j.args[0])
192 # Replace uses of /dev/null with temporary files.
194 for i,arg in enumerate(args):
195 if arg == "/dev/null":
196 f = tempfile.NamedTemporaryFile(delete=False)
198 named_temp_files.append(f.name)
202 procs.append(subprocess.Popen(args, cwd=cmd_shenv.cwd,
203 executable = executable,
208 close_fds = kUseCloseFDs))
210 raise InternalShellError(j, 'Could not create process ({}) due to {}'.format(executable, e))
212 # Immediately close stdin for any process taking stdin from us.
213 if stdin == subprocess.PIPE:
214 procs[-1].stdin.close()
215 procs[-1].stdin = None
217 # Update the current stdin source.
218 if stdout == subprocess.PIPE:
219 input = procs[-1].stdout
221 input = procs[-1].stderr
223 input = subprocess.PIPE
225 # Explicitly close any redirected files. We need to do this now because we
226 # need to release any handles we may have on the temporary files (important
227 # on Win32, for example). Since we have already spawned the subprocess, our
228 # handles have already been transferred so we do not need them anymore.
229 for f in opened_files:
232 # FIXME: There is probably still deadlock potential here. Yawn.
233 procData = [None] * len(procs)
234 procData[-1] = procs[-1].communicate()
236 for i in range(len(procs) - 1):
237 if procs[i].stdout is not None:
238 out = procs[i].stdout.read()
241 if procs[i].stderr is not None:
242 err = procs[i].stderr.read()
245 procData[i] = (out,err)
247 # Read stderr out of the temp files.
248 for i,f in stderrTempFiles:
250 procData[i] = (procData[i][0], f.read())
252 def to_string(bytes):
253 if isinstance(bytes, str):
255 return bytes.encode('utf-8')
258 for i,(out,err) in enumerate(procData):
259 res = procs[i].wait()
260 # Detect Ctrl-C in subprocess.
261 if res == -signal.SIGINT:
262 raise KeyboardInterrupt
264 # Ensure the resulting output is always of string type.
266 out = to_string(out.decode('utf-8'))
270 err = to_string(err.decode('utf-8'))
274 results.append((cmd.commands[i], out, err, res))
276 # Python treats the exit code as a signed char.
280 exitCode = min(exitCode, res)
282 exitCode = max(exitCode, res)
286 # Remove any named temporary files we created.
287 for f in named_temp_files:
294 exitCode = not exitCode
298 def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
302 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
303 test.config.pipefail).parse())
305 return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
309 cmd = ShUtil.Seq(cmd, '&&', c)
313 shenv = ShellEnvironment(cwd, test.config.environment)
314 exitCode = executeShCmd(cmd, shenv, results)
315 except InternalShellError:
316 e = sys.exc_info()[1]
318 results.append((e.command, '', e.message, exitCode))
321 for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
322 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
323 out += 'Command %d Result: %r\n' % (i, res)
324 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
325 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
327 return out, err, exitCode
329 def executeScript(test, litConfig, tmpBase, commands, cwd):
330 bashPath = litConfig.getBashPath();
331 isWin32CMDEXE = (litConfig.isWindows and not bashPath)
332 script = tmpBase + '.script'
338 if litConfig.isWindows and not isWin32CMDEXE:
339 mode += 'b' # Avoid CRLFs when writing bash scripts.
340 f = open(script, mode)
342 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
344 if test.config.pipefail:
345 f.write('set -o pipefail;')
346 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
351 command = ['cmd','/c', script]
354 command = [bashPath, script]
356 command = ['/bin/sh', script]
357 if litConfig.useValgrind:
358 # FIXME: Running valgrind on sh is overkill. We probably could just
359 # run on clang with no real loss.
360 command = litConfig.valgrindArgs + command
362 return lit.util.executeCommand(command, cwd=cwd,
363 env=test.config.environment)
365 def parseIntegratedTestScriptCommands(source_path, keywords):
367 parseIntegratedTestScriptCommands(source_path) -> commands
369 Parse the commands in an integrated test script file into a list of
370 (line_number, command_type, line).
373 # This code is carefully written to be dual compatible with Python 2.5+ and
374 # Python 3 without requiring input files to always have valid codings. The
375 # trick we use is to open the file in binary mode and use the regular
376 # expression library to find the commands, with it scanning strings in
377 # Python2 and bytes in Python3.
379 # Once we find a match, we do require each script line to be decodable to
380 # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
381 # remaining code can work with "strings" agnostic of the executing Python
384 keywords_re = re.compile(
385 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
387 f = open(source_path, 'rb')
389 # Read the entire file contents.
392 # Ensure the data ends with a newline.
393 if not data.endswith(to_bytes('\n')):
394 data = data + to_bytes('\n')
396 # Iterate over the matches.
398 last_match_position = 0
399 for match in keywords_re.finditer(data):
400 # Compute the updated line number by counting the intervening
402 match_position = match.start()
403 line_number += data.count(to_bytes('\n'), last_match_position,
405 last_match_position = match_position
407 # Convert the keyword and line to UTF-8 strings and yield the
408 # command. Note that we take care to return regular strings in
409 # Python 2, to avoid other code having to differentiate between the
410 # str and unicode types.
411 keyword,ln = match.groups()
412 yield (line_number, to_string(keyword[:-1].decode('utf-8')),
413 to_string(ln.decode('utf-8')))
417 def getTempPaths(test):
418 """Get the temporary location, this is always relative to the test suite
419 root, not test source root."""
420 execpath = test.getExecPath()
421 execdir,execbase = os.path.split(execpath)
422 tmpDir = os.path.join(execdir, 'Output')
423 tmpBase = os.path.join(tmpDir, execbase)
424 return tmpDir, tmpBase
426 def getDefaultSubstitutions(test, tmpDir, tmpBase, normalize_slashes=False):
427 sourcepath = test.getSourcePath()
428 sourcedir = os.path.dirname(sourcepath)
430 # Normalize slashes, if requested.
431 if normalize_slashes:
432 sourcepath = sourcepath.replace('\\', '/')
433 sourcedir = sourcedir.replace('\\', '/')
434 tmpDir = tmpDir.replace('\\', '/')
435 tmpBase = tmpBase.replace('\\', '/')
437 # We use #_MARKER_# to hide %% while we do the other substitutions.
439 substitutions.extend([('%%', '#_MARKER_#')])
440 substitutions.extend(test.config.substitutions)
441 substitutions.extend([('%s', sourcepath),
444 ('%{pathsep}', os.pathsep),
445 ('%t', tmpBase + '.tmp'),
447 ('#_MARKER_#', '%')])
449 # "%/[STpst]" should be normalized.
450 substitutions.extend([
451 ('%/s', sourcepath.replace('\\', '/')),
452 ('%/S', sourcedir.replace('\\', '/')),
453 ('%/p', sourcedir.replace('\\', '/')),
454 ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
455 ('%/T', tmpDir.replace('\\', '/')),
459 def applySubstitutions(script, substitutions):
460 """Apply substitutions to the script. Allow full regular expression syntax.
461 Replace each matching occurrence of regular expression pattern a with
462 substitution b in line ln."""
464 # Apply substitutions
465 for a,b in substitutions:
467 b = b.replace("\\","\\\\")
468 ln = re.sub(a, b, ln)
470 # Strip the trailing newline and any extra whitespace.
472 # Note Python 3 map() gives an iterator rather than a list so explicitly
473 # convert to list before returning.
474 return list(map(processLine, script))
476 def parseIntegratedTestScript(test, require_script=True):
477 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
478 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
479 and 'UNSUPPORTED' information. If 'require_script' is False an empty script
480 may be returned. This can be used for test formats where the actual script
481 is optional or ignored.
483 # Collect the test lines from the script.
484 sourcepath = test.getSourcePath()
488 keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
489 for line_number, command_type, ln in \
490 parseIntegratedTestScriptCommands(sourcepath, keywords):
491 if command_type == 'RUN':
492 # Trim trailing whitespace.
495 # Substitute line number expressions
496 ln = re.sub('%\(line\)', str(line_number), ln)
497 def replace_line_number(match):
498 if match.group(1) == '+':
499 return str(line_number + int(match.group(2)))
500 if match.group(1) == '-':
501 return str(line_number - int(match.group(2)))
502 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
504 # Collapse lines with trailing '\\'.
505 if script and script[-1][-1] == '\\':
506 script[-1] = script[-1][:-1] + ln
509 elif command_type == 'XFAIL':
510 test.xfails.extend([s.strip() for s in ln.split(',')])
511 elif command_type == 'REQUIRES':
512 requires.extend([s.strip() for s in ln.split(',')])
513 elif command_type == 'UNSUPPORTED':
514 unsupported.extend([s.strip() for s in ln.split(',')])
515 elif command_type == 'END':
516 # END commands are only honored if the rest of the line is empty.
520 raise ValueError("unknown script command type: %r" % (
523 # Verify the script contains a run line.
524 if require_script and not script:
525 return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
527 # Check for unterminated run lines.
528 if script and script[-1][-1] == '\\':
529 return lit.Test.Result(Test.UNRESOLVED,
530 "Test has unterminated run lines (with '\\')")
532 # Check that we have the required features:
533 missing_required_features = [f for f in requires
534 if f not in test.config.available_features]
535 if missing_required_features:
536 msg = ', '.join(missing_required_features)
537 return lit.Test.Result(Test.UNSUPPORTED,
538 "Test requires the following features: %s" % msg)
539 unsupported_features = [f for f in unsupported
540 if f in test.config.available_features]
541 if unsupported_features:
542 msg = ', '.join(unsupported_features)
543 return lit.Test.Result(Test.UNSUPPORTED,
544 "Test is unsupported with the following features: %s" % msg)
546 unsupported_targets = [f for f in unsupported
547 if f in test.suite.config.target_triple]
548 if unsupported_targets:
549 return lit.Test.Result(Test.UNSUPPORTED,
550 "Test is unsupported with the following triple: %s" % (
551 test.suite.config.target_triple,))
553 if test.config.limit_to_features:
554 # Check that we have one of the limit_to_features features in requires.
555 limit_to_features_tests = [f for f in test.config.limit_to_features
557 if not limit_to_features_tests:
558 msg = ', '.join(test.config.limit_to_features)
559 return lit.Test.Result(Test.UNSUPPORTED,
560 "Test requires one of the limit_to_features features %s" % msg)
564 def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
565 # Create the output directory if it does not already exist.
566 lit.util.mkdir_p(os.path.dirname(tmpBase))
568 execdir = os.path.dirname(test.getExecPath())
570 res = executeScript(test, litConfig, tmpBase, script, execdir)
572 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
573 if isinstance(res, lit.Test.Result):
576 out,err,exitCode = res
582 # Form the output log.
583 output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % (
584 '\n'.join(script), exitCode)
586 # Append the outputs, if present.
588 output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
590 output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
592 return lit.Test.Result(status, output)
595 def executeShTest(test, litConfig, useExternalSh,
596 extra_substitutions=[]):
597 if test.config.unsupported:
598 return (Test.UNSUPPORTED, 'Test is unsupported')
600 script = parseIntegratedTestScript(test)
601 if isinstance(script, lit.Test.Result):
603 if litConfig.noExecute:
604 return lit.Test.Result(Test.PASS)
606 tmpDir, tmpBase = getTempPaths(test)
607 substitutions = list(extra_substitutions)
608 substitutions += getDefaultSubstitutions(test, tmpDir, tmpBase,
609 normalize_slashes=useExternalSh)
610 script = applySubstitutions(script, substitutions)
612 # Re-run failed tests up to test_retry_attempts times.
614 if hasattr(test.config, 'test_retry_attempts'):
615 attempts += test.config.test_retry_attempts
616 for i in range(attempts):
617 res = _runShTest(test, litConfig, useExternalSh, script, tmpBase)
618 if res.code != Test.FAIL:
620 # If we had to run the test more than once, count it as a flaky pass. These
621 # will be printed separately in the test summary.
622 if i > 0 and res.code == Test.PASS:
623 res.code = Test.FLAKYPASS