5 class ResultCode(object):
6 """Test result codes."""
8 # We override __new__ and __getnewargs__ to ensure that pickling still
9 # provides unique ResultCode objects in any particular instance.
11 def __new__(cls, name, isFailure):
12 res = cls._instances.get(name)
14 cls._instances[name] = res = super(ResultCode, cls).__new__(cls)
16 def __getnewargs__(self):
17 return (self.name, self.isFailure)
19 def __init__(self, name, isFailure):
21 self.isFailure = isFailure
24 return '%s%r' % (self.__class__.__name__,
25 (self.name, self.isFailure))
27 PASS = ResultCode('PASS', False)
28 XFAIL = ResultCode('XFAIL', False)
29 FAIL = ResultCode('FAIL', True)
30 XPASS = ResultCode('XPASS', True)
31 UNRESOLVED = ResultCode('UNRESOLVED', True)
32 UNSUPPORTED = ResultCode('UNSUPPORTED', False)
36 class MetricValue(object):
38 raise RuntimeError("abstract method")
40 class IntMetricValue(MetricValue):
41 def __init__(self, value):
45 return str(self.value)
47 class RealMetricValue(MetricValue):
48 def __init__(self, value):
52 return '%.4f' % self.value
57 """Wrapper for the results of executing an individual test."""
59 def __init__(self, code, output='', elapsed=None):
64 # The wall timing to execute the test, if timing.
65 self.elapsed = elapsed
66 # The metrics reported by this test.
69 def addMetric(self, name, value):
71 addMetric(name, value)
73 Attach a test metric to the test result, with the given name and list of
74 values. It is an error to attempt to attach the metrics with the same
77 Each value must be an instance of a MetricValue subclass.
79 if name in self.metrics:
80 raise ValueError("result already includes metrics for %r" % (
82 if not isinstance(value, MetricValue):
83 raise TypeError("unexpected metric value: %r" % (value,))
84 self.metrics[name] = value
89 """TestSuite - Information on a group of tests.
91 A test suite groups together a set of logically related tests.
94 def __init__(self, name, source_root, exec_root, config):
96 self.source_root = source_root
97 self.exec_root = exec_root
98 # The test suite configuration.
101 def getSourcePath(self, components):
102 return os.path.join(self.source_root, *components)
104 def getExecPath(self, components):
105 return os.path.join(self.exec_root, *components)
108 """Test - Information on a single test instance."""
110 def __init__(self, suite, path_in_suite, config):
112 self.path_in_suite = path_in_suite
114 # A list of conditions under which this test is expected to fail. These
115 # can optionally be provided by test format handlers, and will be
116 # honored when the test result is supplied.
118 # The test result, once complete.
121 def setResult(self, result):
122 if self.result is not None:
123 raise ArgumentError("test result already set")
124 if not isinstance(result, Result):
125 raise ArgumentError("unexpected result type")
129 # Apply the XFAIL handling to resolve the result exit code.
130 if self.isExpectedToFail():
131 if self.result.code == PASS:
132 self.result.code = XPASS
133 elif self.result.code == FAIL:
134 self.result.code = XFAIL
136 def getFullName(self):
137 return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
139 def getSourcePath(self):
140 return self.suite.getSourcePath(self.path_in_suite)
142 def getExecPath(self):
143 return self.suite.getExecPath(self.path_in_suite)
145 def isExpectedToFail(self):
147 isExpectedToFail() -> bool
149 Check whether this test is expected to fail in the current
150 configuration. This check relies on the test xfails property which by
151 some test formats may not be computed until the test has first been
155 # Check if any of the xfails match an available feature or the target.
156 for item in self.xfails:
157 # If this is the wildcard, it always fails.
161 # If this is an exact match for one of the features, it fails.
162 if item in self.config.available_features:
165 # If this is a part of the target triple, it fails.
166 if item in self.suite.config.target_triple: