src/llvm/utils/lit/lit/Test.py

   1 import os
   2 from xml.sax.saxutils import escape
   3 from json import JSONEncoder
   4
   5 # Test result codes.
   6
   7 class ResultCode(object):
   8     """Test result codes."""
   9
  10     # We override __new__ and __getnewargs__ to ensure that pickling still
  11     # provides unique ResultCode objects in any particular instance.
  12     _instances = {}
  13     def __new__(cls, name, isFailure):
  14         res = cls._instances.get(name)
  15         if res is None:
  16             cls._instances[name] = res = super(ResultCode, cls).__new__(cls)
  17         return res
  18     def __getnewargs__(self):
  19         return (self.name, self.isFailure)
  20
  21     def __init__(self, name, isFailure):
  22         self.name = name
  23         self.isFailure = isFailure
  24
  25     def __repr__(self):
  26         return '%s%r' % (self.__class__.__name__,
  27                          (self.name, self.isFailure))
  28
  29 PASS        = ResultCode('PASS', False)
  30 XFAIL       = ResultCode('XFAIL', False)
  31 FAIL        = ResultCode('FAIL', True)
  32 XPASS       = ResultCode('XPASS', True)
  33 UNRESOLVED  = ResultCode('UNRESOLVED', True)
  34 UNSUPPORTED = ResultCode('UNSUPPORTED', False)
  35
  36 # Test metric values.
  37
  38 class MetricValue(object):
  39     def format(self):
  40         """
  41         format() -> str
  42
  43         Convert this metric to a string suitable for displaying as part of the
  44         console output.
  45         """
  46         raise RuntimeError("abstract method")
  47
  48     def todata(self):
  49         """
  50         todata() -> json-serializable data
  51
  52         Convert this metric to content suitable for serializing in the JSON test
  53         output.
  54         """
  55         raise RuntimeError("abstract method")
  56
  57 class IntMetricValue(MetricValue):
  58     def __init__(self, value):
  59         self.value = value
  60
  61     def format(self):
  62         return str(self.value)
  63
  64     def todata(self):
  65         return self.value
  66
  67 class RealMetricValue(MetricValue):
  68     def __init__(self, value):
  69         self.value = value
  70
  71     def format(self):
  72         return '%.4f' % self.value
  73
  74     def todata(self):
  75         return self.value
  76
  77 class JSONMetricValue(MetricValue):
  78     """
  79         JSONMetricValue is used for types that are representable in the output
  80         but that are otherwise uninterpreted.
  81     """
  82     def __init__(self, value):
  83         # Ensure the value is a serializable by trying to encode it.
  84         # WARNING: The value may change before it is encoded again, and may
  85         #          not be encodable after the change.
  86         try:
  87             e = JSONEncoder()
  88             e.encode(value)
  89         except TypeError:
  90             raise
  91         self.value = value
  92
  93     def format(self):
  94         return str(self.value)
  95
  96     def todata(self):
  97         return self.value
  98
  99 def toMetricValue(value):
 100     if isinstance(value, MetricValue):
 101         return value
 102     elif isinstance(value, int) or isinstance(value, long):
 103         return IntMetricValue(value)
 104     elif isinstance(value, float):
 105         return RealMetricValue(value)
 106     else:
 107         # Try to create a JSONMetricValue and let the constructor throw
 108         # if value is not a valid type.
 109         return JSONMetricValue(value)
 110
 111
 112 # Test results.
 113
 114 class Result(object):
 115     """Wrapper for the results of executing an individual test."""
 116
 117     def __init__(self, code, output='', elapsed=None):
 118         # The result code.
 119         self.code = code
 120         # The test output.
 121         self.output = output
 122         # The wall timing to execute the test, if timing.
 123         self.elapsed = elapsed
 124         # The metrics reported by this test.
 125         self.metrics = {}
 126
 127     def addMetric(self, name, value):
 128         """
 129         addMetric(name, value)
 130
 131         Attach a test metric to the test result, with the given name and list of
 132         values. It is an error to attempt to attach the metrics with the same
 133         name multiple times.
 134
 135         Each value must be an instance of a MetricValue subclass.
 136         """
 137         if name in self.metrics:
 138             raise ValueError("result already includes metrics for %r" % (
 139                     name,))
 140         if not isinstance(value, MetricValue):
 141             raise TypeError("unexpected metric value: %r" % (value,))
 142         self.metrics[name] = value
 143
 144 # Test classes.
 145
 146 class TestSuite:
 147     """TestSuite - Information on a group of tests.
 148
 149     A test suite groups together a set of logically related tests.
 150     """
 151
 152     def __init__(self, name, source_root, exec_root, config):
 153         self.name = name
 154         self.source_root = source_root
 155         self.exec_root = exec_root
 156         # The test suite configuration.
 157         self.config = config
 158
 159     def getSourcePath(self, components):
 160         return os.path.join(self.source_root, *components)
 161
 162     def getExecPath(self, components):
 163         return os.path.join(self.exec_root, *components)
 164
 165 class Test:
 166     """Test - Information on a single test instance."""
 167
 168     def __init__(self, suite, path_in_suite, config, file_path = None):
 169         self.suite = suite
 170         self.path_in_suite = path_in_suite
 171         self.config = config
 172         self.file_path = file_path
 173         # A list of conditions under which this test is expected to fail. These
 174         # can optionally be provided by test format handlers, and will be
 175         # honored when the test result is supplied.
 176         self.xfails = []
 177         # The test result, once complete.
 178         self.result = None
 179
 180     def setResult(self, result):
 181         if self.result is not None:
 182             raise ArgumentError("test result already set")
 183         if not isinstance(result, Result):
 184             raise ArgumentError("unexpected result type")
 185
 186         self.result = result
 187
 188         # Apply the XFAIL handling to resolve the result exit code.
 189         if self.isExpectedToFail():
 190             if self.result.code == PASS:
 191                 self.result.code = XPASS
 192             elif self.result.code == FAIL:
 193                 self.result.code = XFAIL
 194
 195     def getFullName(self):
 196         return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite)
 197
 198     def getFilePath(self):
 199         if self.file_path:
 200             return self.file_path
 201         return self.getSourcePath()
 202
 203     def getSourcePath(self):
 204         return self.suite.getSourcePath(self.path_in_suite)
 205
 206     def getExecPath(self):
 207         return self.suite.getExecPath(self.path_in_suite)
 208
 209     def isExpectedToFail(self):
 210         """
 211         isExpectedToFail() -> bool
 212
 213         Check whether this test is expected to fail in the current
 214         configuration. This check relies on the test xfails property which by
 215         some test formats may not be computed until the test has first been
 216         executed.
 217         """
 218
 219         # Check if any of the xfails match an available feature or the target.
 220         for item in self.xfails:
 221             # If this is the wildcard, it always fails.
 222             if item == '*':
 223                 return True
 224
 225             # If this is an exact match for one of the features, it fails.
 226             if item in self.config.available_features:
 227                 return True
 228
 229             # If this is a part of the target triple, it fails.
 230             if item in self.suite.config.target_triple:
 231                 return True
 232
 233         return False
 234
 235
 236     def getJUnitXML(self):
 237         test_name = self.path_in_suite[-1]
 238         test_path = self.path_in_suite[:-1]
 239         safe_test_path = [x.replace(".","_") for x in test_path]
 240         safe_name = self.suite.name.replace(".","-")
 241
 242         if safe_test_path:
 243             class_name = safe_name + "." + "/".join(safe_test_path)
 244         else:
 245             class_name = safe_name + "." + safe_name
 246
 247         xml = "<testcase classname='" + class_name + "' name='" + \
 248             test_name + "'"
 249         xml += " time='%.2f'" % (self.result.elapsed,)
 250         if self.result.code.isFailure:
 251             xml += ">\n\t<failure >\n" + escape(self.result.output)
 252             xml += "\n\t</failure>\n</testcase>"
 253         else:
 254             xml += "/>"
 255         return xml