scripts/tracetool/__init__.py

   1 # -*- coding: utf-8 -*-
   2
   3 """
   4 Machinery for generating tracing-related intermediate files.
   5 """
   6
   7 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
   8 __copyright__  = "Copyright 2012-2017, Lluís Vilanova <vilanova@ac.upc.edu>"
   9 __license__    = "GPL version 2 or (at your option) any later version"
  10
  11 __maintainer__ = "Stefan Hajnoczi"
  12 __email__      = "stefanha@redhat.com"
  13
  14
  15 import re
  16 import sys
  17 import weakref
  18
  19 import tracetool.format
  20 import tracetool.backend
  21 import tracetool.transform
  22
  23
  24 def error_write(*lines):
  25     """Write a set of error lines."""
  26     sys.stderr.writelines("\n".join(lines) + "\n")
  27
  28 def error(*lines):
  29     """Write a set of error lines and exit."""
  30     error_write(*lines)
  31     sys.exit(1)
  32
  33
  34 out_filename = '<none>'
  35 out_fobj = sys.stdout
  36
  37 def out_open(filename):
  38     global out_filename, out_fobj
  39     out_filename = filename
  40     out_fobj = open(filename, 'wt')
  41
  42 def out(*lines, **kwargs):
  43     """Write a set of output lines.
  44
  45     You can use kwargs as a shorthand for mapping variables when formatting all
  46     the strings in lines.
  47
  48     The 'out_filename' kwarg is automatically added with the output filename.
  49     """
  50     output = []
  51     for l in lines:
  52         kwargs['out_filename'] = out_filename
  53         output.append(l % kwargs)
  54
  55     out_fobj.writelines("\n".join(output) + "\n")
  56
  57 # We only want to allow standard C types or fixed sized
  58 # integer types. We don't want QEMU specific types
  59 # as we can't assume trace backends can resolve all the
  60 # typedefs
  61 ALLOWED_TYPES = [
  62     "int",
  63     "long",
  64     "short",
  65     "char",
  66     "bool",
  67     "unsigned",
  68     "signed",
  69     "int8_t",
  70     "uint8_t",
  71     "int16_t",
  72     "uint16_t",
  73     "int32_t",
  74     "uint32_t",
  75     "int64_t",
  76     "uint64_t",
  77     "void",
  78     "size_t",
  79     "ssize_t",
  80     "uintptr_t",
  81     "ptrdiff_t",
  82     # Magic substitution is done by tracetool
  83     "TCGv",
  84 ]
  85
  86 def validate_type(name):
  87     bits = name.split(" ")
  88     for bit in bits:
  89         bit = re.sub("\*", "", bit)
  90         if bit == "":
  91             continue
  92         if bit == "const":
  93             continue
  94         if bit not in ALLOWED_TYPES:
  95             raise ValueError("Argument type '%s' is not in whitelist. "
  96                              "Only standard C types and fixed size integer "
  97                              "types should be used. struct, union, and "
  98                              "other complex pointer types should be "
  99                              "declared as 'void *'" % name)
 100
 101 class Arguments:
 102     """Event arguments description."""
 103
 104     def __init__(self, args):
 105         """
 106         Parameters
 107         ----------
 108         args :
 109             List of (type, name) tuples or Arguments objects.
 110         """
 111         self._args = []
 112         for arg in args:
 113             if isinstance(arg, Arguments):
 114                 self._args.extend(arg._args)
 115             else:
 116                 self._args.append(arg)
 117
 118     def copy(self):
 119         """Create a new copy."""
 120         return Arguments(list(self._args))
 121
 122     @staticmethod
 123     def build(arg_str):
 124         """Build and Arguments instance from an argument string.
 125
 126         Parameters
 127         ----------
 128         arg_str : str
 129             String describing the event arguments.
 130         """
 131         res = []
 132         for arg in arg_str.split(","):
 133             arg = arg.strip()
 134             if not arg:
 135                 raise ValueError("Empty argument (did you forget to use 'void'?)")
 136             if arg == 'void':
 137                 continue
 138
 139             if '*' in arg:
 140                 arg_type, identifier = arg.rsplit('*', 1)
 141                 arg_type += '*'
 142                 identifier = identifier.strip()
 143             else:
 144                 arg_type, identifier = arg.rsplit(None, 1)
 145
 146             validate_type(arg_type)
 147             res.append((arg_type, identifier))
 148         return Arguments(res)
 149
 150     def __getitem__(self, index):
 151         if isinstance(index, slice):
 152             return Arguments(self._args[index])
 153         else:
 154             return self._args[index]
 155
 156     def __iter__(self):
 157         """Iterate over the (type, name) pairs."""
 158         return iter(self._args)
 159
 160     def __len__(self):
 161         """Number of arguments."""
 162         return len(self._args)
 163
 164     def __str__(self):
 165         """String suitable for declaring function arguments."""
 166         if len(self._args) == 0:
 167             return "void"
 168         else:
 169             return ", ".join([ " ".join([t, n]) for t,n in self._args ])
 170
 171     def __repr__(self):
 172         """Evaluable string representation for this object."""
 173         return "Arguments(\"%s\")" % str(self)
 174
 175     def names(self):
 176         """List of argument names."""
 177         return [ name for _, name in self._args ]
 178
 179     def types(self):
 180         """List of argument types."""
 181         return [ type_ for type_, _ in self._args ]
 182
 183     def casted(self):
 184         """List of argument names casted to their type."""
 185         return ["(%s)%s" % (type_, name) for type_, name in self._args]
 186
 187     def transform(self, *trans):
 188         """Return a new Arguments instance with transformed types.
 189
 190         The types in the resulting Arguments instance are transformed according
 191         to tracetool.transform.transform_type.
 192         """
 193         res = []
 194         for type_, name in self._args:
 195             res.append((tracetool.transform.transform_type(type_, *trans),
 196                         name))
 197         return Arguments(res)
 198
 199
 200 class Event(object):
 201     """Event description.
 202
 203     Attributes
 204     ----------
 205     name : str
 206         The event name.
 207     fmt : str
 208         The event format string.
 209     properties : set(str)
 210         Properties of the event.
 211     args : Arguments
 212         The event arguments.
 213
 214     """
 215
 216     _CRE = re.compile("((?P<props>[\w\s]+)\s+)?"
 217                       "(?P<name>\w+)"
 218                       "\((?P<args>[^)]*)\)"
 219                       "\s*"
 220                       "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
 221                       "\s*")
 222
 223     _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"])
 224
 225     def __init__(self, name, props, fmt, args, orig=None,
 226                  event_trans=None, event_exec=None):
 227         """
 228         Parameters
 229         ----------
 230         name : string
 231             Event name.
 232         props : list of str
 233             Property names.
 234         fmt : str, list of str
 235             Event printing format string(s).
 236         args : Arguments
 237             Event arguments.
 238         orig : Event or None
 239             Original Event before transformation/generation.
 240         event_trans : Event or None
 241             Generated translation-time event ("tcg" property).
 242         event_exec : Event or None
 243             Generated execution-time event ("tcg" property).
 244
 245         """
 246         self.name = name
 247         self.properties = props
 248         self.fmt = fmt
 249         self.args = args
 250         self.event_trans = event_trans
 251         self.event_exec = event_exec
 252
 253         if len(args) > 10:
 254             raise ValueError("Event '%s' has more than maximum permitted "
 255                              "argument count" % name)
 256
 257         if orig is None:
 258             self.original = weakref.ref(self)
 259         else:
 260             self.original = orig
 261
 262         unknown_props = set(self.properties) - self._VALID_PROPS
 263         if len(unknown_props) > 0:
 264             raise ValueError("Unknown properties: %s"
 265                              % ", ".join(unknown_props))
 266         assert isinstance(self.fmt, str) or len(self.fmt) == 2
 267
 268     def copy(self):
 269         """Create a new copy."""
 270         return Event(self.name, list(self.properties), self.fmt,
 271                      self.args.copy(), self, self.event_trans, self.event_exec)
 272
 273     @staticmethod
 274     def build(line_str):
 275         """Build an Event instance from a string.
 276
 277         Parameters
 278         ----------
 279         line_str : str
 280             Line describing the event.
 281         """
 282         m = Event._CRE.match(line_str)
 283         assert m is not None
 284         groups = m.groupdict('')
 285
 286         name = groups["name"]
 287         props = groups["props"].split()
 288         fmt = groups["fmt"]
 289         fmt_trans = groups["fmt_trans"]
 290         if fmt.find("%m") != -1 or fmt_trans.find("%m") != -1:
 291             raise ValueError("Event format '%m' is forbidden, pass the error "
 292                              "as an explicit trace argument")
 293         if fmt.endswith(r'\n"'):
 294             raise ValueError("Event format must not end with a newline "
 295                              "character")
 296
 297         if len(fmt_trans) > 0:
 298             fmt = [fmt_trans, fmt]
 299         args = Arguments.build(groups["args"])
 300
 301         if "tcg-trans" in props:
 302             raise ValueError("Invalid property 'tcg-trans'")
 303         if "tcg-exec" in props:
 304             raise ValueError("Invalid property 'tcg-exec'")
 305         if "tcg" not in props and not isinstance(fmt, str):
 306             raise ValueError("Only events with 'tcg' property can have two format strings")
 307         if "tcg" in props and isinstance(fmt, str):
 308             raise ValueError("Events with 'tcg' property must have two format strings")
 309
 310         event = Event(name, props, fmt, args)
 311
 312         # add implicit arguments when using the 'vcpu' property
 313         import tracetool.vcpu
 314         event = tracetool.vcpu.transform_event(event)
 315
 316         return event
 317
 318     def __repr__(self):
 319         """Evaluable string representation for this object."""
 320         if isinstance(self.fmt, str):
 321             fmt = self.fmt
 322         else:
 323             fmt = "%s, %s" % (self.fmt[0], self.fmt[1])
 324         return "Event('%s %s(%s) %s')" % (" ".join(self.properties),
 325                                           self.name,
 326                                           self.args,
 327                                           fmt)
 328     # Star matching on PRI is dangerous as one might have multiple
 329     # arguments with that format, hence the non-greedy version of it.
 330     _FMT = re.compile("(%[\d\.]*\w+|%.*?PRI\S+)")
 331
 332     def formats(self):
 333         """List conversion specifiers in the argument print format string."""
 334         assert not isinstance(self.fmt, list)
 335         return self._FMT.findall(self.fmt)
 336
 337     QEMU_TRACE               = "trace_%(name)s"
 338     QEMU_TRACE_NOCHECK       = "_nocheck__" + QEMU_TRACE
 339     QEMU_TRACE_TCG           = QEMU_TRACE + "_tcg"
 340     QEMU_DSTATE              = "_TRACE_%(NAME)s_DSTATE"
 341     QEMU_BACKEND_DSTATE      = "TRACE_%(NAME)s_BACKEND_DSTATE"
 342     QEMU_EVENT               = "_TRACE_%(NAME)s_EVENT"
 343
 344     def api(self, fmt=None):
 345         if fmt is None:
 346             fmt = Event.QEMU_TRACE
 347         return fmt % {"name": self.name, "NAME": self.name.upper()}
 348
 349     def transform(self, *trans):
 350         """Return a new Event with transformed Arguments."""
 351         return Event(self.name,
 352                      list(self.properties),
 353                      self.fmt,
 354                      self.args.transform(*trans),
 355                      self)
 356
 357
 358 def read_events(fobj, fname):
 359     """Generate the output for the given (format, backends) pair.
 360
 361     Parameters
 362     ----------
 363     fobj : file
 364         Event description file.
 365     fname : str
 366         Name of event file
 367
 368     Returns a list of Event objects
 369     """
 370
 371     events = []
 372     for lineno, line in enumerate(fobj, 1):
 373         if line[-1] != '\n':
 374             raise ValueError("%s does not end with a new line" % fname)
 375         if not line.strip():
 376             continue
 377         if line.lstrip().startswith('#'):
 378             continue
 379
 380         try:
 381             event = Event.build(line)
 382         except ValueError as e:
 383             arg0 = 'Error at %s:%d: %s' % (fname, lineno, e.args[0])
 384             e.args = (arg0,) + e.args[1:]
 385             raise
 386
 387         # transform TCG-enabled events
 388         if "tcg" not in event.properties:
 389             events.append(event)
 390         else:
 391             event_trans = event.copy()
 392             event_trans.name += "_trans"
 393             event_trans.properties += ["tcg-trans"]
 394             event_trans.fmt = event.fmt[0]
 395             # ignore TCG arguments
 396             args_trans = []
 397             for atrans, aorig in zip(
 398                     event_trans.transform(tracetool.transform.TCG_2_HOST).args,
 399                     event.args):
 400                 if atrans == aorig:
 401                     args_trans.append(atrans)
 402             event_trans.args = Arguments(args_trans)
 403
 404             event_exec = event.copy()
 405             event_exec.name += "_exec"
 406             event_exec.properties += ["tcg-exec"]
 407             event_exec.fmt = event.fmt[1]
 408             event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST)
 409
 410             new_event = [event_trans, event_exec]
 411             event.event_trans, event.event_exec = new_event
 412
 413             events.extend(new_event)
 414
 415     return events
 416
 417
 418 class TracetoolError (Exception):
 419     """Exception for calls to generate."""
 420     pass
 421
 422
 423 def try_import(mod_name, attr_name=None, attr_default=None):
 424     """Try to import a module and get an attribute from it.
 425
 426     Parameters
 427     ----------
 428     mod_name : str
 429         Module name.
 430     attr_name : str, optional
 431         Name of an attribute in the module.
 432     attr_default : optional
 433         Default value if the attribute does not exist in the module.
 434
 435     Returns
 436     -------
 437     A pair indicating whether the module could be imported and the module or
 438     object or attribute value.
 439     """
 440     try:
 441         module = __import__(mod_name, globals(), locals(), ["__package__"])
 442         if attr_name is None:
 443             return True, module
 444         return True, getattr(module, str(attr_name), attr_default)
 445     except ImportError:
 446         return False, None
 447
 448
 449 def generate(events, group, format, backends,
 450              binary=None, probe_prefix=None):
 451     """Generate the output for the given (format, backends) pair.
 452
 453     Parameters
 454     ----------
 455     events : list
 456         list of Event objects to generate for
 457     group: str
 458         Name of the tracing group
 459     format : str
 460         Output format name.
 461     backends : list
 462         Output backend names.
 463     binary : str or None
 464         See tracetool.backend.dtrace.BINARY.
 465     probe_prefix : str or None
 466         See tracetool.backend.dtrace.PROBEPREFIX.
 467     """
 468     # fix strange python error (UnboundLocalError tracetool)
 469     import tracetool
 470
 471     format = str(format)
 472     if len(format) == 0:
 473         raise TracetoolError("format not set")
 474     if not tracetool.format.exists(format):
 475         raise TracetoolError("unknown format: %s" % format)
 476
 477     if len(backends) == 0:
 478         raise TracetoolError("no backends specified")
 479     for backend in backends:
 480         if not tracetool.backend.exists(backend):
 481             raise TracetoolError("unknown backend: %s" % backend)
 482     backend = tracetool.backend.Wrapper(backends, format)
 483
 484     import tracetool.backend.dtrace
 485     tracetool.backend.dtrace.BINARY = binary
 486     tracetool.backend.dtrace.PROBEPREFIX = probe_prefix
 487
 488     tracetool.format.generate(events, format, backend, group)