scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 events = []
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ""
  62     while parent:
  63         res = ("In file included from %s:%d:\n" % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = "%s:%d" % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ":%s" % self.col
  82         return error_path(self.info) + "%s: %s" % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110             self.optional = False
 111
 112         def append(self, line):
 113             self.content.append(line)
 114
 115         def __repr__(self):
 116             return "\n".join(self.content).strip()
 117
 118     class ArgSection(Section):
 119         def __init__(self, name):
 120             QAPIDoc.Section.__init__(self, name)
 121             self.member = None
 122
 123         def connect(self, member):
 124             self.member = member
 125
 126     def __init__(self, parser, info):
 127         # self.parser is used to report errors with QAPIParseError.  The
 128         # resulting error position depends on the state of the parser.
 129         # It happens to be the beginning of the comment.  More or less
 130         # servicable, but action at a distance.
 131         self.parser = parser
 132         self.info = info
 133         self.symbol = None
 134         self.body = QAPIDoc.Section()
 135         # dict mapping parameter name to ArgSection
 136         self.args = OrderedDict()
 137         # a list of Section
 138         self.sections = []
 139         # the current section
 140         self.section = self.body
 141         # associated expression (to be set by expression parser)
 142         self.expr = None
 143
 144     def has_section(self, name):
 145         """Return True if we have a section with this name."""
 146         for i in self.sections:
 147             if i.name == name:
 148                 return True
 149         return False
 150
 151     def append(self, line):
 152         """Parse a comment line and add it to the documentation."""
 153         line = line[1:]
 154         if not line:
 155             self._append_freeform(line)
 156             return
 157
 158         if line[0] != ' ':
 159             raise QAPIParseError(self.parser, "Missing space after #")
 160         line = line[1:]
 161
 162         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 163         # recognized, and get silently treated as ordinary text
 164         if self.symbol:
 165             self._append_symbol_line(line)
 166         elif not self.body.content and line.startswith("@"):
 167             if not line.endswith(":"):
 168                 raise QAPIParseError(self.parser, "Line should end with :")
 169             self.symbol = line[1:-1]
 170             # FIXME invalid names other than the empty string aren't flagged
 171             if not self.symbol:
 172                 raise QAPIParseError(self.parser, "Invalid name")
 173         else:
 174             self._append_freeform(line)
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith("@") and name.endswith(":"):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ("Returns:", "Since:",
 183                       # those are often singular or plural
 184                       "Note:", "Notes:",
 185                       "Example:", "Examples:",
 186                       "TODO:"):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=""):
 207         if name in ("Returns", "Since") and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self.section = QAPIDoc.Section(name)
 211         self.sections.append(self.section)
 212
 213     def _append_freeform(self, line):
 214         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 215         if (in_arg and self.section.content
 216                 and not self.section.content[-1]
 217                 and line and not line[0].isspace()):
 218             self._start_section()
 219         if (in_arg or not self.section.name
 220                 or not self.section.name.startswith("Example")):
 221             line = line.strip()
 222         self.section.append(line)
 223
 224     def connect_member(self, member):
 225         if member.name not in self.args:
 226             # Undocumented TODO outlaw
 227             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 228         self.args[member.name].connect(member)
 229
 230
 231 class QAPISchemaParser(object):
 232
 233     def __init__(self, fp, previously_included=[], incl_info=None):
 234         abs_fname = os.path.abspath(fp.name)
 235         fname = fp.name
 236         self.fname = fname
 237         previously_included.append(abs_fname)
 238         self.incl_info = incl_info
 239         self.src = fp.read()
 240         if self.src == '' or self.src[-1] != '\n':
 241             self.src += '\n'
 242         self.cursor = 0
 243         self.line = 1
 244         self.line_pos = 0
 245         self.exprs = []
 246         self.docs = []
 247         self.accept()
 248
 249         while self.tok is not None:
 250             info = {'file': fname, 'line': self.line,
 251                     'parent': self.incl_info}
 252             if self.tok == '#':
 253                 doc = self.get_doc(info)
 254                 self.docs.append(doc)
 255                 continue
 256
 257             expr = self.get_expr(False)
 258             if 'include' in expr:
 259                 if len(expr) != 1:
 260                     raise QAPISemError(info, "Invalid 'include' directive")
 261                 include = expr["include"]
 262                 if not isinstance(include, str):
 263                     raise QAPISemError(info,
 264                                        "Value of 'include' must be a string")
 265                 self._include(include, info, os.path.dirname(abs_fname),
 266                               previously_included)
 267             elif "pragma" in expr:
 268                 if len(expr) != 1:
 269                     raise QAPISemError(info, "Invalid 'pragma' directive")
 270                 pragma = expr['pragma']
 271                 if not isinstance(pragma, dict):
 272                     raise QAPISemError(
 273                         info, "Value of 'pragma' must be a dictionary")
 274                 for name, value in pragma.iteritems():
 275                     self._pragma(name, value, info)
 276             else:
 277                 expr_elem = {'expr': expr,
 278                              'info': info}
 279                 if (self.docs
 280                         and self.docs[-1].info['file'] == fname
 281                         and not self.docs[-1].expr):
 282                     self.docs[-1].expr = expr
 283                     expr_elem['doc'] = self.docs[-1]
 284
 285                 self.exprs.append(expr_elem)
 286
 287     def _include(self, include, info, base_dir, previously_included):
 288         incl_abs_fname = os.path.join(base_dir, include)
 289         # catch inclusion cycle
 290         inf = info
 291         while inf:
 292             if incl_abs_fname == os.path.abspath(inf['file']):
 293                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 294             inf = inf['parent']
 295
 296         # skip multiple include of the same file
 297         if incl_abs_fname in previously_included:
 298             return
 299         try:
 300             fobj = open(incl_abs_fname, 'r')
 301         except IOError as e:
 302             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 303         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 304         self.exprs.extend(exprs_include.exprs)
 305         self.docs.extend(exprs_include.docs)
 306
 307     def _pragma(self, name, value, info):
 308         global doc_required, returns_whitelist, name_case_whitelist
 309         if name == 'doc-required':
 310             if not isinstance(value, bool):
 311                 raise QAPISemError(info,
 312                                    "Pragma 'doc-required' must be boolean")
 313             doc_required = value
 314         elif name == 'returns-whitelist':
 315             if (not isinstance(value, list)
 316                     or any([not isinstance(elt, str) for elt in value])):
 317                 raise QAPISemError(info,
 318                                    "Pragma returns-whitelist must be"
 319                                    " a list of strings")
 320             returns_whitelist = value
 321         elif name == 'name-case-whitelist':
 322             if (not isinstance(value, list)
 323                     or any([not isinstance(elt, str) for elt in value])):
 324                 raise QAPISemError(info,
 325                                    "Pragma name-case-whitelist must be"
 326                                    " a list of strings")
 327             name_case_whitelist = value
 328         else:
 329             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 330
 331     def accept(self, skip_comment=True):
 332         while True:
 333             self.tok = self.src[self.cursor]
 334             self.pos = self.cursor
 335             self.cursor += 1
 336             self.val = None
 337
 338             if self.tok == '#':
 339                 if self.src[self.cursor] == '#':
 340                     # Start of doc comment
 341                     skip_comment = False
 342                 self.cursor = self.src.find('\n', self.cursor)
 343                 if not skip_comment:
 344                     self.val = self.src[self.pos:self.cursor]
 345                     return
 346             elif self.tok in "{}:,[]":
 347                 return
 348             elif self.tok == "'":
 349                 string = ''
 350                 esc = False
 351                 while True:
 352                     ch = self.src[self.cursor]
 353                     self.cursor += 1
 354                     if ch == '\n':
 355                         raise QAPIParseError(self, 'Missing terminating "\'"')
 356                     if esc:
 357                         if ch == 'b':
 358                             string += '\b'
 359                         elif ch == 'f':
 360                             string += '\f'
 361                         elif ch == 'n':
 362                             string += '\n'
 363                         elif ch == 'r':
 364                             string += '\r'
 365                         elif ch == 't':
 366                             string += '\t'
 367                         elif ch == 'u':
 368                             value = 0
 369                             for _ in range(0, 4):
 370                                 ch = self.src[self.cursor]
 371                                 self.cursor += 1
 372                                 if ch not in "0123456789abcdefABCDEF":
 373                                     raise QAPIParseError(self,
 374                                                          '\\u escape needs 4 '
 375                                                          'hex digits')
 376                                 value = (value << 4) + int(ch, 16)
 377                             # If Python 2 and 3 didn't disagree so much on
 378                             # how to handle Unicode, then we could allow
 379                             # Unicode string defaults.  But most of QAPI is
 380                             # ASCII-only, so we aren't losing much for now.
 381                             if not value or value > 0x7f:
 382                                 raise QAPIParseError(self,
 383                                                      'For now, \\u escape '
 384                                                      'only supports non-zero '
 385                                                      'values up to \\u007f')
 386                             string += chr(value)
 387                         elif ch in "\\/'\"":
 388                             string += ch
 389                         else:
 390                             raise QAPIParseError(self,
 391                                                  "Unknown escape \\%s" % ch)
 392                         esc = False
 393                     elif ch == "\\":
 394                         esc = True
 395                     elif ch == "'":
 396                         self.val = string
 397                         return
 398                     else:
 399                         string += ch
 400             elif self.src.startswith("true", self.pos):
 401                 self.val = True
 402                 self.cursor += 3
 403                 return
 404             elif self.src.startswith("false", self.pos):
 405                 self.val = False
 406                 self.cursor += 4
 407                 return
 408             elif self.src.startswith("null", self.pos):
 409                 self.val = None
 410                 self.cursor += 3
 411                 return
 412             elif self.tok == '\n':
 413                 if self.cursor == len(self.src):
 414                     self.tok = None
 415                     return
 416                 self.line += 1
 417                 self.line_pos = self.cursor
 418             elif not self.tok.isspace():
 419                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 420
 421     def get_members(self):
 422         expr = OrderedDict()
 423         if self.tok == '}':
 424             self.accept()
 425             return expr
 426         if self.tok != "'":
 427             raise QAPIParseError(self, 'Expected string or "}"')
 428         while True:
 429             key = self.val
 430             self.accept()
 431             if self.tok != ':':
 432                 raise QAPIParseError(self, 'Expected ":"')
 433             self.accept()
 434             if key in expr:
 435                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 436             expr[key] = self.get_expr(True)
 437             if self.tok == '}':
 438                 self.accept()
 439                 return expr
 440             if self.tok != ',':
 441                 raise QAPIParseError(self, 'Expected "," or "}"')
 442             self.accept()
 443             if self.tok != "'":
 444                 raise QAPIParseError(self, 'Expected string')
 445
 446     def get_values(self):
 447         expr = []
 448         if self.tok == ']':
 449             self.accept()
 450             return expr
 451         if self.tok not in "{['tfn":
 452             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 453                                  'boolean or "null"')
 454         while True:
 455             expr.append(self.get_expr(True))
 456             if self.tok == ']':
 457                 self.accept()
 458                 return expr
 459             if self.tok != ',':
 460                 raise QAPIParseError(self, 'Expected "," or "]"')
 461             self.accept()
 462
 463     def get_expr(self, nested):
 464         if self.tok != '{' and not nested:
 465             raise QAPIParseError(self, 'Expected "{"')
 466         if self.tok == '{':
 467             self.accept()
 468             expr = self.get_members()
 469         elif self.tok == '[':
 470             self.accept()
 471             expr = self.get_values()
 472         elif self.tok in "'tfn":
 473             expr = self.val
 474             self.accept()
 475         else:
 476             raise QAPIParseError(self, 'Expected "{", "[" or string')
 477         return expr
 478
 479     def get_doc(self, info):
 480         if self.val != '##':
 481             raise QAPIParseError(self, "Junk after '##' at start of "
 482                                  "documentation comment")
 483
 484         doc = QAPIDoc(self, info)
 485         self.accept(False)
 486         while self.tok == '#':
 487             if self.val.startswith('##'):
 488                 # End of doc comment
 489                 if self.val != '##':
 490                     raise QAPIParseError(self, "Junk after '##' at end of "
 491                                          "documentation comment")
 492                 self.accept()
 493                 return doc
 494             else:
 495                 doc.append(self.val)
 496             self.accept(False)
 497
 498         raise QAPIParseError(self, "Documentation comment must end with '##'")
 499
 500
 501 #
 502 # Semantic analysis of schema expressions
 503 # TODO fold into QAPISchema
 504 # TODO catching name collisions in generated code would be nice
 505 #
 506
 507
 508 def find_base_members(base):
 509     if isinstance(base, dict):
 510         return base
 511     base_struct_define = find_struct(base)
 512     if not base_struct_define:
 513         return None
 514     return base_struct_define['data']
 515
 516
 517 # Return the qtype of an alternate branch, or None on error.
 518 def find_alternate_member_qtype(qapi_type):
 519     if qapi_type in builtin_types:
 520         return builtin_types[qapi_type]
 521     elif find_struct(qapi_type):
 522         return "QTYPE_QDICT"
 523     elif find_enum(qapi_type):
 524         return "QTYPE_QSTRING"
 525     elif find_union(qapi_type):
 526         return "QTYPE_QDICT"
 527     return None
 528
 529
 530 # Return the discriminator enum define if discriminator is specified as an
 531 # enum type, otherwise return None.
 532 def discriminator_find_enum_define(expr):
 533     base = expr.get('base')
 534     discriminator = expr.get('discriminator')
 535
 536     if not (discriminator and base):
 537         return None
 538
 539     base_members = find_base_members(base)
 540     if not base_members:
 541         return None
 542
 543     discriminator_type = base_members.get(discriminator)
 544     if not discriminator_type:
 545         return None
 546
 547     return find_enum(discriminator_type)
 548
 549
 550 # Names must be letters, numbers, -, and _.  They must start with letter,
 551 # except for downstream extensions which must start with __RFQDN_.
 552 # Dots are only valid in the downstream extension prefix.
 553 valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
 554                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 555
 556
 557 def check_name(info, source, name, allow_optional=False,
 558                enum_member=False):
 559     global valid_name
 560     membername = name
 561
 562     if not isinstance(name, str):
 563         raise QAPISemError(info, "%s requires a string name" % source)
 564     if name.startswith('*'):
 565         membername = name[1:]
 566         if not allow_optional:
 567             raise QAPISemError(info, "%s does not allow optional name '%s'"
 568                                % (source, name))
 569     # Enum members can start with a digit, because the generated C
 570     # code always prefixes it with the enum name
 571     if enum_member and membername[0].isdigit():
 572         membername = 'D' + membername
 573     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 574     # and 'q_obj_*' implicit type names.
 575     if not valid_name.match(membername) or \
 576        c_name(membername, False).startswith('q_'):
 577         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 578
 579
 580 def add_name(name, info, meta, implicit=False):
 581     global all_names
 582     check_name(info, "'%s'" % meta, name)
 583     # FIXME should reject names that differ only in '_' vs. '.'
 584     # vs. '-', because they're liable to clash in generated C.
 585     if name in all_names:
 586         raise QAPISemError(info, "%s '%s' is already defined"
 587                            % (all_names[name], name))
 588     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 589         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 590                            % (meta, name, name[-4:]))
 591     all_names[name] = meta
 592
 593
 594 def add_struct(definition, info):
 595     global struct_types
 596     name = definition['struct']
 597     add_name(name, info, 'struct')
 598     struct_types.append(definition)
 599
 600
 601 def find_struct(name):
 602     global struct_types
 603     for struct in struct_types:
 604         if struct['struct'] == name:
 605             return struct
 606     return None
 607
 608
 609 def add_union(definition, info):
 610     global union_types
 611     name = definition['union']
 612     add_name(name, info, 'union')
 613     union_types.append(definition)
 614
 615
 616 def find_union(name):
 617     global union_types
 618     for union in union_types:
 619         if union['union'] == name:
 620             return union
 621     return None
 622
 623
 624 def add_enum(name, info, enum_values=None, implicit=False):
 625     global enum_types
 626     add_name(name, info, 'enum', implicit)
 627     enum_types.append({"enum_name": name, "enum_values": enum_values})
 628
 629
 630 def find_enum(name):
 631     global enum_types
 632     for enum in enum_types:
 633         if enum['enum_name'] == name:
 634             return enum
 635     return None
 636
 637
 638 def is_enum(name):
 639     return find_enum(name) is not None
 640
 641
 642 def check_type(info, source, value, allow_array=False,
 643                allow_dict=False, allow_optional=False,
 644                allow_metas=[]):
 645     global all_names
 646
 647     if value is None:
 648         return
 649
 650     # Check if array type for value is okay
 651     if isinstance(value, list):
 652         if not allow_array:
 653             raise QAPISemError(info, "%s cannot be an array" % source)
 654         if len(value) != 1 or not isinstance(value[0], str):
 655             raise QAPISemError(info,
 656                                "%s: array type must contain single type name" %
 657                                source)
 658         value = value[0]
 659
 660     # Check if type name for value is okay
 661     if isinstance(value, str):
 662         if value not in all_names:
 663             raise QAPISemError(info, "%s uses unknown type '%s'"
 664                                % (source, value))
 665         if not all_names[value] in allow_metas:
 666             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 667                                (source, all_names[value], value))
 668         return
 669
 670     if not allow_dict:
 671         raise QAPISemError(info, "%s should be a type name" % source)
 672
 673     if not isinstance(value, OrderedDict):
 674         raise QAPISemError(info,
 675                            "%s should be a dictionary or type name" % source)
 676
 677     # value is a dictionary, check that each member is okay
 678     for (key, arg) in value.items():
 679         check_name(info, "Member of %s" % source, key,
 680                    allow_optional=allow_optional)
 681         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 682             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 683                                % (source, key))
 684         # Todo: allow dictionaries to represent default values of
 685         # an optional argument.
 686         check_type(info, "Member '%s' of %s" % (key, source), arg,
 687                    allow_array=True,
 688                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 689                                 'enum'])
 690
 691
 692 def check_command(expr, info):
 693     name = expr['command']
 694     boxed = expr.get('boxed', False)
 695
 696     args_meta = ['struct']
 697     if boxed:
 698         args_meta += ['union', 'alternate']
 699     check_type(info, "'data' for command '%s'" % name,
 700                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 701                allow_metas=args_meta)
 702     returns_meta = ['union', 'struct']
 703     if name in returns_whitelist:
 704         returns_meta += ['built-in', 'alternate', 'enum']
 705     check_type(info, "'returns' for command '%s'" % name,
 706                expr.get('returns'), allow_array=True,
 707                allow_optional=True, allow_metas=returns_meta)
 708
 709
 710 def check_event(expr, info):
 711     global events
 712     name = expr['event']
 713     boxed = expr.get('boxed', False)
 714
 715     meta = ['struct']
 716     if boxed:
 717         meta += ['union', 'alternate']
 718     events.append(name)
 719     check_type(info, "'data' for event '%s'" % name,
 720                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 721                allow_metas=meta)
 722
 723
 724 def check_union(expr, info):
 725     name = expr['union']
 726     base = expr.get('base')
 727     discriminator = expr.get('discriminator')
 728     members = expr['data']
 729
 730     # Two types of unions, determined by discriminator.
 731
 732     # With no discriminator it is a simple union.
 733     if discriminator is None:
 734         enum_define = None
 735         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 736         if base is not None:
 737             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 738                                name)
 739
 740     # Else, it's a flat union.
 741     else:
 742         # The object must have a string or dictionary 'base'.
 743         check_type(info, "'base' for union '%s'" % name,
 744                    base, allow_dict=True, allow_optional=True,
 745                    allow_metas=['struct'])
 746         if not base:
 747             raise QAPISemError(info, "Flat union '%s' must have a base"
 748                                % name)
 749         base_members = find_base_members(base)
 750         assert base_members is not None
 751
 752         # The value of member 'discriminator' must name a non-optional
 753         # member of the base struct.
 754         check_name(info, "Discriminator of flat union '%s'" % name,
 755                    discriminator)
 756         discriminator_type = base_members.get(discriminator)
 757         if not discriminator_type:
 758             raise QAPISemError(info,
 759                                "Discriminator '%s' is not a member of base "
 760                                "struct '%s'"
 761                                % (discriminator, base))
 762         enum_define = find_enum(discriminator_type)
 763         allow_metas = ['struct']
 764         # Do not allow string discriminator
 765         if not enum_define:
 766             raise QAPISemError(info,
 767                                "Discriminator '%s' must be of enumeration "
 768                                "type" % discriminator)
 769
 770     # Check every branch; don't allow an empty union
 771     if len(members) == 0:
 772         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 773     for (key, value) in members.items():
 774         check_name(info, "Member of union '%s'" % name, key)
 775
 776         # Each value must name a known type
 777         check_type(info, "Member '%s' of union '%s'" % (key, name),
 778                    value, allow_array=not base, allow_metas=allow_metas)
 779
 780         # If the discriminator names an enum type, then all members
 781         # of 'data' must also be members of the enum type.
 782         if enum_define:
 783             if key not in enum_define['enum_values']:
 784                 raise QAPISemError(info,
 785                                    "Discriminator value '%s' is not found in "
 786                                    "enum '%s'"
 787                                    % (key, enum_define["enum_name"]))
 788
 789     # If discriminator is user-defined, ensure all values are covered
 790     if enum_define:
 791         for value in enum_define['enum_values']:
 792             if value not in members.keys():
 793                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 794                                    % (name, value))
 795
 796
 797 def check_alternate(expr, info):
 798     name = expr['alternate']
 799     members = expr['data']
 800     types_seen = {}
 801
 802     # Check every branch; require at least two branches
 803     if len(members) < 2:
 804         raise QAPISemError(info,
 805                            "Alternate '%s' should have at least two branches "
 806                            "in 'data'" % name)
 807     for (key, value) in members.items():
 808         check_name(info, "Member of alternate '%s'" % name, key)
 809
 810         # Ensure alternates have no type conflicts.
 811         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 812                    value,
 813                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 814         qtype = find_alternate_member_qtype(value)
 815         if not qtype:
 816             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 817                                "type '%s'" % (name, key, value))
 818         if qtype in types_seen:
 819             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 820                                "be distinguished from member '%s'"
 821                                % (name, key, types_seen[qtype]))
 822         types_seen[qtype] = key
 823
 824
 825 def check_enum(expr, info):
 826     name = expr['enum']
 827     members = expr.get('data')
 828     prefix = expr.get('prefix')
 829
 830     if not isinstance(members, list):
 831         raise QAPISemError(info,
 832                            "Enum '%s' requires an array for 'data'" % name)
 833     if prefix is not None and not isinstance(prefix, str):
 834         raise QAPISemError(info,
 835                            "Enum '%s' requires a string for 'prefix'" % name)
 836     for member in members:
 837         check_name(info, "Member of enum '%s'" % name, member,
 838                    enum_member=True)
 839
 840
 841 def check_struct(expr, info):
 842     name = expr['struct']
 843     members = expr['data']
 844
 845     check_type(info, "'data' for struct '%s'" % name, members,
 846                allow_dict=True, allow_optional=True)
 847     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 848                allow_metas=['struct'])
 849
 850
 851 def check_keys(expr_elem, meta, required, optional=[]):
 852     expr = expr_elem['expr']
 853     info = expr_elem['info']
 854     name = expr[meta]
 855     if not isinstance(name, str):
 856         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 857     required = required + [meta]
 858     for (key, value) in expr.items():
 859         if key not in required and key not in optional:
 860             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 861                                % (key, meta, name))
 862         if (key == 'gen' or key == 'success-response') and value is not False:
 863             raise QAPISemError(info,
 864                                "'%s' of %s '%s' should only use false value"
 865                                % (key, meta, name))
 866         if key == 'boxed' and value is not True:
 867             raise QAPISemError(info,
 868                                "'%s' of %s '%s' should only use true value"
 869                                % (key, meta, name))
 870     for key in required:
 871         if key not in expr:
 872             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 873                                % (key, meta, name))
 874
 875
 876 def check_exprs(exprs):
 877     global all_names
 878
 879     # Learn the types and check for valid expression keys
 880     for builtin in builtin_types.keys():
 881         all_names[builtin] = 'built-in'
 882     for expr_elem in exprs:
 883         expr = expr_elem['expr']
 884         info = expr_elem['info']
 885
 886         if 'doc' not in expr_elem and doc_required:
 887             raise QAPISemError(info,
 888                                "Expression missing documentation comment")
 889
 890         if 'enum' in expr:
 891             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 892             add_enum(expr['enum'], info, expr['data'])
 893         elif 'union' in expr:
 894             check_keys(expr_elem, 'union', ['data'],
 895                        ['base', 'discriminator'])
 896             add_union(expr, info)
 897         elif 'alternate' in expr:
 898             check_keys(expr_elem, 'alternate', ['data'])
 899             add_name(expr['alternate'], info, 'alternate')
 900         elif 'struct' in expr:
 901             check_keys(expr_elem, 'struct', ['data'], ['base'])
 902             add_struct(expr, info)
 903         elif 'command' in expr:
 904             check_keys(expr_elem, 'command', [],
 905                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 906             add_name(expr['command'], info, 'command')
 907         elif 'event' in expr:
 908             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 909             add_name(expr['event'], info, 'event')
 910         else:
 911             raise QAPISemError(expr_elem['info'],
 912                                "Expression is missing metatype")
 913
 914     # Try again for hidden UnionKind enum
 915     for expr_elem in exprs:
 916         expr = expr_elem['expr']
 917         if 'union' in expr:
 918             if not discriminator_find_enum_define(expr):
 919                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 920                          implicit=True)
 921         elif 'alternate' in expr:
 922             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 923                      implicit=True)
 924
 925     # Validate that exprs make sense
 926     for expr_elem in exprs:
 927         expr = expr_elem['expr']
 928         info = expr_elem['info']
 929
 930         if 'enum' in expr:
 931             check_enum(expr, info)
 932         elif 'union' in expr:
 933             check_union(expr, info)
 934         elif 'alternate' in expr:
 935             check_alternate(expr, info)
 936         elif 'struct' in expr:
 937             check_struct(expr, info)
 938         elif 'command' in expr:
 939             check_command(expr, info)
 940         elif 'event' in expr:
 941             check_event(expr, info)
 942         else:
 943             assert False, 'unexpected meta type'
 944
 945     return exprs
 946
 947
 948 def check_freeform_doc(doc):
 949     if doc.symbol:
 950         raise QAPISemError(doc.info,
 951                            "Documention for '%s' is not followed"
 952                            " by the definition" % doc.symbol)
 953
 954     body = str(doc.body)
 955     if re.search(r'@\S+:', body, re.MULTILINE):
 956         raise QAPISemError(doc.info,
 957                            "Free-form documentation block must not contain"
 958                            " @NAME: sections")
 959
 960
 961 def check_definition_doc(doc, expr, info):
 962     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 963         if i in expr:
 964             meta = i
 965             break
 966
 967     name = expr[meta]
 968     if doc.symbol != name:
 969         raise QAPISemError(info, "Definition of '%s' follows documentation"
 970                            " for '%s'" % (name, doc.symbol))
 971     if doc.has_section('Returns') and 'command' not in expr:
 972         raise QAPISemError(info, "'Returns:' is only valid for commands")
 973
 974     if meta == 'union':
 975         args = expr.get('base', [])
 976     else:
 977         args = expr.get('data', [])
 978     if isinstance(args, str):
 979         return
 980     if isinstance(args, dict):
 981         args = args.keys()
 982     assert isinstance(args, list)
 983
 984     if (meta == 'alternate'
 985             or (meta == 'union' and not expr.get('discriminator'))):
 986         args.append('type')
 987
 988     for arg in args:
 989         if arg[0] == '*':
 990             opt = True
 991             desc = doc.args.get(arg[1:])
 992         else:
 993             opt = False
 994             desc = doc.args.get(arg)
 995         if not desc:
 996             continue
 997         desc.optional = opt
 998         desc_opt = "#optional" in str(desc)
 999         if desc_opt and not opt:
1000             raise QAPISemError(info, "Description has #optional, "
1001                                "but the declaration doesn't")
1002         if not desc_opt and opt:
1003             # TODO either fix the schema and make this an error,
1004             # or drop #optional entirely
1005             pass
1006
1007     doc_args = set(doc.args.keys())
1008     args = set([name.strip('*') for name in args])
1009     if not doc_args.issubset(args):
1010         raise QAPISemError(info, "The following documented members are not in "
1011                            "the declaration: %s" % ", ".join(doc_args - args))
1012
1013
1014 def check_docs(docs):
1015     for doc in docs:
1016         for section in doc.args.values() + doc.sections:
1017             content = str(section)
1018             if not content or content.isspace():
1019                 raise QAPISemError(doc.info,
1020                                    "Empty doc section '%s'" % section.name)
1021
1022         if not doc.expr:
1023             check_freeform_doc(doc)
1024         else:
1025             check_definition_doc(doc, doc.expr, doc.info)
1026
1027     return docs
1028
1029
1030 #
1031 # Schema compiler frontend
1032 #
1033
1034 class QAPISchemaEntity(object):
1035     def __init__(self, name, info, doc):
1036         assert isinstance(name, str)
1037         self.name = name
1038         # For explicitly defined entities, info points to the (explicit)
1039         # definition.  For builtins (and their arrays), info is None.
1040         # For implicitly defined entities, info points to a place that
1041         # triggered the implicit definition (there may be more than one
1042         # such place).
1043         self.info = info
1044         self.doc = doc
1045
1046     def c_name(self):
1047         return c_name(self.name)
1048
1049     def check(self, schema):
1050         pass
1051
1052     def is_implicit(self):
1053         return not self.info
1054
1055     def visit(self, visitor):
1056         pass
1057
1058
1059 class QAPISchemaVisitor(object):
1060     def visit_begin(self, schema):
1061         pass
1062
1063     def visit_end(self):
1064         pass
1065
1066     def visit_needed(self, entity):
1067         # Default to visiting everything
1068         return True
1069
1070     def visit_builtin_type(self, name, info, json_type):
1071         pass
1072
1073     def visit_enum_type(self, name, info, values, prefix):
1074         pass
1075
1076     def visit_array_type(self, name, info, element_type):
1077         pass
1078
1079     def visit_object_type(self, name, info, base, members, variants):
1080         pass
1081
1082     def visit_object_type_flat(self, name, info, members, variants):
1083         pass
1084
1085     def visit_alternate_type(self, name, info, variants):
1086         pass
1087
1088     def visit_command(self, name, info, arg_type, ret_type,
1089                       gen, success_response, boxed):
1090         pass
1091
1092     def visit_event(self, name, info, arg_type, boxed):
1093         pass
1094
1095
1096 class QAPISchemaType(QAPISchemaEntity):
1097     # Return the C type for common use.
1098     # For the types we commonly box, this is a pointer type.
1099     def c_type(self):
1100         pass
1101
1102     # Return the C type to be used in a parameter list.
1103     def c_param_type(self):
1104         return self.c_type()
1105
1106     # Return the C type to be used where we suppress boxing.
1107     def c_unboxed_type(self):
1108         return self.c_type()
1109
1110     def json_type(self):
1111         pass
1112
1113     def alternate_qtype(self):
1114         json2qtype = {
1115             'string':  'QTYPE_QSTRING',
1116             'number':  'QTYPE_QFLOAT',
1117             'int':     'QTYPE_QINT',
1118             'boolean': 'QTYPE_QBOOL',
1119             'object':  'QTYPE_QDICT'
1120         }
1121         return json2qtype.get(self.json_type())
1122
1123
1124 class QAPISchemaBuiltinType(QAPISchemaType):
1125     def __init__(self, name, json_type, c_type):
1126         QAPISchemaType.__init__(self, name, None, None)
1127         assert not c_type or isinstance(c_type, str)
1128         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1129                              'value')
1130         self._json_type_name = json_type
1131         self._c_type_name = c_type
1132
1133     def c_name(self):
1134         return self.name
1135
1136     def c_type(self):
1137         return self._c_type_name
1138
1139     def c_param_type(self):
1140         if self.name == 'str':
1141             return 'const ' + self._c_type_name
1142         return self._c_type_name
1143
1144     def json_type(self):
1145         return self._json_type_name
1146
1147     def visit(self, visitor):
1148         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1149
1150
1151 class QAPISchemaEnumType(QAPISchemaType):
1152     def __init__(self, name, info, doc, values, prefix):
1153         QAPISchemaType.__init__(self, name, info, doc)
1154         for v in values:
1155             assert isinstance(v, QAPISchemaMember)
1156             v.set_owner(name)
1157         assert prefix is None or isinstance(prefix, str)
1158         self.values = values
1159         self.prefix = prefix
1160
1161     def check(self, schema):
1162         seen = {}
1163         for v in self.values:
1164             v.check_clash(self.info, seen)
1165             if self.doc:
1166                 self.doc.connect_member(v)
1167
1168     def is_implicit(self):
1169         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1170         return self.name.endswith('Kind') or self.name == 'QType'
1171
1172     def c_type(self):
1173         return c_name(self.name)
1174
1175     def member_names(self):
1176         return [v.name for v in self.values]
1177
1178     def json_type(self):
1179         return 'string'
1180
1181     def visit(self, visitor):
1182         visitor.visit_enum_type(self.name, self.info,
1183                                 self.member_names(), self.prefix)
1184
1185
1186 class QAPISchemaArrayType(QAPISchemaType):
1187     def __init__(self, name, info, element_type):
1188         QAPISchemaType.__init__(self, name, info, None)
1189         assert isinstance(element_type, str)
1190         self._element_type_name = element_type
1191         self.element_type = None
1192
1193     def check(self, schema):
1194         self.element_type = schema.lookup_type(self._element_type_name)
1195         assert self.element_type
1196
1197     def is_implicit(self):
1198         return True
1199
1200     def c_type(self):
1201         return c_name(self.name) + pointer_suffix
1202
1203     def json_type(self):
1204         return 'array'
1205
1206     def visit(self, visitor):
1207         visitor.visit_array_type(self.name, self.info, self.element_type)
1208
1209
1210 class QAPISchemaObjectType(QAPISchemaType):
1211     def __init__(self, name, info, doc, base, local_members, variants):
1212         # struct has local_members, optional base, and no variants
1213         # flat union has base, variants, and no local_members
1214         # simple union has local_members, variants, and no base
1215         QAPISchemaType.__init__(self, name, info, doc)
1216         assert base is None or isinstance(base, str)
1217         for m in local_members:
1218             assert isinstance(m, QAPISchemaObjectTypeMember)
1219             m.set_owner(name)
1220         if variants is not None:
1221             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1222             variants.set_owner(name)
1223         self._base_name = base
1224         self.base = None
1225         self.local_members = local_members
1226         self.variants = variants
1227         self.members = None
1228
1229     def check(self, schema):
1230         if self.members is False:               # check for cycles
1231             raise QAPISemError(self.info,
1232                                "Object %s contains itself" % self.name)
1233         if self.members:
1234             return
1235         self.members = False                    # mark as being checked
1236         seen = OrderedDict()
1237         if self._base_name:
1238             self.base = schema.lookup_type(self._base_name)
1239             assert isinstance(self.base, QAPISchemaObjectType)
1240             self.base.check(schema)
1241             self.base.check_clash(schema, self.info, seen)
1242         for m in self.local_members:
1243             m.check(schema)
1244             m.check_clash(self.info, seen)
1245             if self.doc:
1246                 self.doc.connect_member(m)
1247         self.members = seen.values()
1248         if self.variants:
1249             self.variants.check(schema, seen)
1250             assert self.variants.tag_member in self.members
1251             self.variants.check_clash(schema, self.info, seen)
1252
1253     # Check that the members of this type do not cause duplicate JSON members,
1254     # and update seen to track the members seen so far. Report any errors
1255     # on behalf of info, which is not necessarily self.info
1256     def check_clash(self, schema, info, seen):
1257         assert not self.variants       # not implemented
1258         for m in self.members:
1259             m.check_clash(info, seen)
1260
1261     def is_implicit(self):
1262         # See QAPISchema._make_implicit_object_type(), as well as
1263         # _def_predefineds()
1264         return self.name.startswith('q_')
1265
1266     def is_empty(self):
1267         assert self.members is not None
1268         return not self.members and not self.variants
1269
1270     def c_name(self):
1271         assert self.name != 'q_empty'
1272         return QAPISchemaType.c_name(self)
1273
1274     def c_type(self):
1275         assert not self.is_implicit()
1276         return c_name(self.name) + pointer_suffix
1277
1278     def c_unboxed_type(self):
1279         return c_name(self.name)
1280
1281     def json_type(self):
1282         return 'object'
1283
1284     def visit(self, visitor):
1285         visitor.visit_object_type(self.name, self.info,
1286                                   self.base, self.local_members, self.variants)
1287         visitor.visit_object_type_flat(self.name, self.info,
1288                                        self.members, self.variants)
1289
1290
1291 class QAPISchemaMember(object):
1292     role = 'member'
1293
1294     def __init__(self, name):
1295         assert isinstance(name, str)
1296         self.name = name
1297         self.owner = None
1298
1299     def set_owner(self, name):
1300         assert not self.owner
1301         self.owner = name
1302
1303     def check_clash(self, info, seen):
1304         cname = c_name(self.name)
1305         if cname.lower() != cname and self.owner not in name_case_whitelist:
1306             raise QAPISemError(info,
1307                                "%s should not use uppercase" % self.describe())
1308         if cname in seen:
1309             raise QAPISemError(info, "%s collides with %s" %
1310                                (self.describe(), seen[cname].describe()))
1311         seen[cname] = self
1312
1313     def _pretty_owner(self):
1314         owner = self.owner
1315         if owner.startswith('q_obj_'):
1316             # See QAPISchema._make_implicit_object_type() - reverse the
1317             # mapping there to create a nice human-readable description
1318             owner = owner[6:]
1319             if owner.endswith('-arg'):
1320                 return '(parameter of %s)' % owner[:-4]
1321             elif owner.endswith('-base'):
1322                 return '(base of %s)' % owner[:-5]
1323             else:
1324                 assert owner.endswith('-wrapper')
1325                 # Unreachable and not implemented
1326                 assert False
1327         if owner.endswith('Kind'):
1328             # See QAPISchema._make_implicit_enum_type()
1329             return '(branch of %s)' % owner[:-4]
1330         return '(%s of %s)' % (self.role, owner)
1331
1332     def describe(self):
1333         return "'%s' %s" % (self.name, self._pretty_owner())
1334
1335
1336 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1337     def __init__(self, name, typ, optional):
1338         QAPISchemaMember.__init__(self, name)
1339         assert isinstance(typ, str)
1340         assert isinstance(optional, bool)
1341         self._type_name = typ
1342         self.type = None
1343         self.optional = optional
1344
1345     def check(self, schema):
1346         assert self.owner
1347         self.type = schema.lookup_type(self._type_name)
1348         assert self.type
1349
1350
1351 class QAPISchemaObjectTypeVariants(object):
1352     def __init__(self, tag_name, tag_member, variants):
1353         # Flat unions pass tag_name but not tag_member.
1354         # Simple unions and alternates pass tag_member but not tag_name.
1355         # After check(), tag_member is always set, and tag_name remains
1356         # a reliable witness of being used by a flat union.
1357         assert bool(tag_member) != bool(tag_name)
1358         assert (isinstance(tag_name, str) or
1359                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1360         assert len(variants) > 0
1361         for v in variants:
1362             assert isinstance(v, QAPISchemaObjectTypeVariant)
1363         self._tag_name = tag_name
1364         self.tag_member = tag_member
1365         self.variants = variants
1366
1367     def set_owner(self, name):
1368         for v in self.variants:
1369             v.set_owner(name)
1370
1371     def check(self, schema, seen):
1372         if not self.tag_member:    # flat union
1373             self.tag_member = seen[c_name(self._tag_name)]
1374             assert self._tag_name == self.tag_member.name
1375         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1376         for v in self.variants:
1377             v.check(schema)
1378             # Union names must match enum values; alternate names are
1379             # checked separately. Use 'seen' to tell the two apart.
1380             if seen:
1381                 assert v.name in self.tag_member.type.member_names()
1382                 assert isinstance(v.type, QAPISchemaObjectType)
1383                 v.type.check(schema)
1384
1385     def check_clash(self, schema, info, seen):
1386         for v in self.variants:
1387             # Reset seen map for each variant, since qapi names from one
1388             # branch do not affect another branch
1389             assert isinstance(v.type, QAPISchemaObjectType)
1390             v.type.check_clash(schema, info, dict(seen))
1391
1392
1393 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1394     role = 'branch'
1395
1396     def __init__(self, name, typ):
1397         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1398
1399
1400 class QAPISchemaAlternateType(QAPISchemaType):
1401     def __init__(self, name, info, doc, variants):
1402         QAPISchemaType.__init__(self, name, info, doc)
1403         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1404         assert variants.tag_member
1405         variants.set_owner(name)
1406         variants.tag_member.set_owner(self.name)
1407         self.variants = variants
1408
1409     def check(self, schema):
1410         self.variants.tag_member.check(schema)
1411         # Not calling self.variants.check_clash(), because there's nothing
1412         # to clash with
1413         self.variants.check(schema, {})
1414         # Alternate branch names have no relation to the tag enum values;
1415         # so we have to check for potential name collisions ourselves.
1416         seen = {}
1417         for v in self.variants.variants:
1418             v.check_clash(self.info, seen)
1419             if self.doc:
1420                 self.doc.connect_member(v)
1421
1422     def c_type(self):
1423         return c_name(self.name) + pointer_suffix
1424
1425     def json_type(self):
1426         return 'value'
1427
1428     def visit(self, visitor):
1429         visitor.visit_alternate_type(self.name, self.info, self.variants)
1430
1431     def is_empty(self):
1432         return False
1433
1434
1435 class QAPISchemaCommand(QAPISchemaEntity):
1436     def __init__(self, name, info, doc, arg_type, ret_type,
1437                  gen, success_response, boxed):
1438         QAPISchemaEntity.__init__(self, name, info, doc)
1439         assert not arg_type or isinstance(arg_type, str)
1440         assert not ret_type or isinstance(ret_type, str)
1441         self._arg_type_name = arg_type
1442         self.arg_type = None
1443         self._ret_type_name = ret_type
1444         self.ret_type = None
1445         self.gen = gen
1446         self.success_response = success_response
1447         self.boxed = boxed
1448
1449     def check(self, schema):
1450         if self._arg_type_name:
1451             self.arg_type = schema.lookup_type(self._arg_type_name)
1452             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1453                     isinstance(self.arg_type, QAPISchemaAlternateType))
1454             self.arg_type.check(schema)
1455             if self.boxed:
1456                 if self.arg_type.is_empty():
1457                     raise QAPISemError(self.info,
1458                                        "Cannot use 'boxed' with empty type")
1459             else:
1460                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1461                 assert not self.arg_type.variants
1462         elif self.boxed:
1463             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1464         if self._ret_type_name:
1465             self.ret_type = schema.lookup_type(self._ret_type_name)
1466             assert isinstance(self.ret_type, QAPISchemaType)
1467
1468     def visit(self, visitor):
1469         visitor.visit_command(self.name, self.info,
1470                               self.arg_type, self.ret_type,
1471                               self.gen, self.success_response, self.boxed)
1472
1473
1474 class QAPISchemaEvent(QAPISchemaEntity):
1475     def __init__(self, name, info, doc, arg_type, boxed):
1476         QAPISchemaEntity.__init__(self, name, info, doc)
1477         assert not arg_type or isinstance(arg_type, str)
1478         self._arg_type_name = arg_type
1479         self.arg_type = None
1480         self.boxed = boxed
1481
1482     def check(self, schema):
1483         if self._arg_type_name:
1484             self.arg_type = schema.lookup_type(self._arg_type_name)
1485             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1486                     isinstance(self.arg_type, QAPISchemaAlternateType))
1487             self.arg_type.check(schema)
1488             if self.boxed:
1489                 if self.arg_type.is_empty():
1490                     raise QAPISemError(self.info,
1491                                        "Cannot use 'boxed' with empty type")
1492             else:
1493                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1494                 assert not self.arg_type.variants
1495         elif self.boxed:
1496             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1497
1498     def visit(self, visitor):
1499         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1500
1501
1502 class QAPISchema(object):
1503     def __init__(self, fname):
1504         try:
1505             parser = QAPISchemaParser(open(fname, "r"))
1506             self.exprs = check_exprs(parser.exprs)
1507             self.docs = check_docs(parser.docs)
1508             self._entity_dict = {}
1509             self._predefining = True
1510             self._def_predefineds()
1511             self._predefining = False
1512             self._def_exprs()
1513             self.check()
1514         except QAPIError as err:
1515             print >>sys.stderr, err
1516             exit(1)
1517
1518     def _def_entity(self, ent):
1519         # Only the predefined types are allowed to not have info
1520         assert ent.info or self._predefining
1521         assert ent.name not in self._entity_dict
1522         self._entity_dict[ent.name] = ent
1523
1524     def lookup_entity(self, name, typ=None):
1525         ent = self._entity_dict.get(name)
1526         if typ and not isinstance(ent, typ):
1527             return None
1528         return ent
1529
1530     def lookup_type(self, name):
1531         return self.lookup_entity(name, QAPISchemaType)
1532
1533     def _def_builtin_type(self, name, json_type, c_type):
1534         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1535         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1536         # qapi-types.h from a single .c, all arrays of builtins must be
1537         # declared in the first file whether or not they are used.  Nicer
1538         # would be to use lazy instantiation, while figuring out how to
1539         # avoid compilation issues with multiple qapi-types.h.
1540         self._make_array_type(name, None)
1541
1542     def _def_predefineds(self):
1543         for t in [('str',    'string',  'char' + pointer_suffix),
1544                   ('number', 'number',  'double'),
1545                   ('int',    'int',     'int64_t'),
1546                   ('int8',   'int',     'int8_t'),
1547                   ('int16',  'int',     'int16_t'),
1548                   ('int32',  'int',     'int32_t'),
1549                   ('int64',  'int',     'int64_t'),
1550                   ('uint8',  'int',     'uint8_t'),
1551                   ('uint16', 'int',     'uint16_t'),
1552                   ('uint32', 'int',     'uint32_t'),
1553                   ('uint64', 'int',     'uint64_t'),
1554                   ('size',   'int',     'uint64_t'),
1555                   ('bool',   'boolean', 'bool'),
1556                   ('any',    'value',   'QObject' + pointer_suffix)]:
1557             self._def_builtin_type(*t)
1558         self.the_empty_object_type = QAPISchemaObjectType(
1559             'q_empty', None, None, None, [], None)
1560         self._def_entity(self.the_empty_object_type)
1561         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1562                                                 'qstring', 'qdict', 'qlist',
1563                                                 'qfloat', 'qbool'])
1564         self._def_entity(QAPISchemaEnumType('QType', None, None,
1565                                             qtype_values, 'QTYPE'))
1566
1567     def _make_enum_members(self, values):
1568         return [QAPISchemaMember(v) for v in values]
1569
1570     def _make_implicit_enum_type(self, name, info, values):
1571         # See also QAPISchemaObjectTypeMember._pretty_owner()
1572         name = name + 'Kind'   # Use namespace reserved by add_name()
1573         self._def_entity(QAPISchemaEnumType(
1574             name, info, None, self._make_enum_members(values), None))
1575         return name
1576
1577     def _make_array_type(self, element_type, info):
1578         name = element_type + 'List'   # Use namespace reserved by add_name()
1579         if not self.lookup_type(name):
1580             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1581         return name
1582
1583     def _make_implicit_object_type(self, name, info, doc, role, members):
1584         if not members:
1585             return None
1586         # See also QAPISchemaObjectTypeMember._pretty_owner()
1587         name = 'q_obj_%s-%s' % (name, role)
1588         if not self.lookup_entity(name, QAPISchemaObjectType):
1589             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1590                                                   members, None))
1591         return name
1592
1593     def _def_enum_type(self, expr, info, doc):
1594         name = expr['enum']
1595         data = expr['data']
1596         prefix = expr.get('prefix')
1597         self._def_entity(QAPISchemaEnumType(
1598             name, info, doc, self._make_enum_members(data), prefix))
1599
1600     def _make_member(self, name, typ, info):
1601         optional = False
1602         if name.startswith('*'):
1603             name = name[1:]
1604             optional = True
1605         if isinstance(typ, list):
1606             assert len(typ) == 1
1607             typ = self._make_array_type(typ[0], info)
1608         return QAPISchemaObjectTypeMember(name, typ, optional)
1609
1610     def _make_members(self, data, info):
1611         return [self._make_member(key, value, info)
1612                 for (key, value) in data.iteritems()]
1613
1614     def _def_struct_type(self, expr, info, doc):
1615         name = expr['struct']
1616         base = expr.get('base')
1617         data = expr['data']
1618         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1619                                               self._make_members(data, info),
1620                                               None))
1621
1622     def _make_variant(self, case, typ):
1623         return QAPISchemaObjectTypeVariant(case, typ)
1624
1625     def _make_simple_variant(self, case, typ, info):
1626         if isinstance(typ, list):
1627             assert len(typ) == 1
1628             typ = self._make_array_type(typ[0], info)
1629         typ = self._make_implicit_object_type(
1630             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1631         return QAPISchemaObjectTypeVariant(case, typ)
1632
1633     def _def_union_type(self, expr, info, doc):
1634         name = expr['union']
1635         data = expr['data']
1636         base = expr.get('base')
1637         tag_name = expr.get('discriminator')
1638         tag_member = None
1639         if isinstance(base, dict):
1640             base = (self._make_implicit_object_type(
1641                     name, info, doc, 'base', self._make_members(base, info)))
1642         if tag_name:
1643             variants = [self._make_variant(key, value)
1644                         for (key, value) in data.iteritems()]
1645             members = []
1646         else:
1647             variants = [self._make_simple_variant(key, value, info)
1648                         for (key, value) in data.iteritems()]
1649             typ = self._make_implicit_enum_type(name, info,
1650                                                 [v.name for v in variants])
1651             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1652             members = [tag_member]
1653         self._def_entity(
1654             QAPISchemaObjectType(name, info, doc, base, members,
1655                                  QAPISchemaObjectTypeVariants(tag_name,
1656                                                               tag_member,
1657                                                               variants)))
1658
1659     def _def_alternate_type(self, expr, info, doc):
1660         name = expr['alternate']
1661         data = expr['data']
1662         variants = [self._make_variant(key, value)
1663                     for (key, value) in data.iteritems()]
1664         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1665         self._def_entity(
1666             QAPISchemaAlternateType(name, info, doc,
1667                                     QAPISchemaObjectTypeVariants(None,
1668                                                                  tag_member,
1669                                                                  variants)))
1670
1671     def _def_command(self, expr, info, doc):
1672         name = expr['command']
1673         data = expr.get('data')
1674         rets = expr.get('returns')
1675         gen = expr.get('gen', True)
1676         success_response = expr.get('success-response', True)
1677         boxed = expr.get('boxed', False)
1678         if isinstance(data, OrderedDict):
1679             data = self._make_implicit_object_type(
1680                 name, info, doc, 'arg', self._make_members(data, info))
1681         if isinstance(rets, list):
1682             assert len(rets) == 1
1683             rets = self._make_array_type(rets[0], info)
1684         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1685                                            gen, success_response, boxed))
1686
1687     def _def_event(self, expr, info, doc):
1688         name = expr['event']
1689         data = expr.get('data')
1690         boxed = expr.get('boxed', False)
1691         if isinstance(data, OrderedDict):
1692             data = self._make_implicit_object_type(
1693                 name, info, doc, 'arg', self._make_members(data, info))
1694         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1695
1696     def _def_exprs(self):
1697         for expr_elem in self.exprs:
1698             expr = expr_elem['expr']
1699             info = expr_elem['info']
1700             doc = expr_elem.get('doc')
1701             if 'enum' in expr:
1702                 self._def_enum_type(expr, info, doc)
1703             elif 'struct' in expr:
1704                 self._def_struct_type(expr, info, doc)
1705             elif 'union' in expr:
1706                 self._def_union_type(expr, info, doc)
1707             elif 'alternate' in expr:
1708                 self._def_alternate_type(expr, info, doc)
1709             elif 'command' in expr:
1710                 self._def_command(expr, info, doc)
1711             elif 'event' in expr:
1712                 self._def_event(expr, info, doc)
1713             else:
1714                 assert False
1715
1716     def check(self):
1717         for ent in self._entity_dict.values():
1718             ent.check(self)
1719
1720     def visit(self, visitor):
1721         visitor.visit_begin(self)
1722         for (name, entity) in sorted(self._entity_dict.items()):
1723             if visitor.visit_needed(entity):
1724                 entity.visit(visitor)
1725         visitor.visit_end()
1726
1727
1728 #
1729 # Code generation helpers
1730 #
1731
1732 def camel_case(name):
1733     new_name = ''
1734     first = True
1735     for ch in name:
1736         if ch in ['_', '-']:
1737             first = True
1738         elif first:
1739             new_name += ch.upper()
1740             first = False
1741         else:
1742             new_name += ch.lower()
1743     return new_name
1744
1745
1746 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1747 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1748 # ENUM24_Name -> ENUM24_NAME
1749 def camel_to_upper(value):
1750     c_fun_str = c_name(value, False)
1751     if value.isupper():
1752         return c_fun_str
1753
1754     new_name = ''
1755     l = len(c_fun_str)
1756     for i in range(l):
1757         c = c_fun_str[i]
1758         # When c is upper and no "_" appears before, do more checks
1759         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1760             if i < l - 1 and c_fun_str[i + 1].islower():
1761                 new_name += '_'
1762             elif c_fun_str[i - 1].isdigit():
1763                 new_name += '_'
1764         new_name += c
1765     return new_name.lstrip('_').upper()
1766
1767
1768 def c_enum_const(type_name, const_name, prefix=None):
1769     if prefix is not None:
1770         type_name = prefix
1771     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1772
1773 c_name_trans = string.maketrans('.-', '__')
1774
1775
1776 # Map @name to a valid C identifier.
1777 # If @protect, avoid returning certain ticklish identifiers (like
1778 # C keywords) by prepending "q_".
1779 #
1780 # Used for converting 'name' from a 'name':'type' qapi definition
1781 # into a generated struct member, as well as converting type names
1782 # into substrings of a generated C function name.
1783 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1784 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1785 def c_name(name, protect=True):
1786     # ANSI X3J11/88-090, 3.1.1
1787     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1788                      'default', 'do', 'double', 'else', 'enum', 'extern',
1789                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1790                      'return', 'short', 'signed', 'sizeof', 'static',
1791                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1792                      'void', 'volatile', 'while'])
1793     # ISO/IEC 9899:1999, 6.4.1
1794     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1795     # ISO/IEC 9899:2011, 6.4.1
1796     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1797                      '_Noreturn', '_Static_assert', '_Thread_local'])
1798     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1799     # excluding _.*
1800     gcc_words = set(['asm', 'typeof'])
1801     # C++ ISO/IEC 14882:2003 2.11
1802     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1803                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1804                      'namespace', 'new', 'operator', 'private', 'protected',
1805                      'public', 'reinterpret_cast', 'static_cast', 'template',
1806                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1807                      'using', 'virtual', 'wchar_t',
1808                      # alternative representations
1809                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1810                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1811     # namespace pollution:
1812     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1813     name = name.translate(c_name_trans)
1814     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1815                     | cpp_words | polluted_words):
1816         return "q_" + name
1817     return name
1818
1819 eatspace = '\033EATSPACE.'
1820 pointer_suffix = ' *' + eatspace
1821
1822
1823 def genindent(count):
1824     ret = ""
1825     for _ in range(count):
1826         ret += " "
1827     return ret
1828
1829 indent_level = 0
1830
1831
1832 def push_indent(indent_amount=4):
1833     global indent_level
1834     indent_level += indent_amount
1835
1836
1837 def pop_indent(indent_amount=4):
1838     global indent_level
1839     indent_level -= indent_amount
1840
1841
1842 # Generate @code with @kwds interpolated.
1843 # Obey indent_level, and strip eatspace.
1844 def cgen(code, **kwds):
1845     raw = code % kwds
1846     if indent_level:
1847         indent = genindent(indent_level)
1848         # re.subn() lacks flags support before Python 2.7, use re.compile()
1849         raw = re.subn(re.compile("^.", re.MULTILINE),
1850                       indent + r'\g<0>', raw)
1851         raw = raw[0]
1852     return re.sub(re.escape(eatspace) + ' *', '', raw)
1853
1854
1855 def mcgen(code, **kwds):
1856     if code[0] == '\n':
1857         code = code[1:]
1858     return cgen(code, **kwds)
1859
1860
1861 def guardname(filename):
1862     return c_name(filename, protect=False).upper()
1863
1864
1865 def guardstart(name):
1866     return mcgen('''
1867
1868 #ifndef %(name)s
1869 #define %(name)s
1870
1871 ''',
1872                  name=guardname(name))
1873
1874
1875 def guardend(name):
1876     return mcgen('''
1877
1878 #endif /* %(name)s */
1879
1880 ''',
1881                  name=guardname(name))
1882
1883
1884 def gen_enum_lookup(name, values, prefix=None):
1885     ret = mcgen('''
1886
1887 const char *const %(c_name)s_lookup[] = {
1888 ''',
1889                 c_name=c_name(name))
1890     for value in values:
1891         index = c_enum_const(name, value, prefix)
1892         ret += mcgen('''
1893     [%(index)s] = "%(value)s",
1894 ''',
1895                      index=index, value=value)
1896
1897     max_index = c_enum_const(name, '_MAX', prefix)
1898     ret += mcgen('''
1899     [%(max_index)s] = NULL,
1900 };
1901 ''',
1902                  max_index=max_index)
1903     return ret
1904
1905
1906 def gen_enum(name, values, prefix=None):
1907     # append automatically generated _MAX value
1908     enum_values = values + ['_MAX']
1909
1910     ret = mcgen('''
1911
1912 typedef enum %(c_name)s {
1913 ''',
1914                 c_name=c_name(name))
1915
1916     i = 0
1917     for value in enum_values:
1918         ret += mcgen('''
1919     %(c_enum)s = %(i)d,
1920 ''',
1921                      c_enum=c_enum_const(name, value, prefix),
1922                      i=i)
1923         i += 1
1924
1925     ret += mcgen('''
1926 } %(c_name)s;
1927 ''',
1928                  c_name=c_name(name))
1929
1930     ret += mcgen('''
1931
1932 extern const char *const %(c_name)s_lookup[];
1933 ''',
1934                  c_name=c_name(name))
1935     return ret
1936
1937
1938 def gen_params(arg_type, boxed, extra):
1939     if not arg_type:
1940         assert not boxed
1941         return extra
1942     ret = ''
1943     sep = ''
1944     if boxed:
1945         ret += '%s arg' % arg_type.c_param_type()
1946         sep = ', '
1947     else:
1948         assert not arg_type.variants
1949         for memb in arg_type.members:
1950             ret += sep
1951             sep = ', '
1952             if memb.optional:
1953                 ret += 'bool has_%s, ' % c_name(memb.name)
1954             ret += '%s %s' % (memb.type.c_param_type(),
1955                               c_name(memb.name))
1956     if extra:
1957         ret += sep + extra
1958     return ret
1959
1960
1961 #
1962 # Common command line parsing
1963 #
1964
1965
1966 def parse_command_line(extra_options="", extra_long_options=[]):
1967
1968     try:
1969         opts, args = getopt.gnu_getopt(sys.argv[1:],
1970                                        "chp:o:" + extra_options,
1971                                        ["source", "header", "prefix=",
1972                                         "output-dir="] + extra_long_options)
1973     except getopt.GetoptError as err:
1974         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1975         sys.exit(1)
1976
1977     output_dir = ""
1978     prefix = ""
1979     do_c = False
1980     do_h = False
1981     extra_opts = []
1982
1983     for oa in opts:
1984         o, a = oa
1985         if o in ("-p", "--prefix"):
1986             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1987             if match.end() != len(a):
1988                 print >>sys.stderr, \
1989                     "%s: 'funny character '%s' in argument of --prefix" \
1990                     % (sys.argv[0], a[match.end()])
1991                 sys.exit(1)
1992             prefix = a
1993         elif o in ("-o", "--output-dir"):
1994             output_dir = a + "/"
1995         elif o in ("-c", "--source"):
1996             do_c = True
1997         elif o in ("-h", "--header"):
1998             do_h = True
1999         else:
2000             extra_opts.append(oa)
2001
2002     if not do_c and not do_h:
2003         do_c = True
2004         do_h = True
2005
2006     if len(args) != 1:
2007         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2008         sys.exit(1)
2009     fname = args[0]
2010
2011     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2012
2013 #
2014 # Generate output files with boilerplate
2015 #
2016
2017
2018 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2019                 c_comment, h_comment):
2020     guard = guardname(prefix + h_file)
2021     c_file = output_dir + prefix + c_file
2022     h_file = output_dir + prefix + h_file
2023
2024     if output_dir:
2025         try:
2026             os.makedirs(output_dir)
2027         except os.error as e:
2028             if e.errno != errno.EEXIST:
2029                 raise
2030
2031     def maybe_open(really, name, opt):
2032         if really:
2033             return open(name, opt)
2034         else:
2035             import StringIO
2036             return StringIO.StringIO()
2037
2038     fdef = maybe_open(do_c, c_file, 'w')
2039     fdecl = maybe_open(do_h, h_file, 'w')
2040
2041     fdef.write(mcgen('''
2042 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2043 %(comment)s
2044 ''',
2045                      comment=c_comment))
2046
2047     fdecl.write(mcgen('''
2048 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2049 %(comment)s
2050 #ifndef %(guard)s
2051 #define %(guard)s
2052
2053 ''',
2054                       comment=h_comment, guard=guard))
2055
2056     return (fdef, fdecl)
2057
2058
2059 def close_output(fdef, fdecl):
2060     fdecl.write('''
2061 #endif
2062 ''')
2063     fdecl.close()
2064     fdef.close()