scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 events = []
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110             self.optional = False
 111
 112         def append(self, line):
 113             self.content.append(line)
 114
 115         def __repr__(self):
 116             return '\n'.join(self.content).strip()
 117
 118     class ArgSection(Section):
 119         def __init__(self, name):
 120             QAPIDoc.Section.__init__(self, name)
 121             self.member = None
 122
 123         def connect(self, member):
 124             self.member = member
 125
 126     def __init__(self, parser, info):
 127         # self.parser is used to report errors with QAPIParseError.  The
 128         # resulting error position depends on the state of the parser.
 129         # It happens to be the beginning of the comment.  More or less
 130         # servicable, but action at a distance.
 131         self.parser = parser
 132         self.info = info
 133         self.symbol = None
 134         self.body = QAPIDoc.Section()
 135         # dict mapping parameter name to ArgSection
 136         self.args = OrderedDict()
 137         # a list of Section
 138         self.sections = []
 139         # the current section
 140         self.section = self.body
 141         # associated expression (to be set by expression parser)
 142         self.expr = None
 143
 144     def has_section(self, name):
 145         """Return True if we have a section with this name."""
 146         for i in self.sections:
 147             if i.name == name:
 148                 return True
 149         return False
 150
 151     def append(self, line):
 152         """Parse a comment line and add it to the documentation."""
 153         line = line[1:]
 154         if not line:
 155             self._append_freeform(line)
 156             return
 157
 158         if line[0] != ' ':
 159             raise QAPIParseError(self.parser, "Missing space after #")
 160         line = line[1:]
 161
 162         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 163         # recognized, and get silently treated as ordinary text
 164         if self.symbol:
 165             self._append_symbol_line(line)
 166         elif not self.body.content and line.startswith('@'):
 167             if not line.endswith(':'):
 168                 raise QAPIParseError(self.parser, "Line should end with :")
 169             self.symbol = line[1:-1]
 170             # FIXME invalid names other than the empty string aren't flagged
 171             if not self.symbol:
 172                 raise QAPIParseError(self.parser, "Invalid name")
 173         else:
 174             self._append_freeform(line)
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=''):
 207         if name in ('Returns', 'Since') and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self.section = QAPIDoc.Section(name)
 211         self.sections.append(self.section)
 212
 213     def _append_freeform(self, line):
 214         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 215         if (in_arg and self.section.content
 216                 and not self.section.content[-1]
 217                 and line and not line[0].isspace()):
 218             self._start_section()
 219         if (in_arg or not self.section.name
 220                 or not self.section.name.startswith('Example')):
 221             line = line.strip()
 222         # TODO Drop this once the dust has settled
 223         if (isinstance(self.section, QAPIDoc.ArgSection)
 224                 and '#optional' in line):
 225             raise QAPISemError(self.info, "Please drop the #optional tag")
 226         self.section.append(line)
 227
 228     def connect_member(self, member):
 229         if member.name not in self.args:
 230             # Undocumented TODO outlaw
 231             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 232         self.args[member.name].connect(member)
 233
 234
 235 class QAPISchemaParser(object):
 236
 237     def __init__(self, fp, previously_included=[], incl_info=None):
 238         abs_fname = os.path.abspath(fp.name)
 239         fname = fp.name
 240         self.fname = fname
 241         previously_included.append(abs_fname)
 242         self.incl_info = incl_info
 243         self.src = fp.read()
 244         if self.src == '' or self.src[-1] != '\n':
 245             self.src += '\n'
 246         self.cursor = 0
 247         self.line = 1
 248         self.line_pos = 0
 249         self.exprs = []
 250         self.docs = []
 251         self.accept()
 252
 253         while self.tok is not None:
 254             info = {'file': fname, 'line': self.line,
 255                     'parent': self.incl_info}
 256             if self.tok == '#':
 257                 doc = self.get_doc(info)
 258                 self.docs.append(doc)
 259                 continue
 260
 261             expr = self.get_expr(False)
 262             if 'include' in expr:
 263                 if len(expr) != 1:
 264                     raise QAPISemError(info, "Invalid 'include' directive")
 265                 include = expr['include']
 266                 if not isinstance(include, str):
 267                     raise QAPISemError(info,
 268                                        "Value of 'include' must be a string")
 269                 self._include(include, info, os.path.dirname(abs_fname),
 270                               previously_included)
 271             elif "pragma" in expr:
 272                 if len(expr) != 1:
 273                     raise QAPISemError(info, "Invalid 'pragma' directive")
 274                 pragma = expr['pragma']
 275                 if not isinstance(pragma, dict):
 276                     raise QAPISemError(
 277                         info, "Value of 'pragma' must be a dictionary")
 278                 for name, value in pragma.iteritems():
 279                     self._pragma(name, value, info)
 280             else:
 281                 expr_elem = {'expr': expr,
 282                              'info': info}
 283                 if (self.docs
 284                         and self.docs[-1].info['file'] == fname
 285                         and not self.docs[-1].expr):
 286                     self.docs[-1].expr = expr
 287                     expr_elem['doc'] = self.docs[-1]
 288
 289                 self.exprs.append(expr_elem)
 290
 291     def _include(self, include, info, base_dir, previously_included):
 292         incl_abs_fname = os.path.join(base_dir, include)
 293         # catch inclusion cycle
 294         inf = info
 295         while inf:
 296             if incl_abs_fname == os.path.abspath(inf['file']):
 297                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 298             inf = inf['parent']
 299
 300         # skip multiple include of the same file
 301         if incl_abs_fname in previously_included:
 302             return
 303         try:
 304             fobj = open(incl_abs_fname, 'r')
 305         except IOError as e:
 306             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 307         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 308         self.exprs.extend(exprs_include.exprs)
 309         self.docs.extend(exprs_include.docs)
 310
 311     def _pragma(self, name, value, info):
 312         global doc_required, returns_whitelist, name_case_whitelist
 313         if name == 'doc-required':
 314             if not isinstance(value, bool):
 315                 raise QAPISemError(info,
 316                                    "Pragma 'doc-required' must be boolean")
 317             doc_required = value
 318         elif name == 'returns-whitelist':
 319             if (not isinstance(value, list)
 320                     or any([not isinstance(elt, str) for elt in value])):
 321                 raise QAPISemError(info,
 322                                    "Pragma returns-whitelist must be"
 323                                    " a list of strings")
 324             returns_whitelist = value
 325         elif name == 'name-case-whitelist':
 326             if (not isinstance(value, list)
 327                     or any([not isinstance(elt, str) for elt in value])):
 328                 raise QAPISemError(info,
 329                                    "Pragma name-case-whitelist must be"
 330                                    " a list of strings")
 331             name_case_whitelist = value
 332         else:
 333             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 334
 335     def accept(self, skip_comment=True):
 336         while True:
 337             self.tok = self.src[self.cursor]
 338             self.pos = self.cursor
 339             self.cursor += 1
 340             self.val = None
 341
 342             if self.tok == '#':
 343                 if self.src[self.cursor] == '#':
 344                     # Start of doc comment
 345                     skip_comment = False
 346                 self.cursor = self.src.find('\n', self.cursor)
 347                 if not skip_comment:
 348                     self.val = self.src[self.pos:self.cursor]
 349                     return
 350             elif self.tok in '{}:,[]':
 351                 return
 352             elif self.tok == "'":
 353                 string = ''
 354                 esc = False
 355                 while True:
 356                     ch = self.src[self.cursor]
 357                     self.cursor += 1
 358                     if ch == '\n':
 359                         raise QAPIParseError(self, 'Missing terminating "\'"')
 360                     if esc:
 361                         if ch == 'b':
 362                             string += '\b'
 363                         elif ch == 'f':
 364                             string += '\f'
 365                         elif ch == 'n':
 366                             string += '\n'
 367                         elif ch == 'r':
 368                             string += '\r'
 369                         elif ch == 't':
 370                             string += '\t'
 371                         elif ch == 'u':
 372                             value = 0
 373                             for _ in range(0, 4):
 374                                 ch = self.src[self.cursor]
 375                                 self.cursor += 1
 376                                 if ch not in '0123456789abcdefABCDEF':
 377                                     raise QAPIParseError(self,
 378                                                          '\\u escape needs 4 '
 379                                                          'hex digits')
 380                                 value = (value << 4) + int(ch, 16)
 381                             # If Python 2 and 3 didn't disagree so much on
 382                             # how to handle Unicode, then we could allow
 383                             # Unicode string defaults.  But most of QAPI is
 384                             # ASCII-only, so we aren't losing much for now.
 385                             if not value or value > 0x7f:
 386                                 raise QAPIParseError(self,
 387                                                      'For now, \\u escape '
 388                                                      'only supports non-zero '
 389                                                      'values up to \\u007f')
 390                             string += chr(value)
 391                         elif ch in '\\/\'"':
 392                             string += ch
 393                         else:
 394                             raise QAPIParseError(self,
 395                                                  "Unknown escape \\%s" % ch)
 396                         esc = False
 397                     elif ch == '\\':
 398                         esc = True
 399                     elif ch == "'":
 400                         self.val = string
 401                         return
 402                     else:
 403                         string += ch
 404             elif self.src.startswith('true', self.pos):
 405                 self.val = True
 406                 self.cursor += 3
 407                 return
 408             elif self.src.startswith('false', self.pos):
 409                 self.val = False
 410                 self.cursor += 4
 411                 return
 412             elif self.src.startswith('null', self.pos):
 413                 self.val = None
 414                 self.cursor += 3
 415                 return
 416             elif self.tok == '\n':
 417                 if self.cursor == len(self.src):
 418                     self.tok = None
 419                     return
 420                 self.line += 1
 421                 self.line_pos = self.cursor
 422             elif not self.tok.isspace():
 423                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 424
 425     def get_members(self):
 426         expr = OrderedDict()
 427         if self.tok == '}':
 428             self.accept()
 429             return expr
 430         if self.tok != "'":
 431             raise QAPIParseError(self, 'Expected string or "}"')
 432         while True:
 433             key = self.val
 434             self.accept()
 435             if self.tok != ':':
 436                 raise QAPIParseError(self, 'Expected ":"')
 437             self.accept()
 438             if key in expr:
 439                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 440             expr[key] = self.get_expr(True)
 441             if self.tok == '}':
 442                 self.accept()
 443                 return expr
 444             if self.tok != ',':
 445                 raise QAPIParseError(self, 'Expected "," or "}"')
 446             self.accept()
 447             if self.tok != "'":
 448                 raise QAPIParseError(self, 'Expected string')
 449
 450     def get_values(self):
 451         expr = []
 452         if self.tok == ']':
 453             self.accept()
 454             return expr
 455         if self.tok not in "{['tfn":
 456             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 457                                  'boolean or "null"')
 458         while True:
 459             expr.append(self.get_expr(True))
 460             if self.tok == ']':
 461                 self.accept()
 462                 return expr
 463             if self.tok != ',':
 464                 raise QAPIParseError(self, 'Expected "," or "]"')
 465             self.accept()
 466
 467     def get_expr(self, nested):
 468         if self.tok != '{' and not nested:
 469             raise QAPIParseError(self, 'Expected "{"')
 470         if self.tok == '{':
 471             self.accept()
 472             expr = self.get_members()
 473         elif self.tok == '[':
 474             self.accept()
 475             expr = self.get_values()
 476         elif self.tok in "'tfn":
 477             expr = self.val
 478             self.accept()
 479         else:
 480             raise QAPIParseError(self, 'Expected "{", "[" or string')
 481         return expr
 482
 483     def get_doc(self, info):
 484         if self.val != '##':
 485             raise QAPIParseError(self, "Junk after '##' at start of "
 486                                  "documentation comment")
 487
 488         doc = QAPIDoc(self, info)
 489         self.accept(False)
 490         while self.tok == '#':
 491             if self.val.startswith('##'):
 492                 # End of doc comment
 493                 if self.val != '##':
 494                     raise QAPIParseError(self, "Junk after '##' at end of "
 495                                          "documentation comment")
 496                 self.accept()
 497                 return doc
 498             else:
 499                 doc.append(self.val)
 500             self.accept(False)
 501
 502         raise QAPIParseError(self, "Documentation comment must end with '##'")
 503
 504
 505 #
 506 # Semantic analysis of schema expressions
 507 # TODO fold into QAPISchema
 508 # TODO catching name collisions in generated code would be nice
 509 #
 510
 511
 512 def find_base_members(base):
 513     if isinstance(base, dict):
 514         return base
 515     base_struct_define = find_struct(base)
 516     if not base_struct_define:
 517         return None
 518     return base_struct_define['data']
 519
 520
 521 # Return the qtype of an alternate branch, or None on error.
 522 def find_alternate_member_qtype(qapi_type):
 523     if qapi_type in builtin_types:
 524         return builtin_types[qapi_type]
 525     elif find_struct(qapi_type):
 526         return 'QTYPE_QDICT'
 527     elif find_enum(qapi_type):
 528         return 'QTYPE_QSTRING'
 529     elif find_union(qapi_type):
 530         return 'QTYPE_QDICT'
 531     return None
 532
 533
 534 # Return the discriminator enum define if discriminator is specified as an
 535 # enum type, otherwise return None.
 536 def discriminator_find_enum_define(expr):
 537     base = expr.get('base')
 538     discriminator = expr.get('discriminator')
 539
 540     if not (discriminator and base):
 541         return None
 542
 543     base_members = find_base_members(base)
 544     if not base_members:
 545         return None
 546
 547     discriminator_type = base_members.get(discriminator)
 548     if not discriminator_type:
 549         return None
 550
 551     return find_enum(discriminator_type)
 552
 553
 554 # Names must be letters, numbers, -, and _.  They must start with letter,
 555 # except for downstream extensions which must start with __RFQDN_.
 556 # Dots are only valid in the downstream extension prefix.
 557 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 558                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 559
 560
 561 def check_name(info, source, name, allow_optional=False,
 562                enum_member=False):
 563     global valid_name
 564     membername = name
 565
 566     if not isinstance(name, str):
 567         raise QAPISemError(info, "%s requires a string name" % source)
 568     if name.startswith('*'):
 569         membername = name[1:]
 570         if not allow_optional:
 571             raise QAPISemError(info, "%s does not allow optional name '%s'"
 572                                % (source, name))
 573     # Enum members can start with a digit, because the generated C
 574     # code always prefixes it with the enum name
 575     if enum_member and membername[0].isdigit():
 576         membername = 'D' + membername
 577     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 578     # and 'q_obj_*' implicit type names.
 579     if not valid_name.match(membername) or \
 580        c_name(membername, False).startswith('q_'):
 581         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 582
 583
 584 def add_name(name, info, meta, implicit=False):
 585     global all_names
 586     check_name(info, "'%s'" % meta, name)
 587     # FIXME should reject names that differ only in '_' vs. '.'
 588     # vs. '-', because they're liable to clash in generated C.
 589     if name in all_names:
 590         raise QAPISemError(info, "%s '%s' is already defined"
 591                            % (all_names[name], name))
 592     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 593         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 594                            % (meta, name, name[-4:]))
 595     all_names[name] = meta
 596
 597
 598 def add_struct(definition, info):
 599     global struct_types
 600     name = definition['struct']
 601     add_name(name, info, 'struct')
 602     struct_types.append(definition)
 603
 604
 605 def find_struct(name):
 606     global struct_types
 607     for struct in struct_types:
 608         if struct['struct'] == name:
 609             return struct
 610     return None
 611
 612
 613 def add_union(definition, info):
 614     global union_types
 615     name = definition['union']
 616     add_name(name, info, 'union')
 617     union_types.append(definition)
 618
 619
 620 def find_union(name):
 621     global union_types
 622     for union in union_types:
 623         if union['union'] == name:
 624             return union
 625     return None
 626
 627
 628 def add_enum(name, info, enum_values=None, implicit=False):
 629     global enum_types
 630     add_name(name, info, 'enum', implicit)
 631     enum_types.append({'enum_name': name, 'enum_values': enum_values})
 632
 633
 634 def find_enum(name):
 635     global enum_types
 636     for enum in enum_types:
 637         if enum['enum_name'] == name:
 638             return enum
 639     return None
 640
 641
 642 def is_enum(name):
 643     return find_enum(name) is not None
 644
 645
 646 def check_type(info, source, value, allow_array=False,
 647                allow_dict=False, allow_optional=False,
 648                allow_metas=[]):
 649     global all_names
 650
 651     if value is None:
 652         return
 653
 654     # Check if array type for value is okay
 655     if isinstance(value, list):
 656         if not allow_array:
 657             raise QAPISemError(info, "%s cannot be an array" % source)
 658         if len(value) != 1 or not isinstance(value[0], str):
 659             raise QAPISemError(info,
 660                                "%s: array type must contain single type name" %
 661                                source)
 662         value = value[0]
 663
 664     # Check if type name for value is okay
 665     if isinstance(value, str):
 666         if value not in all_names:
 667             raise QAPISemError(info, "%s uses unknown type '%s'"
 668                                % (source, value))
 669         if not all_names[value] in allow_metas:
 670             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 671                                (source, all_names[value], value))
 672         return
 673
 674     if not allow_dict:
 675         raise QAPISemError(info, "%s should be a type name" % source)
 676
 677     if not isinstance(value, OrderedDict):
 678         raise QAPISemError(info,
 679                            "%s should be a dictionary or type name" % source)
 680
 681     # value is a dictionary, check that each member is okay
 682     for (key, arg) in value.items():
 683         check_name(info, "Member of %s" % source, key,
 684                    allow_optional=allow_optional)
 685         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 686             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 687                                % (source, key))
 688         # Todo: allow dictionaries to represent default values of
 689         # an optional argument.
 690         check_type(info, "Member '%s' of %s" % (key, source), arg,
 691                    allow_array=True,
 692                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 693                                 'enum'])
 694
 695
 696 def check_command(expr, info):
 697     name = expr['command']
 698     boxed = expr.get('boxed', False)
 699
 700     args_meta = ['struct']
 701     if boxed:
 702         args_meta += ['union', 'alternate']
 703     check_type(info, "'data' for command '%s'" % name,
 704                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 705                allow_metas=args_meta)
 706     returns_meta = ['union', 'struct']
 707     if name in returns_whitelist:
 708         returns_meta += ['built-in', 'alternate', 'enum']
 709     check_type(info, "'returns' for command '%s'" % name,
 710                expr.get('returns'), allow_array=True,
 711                allow_optional=True, allow_metas=returns_meta)
 712
 713
 714 def check_event(expr, info):
 715     global events
 716     name = expr['event']
 717     boxed = expr.get('boxed', False)
 718
 719     meta = ['struct']
 720     if boxed:
 721         meta += ['union', 'alternate']
 722     events.append(name)
 723     check_type(info, "'data' for event '%s'" % name,
 724                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 725                allow_metas=meta)
 726
 727
 728 def check_union(expr, info):
 729     name = expr['union']
 730     base = expr.get('base')
 731     discriminator = expr.get('discriminator')
 732     members = expr['data']
 733
 734     # Two types of unions, determined by discriminator.
 735
 736     # With no discriminator it is a simple union.
 737     if discriminator is None:
 738         enum_define = None
 739         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 740         if base is not None:
 741             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 742                                name)
 743
 744     # Else, it's a flat union.
 745     else:
 746         # The object must have a string or dictionary 'base'.
 747         check_type(info, "'base' for union '%s'" % name,
 748                    base, allow_dict=True, allow_optional=True,
 749                    allow_metas=['struct'])
 750         if not base:
 751             raise QAPISemError(info, "Flat union '%s' must have a base"
 752                                % name)
 753         base_members = find_base_members(base)
 754         assert base_members is not None
 755
 756         # The value of member 'discriminator' must name a non-optional
 757         # member of the base struct.
 758         check_name(info, "Discriminator of flat union '%s'" % name,
 759                    discriminator)
 760         discriminator_type = base_members.get(discriminator)
 761         if not discriminator_type:
 762             raise QAPISemError(info,
 763                                "Discriminator '%s' is not a member of base "
 764                                "struct '%s'"
 765                                % (discriminator, base))
 766         enum_define = find_enum(discriminator_type)
 767         allow_metas = ['struct']
 768         # Do not allow string discriminator
 769         if not enum_define:
 770             raise QAPISemError(info,
 771                                "Discriminator '%s' must be of enumeration "
 772                                "type" % discriminator)
 773
 774     # Check every branch; don't allow an empty union
 775     if len(members) == 0:
 776         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 777     for (key, value) in members.items():
 778         check_name(info, "Member of union '%s'" % name, key)
 779
 780         # Each value must name a known type
 781         check_type(info, "Member '%s' of union '%s'" % (key, name),
 782                    value, allow_array=not base, allow_metas=allow_metas)
 783
 784         # If the discriminator names an enum type, then all members
 785         # of 'data' must also be members of the enum type.
 786         if enum_define:
 787             if key not in enum_define['enum_values']:
 788                 raise QAPISemError(info,
 789                                    "Discriminator value '%s' is not found in "
 790                                    "enum '%s'"
 791                                    % (key, enum_define['enum_name']))
 792
 793     # If discriminator is user-defined, ensure all values are covered
 794     if enum_define:
 795         for value in enum_define['enum_values']:
 796             if value not in members.keys():
 797                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 798                                    % (name, value))
 799
 800
 801 def check_alternate(expr, info):
 802     name = expr['alternate']
 803     members = expr['data']
 804     types_seen = {}
 805
 806     # Check every branch; require at least two branches
 807     if len(members) < 2:
 808         raise QAPISemError(info,
 809                            "Alternate '%s' should have at least two branches "
 810                            "in 'data'" % name)
 811     for (key, value) in members.items():
 812         check_name(info, "Member of alternate '%s'" % name, key)
 813
 814         # Ensure alternates have no type conflicts.
 815         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 816                    value,
 817                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 818         qtype = find_alternate_member_qtype(value)
 819         if not qtype:
 820             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 821                                "type '%s'" % (name, key, value))
 822         if qtype in types_seen:
 823             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 824                                "be distinguished from member '%s'"
 825                                % (name, key, types_seen[qtype]))
 826         types_seen[qtype] = key
 827
 828
 829 def check_enum(expr, info):
 830     name = expr['enum']
 831     members = expr.get('data')
 832     prefix = expr.get('prefix')
 833
 834     if not isinstance(members, list):
 835         raise QAPISemError(info,
 836                            "Enum '%s' requires an array for 'data'" % name)
 837     if prefix is not None and not isinstance(prefix, str):
 838         raise QAPISemError(info,
 839                            "Enum '%s' requires a string for 'prefix'" % name)
 840     for member in members:
 841         check_name(info, "Member of enum '%s'" % name, member,
 842                    enum_member=True)
 843
 844
 845 def check_struct(expr, info):
 846     name = expr['struct']
 847     members = expr['data']
 848
 849     check_type(info, "'data' for struct '%s'" % name, members,
 850                allow_dict=True, allow_optional=True)
 851     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 852                allow_metas=['struct'])
 853
 854
 855 def check_keys(expr_elem, meta, required, optional=[]):
 856     expr = expr_elem['expr']
 857     info = expr_elem['info']
 858     name = expr[meta]
 859     if not isinstance(name, str):
 860         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 861     required = required + [meta]
 862     for (key, value) in expr.items():
 863         if key not in required and key not in optional:
 864             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 865                                % (key, meta, name))
 866         if (key == 'gen' or key == 'success-response') and value is not False:
 867             raise QAPISemError(info,
 868                                "'%s' of %s '%s' should only use false value"
 869                                % (key, meta, name))
 870         if key == 'boxed' and value is not True:
 871             raise QAPISemError(info,
 872                                "'%s' of %s '%s' should only use true value"
 873                                % (key, meta, name))
 874     for key in required:
 875         if key not in expr:
 876             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 877                                % (key, meta, name))
 878
 879
 880 def check_exprs(exprs):
 881     global all_names
 882
 883     # Learn the types and check for valid expression keys
 884     for builtin in builtin_types.keys():
 885         all_names[builtin] = 'built-in'
 886     for expr_elem in exprs:
 887         expr = expr_elem['expr']
 888         info = expr_elem['info']
 889
 890         if 'doc' not in expr_elem and doc_required:
 891             raise QAPISemError(info,
 892                                "Expression missing documentation comment")
 893
 894         if 'enum' in expr:
 895             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 896             add_enum(expr['enum'], info, expr['data'])
 897         elif 'union' in expr:
 898             check_keys(expr_elem, 'union', ['data'],
 899                        ['base', 'discriminator'])
 900             add_union(expr, info)
 901         elif 'alternate' in expr:
 902             check_keys(expr_elem, 'alternate', ['data'])
 903             add_name(expr['alternate'], info, 'alternate')
 904         elif 'struct' in expr:
 905             check_keys(expr_elem, 'struct', ['data'], ['base'])
 906             add_struct(expr, info)
 907         elif 'command' in expr:
 908             check_keys(expr_elem, 'command', [],
 909                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 910             add_name(expr['command'], info, 'command')
 911         elif 'event' in expr:
 912             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 913             add_name(expr['event'], info, 'event')
 914         else:
 915             raise QAPISemError(expr_elem['info'],
 916                                "Expression is missing metatype")
 917
 918     # Try again for hidden UnionKind enum
 919     for expr_elem in exprs:
 920         expr = expr_elem['expr']
 921         if 'union' in expr:
 922             if not discriminator_find_enum_define(expr):
 923                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 924                          implicit=True)
 925         elif 'alternate' in expr:
 926             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 927                      implicit=True)
 928
 929     # Validate that exprs make sense
 930     for expr_elem in exprs:
 931         expr = expr_elem['expr']
 932         info = expr_elem['info']
 933
 934         if 'enum' in expr:
 935             check_enum(expr, info)
 936         elif 'union' in expr:
 937             check_union(expr, info)
 938         elif 'alternate' in expr:
 939             check_alternate(expr, info)
 940         elif 'struct' in expr:
 941             check_struct(expr, info)
 942         elif 'command' in expr:
 943             check_command(expr, info)
 944         elif 'event' in expr:
 945             check_event(expr, info)
 946         else:
 947             assert False, 'unexpected meta type'
 948
 949     return exprs
 950
 951
 952 def check_freeform_doc(doc):
 953     if doc.symbol:
 954         raise QAPISemError(doc.info,
 955                            "Documention for '%s' is not followed"
 956                            " by the definition" % doc.symbol)
 957
 958     body = str(doc.body)
 959     if re.search(r'@\S+:', body, re.MULTILINE):
 960         raise QAPISemError(doc.info,
 961                            "Free-form documentation block must not contain"
 962                            " @NAME: sections")
 963
 964
 965 def check_definition_doc(doc, expr, info):
 966     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 967         if i in expr:
 968             meta = i
 969             break
 970
 971     name = expr[meta]
 972     if doc.symbol != name:
 973         raise QAPISemError(info, "Definition of '%s' follows documentation"
 974                            " for '%s'" % (name, doc.symbol))
 975     if doc.has_section('Returns') and 'command' not in expr:
 976         raise QAPISemError(info, "'Returns:' is only valid for commands")
 977
 978     if meta == 'union':
 979         args = expr.get('base', [])
 980     else:
 981         args = expr.get('data', [])
 982     if isinstance(args, str):
 983         return
 984     if isinstance(args, dict):
 985         args = args.keys()
 986     assert isinstance(args, list)
 987
 988     if (meta == 'alternate'
 989             or (meta == 'union' and not expr.get('discriminator'))):
 990         args.append('type')
 991
 992     doc_args = set(doc.args.keys())
 993     args = set([name.strip('*') for name in args])
 994     if not doc_args.issubset(args):
 995         raise QAPISemError(info, "The following documented members are not in "
 996                            "the declaration: %s" % ', '.join(doc_args - args))
 997
 998
 999 def check_docs(docs):
1000     for doc in docs:
1001         for section in doc.args.values() + doc.sections:
1002             content = str(section)
1003             if not content or content.isspace():
1004                 raise QAPISemError(doc.info,
1005                                    "Empty doc section '%s'" % section.name)
1006
1007         if not doc.expr:
1008             check_freeform_doc(doc)
1009         else:
1010             check_definition_doc(doc, doc.expr, doc.info)
1011
1012     return docs
1013
1014
1015 #
1016 # Schema compiler frontend
1017 #
1018
1019 class QAPISchemaEntity(object):
1020     def __init__(self, name, info, doc):
1021         assert isinstance(name, str)
1022         self.name = name
1023         # For explicitly defined entities, info points to the (explicit)
1024         # definition.  For builtins (and their arrays), info is None.
1025         # For implicitly defined entities, info points to a place that
1026         # triggered the implicit definition (there may be more than one
1027         # such place).
1028         self.info = info
1029         self.doc = doc
1030
1031     def c_name(self):
1032         return c_name(self.name)
1033
1034     def check(self, schema):
1035         pass
1036
1037     def is_implicit(self):
1038         return not self.info
1039
1040     def visit(self, visitor):
1041         pass
1042
1043
1044 class QAPISchemaVisitor(object):
1045     def visit_begin(self, schema):
1046         pass
1047
1048     def visit_end(self):
1049         pass
1050
1051     def visit_needed(self, entity):
1052         # Default to visiting everything
1053         return True
1054
1055     def visit_builtin_type(self, name, info, json_type):
1056         pass
1057
1058     def visit_enum_type(self, name, info, values, prefix):
1059         pass
1060
1061     def visit_array_type(self, name, info, element_type):
1062         pass
1063
1064     def visit_object_type(self, name, info, base, members, variants):
1065         pass
1066
1067     def visit_object_type_flat(self, name, info, members, variants):
1068         pass
1069
1070     def visit_alternate_type(self, name, info, variants):
1071         pass
1072
1073     def visit_command(self, name, info, arg_type, ret_type,
1074                       gen, success_response, boxed):
1075         pass
1076
1077     def visit_event(self, name, info, arg_type, boxed):
1078         pass
1079
1080
1081 class QAPISchemaType(QAPISchemaEntity):
1082     # Return the C type for common use.
1083     # For the types we commonly box, this is a pointer type.
1084     def c_type(self):
1085         pass
1086
1087     # Return the C type to be used in a parameter list.
1088     def c_param_type(self):
1089         return self.c_type()
1090
1091     # Return the C type to be used where we suppress boxing.
1092     def c_unboxed_type(self):
1093         return self.c_type()
1094
1095     def json_type(self):
1096         pass
1097
1098     def alternate_qtype(self):
1099         json2qtype = {
1100             'string':  'QTYPE_QSTRING',
1101             'number':  'QTYPE_QFLOAT',
1102             'int':     'QTYPE_QINT',
1103             'boolean': 'QTYPE_QBOOL',
1104             'object':  'QTYPE_QDICT'
1105         }
1106         return json2qtype.get(self.json_type())
1107
1108     def doc_type(self):
1109         if self.is_implicit():
1110             return None
1111         return self.name
1112
1113
1114 class QAPISchemaBuiltinType(QAPISchemaType):
1115     def __init__(self, name, json_type, c_type):
1116         QAPISchemaType.__init__(self, name, None, None)
1117         assert not c_type or isinstance(c_type, str)
1118         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1119                              'value')
1120         self._json_type_name = json_type
1121         self._c_type_name = c_type
1122
1123     def c_name(self):
1124         return self.name
1125
1126     def c_type(self):
1127         return self._c_type_name
1128
1129     def c_param_type(self):
1130         if self.name == 'str':
1131             return 'const ' + self._c_type_name
1132         return self._c_type_name
1133
1134     def json_type(self):
1135         return self._json_type_name
1136
1137     def doc_type(self):
1138         return self.json_type()
1139
1140     def visit(self, visitor):
1141         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1142
1143
1144 class QAPISchemaEnumType(QAPISchemaType):
1145     def __init__(self, name, info, doc, values, prefix):
1146         QAPISchemaType.__init__(self, name, info, doc)
1147         for v in values:
1148             assert isinstance(v, QAPISchemaMember)
1149             v.set_owner(name)
1150         assert prefix is None or isinstance(prefix, str)
1151         self.values = values
1152         self.prefix = prefix
1153
1154     def check(self, schema):
1155         seen = {}
1156         for v in self.values:
1157             v.check_clash(self.info, seen)
1158             if self.doc:
1159                 self.doc.connect_member(v)
1160
1161     def is_implicit(self):
1162         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1163         return self.name.endswith('Kind') or self.name == 'QType'
1164
1165     def c_type(self):
1166         return c_name(self.name)
1167
1168     def member_names(self):
1169         return [v.name for v in self.values]
1170
1171     def json_type(self):
1172         return 'string'
1173
1174     def visit(self, visitor):
1175         visitor.visit_enum_type(self.name, self.info,
1176                                 self.member_names(), self.prefix)
1177
1178
1179 class QAPISchemaArrayType(QAPISchemaType):
1180     def __init__(self, name, info, element_type):
1181         QAPISchemaType.__init__(self, name, info, None)
1182         assert isinstance(element_type, str)
1183         self._element_type_name = element_type
1184         self.element_type = None
1185
1186     def check(self, schema):
1187         self.element_type = schema.lookup_type(self._element_type_name)
1188         assert self.element_type
1189
1190     def is_implicit(self):
1191         return True
1192
1193     def c_type(self):
1194         return c_name(self.name) + pointer_suffix
1195
1196     def json_type(self):
1197         return 'array'
1198
1199     def doc_type(self):
1200         elt_doc_type = self.element_type.doc_type()
1201         if not elt_doc_type:
1202             return None
1203         return 'array of ' + elt_doc_type
1204
1205     def visit(self, visitor):
1206         visitor.visit_array_type(self.name, self.info, self.element_type)
1207
1208
1209 class QAPISchemaObjectType(QAPISchemaType):
1210     def __init__(self, name, info, doc, base, local_members, variants):
1211         # struct has local_members, optional base, and no variants
1212         # flat union has base, variants, and no local_members
1213         # simple union has local_members, variants, and no base
1214         QAPISchemaType.__init__(self, name, info, doc)
1215         assert base is None or isinstance(base, str)
1216         for m in local_members:
1217             assert isinstance(m, QAPISchemaObjectTypeMember)
1218             m.set_owner(name)
1219         if variants is not None:
1220             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1221             variants.set_owner(name)
1222         self._base_name = base
1223         self.base = None
1224         self.local_members = local_members
1225         self.variants = variants
1226         self.members = None
1227
1228     def check(self, schema):
1229         if self.members is False:               # check for cycles
1230             raise QAPISemError(self.info,
1231                                "Object %s contains itself" % self.name)
1232         if self.members:
1233             return
1234         self.members = False                    # mark as being checked
1235         seen = OrderedDict()
1236         if self._base_name:
1237             self.base = schema.lookup_type(self._base_name)
1238             assert isinstance(self.base, QAPISchemaObjectType)
1239             self.base.check(schema)
1240             self.base.check_clash(schema, self.info, seen)
1241         for m in self.local_members:
1242             m.check(schema)
1243             m.check_clash(self.info, seen)
1244             if self.doc:
1245                 self.doc.connect_member(m)
1246         self.members = seen.values()
1247         if self.variants:
1248             self.variants.check(schema, seen)
1249             assert self.variants.tag_member in self.members
1250             self.variants.check_clash(schema, self.info, seen)
1251
1252     # Check that the members of this type do not cause duplicate JSON members,
1253     # and update seen to track the members seen so far. Report any errors
1254     # on behalf of info, which is not necessarily self.info
1255     def check_clash(self, schema, info, seen):
1256         assert not self.variants       # not implemented
1257         for m in self.members:
1258             m.check_clash(info, seen)
1259
1260     def is_implicit(self):
1261         # See QAPISchema._make_implicit_object_type(), as well as
1262         # _def_predefineds()
1263         return self.name.startswith('q_')
1264
1265     def is_empty(self):
1266         assert self.members is not None
1267         return not self.members and not self.variants
1268
1269     def c_name(self):
1270         assert self.name != 'q_empty'
1271         return QAPISchemaType.c_name(self)
1272
1273     def c_type(self):
1274         assert not self.is_implicit()
1275         return c_name(self.name) + pointer_suffix
1276
1277     def c_unboxed_type(self):
1278         return c_name(self.name)
1279
1280     def json_type(self):
1281         return 'object'
1282
1283     def visit(self, visitor):
1284         visitor.visit_object_type(self.name, self.info,
1285                                   self.base, self.local_members, self.variants)
1286         visitor.visit_object_type_flat(self.name, self.info,
1287                                        self.members, self.variants)
1288
1289
1290 class QAPISchemaMember(object):
1291     role = 'member'
1292
1293     def __init__(self, name):
1294         assert isinstance(name, str)
1295         self.name = name
1296         self.owner = None
1297
1298     def set_owner(self, name):
1299         assert not self.owner
1300         self.owner = name
1301
1302     def check_clash(self, info, seen):
1303         cname = c_name(self.name)
1304         if cname.lower() != cname and self.owner not in name_case_whitelist:
1305             raise QAPISemError(info,
1306                                "%s should not use uppercase" % self.describe())
1307         if cname in seen:
1308             raise QAPISemError(info, "%s collides with %s" %
1309                                (self.describe(), seen[cname].describe()))
1310         seen[cname] = self
1311
1312     def _pretty_owner(self):
1313         owner = self.owner
1314         if owner.startswith('q_obj_'):
1315             # See QAPISchema._make_implicit_object_type() - reverse the
1316             # mapping there to create a nice human-readable description
1317             owner = owner[6:]
1318             if owner.endswith('-arg'):
1319                 return '(parameter of %s)' % owner[:-4]
1320             elif owner.endswith('-base'):
1321                 return '(base of %s)' % owner[:-5]
1322             else:
1323                 assert owner.endswith('-wrapper')
1324                 # Unreachable and not implemented
1325                 assert False
1326         if owner.endswith('Kind'):
1327             # See QAPISchema._make_implicit_enum_type()
1328             return '(branch of %s)' % owner[:-4]
1329         return '(%s of %s)' % (self.role, owner)
1330
1331     def describe(self):
1332         return "'%s' %s" % (self.name, self._pretty_owner())
1333
1334
1335 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1336     def __init__(self, name, typ, optional):
1337         QAPISchemaMember.__init__(self, name)
1338         assert isinstance(typ, str)
1339         assert isinstance(optional, bool)
1340         self._type_name = typ
1341         self.type = None
1342         self.optional = optional
1343
1344     def check(self, schema):
1345         assert self.owner
1346         self.type = schema.lookup_type(self._type_name)
1347         assert self.type
1348
1349
1350 class QAPISchemaObjectTypeVariants(object):
1351     def __init__(self, tag_name, tag_member, variants):
1352         # Flat unions pass tag_name but not tag_member.
1353         # Simple unions and alternates pass tag_member but not tag_name.
1354         # After check(), tag_member is always set, and tag_name remains
1355         # a reliable witness of being used by a flat union.
1356         assert bool(tag_member) != bool(tag_name)
1357         assert (isinstance(tag_name, str) or
1358                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1359         assert len(variants) > 0
1360         for v in variants:
1361             assert isinstance(v, QAPISchemaObjectTypeVariant)
1362         self._tag_name = tag_name
1363         self.tag_member = tag_member
1364         self.variants = variants
1365
1366     def set_owner(self, name):
1367         for v in self.variants:
1368             v.set_owner(name)
1369
1370     def check(self, schema, seen):
1371         if not self.tag_member:    # flat union
1372             self.tag_member = seen[c_name(self._tag_name)]
1373             assert self._tag_name == self.tag_member.name
1374         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1375         for v in self.variants:
1376             v.check(schema)
1377             # Union names must match enum values; alternate names are
1378             # checked separately. Use 'seen' to tell the two apart.
1379             if seen:
1380                 assert v.name in self.tag_member.type.member_names()
1381                 assert isinstance(v.type, QAPISchemaObjectType)
1382                 v.type.check(schema)
1383
1384     def check_clash(self, schema, info, seen):
1385         for v in self.variants:
1386             # Reset seen map for each variant, since qapi names from one
1387             # branch do not affect another branch
1388             assert isinstance(v.type, QAPISchemaObjectType)
1389             v.type.check_clash(schema, info, dict(seen))
1390
1391
1392 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1393     role = 'branch'
1394
1395     def __init__(self, name, typ):
1396         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1397
1398
1399 class QAPISchemaAlternateType(QAPISchemaType):
1400     def __init__(self, name, info, doc, variants):
1401         QAPISchemaType.__init__(self, name, info, doc)
1402         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1403         assert variants.tag_member
1404         variants.set_owner(name)
1405         variants.tag_member.set_owner(self.name)
1406         self.variants = variants
1407
1408     def check(self, schema):
1409         self.variants.tag_member.check(schema)
1410         # Not calling self.variants.check_clash(), because there's nothing
1411         # to clash with
1412         self.variants.check(schema, {})
1413         # Alternate branch names have no relation to the tag enum values;
1414         # so we have to check for potential name collisions ourselves.
1415         seen = {}
1416         for v in self.variants.variants:
1417             v.check_clash(self.info, seen)
1418             if self.doc:
1419                 self.doc.connect_member(v)
1420
1421     def c_type(self):
1422         return c_name(self.name) + pointer_suffix
1423
1424     def json_type(self):
1425         return 'value'
1426
1427     def visit(self, visitor):
1428         visitor.visit_alternate_type(self.name, self.info, self.variants)
1429
1430     def is_empty(self):
1431         return False
1432
1433
1434 class QAPISchemaCommand(QAPISchemaEntity):
1435     def __init__(self, name, info, doc, arg_type, ret_type,
1436                  gen, success_response, boxed):
1437         QAPISchemaEntity.__init__(self, name, info, doc)
1438         assert not arg_type or isinstance(arg_type, str)
1439         assert not ret_type or isinstance(ret_type, str)
1440         self._arg_type_name = arg_type
1441         self.arg_type = None
1442         self._ret_type_name = ret_type
1443         self.ret_type = None
1444         self.gen = gen
1445         self.success_response = success_response
1446         self.boxed = boxed
1447
1448     def check(self, schema):
1449         if self._arg_type_name:
1450             self.arg_type = schema.lookup_type(self._arg_type_name)
1451             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1452                     isinstance(self.arg_type, QAPISchemaAlternateType))
1453             self.arg_type.check(schema)
1454             if self.boxed:
1455                 if self.arg_type.is_empty():
1456                     raise QAPISemError(self.info,
1457                                        "Cannot use 'boxed' with empty type")
1458             else:
1459                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1460                 assert not self.arg_type.variants
1461         elif self.boxed:
1462             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1463         if self._ret_type_name:
1464             self.ret_type = schema.lookup_type(self._ret_type_name)
1465             assert isinstance(self.ret_type, QAPISchemaType)
1466
1467     def visit(self, visitor):
1468         visitor.visit_command(self.name, self.info,
1469                               self.arg_type, self.ret_type,
1470                               self.gen, self.success_response, self.boxed)
1471
1472
1473 class QAPISchemaEvent(QAPISchemaEntity):
1474     def __init__(self, name, info, doc, arg_type, boxed):
1475         QAPISchemaEntity.__init__(self, name, info, doc)
1476         assert not arg_type or isinstance(arg_type, str)
1477         self._arg_type_name = arg_type
1478         self.arg_type = None
1479         self.boxed = boxed
1480
1481     def check(self, schema):
1482         if self._arg_type_name:
1483             self.arg_type = schema.lookup_type(self._arg_type_name)
1484             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1485                     isinstance(self.arg_type, QAPISchemaAlternateType))
1486             self.arg_type.check(schema)
1487             if self.boxed:
1488                 if self.arg_type.is_empty():
1489                     raise QAPISemError(self.info,
1490                                        "Cannot use 'boxed' with empty type")
1491             else:
1492                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1493                 assert not self.arg_type.variants
1494         elif self.boxed:
1495             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1496
1497     def visit(self, visitor):
1498         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1499
1500
1501 class QAPISchema(object):
1502     def __init__(self, fname):
1503         try:
1504             parser = QAPISchemaParser(open(fname, 'r'))
1505             self.exprs = check_exprs(parser.exprs)
1506             self.docs = check_docs(parser.docs)
1507             self._entity_dict = {}
1508             self._predefining = True
1509             self._def_predefineds()
1510             self._predefining = False
1511             self._def_exprs()
1512             self.check()
1513         except QAPIError as err:
1514             print >>sys.stderr, err
1515             exit(1)
1516
1517     def _def_entity(self, ent):
1518         # Only the predefined types are allowed to not have info
1519         assert ent.info or self._predefining
1520         assert ent.name not in self._entity_dict
1521         self._entity_dict[ent.name] = ent
1522
1523     def lookup_entity(self, name, typ=None):
1524         ent = self._entity_dict.get(name)
1525         if typ and not isinstance(ent, typ):
1526             return None
1527         return ent
1528
1529     def lookup_type(self, name):
1530         return self.lookup_entity(name, QAPISchemaType)
1531
1532     def _def_builtin_type(self, name, json_type, c_type):
1533         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1534         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1535         # qapi-types.h from a single .c, all arrays of builtins must be
1536         # declared in the first file whether or not they are used.  Nicer
1537         # would be to use lazy instantiation, while figuring out how to
1538         # avoid compilation issues with multiple qapi-types.h.
1539         self._make_array_type(name, None)
1540
1541     def _def_predefineds(self):
1542         for t in [('str',    'string',  'char' + pointer_suffix),
1543                   ('number', 'number',  'double'),
1544                   ('int',    'int',     'int64_t'),
1545                   ('int8',   'int',     'int8_t'),
1546                   ('int16',  'int',     'int16_t'),
1547                   ('int32',  'int',     'int32_t'),
1548                   ('int64',  'int',     'int64_t'),
1549                   ('uint8',  'int',     'uint8_t'),
1550                   ('uint16', 'int',     'uint16_t'),
1551                   ('uint32', 'int',     'uint32_t'),
1552                   ('uint64', 'int',     'uint64_t'),
1553                   ('size',   'int',     'uint64_t'),
1554                   ('bool',   'boolean', 'bool'),
1555                   ('any',    'value',   'QObject' + pointer_suffix)]:
1556             self._def_builtin_type(*t)
1557         self.the_empty_object_type = QAPISchemaObjectType(
1558             'q_empty', None, None, None, [], None)
1559         self._def_entity(self.the_empty_object_type)
1560         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1561                                                 'qstring', 'qdict', 'qlist',
1562                                                 'qfloat', 'qbool'])
1563         self._def_entity(QAPISchemaEnumType('QType', None, None,
1564                                             qtype_values, 'QTYPE'))
1565
1566     def _make_enum_members(self, values):
1567         return [QAPISchemaMember(v) for v in values]
1568
1569     def _make_implicit_enum_type(self, name, info, values):
1570         # See also QAPISchemaObjectTypeMember._pretty_owner()
1571         name = name + 'Kind'   # Use namespace reserved by add_name()
1572         self._def_entity(QAPISchemaEnumType(
1573             name, info, None, self._make_enum_members(values), None))
1574         return name
1575
1576     def _make_array_type(self, element_type, info):
1577         name = element_type + 'List'   # Use namespace reserved by add_name()
1578         if not self.lookup_type(name):
1579             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1580         return name
1581
1582     def _make_implicit_object_type(self, name, info, doc, role, members):
1583         if not members:
1584             return None
1585         # See also QAPISchemaObjectTypeMember._pretty_owner()
1586         name = 'q_obj_%s-%s' % (name, role)
1587         if not self.lookup_entity(name, QAPISchemaObjectType):
1588             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1589                                                   members, None))
1590         return name
1591
1592     def _def_enum_type(self, expr, info, doc):
1593         name = expr['enum']
1594         data = expr['data']
1595         prefix = expr.get('prefix')
1596         self._def_entity(QAPISchemaEnumType(
1597             name, info, doc, self._make_enum_members(data), prefix))
1598
1599     def _make_member(self, name, typ, info):
1600         optional = False
1601         if name.startswith('*'):
1602             name = name[1:]
1603             optional = True
1604         if isinstance(typ, list):
1605             assert len(typ) == 1
1606             typ = self._make_array_type(typ[0], info)
1607         return QAPISchemaObjectTypeMember(name, typ, optional)
1608
1609     def _make_members(self, data, info):
1610         return [self._make_member(key, value, info)
1611                 for (key, value) in data.iteritems()]
1612
1613     def _def_struct_type(self, expr, info, doc):
1614         name = expr['struct']
1615         base = expr.get('base')
1616         data = expr['data']
1617         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1618                                               self._make_members(data, info),
1619                                               None))
1620
1621     def _make_variant(self, case, typ):
1622         return QAPISchemaObjectTypeVariant(case, typ)
1623
1624     def _make_simple_variant(self, case, typ, info):
1625         if isinstance(typ, list):
1626             assert len(typ) == 1
1627             typ = self._make_array_type(typ[0], info)
1628         typ = self._make_implicit_object_type(
1629             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1630         return QAPISchemaObjectTypeVariant(case, typ)
1631
1632     def _def_union_type(self, expr, info, doc):
1633         name = expr['union']
1634         data = expr['data']
1635         base = expr.get('base')
1636         tag_name = expr.get('discriminator')
1637         tag_member = None
1638         if isinstance(base, dict):
1639             base = (self._make_implicit_object_type(
1640                     name, info, doc, 'base', self._make_members(base, info)))
1641         if tag_name:
1642             variants = [self._make_variant(key, value)
1643                         for (key, value) in data.iteritems()]
1644             members = []
1645         else:
1646             variants = [self._make_simple_variant(key, value, info)
1647                         for (key, value) in data.iteritems()]
1648             typ = self._make_implicit_enum_type(name, info,
1649                                                 [v.name for v in variants])
1650             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1651             members = [tag_member]
1652         self._def_entity(
1653             QAPISchemaObjectType(name, info, doc, base, members,
1654                                  QAPISchemaObjectTypeVariants(tag_name,
1655                                                               tag_member,
1656                                                               variants)))
1657
1658     def _def_alternate_type(self, expr, info, doc):
1659         name = expr['alternate']
1660         data = expr['data']
1661         variants = [self._make_variant(key, value)
1662                     for (key, value) in data.iteritems()]
1663         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1664         self._def_entity(
1665             QAPISchemaAlternateType(name, info, doc,
1666                                     QAPISchemaObjectTypeVariants(None,
1667                                                                  tag_member,
1668                                                                  variants)))
1669
1670     def _def_command(self, expr, info, doc):
1671         name = expr['command']
1672         data = expr.get('data')
1673         rets = expr.get('returns')
1674         gen = expr.get('gen', True)
1675         success_response = expr.get('success-response', True)
1676         boxed = expr.get('boxed', False)
1677         if isinstance(data, OrderedDict):
1678             data = self._make_implicit_object_type(
1679                 name, info, doc, 'arg', self._make_members(data, info))
1680         if isinstance(rets, list):
1681             assert len(rets) == 1
1682             rets = self._make_array_type(rets[0], info)
1683         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1684                                            gen, success_response, boxed))
1685
1686     def _def_event(self, expr, info, doc):
1687         name = expr['event']
1688         data = expr.get('data')
1689         boxed = expr.get('boxed', False)
1690         if isinstance(data, OrderedDict):
1691             data = self._make_implicit_object_type(
1692                 name, info, doc, 'arg', self._make_members(data, info))
1693         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1694
1695     def _def_exprs(self):
1696         for expr_elem in self.exprs:
1697             expr = expr_elem['expr']
1698             info = expr_elem['info']
1699             doc = expr_elem.get('doc')
1700             if 'enum' in expr:
1701                 self._def_enum_type(expr, info, doc)
1702             elif 'struct' in expr:
1703                 self._def_struct_type(expr, info, doc)
1704             elif 'union' in expr:
1705                 self._def_union_type(expr, info, doc)
1706             elif 'alternate' in expr:
1707                 self._def_alternate_type(expr, info, doc)
1708             elif 'command' in expr:
1709                 self._def_command(expr, info, doc)
1710             elif 'event' in expr:
1711                 self._def_event(expr, info, doc)
1712             else:
1713                 assert False
1714
1715     def check(self):
1716         for ent in self._entity_dict.values():
1717             ent.check(self)
1718
1719     def visit(self, visitor):
1720         visitor.visit_begin(self)
1721         for (name, entity) in sorted(self._entity_dict.items()):
1722             if visitor.visit_needed(entity):
1723                 entity.visit(visitor)
1724         visitor.visit_end()
1725
1726
1727 #
1728 # Code generation helpers
1729 #
1730
1731 def camel_case(name):
1732     new_name = ''
1733     first = True
1734     for ch in name:
1735         if ch in ['_', '-']:
1736             first = True
1737         elif first:
1738             new_name += ch.upper()
1739             first = False
1740         else:
1741             new_name += ch.lower()
1742     return new_name
1743
1744
1745 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1746 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1747 # ENUM24_Name -> ENUM24_NAME
1748 def camel_to_upper(value):
1749     c_fun_str = c_name(value, False)
1750     if value.isupper():
1751         return c_fun_str
1752
1753     new_name = ''
1754     l = len(c_fun_str)
1755     for i in range(l):
1756         c = c_fun_str[i]
1757         # When c is upper and no '_' appears before, do more checks
1758         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1759             if i < l - 1 and c_fun_str[i + 1].islower():
1760                 new_name += '_'
1761             elif c_fun_str[i - 1].isdigit():
1762                 new_name += '_'
1763         new_name += c
1764     return new_name.lstrip('_').upper()
1765
1766
1767 def c_enum_const(type_name, const_name, prefix=None):
1768     if prefix is not None:
1769         type_name = prefix
1770     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1771
1772 c_name_trans = string.maketrans('.-', '__')
1773
1774
1775 # Map @name to a valid C identifier.
1776 # If @protect, avoid returning certain ticklish identifiers (like
1777 # C keywords) by prepending 'q_'.
1778 #
1779 # Used for converting 'name' from a 'name':'type' qapi definition
1780 # into a generated struct member, as well as converting type names
1781 # into substrings of a generated C function name.
1782 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1783 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1784 def c_name(name, protect=True):
1785     # ANSI X3J11/88-090, 3.1.1
1786     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1787                      'default', 'do', 'double', 'else', 'enum', 'extern',
1788                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1789                      'return', 'short', 'signed', 'sizeof', 'static',
1790                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1791                      'void', 'volatile', 'while'])
1792     # ISO/IEC 9899:1999, 6.4.1
1793     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1794     # ISO/IEC 9899:2011, 6.4.1
1795     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1796                      '_Noreturn', '_Static_assert', '_Thread_local'])
1797     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1798     # excluding _.*
1799     gcc_words = set(['asm', 'typeof'])
1800     # C++ ISO/IEC 14882:2003 2.11
1801     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1802                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1803                      'namespace', 'new', 'operator', 'private', 'protected',
1804                      'public', 'reinterpret_cast', 'static_cast', 'template',
1805                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1806                      'using', 'virtual', 'wchar_t',
1807                      # alternative representations
1808                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1809                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1810     # namespace pollution:
1811     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1812     name = name.translate(c_name_trans)
1813     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1814                     | cpp_words | polluted_words):
1815         return 'q_' + name
1816     return name
1817
1818 eatspace = '\033EATSPACE.'
1819 pointer_suffix = ' *' + eatspace
1820
1821
1822 def genindent(count):
1823     ret = ''
1824     for _ in range(count):
1825         ret += ' '
1826     return ret
1827
1828 indent_level = 0
1829
1830
1831 def push_indent(indent_amount=4):
1832     global indent_level
1833     indent_level += indent_amount
1834
1835
1836 def pop_indent(indent_amount=4):
1837     global indent_level
1838     indent_level -= indent_amount
1839
1840
1841 # Generate @code with @kwds interpolated.
1842 # Obey indent_level, and strip eatspace.
1843 def cgen(code, **kwds):
1844     raw = code % kwds
1845     if indent_level:
1846         indent = genindent(indent_level)
1847         # re.subn() lacks flags support before Python 2.7, use re.compile()
1848         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1849                       indent + r'\g<0>', raw)
1850         raw = raw[0]
1851     return re.sub(re.escape(eatspace) + r' *', '', raw)
1852
1853
1854 def mcgen(code, **kwds):
1855     if code[0] == '\n':
1856         code = code[1:]
1857     return cgen(code, **kwds)
1858
1859
1860 def guardname(filename):
1861     return c_name(filename, protect=False).upper()
1862
1863
1864 def guardstart(name):
1865     return mcgen('''
1866
1867 #ifndef %(name)s
1868 #define %(name)s
1869
1870 ''',
1871                  name=guardname(name))
1872
1873
1874 def guardend(name):
1875     return mcgen('''
1876
1877 #endif /* %(name)s */
1878
1879 ''',
1880                  name=guardname(name))
1881
1882
1883 def gen_enum_lookup(name, values, prefix=None):
1884     ret = mcgen('''
1885
1886 const char *const %(c_name)s_lookup[] = {
1887 ''',
1888                 c_name=c_name(name))
1889     for value in values:
1890         index = c_enum_const(name, value, prefix)
1891         ret += mcgen('''
1892     [%(index)s] = "%(value)s",
1893 ''',
1894                      index=index, value=value)
1895
1896     max_index = c_enum_const(name, '_MAX', prefix)
1897     ret += mcgen('''
1898     [%(max_index)s] = NULL,
1899 };
1900 ''',
1901                  max_index=max_index)
1902     return ret
1903
1904
1905 def gen_enum(name, values, prefix=None):
1906     # append automatically generated _MAX value
1907     enum_values = values + ['_MAX']
1908
1909     ret = mcgen('''
1910
1911 typedef enum %(c_name)s {
1912 ''',
1913                 c_name=c_name(name))
1914
1915     i = 0
1916     for value in enum_values:
1917         ret += mcgen('''
1918     %(c_enum)s = %(i)d,
1919 ''',
1920                      c_enum=c_enum_const(name, value, prefix),
1921                      i=i)
1922         i += 1
1923
1924     ret += mcgen('''
1925 } %(c_name)s;
1926 ''',
1927                  c_name=c_name(name))
1928
1929     ret += mcgen('''
1930
1931 extern const char *const %(c_name)s_lookup[];
1932 ''',
1933                  c_name=c_name(name))
1934     return ret
1935
1936
1937 def gen_params(arg_type, boxed, extra):
1938     if not arg_type:
1939         assert not boxed
1940         return extra
1941     ret = ''
1942     sep = ''
1943     if boxed:
1944         ret += '%s arg' % arg_type.c_param_type()
1945         sep = ', '
1946     else:
1947         assert not arg_type.variants
1948         for memb in arg_type.members:
1949             ret += sep
1950             sep = ', '
1951             if memb.optional:
1952                 ret += 'bool has_%s, ' % c_name(memb.name)
1953             ret += '%s %s' % (memb.type.c_param_type(),
1954                               c_name(memb.name))
1955     if extra:
1956         ret += sep + extra
1957     return ret
1958
1959
1960 #
1961 # Common command line parsing
1962 #
1963
1964
1965 def parse_command_line(extra_options='', extra_long_options=[]):
1966
1967     try:
1968         opts, args = getopt.gnu_getopt(sys.argv[1:],
1969                                        'chp:o:' + extra_options,
1970                                        ['source', 'header', 'prefix=',
1971                                         'output-dir='] + extra_long_options)
1972     except getopt.GetoptError as err:
1973         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1974         sys.exit(1)
1975
1976     output_dir = ''
1977     prefix = ''
1978     do_c = False
1979     do_h = False
1980     extra_opts = []
1981
1982     for oa in opts:
1983         o, a = oa
1984         if o in ('-p', '--prefix'):
1985             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1986             if match.end() != len(a):
1987                 print >>sys.stderr, \
1988                     "%s: 'funny character '%s' in argument of --prefix" \
1989                     % (sys.argv[0], a[match.end()])
1990                 sys.exit(1)
1991             prefix = a
1992         elif o in ('-o', '--output-dir'):
1993             output_dir = a + '/'
1994         elif o in ('-c', '--source'):
1995             do_c = True
1996         elif o in ('-h', '--header'):
1997             do_h = True
1998         else:
1999             extra_opts.append(oa)
2000
2001     if not do_c and not do_h:
2002         do_c = True
2003         do_h = True
2004
2005     if len(args) != 1:
2006         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2007         sys.exit(1)
2008     fname = args[0]
2009
2010     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2011
2012 #
2013 # Generate output files with boilerplate
2014 #
2015
2016
2017 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2018                 c_comment, h_comment):
2019     guard = guardname(prefix + h_file)
2020     c_file = output_dir + prefix + c_file
2021     h_file = output_dir + prefix + h_file
2022
2023     if output_dir:
2024         try:
2025             os.makedirs(output_dir)
2026         except os.error as e:
2027             if e.errno != errno.EEXIST:
2028                 raise
2029
2030     def maybe_open(really, name, opt):
2031         if really:
2032             return open(name, opt)
2033         else:
2034             import StringIO
2035             return StringIO.StringIO()
2036
2037     fdef = maybe_open(do_c, c_file, 'w')
2038     fdecl = maybe_open(do_h, h_file, 'w')
2039
2040     fdef.write(mcgen('''
2041 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2042 %(comment)s
2043 ''',
2044                      comment=c_comment))
2045
2046     fdecl.write(mcgen('''
2047 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2048 %(comment)s
2049 #ifndef %(guard)s
2050 #define %(guard)s
2051
2052 ''',
2053                       comment=h_comment, guard=guard))
2054
2055     return (fdef, fdecl)
2056
2057
2058 def close_output(fdef, fdecl):
2059     fdecl.write('''
2060 #endif
2061 ''')
2062     fdecl.close()
2063     fdef.close()