scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 all_names = {}
  53
  54 #
  55 # Parsing the schema into expressions
  56 #
  57
  58
  59 def error_path(parent):
  60     res = ''
  61     while parent:
  62         res = ('In file included from %s:%d:\n' % (parent['file'],
  63                                                    parent['line'])) + res
  64         parent = parent['parent']
  65     return res
  66
  67
  68 class QAPIError(Exception):
  69     def __init__(self, fname, line, col, incl_info, msg):
  70         Exception.__init__(self)
  71         self.fname = fname
  72         self.line = line
  73         self.col = col
  74         self.info = incl_info
  75         self.msg = msg
  76
  77     def __str__(self):
  78         loc = '%s:%d' % (self.fname, self.line)
  79         if self.col is not None:
  80             loc += ':%s' % self.col
  81         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  82
  83
  84 class QAPIParseError(QAPIError):
  85     def __init__(self, parser, msg):
  86         col = 1
  87         for ch in parser.src[parser.line_pos:parser.pos]:
  88             if ch == '\t':
  89                 col = (col + 7) % 8 + 1
  90             else:
  91                 col += 1
  92         QAPIError.__init__(self, parser.fname, parser.line, col,
  93                            parser.incl_info, msg)
  94
  95
  96 class QAPISemError(QAPIError):
  97     def __init__(self, info, msg):
  98         QAPIError.__init__(self, info['file'], info['line'], None,
  99                            info['parent'], msg)
 100
 101
 102 class QAPIDoc(object):
 103     class Section(object):
 104         def __init__(self, name=None):
 105             # optional section name (argument/member or section name)
 106             self.name = name
 107             # the list of lines for this section
 108             self.content = []
 109             self.optional = False
 110
 111         def append(self, line):
 112             self.content.append(line)
 113
 114         def __repr__(self):
 115             return '\n'.join(self.content).strip()
 116
 117     class ArgSection(Section):
 118         def __init__(self, name):
 119             QAPIDoc.Section.__init__(self, name)
 120             self.member = None
 121
 122         def connect(self, member):
 123             self.member = member
 124
 125     def __init__(self, parser, info):
 126         # self.parser is used to report errors with QAPIParseError.  The
 127         # resulting error position depends on the state of the parser.
 128         # It happens to be the beginning of the comment.  More or less
 129         # servicable, but action at a distance.
 130         self.parser = parser
 131         self.info = info
 132         self.symbol = None
 133         self.body = QAPIDoc.Section()
 134         # dict mapping parameter name to ArgSection
 135         self.args = OrderedDict()
 136         # a list of Section
 137         self.sections = []
 138         # the current section
 139         self.section = self.body
 140
 141     def has_section(self, name):
 142         """Return True if we have a section with this name."""
 143         for i in self.sections:
 144             if i.name == name:
 145                 return True
 146         return False
 147
 148     def append(self, line):
 149         """Parse a comment line and add it to the documentation."""
 150         line = line[1:]
 151         if not line:
 152             self._append_freeform(line)
 153             return
 154
 155         if line[0] != ' ':
 156             raise QAPIParseError(self.parser, "Missing space after #")
 157         line = line[1:]
 158
 159         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 160         # recognized, and get silently treated as ordinary text
 161         if self.symbol:
 162             self._append_symbol_line(line)
 163         elif not self.body.content and line.startswith('@'):
 164             if not line.endswith(':'):
 165                 raise QAPIParseError(self.parser, "Line should end with :")
 166             self.symbol = line[1:-1]
 167             # FIXME invalid names other than the empty string aren't flagged
 168             if not self.symbol:
 169                 raise QAPIParseError(self.parser, "Invalid name")
 170         else:
 171             self._append_freeform(line)
 172
 173     def end_comment(self):
 174         self._end_section()
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self._end_section()
 204         self.section = QAPIDoc.ArgSection(name)
 205         self.args[name] = self.section
 206
 207     def _start_section(self, name=''):
 208         if name in ('Returns', 'Since') and self.has_section(name):
 209             raise QAPIParseError(self.parser,
 210                                  "Duplicated '%s' section" % name)
 211         self._end_section()
 212         self.section = QAPIDoc.Section(name)
 213         self.sections.append(self.section)
 214
 215     def _end_section(self):
 216         if self.section:
 217             contents = str(self.section)
 218             if self.section.name and (not contents or contents.isspace()):
 219                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 220                                      % self.section.name)
 221             self.section = None
 222
 223     def _append_freeform(self, line):
 224         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 225         if (in_arg and self.section.content
 226                 and not self.section.content[-1]
 227                 and line and not line[0].isspace()):
 228             self._start_section()
 229         if (in_arg or not self.section.name
 230                 or not self.section.name.startswith('Example')):
 231             line = line.strip()
 232         match = re.match(r'(@\S+:)', line)
 233         if match:
 234             raise QAPIParseError(self.parser,
 235                                  "'%s' not allowed in free-form documentation"
 236                                  % match.group(1))
 237         # TODO Drop this once the dust has settled
 238         if (isinstance(self.section, QAPIDoc.ArgSection)
 239                 and '#optional' in line):
 240             raise QAPISemError(self.info, "Please drop the #optional tag")
 241         self.section.append(line)
 242
 243     def connect_member(self, member):
 244         if member.name not in self.args:
 245             # Undocumented TODO outlaw
 246             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 247         self.args[member.name].connect(member)
 248
 249     def check_expr(self, expr):
 250         if self.has_section('Returns') and 'command' not in expr:
 251             raise QAPISemError(self.info,
 252                                "'Returns:' is only valid for commands")
 253
 254     def check(self):
 255         bogus = [name for name, section in self.args.iteritems()
 256                  if not section.member]
 257         if bogus:
 258             raise QAPISemError(
 259                 self.info,
 260                 "The following documented members are not in "
 261                 "the declaration: %s" % ", ".join(bogus))
 262
 263
 264 class QAPISchemaParser(object):
 265
 266     def __init__(self, fp, previously_included=[], incl_info=None):
 267         abs_fname = os.path.abspath(fp.name)
 268         fname = fp.name
 269         self.fname = fname
 270         previously_included.append(abs_fname)
 271         self.incl_info = incl_info
 272         self.src = fp.read()
 273         if self.src == '' or self.src[-1] != '\n':
 274             self.src += '\n'
 275         self.cursor = 0
 276         self.line = 1
 277         self.line_pos = 0
 278         self.exprs = []
 279         self.docs = []
 280         self.cur_doc = None
 281         self.accept()
 282
 283         while self.tok is not None:
 284             info = {'file': fname, 'line': self.line,
 285                     'parent': self.incl_info}
 286             if self.tok == '#':
 287                 self.reject_expr_doc()
 288                 self.cur_doc = self.get_doc(info)
 289                 self.docs.append(self.cur_doc)
 290                 continue
 291
 292             expr = self.get_expr(False)
 293             if 'include' in expr:
 294                 self.reject_expr_doc()
 295                 if len(expr) != 1:
 296                     raise QAPISemError(info, "Invalid 'include' directive")
 297                 include = expr['include']
 298                 if not isinstance(include, str):
 299                     raise QAPISemError(info,
 300                                        "Value of 'include' must be a string")
 301                 self._include(include, info, os.path.dirname(abs_fname),
 302                               previously_included)
 303             elif "pragma" in expr:
 304                 self.reject_expr_doc()
 305                 if len(expr) != 1:
 306                     raise QAPISemError(info, "Invalid 'pragma' directive")
 307                 pragma = expr['pragma']
 308                 if not isinstance(pragma, dict):
 309                     raise QAPISemError(
 310                         info, "Value of 'pragma' must be a dictionary")
 311                 for name, value in pragma.iteritems():
 312                     self._pragma(name, value, info)
 313             else:
 314                 expr_elem = {'expr': expr,
 315                              'info': info}
 316                 if self.cur_doc:
 317                     if not self.cur_doc.symbol:
 318                         raise QAPISemError(
 319                             self.cur_doc.info,
 320                             "Expression documentation required")
 321                     expr_elem['doc'] = self.cur_doc
 322                 self.exprs.append(expr_elem)
 323             self.cur_doc = None
 324         self.reject_expr_doc()
 325
 326     def reject_expr_doc(self):
 327         if self.cur_doc and self.cur_doc.symbol:
 328             raise QAPISemError(
 329                 self.cur_doc.info,
 330                 "Documentation for '%s' is not followed by the definition"
 331                 % self.cur_doc.symbol)
 332
 333     def _include(self, include, info, base_dir, previously_included):
 334         incl_abs_fname = os.path.join(base_dir, include)
 335         # catch inclusion cycle
 336         inf = info
 337         while inf:
 338             if incl_abs_fname == os.path.abspath(inf['file']):
 339                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 340             inf = inf['parent']
 341
 342         # skip multiple include of the same file
 343         if incl_abs_fname in previously_included:
 344             return
 345         try:
 346             fobj = open(incl_abs_fname, 'r')
 347         except IOError as e:
 348             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 349         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 350         self.exprs.extend(exprs_include.exprs)
 351         self.docs.extend(exprs_include.docs)
 352
 353     def _pragma(self, name, value, info):
 354         global doc_required, returns_whitelist, name_case_whitelist
 355         if name == 'doc-required':
 356             if not isinstance(value, bool):
 357                 raise QAPISemError(info,
 358                                    "Pragma 'doc-required' must be boolean")
 359             doc_required = value
 360         elif name == 'returns-whitelist':
 361             if (not isinstance(value, list)
 362                     or any([not isinstance(elt, str) for elt in value])):
 363                 raise QAPISemError(info,
 364                                    "Pragma returns-whitelist must be"
 365                                    " a list of strings")
 366             returns_whitelist = value
 367         elif name == 'name-case-whitelist':
 368             if (not isinstance(value, list)
 369                     or any([not isinstance(elt, str) for elt in value])):
 370                 raise QAPISemError(info,
 371                                    "Pragma name-case-whitelist must be"
 372                                    " a list of strings")
 373             name_case_whitelist = value
 374         else:
 375             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 376
 377     def accept(self, skip_comment=True):
 378         while True:
 379             self.tok = self.src[self.cursor]
 380             self.pos = self.cursor
 381             self.cursor += 1
 382             self.val = None
 383
 384             if self.tok == '#':
 385                 if self.src[self.cursor] == '#':
 386                     # Start of doc comment
 387                     skip_comment = False
 388                 self.cursor = self.src.find('\n', self.cursor)
 389                 if not skip_comment:
 390                     self.val = self.src[self.pos:self.cursor]
 391                     return
 392             elif self.tok in '{}:,[]':
 393                 return
 394             elif self.tok == "'":
 395                 string = ''
 396                 esc = False
 397                 while True:
 398                     ch = self.src[self.cursor]
 399                     self.cursor += 1
 400                     if ch == '\n':
 401                         raise QAPIParseError(self, 'Missing terminating "\'"')
 402                     if esc:
 403                         if ch == 'b':
 404                             string += '\b'
 405                         elif ch == 'f':
 406                             string += '\f'
 407                         elif ch == 'n':
 408                             string += '\n'
 409                         elif ch == 'r':
 410                             string += '\r'
 411                         elif ch == 't':
 412                             string += '\t'
 413                         elif ch == 'u':
 414                             value = 0
 415                             for _ in range(0, 4):
 416                                 ch = self.src[self.cursor]
 417                                 self.cursor += 1
 418                                 if ch not in '0123456789abcdefABCDEF':
 419                                     raise QAPIParseError(self,
 420                                                          '\\u escape needs 4 '
 421                                                          'hex digits')
 422                                 value = (value << 4) + int(ch, 16)
 423                             # If Python 2 and 3 didn't disagree so much on
 424                             # how to handle Unicode, then we could allow
 425                             # Unicode string defaults.  But most of QAPI is
 426                             # ASCII-only, so we aren't losing much for now.
 427                             if not value or value > 0x7f:
 428                                 raise QAPIParseError(self,
 429                                                      'For now, \\u escape '
 430                                                      'only supports non-zero '
 431                                                      'values up to \\u007f')
 432                             string += chr(value)
 433                         elif ch in '\\/\'"':
 434                             string += ch
 435                         else:
 436                             raise QAPIParseError(self,
 437                                                  "Unknown escape \\%s" % ch)
 438                         esc = False
 439                     elif ch == '\\':
 440                         esc = True
 441                     elif ch == "'":
 442                         self.val = string
 443                         return
 444                     else:
 445                         string += ch
 446             elif self.src.startswith('true', self.pos):
 447                 self.val = True
 448                 self.cursor += 3
 449                 return
 450             elif self.src.startswith('false', self.pos):
 451                 self.val = False
 452                 self.cursor += 4
 453                 return
 454             elif self.src.startswith('null', self.pos):
 455                 self.val = None
 456                 self.cursor += 3
 457                 return
 458             elif self.tok == '\n':
 459                 if self.cursor == len(self.src):
 460                     self.tok = None
 461                     return
 462                 self.line += 1
 463                 self.line_pos = self.cursor
 464             elif not self.tok.isspace():
 465                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 466
 467     def get_members(self):
 468         expr = OrderedDict()
 469         if self.tok == '}':
 470             self.accept()
 471             return expr
 472         if self.tok != "'":
 473             raise QAPIParseError(self, 'Expected string or "}"')
 474         while True:
 475             key = self.val
 476             self.accept()
 477             if self.tok != ':':
 478                 raise QAPIParseError(self, 'Expected ":"')
 479             self.accept()
 480             if key in expr:
 481                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 482             expr[key] = self.get_expr(True)
 483             if self.tok == '}':
 484                 self.accept()
 485                 return expr
 486             if self.tok != ',':
 487                 raise QAPIParseError(self, 'Expected "," or "}"')
 488             self.accept()
 489             if self.tok != "'":
 490                 raise QAPIParseError(self, 'Expected string')
 491
 492     def get_values(self):
 493         expr = []
 494         if self.tok == ']':
 495             self.accept()
 496             return expr
 497         if self.tok not in "{['tfn":
 498             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 499                                  'boolean or "null"')
 500         while True:
 501             expr.append(self.get_expr(True))
 502             if self.tok == ']':
 503                 self.accept()
 504                 return expr
 505             if self.tok != ',':
 506                 raise QAPIParseError(self, 'Expected "," or "]"')
 507             self.accept()
 508
 509     def get_expr(self, nested):
 510         if self.tok != '{' and not nested:
 511             raise QAPIParseError(self, 'Expected "{"')
 512         if self.tok == '{':
 513             self.accept()
 514             expr = self.get_members()
 515         elif self.tok == '[':
 516             self.accept()
 517             expr = self.get_values()
 518         elif self.tok in "'tfn":
 519             expr = self.val
 520             self.accept()
 521         else:
 522             raise QAPIParseError(self, 'Expected "{", "[" or string')
 523         return expr
 524
 525     def get_doc(self, info):
 526         if self.val != '##':
 527             raise QAPIParseError(self, "Junk after '##' at start of "
 528                                  "documentation comment")
 529
 530         doc = QAPIDoc(self, info)
 531         self.accept(False)
 532         while self.tok == '#':
 533             if self.val.startswith('##'):
 534                 # End of doc comment
 535                 if self.val != '##':
 536                     raise QAPIParseError(self, "Junk after '##' at end of "
 537                                          "documentation comment")
 538                 doc.end_comment()
 539                 self.accept()
 540                 return doc
 541             else:
 542                 doc.append(self.val)
 543             self.accept(False)
 544
 545         raise QAPIParseError(self, "Documentation comment must end with '##'")
 546
 547
 548 #
 549 # Semantic analysis of schema expressions
 550 # TODO fold into QAPISchema
 551 # TODO catching name collisions in generated code would be nice
 552 #
 553
 554
 555 def find_base_members(base):
 556     if isinstance(base, dict):
 557         return base
 558     base_struct_define = find_struct(base)
 559     if not base_struct_define:
 560         return None
 561     return base_struct_define['data']
 562
 563
 564 # Return the qtype of an alternate branch, or None on error.
 565 def find_alternate_member_qtype(qapi_type):
 566     if qapi_type in builtin_types:
 567         return builtin_types[qapi_type]
 568     elif find_struct(qapi_type):
 569         return 'QTYPE_QDICT'
 570     elif find_enum(qapi_type):
 571         return 'QTYPE_QSTRING'
 572     elif find_union(qapi_type):
 573         return 'QTYPE_QDICT'
 574     return None
 575
 576
 577 # Return the discriminator enum define if discriminator is specified as an
 578 # enum type, otherwise return None.
 579 def discriminator_find_enum_define(expr):
 580     base = expr.get('base')
 581     discriminator = expr.get('discriminator')
 582
 583     if not (discriminator and base):
 584         return None
 585
 586     base_members = find_base_members(base)
 587     if not base_members:
 588         return None
 589
 590     discriminator_type = base_members.get(discriminator)
 591     if not discriminator_type:
 592         return None
 593
 594     return find_enum(discriminator_type)
 595
 596
 597 # Names must be letters, numbers, -, and _.  They must start with letter,
 598 # except for downstream extensions which must start with __RFQDN_.
 599 # Dots are only valid in the downstream extension prefix.
 600 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 601                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 602
 603
 604 def check_name(info, source, name, allow_optional=False,
 605                enum_member=False):
 606     global valid_name
 607     membername = name
 608
 609     if not isinstance(name, str):
 610         raise QAPISemError(info, "%s requires a string name" % source)
 611     if name.startswith('*'):
 612         membername = name[1:]
 613         if not allow_optional:
 614             raise QAPISemError(info, "%s does not allow optional name '%s'"
 615                                % (source, name))
 616     # Enum members can start with a digit, because the generated C
 617     # code always prefixes it with the enum name
 618     if enum_member and membername[0].isdigit():
 619         membername = 'D' + membername
 620     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 621     # and 'q_obj_*' implicit type names.
 622     if not valid_name.match(membername) or \
 623        c_name(membername, False).startswith('q_'):
 624         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 625
 626
 627 def add_name(name, info, meta, implicit=False):
 628     global all_names
 629     check_name(info, "'%s'" % meta, name)
 630     # FIXME should reject names that differ only in '_' vs. '.'
 631     # vs. '-', because they're liable to clash in generated C.
 632     if name in all_names:
 633         raise QAPISemError(info, "%s '%s' is already defined"
 634                            % (all_names[name], name))
 635     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 636         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 637                            % (meta, name, name[-4:]))
 638     all_names[name] = meta
 639
 640
 641 def add_struct(definition, info):
 642     global struct_types
 643     name = definition['struct']
 644     add_name(name, info, 'struct')
 645     struct_types.append(definition)
 646
 647
 648 def find_struct(name):
 649     global struct_types
 650     for struct in struct_types:
 651         if struct['struct'] == name:
 652             return struct
 653     return None
 654
 655
 656 def add_union(definition, info):
 657     global union_types
 658     name = definition['union']
 659     add_name(name, info, 'union')
 660     union_types.append(definition)
 661
 662
 663 def find_union(name):
 664     global union_types
 665     for union in union_types:
 666         if union['union'] == name:
 667             return union
 668     return None
 669
 670
 671 def add_enum(definition, info):
 672     global enum_types
 673     name = definition['enum']
 674     add_name(name, info, 'enum', 'data' not in definition)
 675     enum_types.append(definition)
 676
 677
 678 def find_enum(name):
 679     global enum_types
 680     for enum in enum_types:
 681         if enum['enum'] == name:
 682             return enum
 683     return None
 684
 685
 686 def is_enum(name):
 687     return find_enum(name) is not None
 688
 689
 690 def check_type(info, source, value, allow_array=False,
 691                allow_dict=False, allow_optional=False,
 692                allow_metas=[]):
 693     global all_names
 694
 695     if value is None:
 696         return
 697
 698     # Check if array type for value is okay
 699     if isinstance(value, list):
 700         if not allow_array:
 701             raise QAPISemError(info, "%s cannot be an array" % source)
 702         if len(value) != 1 or not isinstance(value[0], str):
 703             raise QAPISemError(info,
 704                                "%s: array type must contain single type name" %
 705                                source)
 706         value = value[0]
 707
 708     # Check if type name for value is okay
 709     if isinstance(value, str):
 710         if value not in all_names:
 711             raise QAPISemError(info, "%s uses unknown type '%s'"
 712                                % (source, value))
 713         if not all_names[value] in allow_metas:
 714             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 715                                (source, all_names[value], value))
 716         return
 717
 718     if not allow_dict:
 719         raise QAPISemError(info, "%s should be a type name" % source)
 720
 721     if not isinstance(value, OrderedDict):
 722         raise QAPISemError(info,
 723                            "%s should be a dictionary or type name" % source)
 724
 725     # value is a dictionary, check that each member is okay
 726     for (key, arg) in value.items():
 727         check_name(info, "Member of %s" % source, key,
 728                    allow_optional=allow_optional)
 729         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 730             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 731                                % (source, key))
 732         # Todo: allow dictionaries to represent default values of
 733         # an optional argument.
 734         check_type(info, "Member '%s' of %s" % (key, source), arg,
 735                    allow_array=True,
 736                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 737                                 'enum'])
 738
 739
 740 def check_command(expr, info):
 741     name = expr['command']
 742     boxed = expr.get('boxed', False)
 743
 744     args_meta = ['struct']
 745     if boxed:
 746         args_meta += ['union', 'alternate']
 747     check_type(info, "'data' for command '%s'" % name,
 748                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 749                allow_metas=args_meta)
 750     returns_meta = ['union', 'struct']
 751     if name in returns_whitelist:
 752         returns_meta += ['built-in', 'alternate', 'enum']
 753     check_type(info, "'returns' for command '%s'" % name,
 754                expr.get('returns'), allow_array=True,
 755                allow_optional=True, allow_metas=returns_meta)
 756
 757
 758 def check_event(expr, info):
 759     name = expr['event']
 760     boxed = expr.get('boxed', False)
 761
 762     meta = ['struct']
 763     if boxed:
 764         meta += ['union', 'alternate']
 765     check_type(info, "'data' for event '%s'" % name,
 766                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 767                allow_metas=meta)
 768
 769
 770 def check_union(expr, info):
 771     name = expr['union']
 772     base = expr.get('base')
 773     discriminator = expr.get('discriminator')
 774     members = expr['data']
 775
 776     # Two types of unions, determined by discriminator.
 777
 778     # With no discriminator it is a simple union.
 779     if discriminator is None:
 780         enum_define = None
 781         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 782         if base is not None:
 783             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 784                                name)
 785
 786     # Else, it's a flat union.
 787     else:
 788         # The object must have a string or dictionary 'base'.
 789         check_type(info, "'base' for union '%s'" % name,
 790                    base, allow_dict=True, allow_optional=True,
 791                    allow_metas=['struct'])
 792         if not base:
 793             raise QAPISemError(info, "Flat union '%s' must have a base"
 794                                % name)
 795         base_members = find_base_members(base)
 796         assert base_members is not None
 797
 798         # The value of member 'discriminator' must name a non-optional
 799         # member of the base struct.
 800         check_name(info, "Discriminator of flat union '%s'" % name,
 801                    discriminator)
 802         discriminator_type = base_members.get(discriminator)
 803         if not discriminator_type:
 804             raise QAPISemError(info,
 805                                "Discriminator '%s' is not a member of base "
 806                                "struct '%s'"
 807                                % (discriminator, base))
 808         enum_define = find_enum(discriminator_type)
 809         allow_metas = ['struct']
 810         # Do not allow string discriminator
 811         if not enum_define:
 812             raise QAPISemError(info,
 813                                "Discriminator '%s' must be of enumeration "
 814                                "type" % discriminator)
 815
 816     # Check every branch; don't allow an empty union
 817     if len(members) == 0:
 818         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 819     for (key, value) in members.items():
 820         check_name(info, "Member of union '%s'" % name, key)
 821
 822         # Each value must name a known type
 823         check_type(info, "Member '%s' of union '%s'" % (key, name),
 824                    value, allow_array=not base, allow_metas=allow_metas)
 825
 826         # If the discriminator names an enum type, then all members
 827         # of 'data' must also be members of the enum type.
 828         if enum_define:
 829             if key not in enum_define['data']:
 830                 raise QAPISemError(info,
 831                                    "Discriminator value '%s' is not found in "
 832                                    "enum '%s'"
 833                                    % (key, enum_define['enum']))
 834
 835     # If discriminator is user-defined, ensure all values are covered
 836     if enum_define:
 837         for value in enum_define['data']:
 838             if value not in members.keys():
 839                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 840                                    % (name, value))
 841
 842
 843 def check_alternate(expr, info):
 844     name = expr['alternate']
 845     members = expr['data']
 846     types_seen = {}
 847
 848     # Check every branch; require at least two branches
 849     if len(members) < 2:
 850         raise QAPISemError(info,
 851                            "Alternate '%s' should have at least two branches "
 852                            "in 'data'" % name)
 853     for (key, value) in members.items():
 854         check_name(info, "Member of alternate '%s'" % name, key)
 855
 856         # Ensure alternates have no type conflicts.
 857         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 858                    value,
 859                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 860         qtype = find_alternate_member_qtype(value)
 861         if not qtype:
 862             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 863                                "type '%s'" % (name, key, value))
 864         if qtype in types_seen:
 865             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 866                                "be distinguished from member '%s'"
 867                                % (name, key, types_seen[qtype]))
 868         types_seen[qtype] = key
 869
 870
 871 def check_enum(expr, info):
 872     name = expr['enum']
 873     members = expr.get('data')
 874     prefix = expr.get('prefix')
 875
 876     if not isinstance(members, list):
 877         raise QAPISemError(info,
 878                            "Enum '%s' requires an array for 'data'" % name)
 879     if prefix is not None and not isinstance(prefix, str):
 880         raise QAPISemError(info,
 881                            "Enum '%s' requires a string for 'prefix'" % name)
 882     for member in members:
 883         check_name(info, "Member of enum '%s'" % name, member,
 884                    enum_member=True)
 885
 886
 887 def check_struct(expr, info):
 888     name = expr['struct']
 889     members = expr['data']
 890
 891     check_type(info, "'data' for struct '%s'" % name, members,
 892                allow_dict=True, allow_optional=True)
 893     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 894                allow_metas=['struct'])
 895
 896
 897 def check_keys(expr_elem, meta, required, optional=[]):
 898     expr = expr_elem['expr']
 899     info = expr_elem['info']
 900     name = expr[meta]
 901     if not isinstance(name, str):
 902         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 903     required = required + [meta]
 904     for (key, value) in expr.items():
 905         if key not in required and key not in optional:
 906             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 907                                % (key, meta, name))
 908         if (key == 'gen' or key == 'success-response') and value is not False:
 909             raise QAPISemError(info,
 910                                "'%s' of %s '%s' should only use false value"
 911                                % (key, meta, name))
 912         if key == 'boxed' and value is not True:
 913             raise QAPISemError(info,
 914                                "'%s' of %s '%s' should only use true value"
 915                                % (key, meta, name))
 916     for key in required:
 917         if key not in expr:
 918             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 919                                % (key, meta, name))
 920
 921
 922 def check_exprs(exprs):
 923     global all_names
 924
 925     # Populate name table with names of built-in types
 926     for builtin in builtin_types.keys():
 927         all_names[builtin] = 'built-in'
 928
 929     # Learn the types and check for valid expression keys
 930     for expr_elem in exprs:
 931         expr = expr_elem['expr']
 932         info = expr_elem['info']
 933         doc = expr_elem.get('doc')
 934
 935         if not doc and doc_required:
 936             raise QAPISemError(info,
 937                                "Expression missing documentation comment")
 938
 939         if 'enum' in expr:
 940             name = expr['enum']
 941             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 942             add_enum(expr, info)
 943         elif 'union' in expr:
 944             name = expr['union']
 945             check_keys(expr_elem, 'union', ['data'],
 946                        ['base', 'discriminator'])
 947             add_union(expr, info)
 948         elif 'alternate' in expr:
 949             name = expr['alternate']
 950             check_keys(expr_elem, 'alternate', ['data'])
 951             add_name(name, info, 'alternate')
 952         elif 'struct' in expr:
 953             name = expr['struct']
 954             check_keys(expr_elem, 'struct', ['data'], ['base'])
 955             add_struct(expr, info)
 956         elif 'command' in expr:
 957             name = expr['command']
 958             check_keys(expr_elem, 'command', [],
 959                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 960             add_name(name, info, 'command')
 961         elif 'event' in expr:
 962             name = expr['event']
 963             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 964             add_name(name, info, 'event')
 965         else:
 966             raise QAPISemError(expr_elem['info'],
 967                                "Expression is missing metatype")
 968         if doc and doc.symbol != name:
 969             raise QAPISemError(info, "Definition of '%s' follows documentation"
 970                                " for '%s'" % (name, doc.symbol))
 971
 972     # Try again for hidden UnionKind enum
 973     for expr_elem in exprs:
 974         expr = expr_elem['expr']
 975         if 'union' in expr and not discriminator_find_enum_define(expr):
 976             name = '%sKind' % expr['union']
 977         elif 'alternate' in expr:
 978             name = '%sKind' % expr['alternate']
 979         else:
 980             continue
 981         add_enum({ 'enum': name }, expr_elem['info'])
 982
 983     # Validate that exprs make sense
 984     for expr_elem in exprs:
 985         expr = expr_elem['expr']
 986         info = expr_elem['info']
 987         doc = expr_elem.get('doc')
 988
 989         if 'enum' in expr:
 990             check_enum(expr, info)
 991         elif 'union' in expr:
 992             check_union(expr, info)
 993         elif 'alternate' in expr:
 994             check_alternate(expr, info)
 995         elif 'struct' in expr:
 996             check_struct(expr, info)
 997         elif 'command' in expr:
 998             check_command(expr, info)
 999         elif 'event' in expr:
1000             check_event(expr, info)
1001         else:
1002             assert False, 'unexpected meta type'
1003
1004         if doc:
1005             doc.check_expr(expr)
1006
1007     return exprs
1008
1009
1010 #
1011 # Schema compiler frontend
1012 #
1013
1014 class QAPISchemaEntity(object):
1015     def __init__(self, name, info, doc):
1016         assert isinstance(name, str)
1017         self.name = name
1018         # For explicitly defined entities, info points to the (explicit)
1019         # definition.  For builtins (and their arrays), info is None.
1020         # For implicitly defined entities, info points to a place that
1021         # triggered the implicit definition (there may be more than one
1022         # such place).
1023         self.info = info
1024         self.doc = doc
1025
1026     def c_name(self):
1027         return c_name(self.name)
1028
1029     def check(self, schema):
1030         pass
1031
1032     def is_implicit(self):
1033         return not self.info
1034
1035     def visit(self, visitor):
1036         pass
1037
1038
1039 class QAPISchemaVisitor(object):
1040     def visit_begin(self, schema):
1041         pass
1042
1043     def visit_end(self):
1044         pass
1045
1046     def visit_needed(self, entity):
1047         # Default to visiting everything
1048         return True
1049
1050     def visit_builtin_type(self, name, info, json_type):
1051         pass
1052
1053     def visit_enum_type(self, name, info, values, prefix):
1054         pass
1055
1056     def visit_array_type(self, name, info, element_type):
1057         pass
1058
1059     def visit_object_type(self, name, info, base, members, variants):
1060         pass
1061
1062     def visit_object_type_flat(self, name, info, members, variants):
1063         pass
1064
1065     def visit_alternate_type(self, name, info, variants):
1066         pass
1067
1068     def visit_command(self, name, info, arg_type, ret_type,
1069                       gen, success_response, boxed):
1070         pass
1071
1072     def visit_event(self, name, info, arg_type, boxed):
1073         pass
1074
1075
1076 class QAPISchemaType(QAPISchemaEntity):
1077     # Return the C type for common use.
1078     # For the types we commonly box, this is a pointer type.
1079     def c_type(self):
1080         pass
1081
1082     # Return the C type to be used in a parameter list.
1083     def c_param_type(self):
1084         return self.c_type()
1085
1086     # Return the C type to be used where we suppress boxing.
1087     def c_unboxed_type(self):
1088         return self.c_type()
1089
1090     def json_type(self):
1091         pass
1092
1093     def alternate_qtype(self):
1094         json2qtype = {
1095             'string':  'QTYPE_QSTRING',
1096             'number':  'QTYPE_QFLOAT',
1097             'int':     'QTYPE_QINT',
1098             'boolean': 'QTYPE_QBOOL',
1099             'object':  'QTYPE_QDICT'
1100         }
1101         return json2qtype.get(self.json_type())
1102
1103     def doc_type(self):
1104         if self.is_implicit():
1105             return None
1106         return self.name
1107
1108
1109 class QAPISchemaBuiltinType(QAPISchemaType):
1110     def __init__(self, name, json_type, c_type):
1111         QAPISchemaType.__init__(self, name, None, None)
1112         assert not c_type or isinstance(c_type, str)
1113         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1114                              'value')
1115         self._json_type_name = json_type
1116         self._c_type_name = c_type
1117
1118     def c_name(self):
1119         return self.name
1120
1121     def c_type(self):
1122         return self._c_type_name
1123
1124     def c_param_type(self):
1125         if self.name == 'str':
1126             return 'const ' + self._c_type_name
1127         return self._c_type_name
1128
1129     def json_type(self):
1130         return self._json_type_name
1131
1132     def doc_type(self):
1133         return self.json_type()
1134
1135     def visit(self, visitor):
1136         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1137
1138
1139 class QAPISchemaEnumType(QAPISchemaType):
1140     def __init__(self, name, info, doc, values, prefix):
1141         QAPISchemaType.__init__(self, name, info, doc)
1142         for v in values:
1143             assert isinstance(v, QAPISchemaMember)
1144             v.set_owner(name)
1145         assert prefix is None or isinstance(prefix, str)
1146         self.values = values
1147         self.prefix = prefix
1148
1149     def check(self, schema):
1150         seen = {}
1151         for v in self.values:
1152             v.check_clash(self.info, seen)
1153             if self.doc:
1154                 self.doc.connect_member(v)
1155
1156     def is_implicit(self):
1157         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1158         return self.name.endswith('Kind') or self.name == 'QType'
1159
1160     def c_type(self):
1161         return c_name(self.name)
1162
1163     def member_names(self):
1164         return [v.name for v in self.values]
1165
1166     def json_type(self):
1167         return 'string'
1168
1169     def visit(self, visitor):
1170         visitor.visit_enum_type(self.name, self.info,
1171                                 self.member_names(), self.prefix)
1172
1173
1174 class QAPISchemaArrayType(QAPISchemaType):
1175     def __init__(self, name, info, element_type):
1176         QAPISchemaType.__init__(self, name, info, None)
1177         assert isinstance(element_type, str)
1178         self._element_type_name = element_type
1179         self.element_type = None
1180
1181     def check(self, schema):
1182         self.element_type = schema.lookup_type(self._element_type_name)
1183         assert self.element_type
1184
1185     def is_implicit(self):
1186         return True
1187
1188     def c_type(self):
1189         return c_name(self.name) + pointer_suffix
1190
1191     def json_type(self):
1192         return 'array'
1193
1194     def doc_type(self):
1195         elt_doc_type = self.element_type.doc_type()
1196         if not elt_doc_type:
1197             return None
1198         return 'array of ' + elt_doc_type
1199
1200     def visit(self, visitor):
1201         visitor.visit_array_type(self.name, self.info, self.element_type)
1202
1203
1204 class QAPISchemaObjectType(QAPISchemaType):
1205     def __init__(self, name, info, doc, base, local_members, variants):
1206         # struct has local_members, optional base, and no variants
1207         # flat union has base, variants, and no local_members
1208         # simple union has local_members, variants, and no base
1209         QAPISchemaType.__init__(self, name, info, doc)
1210         assert base is None or isinstance(base, str)
1211         for m in local_members:
1212             assert isinstance(m, QAPISchemaObjectTypeMember)
1213             m.set_owner(name)
1214         if variants is not None:
1215             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1216             variants.set_owner(name)
1217         self._base_name = base
1218         self.base = None
1219         self.local_members = local_members
1220         self.variants = variants
1221         self.members = None
1222
1223     def check(self, schema):
1224         if self.members is False:               # check for cycles
1225             raise QAPISemError(self.info,
1226                                "Object %s contains itself" % self.name)
1227         if self.members:
1228             return
1229         self.members = False                    # mark as being checked
1230         seen = OrderedDict()
1231         if self._base_name:
1232             self.base = schema.lookup_type(self._base_name)
1233             assert isinstance(self.base, QAPISchemaObjectType)
1234             self.base.check(schema)
1235             self.base.check_clash(schema, self.info, seen)
1236         for m in self.local_members:
1237             m.check(schema)
1238             m.check_clash(self.info, seen)
1239             if self.doc:
1240                 self.doc.connect_member(m)
1241         self.members = seen.values()
1242         if self.variants:
1243             self.variants.check(schema, seen)
1244             assert self.variants.tag_member in self.members
1245             self.variants.check_clash(schema, self.info, seen)
1246         if self.doc:
1247             self.doc.check()
1248
1249     # Check that the members of this type do not cause duplicate JSON members,
1250     # and update seen to track the members seen so far. Report any errors
1251     # on behalf of info, which is not necessarily self.info
1252     def check_clash(self, schema, info, seen):
1253         assert not self.variants       # not implemented
1254         for m in self.members:
1255             m.check_clash(info, seen)
1256
1257     def is_implicit(self):
1258         # See QAPISchema._make_implicit_object_type(), as well as
1259         # _def_predefineds()
1260         return self.name.startswith('q_')
1261
1262     def is_empty(self):
1263         assert self.members is not None
1264         return not self.members and not self.variants
1265
1266     def c_name(self):
1267         assert self.name != 'q_empty'
1268         return QAPISchemaType.c_name(self)
1269
1270     def c_type(self):
1271         assert not self.is_implicit()
1272         return c_name(self.name) + pointer_suffix
1273
1274     def c_unboxed_type(self):
1275         return c_name(self.name)
1276
1277     def json_type(self):
1278         return 'object'
1279
1280     def visit(self, visitor):
1281         visitor.visit_object_type(self.name, self.info,
1282                                   self.base, self.local_members, self.variants)
1283         visitor.visit_object_type_flat(self.name, self.info,
1284                                        self.members, self.variants)
1285
1286
1287 class QAPISchemaMember(object):
1288     role = 'member'
1289
1290     def __init__(self, name):
1291         assert isinstance(name, str)
1292         self.name = name
1293         self.owner = None
1294
1295     def set_owner(self, name):
1296         assert not self.owner
1297         self.owner = name
1298
1299     def check_clash(self, info, seen):
1300         cname = c_name(self.name)
1301         if cname.lower() != cname and self.owner not in name_case_whitelist:
1302             raise QAPISemError(info,
1303                                "%s should not use uppercase" % self.describe())
1304         if cname in seen:
1305             raise QAPISemError(info, "%s collides with %s" %
1306                                (self.describe(), seen[cname].describe()))
1307         seen[cname] = self
1308
1309     def _pretty_owner(self):
1310         owner = self.owner
1311         if owner.startswith('q_obj_'):
1312             # See QAPISchema._make_implicit_object_type() - reverse the
1313             # mapping there to create a nice human-readable description
1314             owner = owner[6:]
1315             if owner.endswith('-arg'):
1316                 return '(parameter of %s)' % owner[:-4]
1317             elif owner.endswith('-base'):
1318                 return '(base of %s)' % owner[:-5]
1319             else:
1320                 assert owner.endswith('-wrapper')
1321                 # Unreachable and not implemented
1322                 assert False
1323         if owner.endswith('Kind'):
1324             # See QAPISchema._make_implicit_enum_type()
1325             return '(branch of %s)' % owner[:-4]
1326         return '(%s of %s)' % (self.role, owner)
1327
1328     def describe(self):
1329         return "'%s' %s" % (self.name, self._pretty_owner())
1330
1331
1332 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1333     def __init__(self, name, typ, optional):
1334         QAPISchemaMember.__init__(self, name)
1335         assert isinstance(typ, str)
1336         assert isinstance(optional, bool)
1337         self._type_name = typ
1338         self.type = None
1339         self.optional = optional
1340
1341     def check(self, schema):
1342         assert self.owner
1343         self.type = schema.lookup_type(self._type_name)
1344         assert self.type
1345
1346
1347 class QAPISchemaObjectTypeVariants(object):
1348     def __init__(self, tag_name, tag_member, variants):
1349         # Flat unions pass tag_name but not tag_member.
1350         # Simple unions and alternates pass tag_member but not tag_name.
1351         # After check(), tag_member is always set, and tag_name remains
1352         # a reliable witness of being used by a flat union.
1353         assert bool(tag_member) != bool(tag_name)
1354         assert (isinstance(tag_name, str) or
1355                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1356         assert len(variants) > 0
1357         for v in variants:
1358             assert isinstance(v, QAPISchemaObjectTypeVariant)
1359         self._tag_name = tag_name
1360         self.tag_member = tag_member
1361         self.variants = variants
1362
1363     def set_owner(self, name):
1364         for v in self.variants:
1365             v.set_owner(name)
1366
1367     def check(self, schema, seen):
1368         if not self.tag_member:    # flat union
1369             self.tag_member = seen[c_name(self._tag_name)]
1370             assert self._tag_name == self.tag_member.name
1371         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1372         for v in self.variants:
1373             v.check(schema)
1374             # Union names must match enum values; alternate names are
1375             # checked separately. Use 'seen' to tell the two apart.
1376             if seen:
1377                 assert v.name in self.tag_member.type.member_names()
1378                 assert isinstance(v.type, QAPISchemaObjectType)
1379                 v.type.check(schema)
1380
1381     def check_clash(self, schema, info, seen):
1382         for v in self.variants:
1383             # Reset seen map for each variant, since qapi names from one
1384             # branch do not affect another branch
1385             assert isinstance(v.type, QAPISchemaObjectType)
1386             v.type.check_clash(schema, info, dict(seen))
1387
1388
1389 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1390     role = 'branch'
1391
1392     def __init__(self, name, typ):
1393         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1394
1395
1396 class QAPISchemaAlternateType(QAPISchemaType):
1397     def __init__(self, name, info, doc, variants):
1398         QAPISchemaType.__init__(self, name, info, doc)
1399         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1400         assert variants.tag_member
1401         variants.set_owner(name)
1402         variants.tag_member.set_owner(self.name)
1403         self.variants = variants
1404
1405     def check(self, schema):
1406         self.variants.tag_member.check(schema)
1407         # Not calling self.variants.check_clash(), because there's nothing
1408         # to clash with
1409         self.variants.check(schema, {})
1410         # Alternate branch names have no relation to the tag enum values;
1411         # so we have to check for potential name collisions ourselves.
1412         seen = {}
1413         for v in self.variants.variants:
1414             v.check_clash(self.info, seen)
1415             if self.doc:
1416                 self.doc.connect_member(v)
1417         if self.doc:
1418             self.doc.check()
1419
1420     def c_type(self):
1421         return c_name(self.name) + pointer_suffix
1422
1423     def json_type(self):
1424         return 'value'
1425
1426     def visit(self, visitor):
1427         visitor.visit_alternate_type(self.name, self.info, self.variants)
1428
1429     def is_empty(self):
1430         return False
1431
1432
1433 class QAPISchemaCommand(QAPISchemaEntity):
1434     def __init__(self, name, info, doc, arg_type, ret_type,
1435                  gen, success_response, boxed):
1436         QAPISchemaEntity.__init__(self, name, info, doc)
1437         assert not arg_type or isinstance(arg_type, str)
1438         assert not ret_type or isinstance(ret_type, str)
1439         self._arg_type_name = arg_type
1440         self.arg_type = None
1441         self._ret_type_name = ret_type
1442         self.ret_type = None
1443         self.gen = gen
1444         self.success_response = success_response
1445         self.boxed = boxed
1446
1447     def check(self, schema):
1448         if self._arg_type_name:
1449             self.arg_type = schema.lookup_type(self._arg_type_name)
1450             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1451                     isinstance(self.arg_type, QAPISchemaAlternateType))
1452             self.arg_type.check(schema)
1453             if self.boxed:
1454                 if self.arg_type.is_empty():
1455                     raise QAPISemError(self.info,
1456                                        "Cannot use 'boxed' with empty type")
1457             else:
1458                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1459                 assert not self.arg_type.variants
1460         elif self.boxed:
1461             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1462         if self._ret_type_name:
1463             self.ret_type = schema.lookup_type(self._ret_type_name)
1464             assert isinstance(self.ret_type, QAPISchemaType)
1465
1466     def visit(self, visitor):
1467         visitor.visit_command(self.name, self.info,
1468                               self.arg_type, self.ret_type,
1469                               self.gen, self.success_response, self.boxed)
1470
1471
1472 class QAPISchemaEvent(QAPISchemaEntity):
1473     def __init__(self, name, info, doc, arg_type, boxed):
1474         QAPISchemaEntity.__init__(self, name, info, doc)
1475         assert not arg_type or isinstance(arg_type, str)
1476         self._arg_type_name = arg_type
1477         self.arg_type = None
1478         self.boxed = boxed
1479
1480     def check(self, schema):
1481         if self._arg_type_name:
1482             self.arg_type = schema.lookup_type(self._arg_type_name)
1483             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1484                     isinstance(self.arg_type, QAPISchemaAlternateType))
1485             self.arg_type.check(schema)
1486             if self.boxed:
1487                 if self.arg_type.is_empty():
1488                     raise QAPISemError(self.info,
1489                                        "Cannot use 'boxed' with empty type")
1490             else:
1491                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1492                 assert not self.arg_type.variants
1493         elif self.boxed:
1494             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1495
1496     def visit(self, visitor):
1497         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1498
1499
1500 class QAPISchema(object):
1501     def __init__(self, fname):
1502         try:
1503             parser = QAPISchemaParser(open(fname, 'r'))
1504             self.exprs = check_exprs(parser.exprs)
1505             self.docs = parser.docs
1506             self._entity_dict = {}
1507             self._predefining = True
1508             self._def_predefineds()
1509             self._predefining = False
1510             self._def_exprs()
1511             self.check()
1512         except QAPIError as err:
1513             print >>sys.stderr, err
1514             exit(1)
1515
1516     def _def_entity(self, ent):
1517         # Only the predefined types are allowed to not have info
1518         assert ent.info or self._predefining
1519         assert ent.name not in self._entity_dict
1520         self._entity_dict[ent.name] = ent
1521
1522     def lookup_entity(self, name, typ=None):
1523         ent = self._entity_dict.get(name)
1524         if typ and not isinstance(ent, typ):
1525             return None
1526         return ent
1527
1528     def lookup_type(self, name):
1529         return self.lookup_entity(name, QAPISchemaType)
1530
1531     def _def_builtin_type(self, name, json_type, c_type):
1532         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1533         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1534         # qapi-types.h from a single .c, all arrays of builtins must be
1535         # declared in the first file whether or not they are used.  Nicer
1536         # would be to use lazy instantiation, while figuring out how to
1537         # avoid compilation issues with multiple qapi-types.h.
1538         self._make_array_type(name, None)
1539
1540     def _def_predefineds(self):
1541         for t in [('str',    'string',  'char' + pointer_suffix),
1542                   ('number', 'number',  'double'),
1543                   ('int',    'int',     'int64_t'),
1544                   ('int8',   'int',     'int8_t'),
1545                   ('int16',  'int',     'int16_t'),
1546                   ('int32',  'int',     'int32_t'),
1547                   ('int64',  'int',     'int64_t'),
1548                   ('uint8',  'int',     'uint8_t'),
1549                   ('uint16', 'int',     'uint16_t'),
1550                   ('uint32', 'int',     'uint32_t'),
1551                   ('uint64', 'int',     'uint64_t'),
1552                   ('size',   'int',     'uint64_t'),
1553                   ('bool',   'boolean', 'bool'),
1554                   ('any',    'value',   'QObject' + pointer_suffix)]:
1555             self._def_builtin_type(*t)
1556         self.the_empty_object_type = QAPISchemaObjectType(
1557             'q_empty', None, None, None, [], None)
1558         self._def_entity(self.the_empty_object_type)
1559         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1560                                                 'qstring', 'qdict', 'qlist',
1561                                                 'qfloat', 'qbool'])
1562         self._def_entity(QAPISchemaEnumType('QType', None, None,
1563                                             qtype_values, 'QTYPE'))
1564
1565     def _make_enum_members(self, values):
1566         return [QAPISchemaMember(v) for v in values]
1567
1568     def _make_implicit_enum_type(self, name, info, values):
1569         # See also QAPISchemaObjectTypeMember._pretty_owner()
1570         name = name + 'Kind'   # Use namespace reserved by add_name()
1571         self._def_entity(QAPISchemaEnumType(
1572             name, info, None, self._make_enum_members(values), None))
1573         return name
1574
1575     def _make_array_type(self, element_type, info):
1576         name = element_type + 'List'   # Use namespace reserved by add_name()
1577         if not self.lookup_type(name):
1578             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1579         return name
1580
1581     def _make_implicit_object_type(self, name, info, doc, role, members):
1582         if not members:
1583             return None
1584         # See also QAPISchemaObjectTypeMember._pretty_owner()
1585         name = 'q_obj_%s-%s' % (name, role)
1586         if not self.lookup_entity(name, QAPISchemaObjectType):
1587             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1588                                                   members, None))
1589         return name
1590
1591     def _def_enum_type(self, expr, info, doc):
1592         name = expr['enum']
1593         data = expr['data']
1594         prefix = expr.get('prefix')
1595         self._def_entity(QAPISchemaEnumType(
1596             name, info, doc, self._make_enum_members(data), prefix))
1597
1598     def _make_member(self, name, typ, info):
1599         optional = False
1600         if name.startswith('*'):
1601             name = name[1:]
1602             optional = True
1603         if isinstance(typ, list):
1604             assert len(typ) == 1
1605             typ = self._make_array_type(typ[0], info)
1606         return QAPISchemaObjectTypeMember(name, typ, optional)
1607
1608     def _make_members(self, data, info):
1609         return [self._make_member(key, value, info)
1610                 for (key, value) in data.iteritems()]
1611
1612     def _def_struct_type(self, expr, info, doc):
1613         name = expr['struct']
1614         base = expr.get('base')
1615         data = expr['data']
1616         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1617                                               self._make_members(data, info),
1618                                               None))
1619
1620     def _make_variant(self, case, typ):
1621         return QAPISchemaObjectTypeVariant(case, typ)
1622
1623     def _make_simple_variant(self, case, typ, info):
1624         if isinstance(typ, list):
1625             assert len(typ) == 1
1626             typ = self._make_array_type(typ[0], info)
1627         typ = self._make_implicit_object_type(
1628             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1629         return QAPISchemaObjectTypeVariant(case, typ)
1630
1631     def _def_union_type(self, expr, info, doc):
1632         name = expr['union']
1633         data = expr['data']
1634         base = expr.get('base')
1635         tag_name = expr.get('discriminator')
1636         tag_member = None
1637         if isinstance(base, dict):
1638             base = (self._make_implicit_object_type(
1639                     name, info, doc, 'base', self._make_members(base, info)))
1640         if tag_name:
1641             variants = [self._make_variant(key, value)
1642                         for (key, value) in data.iteritems()]
1643             members = []
1644         else:
1645             variants = [self._make_simple_variant(key, value, info)
1646                         for (key, value) in data.iteritems()]
1647             typ = self._make_implicit_enum_type(name, info,
1648                                                 [v.name for v in variants])
1649             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1650             members = [tag_member]
1651         self._def_entity(
1652             QAPISchemaObjectType(name, info, doc, base, members,
1653                                  QAPISchemaObjectTypeVariants(tag_name,
1654                                                               tag_member,
1655                                                               variants)))
1656
1657     def _def_alternate_type(self, expr, info, doc):
1658         name = expr['alternate']
1659         data = expr['data']
1660         variants = [self._make_variant(key, value)
1661                     for (key, value) in data.iteritems()]
1662         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1663         self._def_entity(
1664             QAPISchemaAlternateType(name, info, doc,
1665                                     QAPISchemaObjectTypeVariants(None,
1666                                                                  tag_member,
1667                                                                  variants)))
1668
1669     def _def_command(self, expr, info, doc):
1670         name = expr['command']
1671         data = expr.get('data')
1672         rets = expr.get('returns')
1673         gen = expr.get('gen', True)
1674         success_response = expr.get('success-response', True)
1675         boxed = expr.get('boxed', False)
1676         if isinstance(data, OrderedDict):
1677             data = self._make_implicit_object_type(
1678                 name, info, doc, 'arg', self._make_members(data, info))
1679         if isinstance(rets, list):
1680             assert len(rets) == 1
1681             rets = self._make_array_type(rets[0], info)
1682         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1683                                            gen, success_response, boxed))
1684
1685     def _def_event(self, expr, info, doc):
1686         name = expr['event']
1687         data = expr.get('data')
1688         boxed = expr.get('boxed', False)
1689         if isinstance(data, OrderedDict):
1690             data = self._make_implicit_object_type(
1691                 name, info, doc, 'arg', self._make_members(data, info))
1692         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1693
1694     def _def_exprs(self):
1695         for expr_elem in self.exprs:
1696             expr = expr_elem['expr']
1697             info = expr_elem['info']
1698             doc = expr_elem.get('doc')
1699             if 'enum' in expr:
1700                 self._def_enum_type(expr, info, doc)
1701             elif 'struct' in expr:
1702                 self._def_struct_type(expr, info, doc)
1703             elif 'union' in expr:
1704                 self._def_union_type(expr, info, doc)
1705             elif 'alternate' in expr:
1706                 self._def_alternate_type(expr, info, doc)
1707             elif 'command' in expr:
1708                 self._def_command(expr, info, doc)
1709             elif 'event' in expr:
1710                 self._def_event(expr, info, doc)
1711             else:
1712                 assert False
1713
1714     def check(self):
1715         for ent in self._entity_dict.values():
1716             ent.check(self)
1717
1718     def visit(self, visitor):
1719         visitor.visit_begin(self)
1720         for (name, entity) in sorted(self._entity_dict.items()):
1721             if visitor.visit_needed(entity):
1722                 entity.visit(visitor)
1723         visitor.visit_end()
1724
1725
1726 #
1727 # Code generation helpers
1728 #
1729
1730 def camel_case(name):
1731     new_name = ''
1732     first = True
1733     for ch in name:
1734         if ch in ['_', '-']:
1735             first = True
1736         elif first:
1737             new_name += ch.upper()
1738             first = False
1739         else:
1740             new_name += ch.lower()
1741     return new_name
1742
1743
1744 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1745 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1746 # ENUM24_Name -> ENUM24_NAME
1747 def camel_to_upper(value):
1748     c_fun_str = c_name(value, False)
1749     if value.isupper():
1750         return c_fun_str
1751
1752     new_name = ''
1753     l = len(c_fun_str)
1754     for i in range(l):
1755         c = c_fun_str[i]
1756         # When c is upper and no '_' appears before, do more checks
1757         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1758             if i < l - 1 and c_fun_str[i + 1].islower():
1759                 new_name += '_'
1760             elif c_fun_str[i - 1].isdigit():
1761                 new_name += '_'
1762         new_name += c
1763     return new_name.lstrip('_').upper()
1764
1765
1766 def c_enum_const(type_name, const_name, prefix=None):
1767     if prefix is not None:
1768         type_name = prefix
1769     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1770
1771 c_name_trans = string.maketrans('.-', '__')
1772
1773
1774 # Map @name to a valid C identifier.
1775 # If @protect, avoid returning certain ticklish identifiers (like
1776 # C keywords) by prepending 'q_'.
1777 #
1778 # Used for converting 'name' from a 'name':'type' qapi definition
1779 # into a generated struct member, as well as converting type names
1780 # into substrings of a generated C function name.
1781 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1782 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1783 def c_name(name, protect=True):
1784     # ANSI X3J11/88-090, 3.1.1
1785     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1786                      'default', 'do', 'double', 'else', 'enum', 'extern',
1787                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1788                      'return', 'short', 'signed', 'sizeof', 'static',
1789                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1790                      'void', 'volatile', 'while'])
1791     # ISO/IEC 9899:1999, 6.4.1
1792     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1793     # ISO/IEC 9899:2011, 6.4.1
1794     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1795                      '_Noreturn', '_Static_assert', '_Thread_local'])
1796     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1797     # excluding _.*
1798     gcc_words = set(['asm', 'typeof'])
1799     # C++ ISO/IEC 14882:2003 2.11
1800     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1801                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1802                      'namespace', 'new', 'operator', 'private', 'protected',
1803                      'public', 'reinterpret_cast', 'static_cast', 'template',
1804                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1805                      'using', 'virtual', 'wchar_t',
1806                      # alternative representations
1807                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1808                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1809     # namespace pollution:
1810     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1811     name = name.translate(c_name_trans)
1812     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1813                     | cpp_words | polluted_words):
1814         return 'q_' + name
1815     return name
1816
1817 eatspace = '\033EATSPACE.'
1818 pointer_suffix = ' *' + eatspace
1819
1820
1821 def genindent(count):
1822     ret = ''
1823     for _ in range(count):
1824         ret += ' '
1825     return ret
1826
1827 indent_level = 0
1828
1829
1830 def push_indent(indent_amount=4):
1831     global indent_level
1832     indent_level += indent_amount
1833
1834
1835 def pop_indent(indent_amount=4):
1836     global indent_level
1837     indent_level -= indent_amount
1838
1839
1840 # Generate @code with @kwds interpolated.
1841 # Obey indent_level, and strip eatspace.
1842 def cgen(code, **kwds):
1843     raw = code % kwds
1844     if indent_level:
1845         indent = genindent(indent_level)
1846         # re.subn() lacks flags support before Python 2.7, use re.compile()
1847         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1848                       indent + r'\g<0>', raw)
1849         raw = raw[0]
1850     return re.sub(re.escape(eatspace) + r' *', '', raw)
1851
1852
1853 def mcgen(code, **kwds):
1854     if code[0] == '\n':
1855         code = code[1:]
1856     return cgen(code, **kwds)
1857
1858
1859 def guardname(filename):
1860     return c_name(filename, protect=False).upper()
1861
1862
1863 def guardstart(name):
1864     return mcgen('''
1865
1866 #ifndef %(name)s
1867 #define %(name)s
1868
1869 ''',
1870                  name=guardname(name))
1871
1872
1873 def guardend(name):
1874     return mcgen('''
1875
1876 #endif /* %(name)s */
1877
1878 ''',
1879                  name=guardname(name))
1880
1881
1882 def gen_enum_lookup(name, values, prefix=None):
1883     ret = mcgen('''
1884
1885 const char *const %(c_name)s_lookup[] = {
1886 ''',
1887                 c_name=c_name(name))
1888     for value in values:
1889         index = c_enum_const(name, value, prefix)
1890         ret += mcgen('''
1891     [%(index)s] = "%(value)s",
1892 ''',
1893                      index=index, value=value)
1894
1895     max_index = c_enum_const(name, '_MAX', prefix)
1896     ret += mcgen('''
1897     [%(max_index)s] = NULL,
1898 };
1899 ''',
1900                  max_index=max_index)
1901     return ret
1902
1903
1904 def gen_enum(name, values, prefix=None):
1905     # append automatically generated _MAX value
1906     enum_values = values + ['_MAX']
1907
1908     ret = mcgen('''
1909
1910 typedef enum %(c_name)s {
1911 ''',
1912                 c_name=c_name(name))
1913
1914     i = 0
1915     for value in enum_values:
1916         ret += mcgen('''
1917     %(c_enum)s = %(i)d,
1918 ''',
1919                      c_enum=c_enum_const(name, value, prefix),
1920                      i=i)
1921         i += 1
1922
1923     ret += mcgen('''
1924 } %(c_name)s;
1925 ''',
1926                  c_name=c_name(name))
1927
1928     ret += mcgen('''
1929
1930 extern const char *const %(c_name)s_lookup[];
1931 ''',
1932                  c_name=c_name(name))
1933     return ret
1934
1935
1936 def gen_params(arg_type, boxed, extra):
1937     if not arg_type:
1938         assert not boxed
1939         return extra
1940     ret = ''
1941     sep = ''
1942     if boxed:
1943         ret += '%s arg' % arg_type.c_param_type()
1944         sep = ', '
1945     else:
1946         assert not arg_type.variants
1947         for memb in arg_type.members:
1948             ret += sep
1949             sep = ', '
1950             if memb.optional:
1951                 ret += 'bool has_%s, ' % c_name(memb.name)
1952             ret += '%s %s' % (memb.type.c_param_type(),
1953                               c_name(memb.name))
1954     if extra:
1955         ret += sep + extra
1956     return ret
1957
1958
1959 #
1960 # Common command line parsing
1961 #
1962
1963
1964 def parse_command_line(extra_options='', extra_long_options=[]):
1965
1966     try:
1967         opts, args = getopt.gnu_getopt(sys.argv[1:],
1968                                        'chp:o:' + extra_options,
1969                                        ['source', 'header', 'prefix=',
1970                                         'output-dir='] + extra_long_options)
1971     except getopt.GetoptError as err:
1972         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1973         sys.exit(1)
1974
1975     output_dir = ''
1976     prefix = ''
1977     do_c = False
1978     do_h = False
1979     extra_opts = []
1980
1981     for oa in opts:
1982         o, a = oa
1983         if o in ('-p', '--prefix'):
1984             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1985             if match.end() != len(a):
1986                 print >>sys.stderr, \
1987                     "%s: 'funny character '%s' in argument of --prefix" \
1988                     % (sys.argv[0], a[match.end()])
1989                 sys.exit(1)
1990             prefix = a
1991         elif o in ('-o', '--output-dir'):
1992             output_dir = a + '/'
1993         elif o in ('-c', '--source'):
1994             do_c = True
1995         elif o in ('-h', '--header'):
1996             do_h = True
1997         else:
1998             extra_opts.append(oa)
1999
2000     if not do_c and not do_h:
2001         do_c = True
2002         do_h = True
2003
2004     if len(args) != 1:
2005         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2006         sys.exit(1)
2007     fname = args[0]
2008
2009     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2010
2011 #
2012 # Generate output files with boilerplate
2013 #
2014
2015
2016 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2017                 c_comment, h_comment):
2018     guard = guardname(prefix + h_file)
2019     c_file = output_dir + prefix + c_file
2020     h_file = output_dir + prefix + h_file
2021
2022     if output_dir:
2023         try:
2024             os.makedirs(output_dir)
2025         except os.error as e:
2026             if e.errno != errno.EEXIST:
2027                 raise
2028
2029     def maybe_open(really, name, opt):
2030         if really:
2031             return open(name, opt)
2032         else:
2033             import StringIO
2034             return StringIO.StringIO()
2035
2036     fdef = maybe_open(do_c, c_file, 'w')
2037     fdecl = maybe_open(do_h, h_file, 'w')
2038
2039     fdef.write(mcgen('''
2040 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2041 %(comment)s
2042 ''',
2043                      comment=c_comment))
2044
2045     fdecl.write(mcgen('''
2046 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2047 %(comment)s
2048 #ifndef %(guard)s
2049 #define %(guard)s
2050
2051 ''',
2052                       comment=h_comment, guard=guard))
2053
2054     return (fdef, fdecl)
2055
2056
2057 def close_output(fdef, fdecl):
2058     fdecl.write('''
2059 #endif
2060 ''')
2061     fdecl.close()
2062     fdef.close()