scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 events = []
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110             self.optional = False
 111
 112         def append(self, line):
 113             self.content.append(line)
 114
 115         def __repr__(self):
 116             return '\n'.join(self.content).strip()
 117
 118     class ArgSection(Section):
 119         def __init__(self, name):
 120             QAPIDoc.Section.__init__(self, name)
 121             self.member = None
 122
 123         def connect(self, member):
 124             self.member = member
 125
 126     def __init__(self, parser, info):
 127         # self.parser is used to report errors with QAPIParseError.  The
 128         # resulting error position depends on the state of the parser.
 129         # It happens to be the beginning of the comment.  More or less
 130         # servicable, but action at a distance.
 131         self.parser = parser
 132         self.info = info
 133         self.symbol = None
 134         self.body = QAPIDoc.Section()
 135         # dict mapping parameter name to ArgSection
 136         self.args = OrderedDict()
 137         # a list of Section
 138         self.sections = []
 139         # the current section
 140         self.section = self.body
 141         # associated expression (to be set by expression parser)
 142         self.expr = None
 143
 144     def has_section(self, name):
 145         """Return True if we have a section with this name."""
 146         for i in self.sections:
 147             if i.name == name:
 148                 return True
 149         return False
 150
 151     def append(self, line):
 152         """Parse a comment line and add it to the documentation."""
 153         line = line[1:]
 154         if not line:
 155             self._append_freeform(line)
 156             return
 157
 158         if line[0] != ' ':
 159             raise QAPIParseError(self.parser, "Missing space after #")
 160         line = line[1:]
 161
 162         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 163         # recognized, and get silently treated as ordinary text
 164         if self.symbol:
 165             self._append_symbol_line(line)
 166         elif not self.body.content and line.startswith('@'):
 167             if not line.endswith(':'):
 168                 raise QAPIParseError(self.parser, "Line should end with :")
 169             self.symbol = line[1:-1]
 170             # FIXME invalid names other than the empty string aren't flagged
 171             if not self.symbol:
 172                 raise QAPIParseError(self.parser, "Invalid name")
 173         else:
 174             self._append_freeform(line)
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith('@') and name.endswith(':'):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ('Returns:', 'Since:',
 183                       # those are often singular or plural
 184                       'Note:', 'Notes:',
 185                       'Example:', 'Examples:',
 186                       'TODO:'):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=''):
 207         if name in ('Returns', 'Since') and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self.section = QAPIDoc.Section(name)
 211         self.sections.append(self.section)
 212
 213     def _append_freeform(self, line):
 214         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 215         if (in_arg and self.section.content
 216                 and not self.section.content[-1]
 217                 and line and not line[0].isspace()):
 218             self._start_section()
 219         if (in_arg or not self.section.name
 220                 or not self.section.name.startswith('Example')):
 221             line = line.strip()
 222         # TODO Drop this once the dust has settled
 223         if (isinstance(self.section, QAPIDoc.ArgSection)
 224                 and '#optional' in line):
 225             raise QAPISemError(self.info, "Please drop the #optional tag")
 226         self.section.append(line)
 227
 228     def connect_member(self, member):
 229         if member.name not in self.args:
 230             # Undocumented TODO outlaw
 231             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 232         self.args[member.name].connect(member)
 233
 234
 235 class QAPISchemaParser(object):
 236
 237     def __init__(self, fp, previously_included=[], incl_info=None):
 238         abs_fname = os.path.abspath(fp.name)
 239         fname = fp.name
 240         self.fname = fname
 241         previously_included.append(abs_fname)
 242         self.incl_info = incl_info
 243         self.src = fp.read()
 244         if self.src == '' or self.src[-1] != '\n':
 245             self.src += '\n'
 246         self.cursor = 0
 247         self.line = 1
 248         self.line_pos = 0
 249         self.exprs = []
 250         self.docs = []
 251         self.cur_doc = None
 252         self.accept()
 253
 254         while self.tok is not None:
 255             info = {'file': fname, 'line': self.line,
 256                     'parent': self.incl_info}
 257             if self.tok == '#':
 258                 self.reject_expr_doc()
 259                 self.cur_doc = self.get_doc(info)
 260                 self.docs.append(self.cur_doc)
 261                 continue
 262
 263             expr = self.get_expr(False)
 264             if 'include' in expr:
 265                 self.reject_expr_doc()
 266                 if len(expr) != 1:
 267                     raise QAPISemError(info, "Invalid 'include' directive")
 268                 include = expr['include']
 269                 if not isinstance(include, str):
 270                     raise QAPISemError(info,
 271                                        "Value of 'include' must be a string")
 272                 self._include(include, info, os.path.dirname(abs_fname),
 273                               previously_included)
 274             elif "pragma" in expr:
 275                 self.reject_expr_doc()
 276                 if len(expr) != 1:
 277                     raise QAPISemError(info, "Invalid 'pragma' directive")
 278                 pragma = expr['pragma']
 279                 if not isinstance(pragma, dict):
 280                     raise QAPISemError(
 281                         info, "Value of 'pragma' must be a dictionary")
 282                 for name, value in pragma.iteritems():
 283                     self._pragma(name, value, info)
 284             else:
 285                 expr_elem = {'expr': expr,
 286                              'info': info}
 287                 if self.cur_doc:
 288                     if not self.cur_doc.symbol:
 289                         raise QAPISemError(
 290                             self.cur_doc.info,
 291                             "Expression documentation required")
 292                     self.cur_doc.expr = expr
 293                     expr_elem['doc'] = self.cur_doc
 294                 self.exprs.append(expr_elem)
 295             self.cur_doc = None
 296         self.reject_expr_doc()
 297
 298     def reject_expr_doc(self):
 299         if self.cur_doc and self.cur_doc.symbol:
 300             raise QAPISemError(
 301                 self.cur_doc.info,
 302                 "Documentation for '%s' is not followed by the definition"
 303                 % self.cur_doc.symbol)
 304
 305     def _include(self, include, info, base_dir, previously_included):
 306         incl_abs_fname = os.path.join(base_dir, include)
 307         # catch inclusion cycle
 308         inf = info
 309         while inf:
 310             if incl_abs_fname == os.path.abspath(inf['file']):
 311                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 312             inf = inf['parent']
 313
 314         # skip multiple include of the same file
 315         if incl_abs_fname in previously_included:
 316             return
 317         try:
 318             fobj = open(incl_abs_fname, 'r')
 319         except IOError as e:
 320             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 321         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 322         self.exprs.extend(exprs_include.exprs)
 323         self.docs.extend(exprs_include.docs)
 324
 325     def _pragma(self, name, value, info):
 326         global doc_required, returns_whitelist, name_case_whitelist
 327         if name == 'doc-required':
 328             if not isinstance(value, bool):
 329                 raise QAPISemError(info,
 330                                    "Pragma 'doc-required' must be boolean")
 331             doc_required = value
 332         elif name == 'returns-whitelist':
 333             if (not isinstance(value, list)
 334                     or any([not isinstance(elt, str) for elt in value])):
 335                 raise QAPISemError(info,
 336                                    "Pragma returns-whitelist must be"
 337                                    " a list of strings")
 338             returns_whitelist = value
 339         elif name == 'name-case-whitelist':
 340             if (not isinstance(value, list)
 341                     or any([not isinstance(elt, str) for elt in value])):
 342                 raise QAPISemError(info,
 343                                    "Pragma name-case-whitelist must be"
 344                                    " a list of strings")
 345             name_case_whitelist = value
 346         else:
 347             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 348
 349     def accept(self, skip_comment=True):
 350         while True:
 351             self.tok = self.src[self.cursor]
 352             self.pos = self.cursor
 353             self.cursor += 1
 354             self.val = None
 355
 356             if self.tok == '#':
 357                 if self.src[self.cursor] == '#':
 358                     # Start of doc comment
 359                     skip_comment = False
 360                 self.cursor = self.src.find('\n', self.cursor)
 361                 if not skip_comment:
 362                     self.val = self.src[self.pos:self.cursor]
 363                     return
 364             elif self.tok in '{}:,[]':
 365                 return
 366             elif self.tok == "'":
 367                 string = ''
 368                 esc = False
 369                 while True:
 370                     ch = self.src[self.cursor]
 371                     self.cursor += 1
 372                     if ch == '\n':
 373                         raise QAPIParseError(self, 'Missing terminating "\'"')
 374                     if esc:
 375                         if ch == 'b':
 376                             string += '\b'
 377                         elif ch == 'f':
 378                             string += '\f'
 379                         elif ch == 'n':
 380                             string += '\n'
 381                         elif ch == 'r':
 382                             string += '\r'
 383                         elif ch == 't':
 384                             string += '\t'
 385                         elif ch == 'u':
 386                             value = 0
 387                             for _ in range(0, 4):
 388                                 ch = self.src[self.cursor]
 389                                 self.cursor += 1
 390                                 if ch not in '0123456789abcdefABCDEF':
 391                                     raise QAPIParseError(self,
 392                                                          '\\u escape needs 4 '
 393                                                          'hex digits')
 394                                 value = (value << 4) + int(ch, 16)
 395                             # If Python 2 and 3 didn't disagree so much on
 396                             # how to handle Unicode, then we could allow
 397                             # Unicode string defaults.  But most of QAPI is
 398                             # ASCII-only, so we aren't losing much for now.
 399                             if not value or value > 0x7f:
 400                                 raise QAPIParseError(self,
 401                                                      'For now, \\u escape '
 402                                                      'only supports non-zero '
 403                                                      'values up to \\u007f')
 404                             string += chr(value)
 405                         elif ch in '\\/\'"':
 406                             string += ch
 407                         else:
 408                             raise QAPIParseError(self,
 409                                                  "Unknown escape \\%s" % ch)
 410                         esc = False
 411                     elif ch == '\\':
 412                         esc = True
 413                     elif ch == "'":
 414                         self.val = string
 415                         return
 416                     else:
 417                         string += ch
 418             elif self.src.startswith('true', self.pos):
 419                 self.val = True
 420                 self.cursor += 3
 421                 return
 422             elif self.src.startswith('false', self.pos):
 423                 self.val = False
 424                 self.cursor += 4
 425                 return
 426             elif self.src.startswith('null', self.pos):
 427                 self.val = None
 428                 self.cursor += 3
 429                 return
 430             elif self.tok == '\n':
 431                 if self.cursor == len(self.src):
 432                     self.tok = None
 433                     return
 434                 self.line += 1
 435                 self.line_pos = self.cursor
 436             elif not self.tok.isspace():
 437                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 438
 439     def get_members(self):
 440         expr = OrderedDict()
 441         if self.tok == '}':
 442             self.accept()
 443             return expr
 444         if self.tok != "'":
 445             raise QAPIParseError(self, 'Expected string or "}"')
 446         while True:
 447             key = self.val
 448             self.accept()
 449             if self.tok != ':':
 450                 raise QAPIParseError(self, 'Expected ":"')
 451             self.accept()
 452             if key in expr:
 453                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 454             expr[key] = self.get_expr(True)
 455             if self.tok == '}':
 456                 self.accept()
 457                 return expr
 458             if self.tok != ',':
 459                 raise QAPIParseError(self, 'Expected "," or "}"')
 460             self.accept()
 461             if self.tok != "'":
 462                 raise QAPIParseError(self, 'Expected string')
 463
 464     def get_values(self):
 465         expr = []
 466         if self.tok == ']':
 467             self.accept()
 468             return expr
 469         if self.tok not in "{['tfn":
 470             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 471                                  'boolean or "null"')
 472         while True:
 473             expr.append(self.get_expr(True))
 474             if self.tok == ']':
 475                 self.accept()
 476                 return expr
 477             if self.tok != ',':
 478                 raise QAPIParseError(self, 'Expected "," or "]"')
 479             self.accept()
 480
 481     def get_expr(self, nested):
 482         if self.tok != '{' and not nested:
 483             raise QAPIParseError(self, 'Expected "{"')
 484         if self.tok == '{':
 485             self.accept()
 486             expr = self.get_members()
 487         elif self.tok == '[':
 488             self.accept()
 489             expr = self.get_values()
 490         elif self.tok in "'tfn":
 491             expr = self.val
 492             self.accept()
 493         else:
 494             raise QAPIParseError(self, 'Expected "{", "[" or string')
 495         return expr
 496
 497     def get_doc(self, info):
 498         if self.val != '##':
 499             raise QAPIParseError(self, "Junk after '##' at start of "
 500                                  "documentation comment")
 501
 502         doc = QAPIDoc(self, info)
 503         self.accept(False)
 504         while self.tok == '#':
 505             if self.val.startswith('##'):
 506                 # End of doc comment
 507                 if self.val != '##':
 508                     raise QAPIParseError(self, "Junk after '##' at end of "
 509                                          "documentation comment")
 510                 self.accept()
 511                 return doc
 512             else:
 513                 doc.append(self.val)
 514             self.accept(False)
 515
 516         raise QAPIParseError(self, "Documentation comment must end with '##'")
 517
 518
 519 #
 520 # Semantic analysis of schema expressions
 521 # TODO fold into QAPISchema
 522 # TODO catching name collisions in generated code would be nice
 523 #
 524
 525
 526 def find_base_members(base):
 527     if isinstance(base, dict):
 528         return base
 529     base_struct_define = find_struct(base)
 530     if not base_struct_define:
 531         return None
 532     return base_struct_define['data']
 533
 534
 535 # Return the qtype of an alternate branch, or None on error.
 536 def find_alternate_member_qtype(qapi_type):
 537     if qapi_type in builtin_types:
 538         return builtin_types[qapi_type]
 539     elif find_struct(qapi_type):
 540         return 'QTYPE_QDICT'
 541     elif find_enum(qapi_type):
 542         return 'QTYPE_QSTRING'
 543     elif find_union(qapi_type):
 544         return 'QTYPE_QDICT'
 545     return None
 546
 547
 548 # Return the discriminator enum define if discriminator is specified as an
 549 # enum type, otherwise return None.
 550 def discriminator_find_enum_define(expr):
 551     base = expr.get('base')
 552     discriminator = expr.get('discriminator')
 553
 554     if not (discriminator and base):
 555         return None
 556
 557     base_members = find_base_members(base)
 558     if not base_members:
 559         return None
 560
 561     discriminator_type = base_members.get(discriminator)
 562     if not discriminator_type:
 563         return None
 564
 565     return find_enum(discriminator_type)
 566
 567
 568 # Names must be letters, numbers, -, and _.  They must start with letter,
 569 # except for downstream extensions which must start with __RFQDN_.
 570 # Dots are only valid in the downstream extension prefix.
 571 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 572                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 573
 574
 575 def check_name(info, source, name, allow_optional=False,
 576                enum_member=False):
 577     global valid_name
 578     membername = name
 579
 580     if not isinstance(name, str):
 581         raise QAPISemError(info, "%s requires a string name" % source)
 582     if name.startswith('*'):
 583         membername = name[1:]
 584         if not allow_optional:
 585             raise QAPISemError(info, "%s does not allow optional name '%s'"
 586                                % (source, name))
 587     # Enum members can start with a digit, because the generated C
 588     # code always prefixes it with the enum name
 589     if enum_member and membername[0].isdigit():
 590         membername = 'D' + membername
 591     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 592     # and 'q_obj_*' implicit type names.
 593     if not valid_name.match(membername) or \
 594        c_name(membername, False).startswith('q_'):
 595         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 596
 597
 598 def add_name(name, info, meta, implicit=False):
 599     global all_names
 600     check_name(info, "'%s'" % meta, name)
 601     # FIXME should reject names that differ only in '_' vs. '.'
 602     # vs. '-', because they're liable to clash in generated C.
 603     if name in all_names:
 604         raise QAPISemError(info, "%s '%s' is already defined"
 605                            % (all_names[name], name))
 606     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 607         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 608                            % (meta, name, name[-4:]))
 609     all_names[name] = meta
 610
 611
 612 def add_struct(definition, info):
 613     global struct_types
 614     name = definition['struct']
 615     add_name(name, info, 'struct')
 616     struct_types.append(definition)
 617
 618
 619 def find_struct(name):
 620     global struct_types
 621     for struct in struct_types:
 622         if struct['struct'] == name:
 623             return struct
 624     return None
 625
 626
 627 def add_union(definition, info):
 628     global union_types
 629     name = definition['union']
 630     add_name(name, info, 'union')
 631     union_types.append(definition)
 632
 633
 634 def find_union(name):
 635     global union_types
 636     for union in union_types:
 637         if union['union'] == name:
 638             return union
 639     return None
 640
 641
 642 def add_enum(name, info, enum_values=None, implicit=False):
 643     global enum_types
 644     add_name(name, info, 'enum', implicit)
 645     enum_types.append({'enum_name': name, 'enum_values': enum_values})
 646
 647
 648 def find_enum(name):
 649     global enum_types
 650     for enum in enum_types:
 651         if enum['enum_name'] == name:
 652             return enum
 653     return None
 654
 655
 656 def is_enum(name):
 657     return find_enum(name) is not None
 658
 659
 660 def check_type(info, source, value, allow_array=False,
 661                allow_dict=False, allow_optional=False,
 662                allow_metas=[]):
 663     global all_names
 664
 665     if value is None:
 666         return
 667
 668     # Check if array type for value is okay
 669     if isinstance(value, list):
 670         if not allow_array:
 671             raise QAPISemError(info, "%s cannot be an array" % source)
 672         if len(value) != 1 or not isinstance(value[0], str):
 673             raise QAPISemError(info,
 674                                "%s: array type must contain single type name" %
 675                                source)
 676         value = value[0]
 677
 678     # Check if type name for value is okay
 679     if isinstance(value, str):
 680         if value not in all_names:
 681             raise QAPISemError(info, "%s uses unknown type '%s'"
 682                                % (source, value))
 683         if not all_names[value] in allow_metas:
 684             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 685                                (source, all_names[value], value))
 686         return
 687
 688     if not allow_dict:
 689         raise QAPISemError(info, "%s should be a type name" % source)
 690
 691     if not isinstance(value, OrderedDict):
 692         raise QAPISemError(info,
 693                            "%s should be a dictionary or type name" % source)
 694
 695     # value is a dictionary, check that each member is okay
 696     for (key, arg) in value.items():
 697         check_name(info, "Member of %s" % source, key,
 698                    allow_optional=allow_optional)
 699         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 700             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 701                                % (source, key))
 702         # Todo: allow dictionaries to represent default values of
 703         # an optional argument.
 704         check_type(info, "Member '%s' of %s" % (key, source), arg,
 705                    allow_array=True,
 706                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 707                                 'enum'])
 708
 709
 710 def check_command(expr, info):
 711     name = expr['command']
 712     boxed = expr.get('boxed', False)
 713
 714     args_meta = ['struct']
 715     if boxed:
 716         args_meta += ['union', 'alternate']
 717     check_type(info, "'data' for command '%s'" % name,
 718                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 719                allow_metas=args_meta)
 720     returns_meta = ['union', 'struct']
 721     if name in returns_whitelist:
 722         returns_meta += ['built-in', 'alternate', 'enum']
 723     check_type(info, "'returns' for command '%s'" % name,
 724                expr.get('returns'), allow_array=True,
 725                allow_optional=True, allow_metas=returns_meta)
 726
 727
 728 def check_event(expr, info):
 729     global events
 730     name = expr['event']
 731     boxed = expr.get('boxed', False)
 732
 733     meta = ['struct']
 734     if boxed:
 735         meta += ['union', 'alternate']
 736     events.append(name)
 737     check_type(info, "'data' for event '%s'" % name,
 738                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 739                allow_metas=meta)
 740
 741
 742 def check_union(expr, info):
 743     name = expr['union']
 744     base = expr.get('base')
 745     discriminator = expr.get('discriminator')
 746     members = expr['data']
 747
 748     # Two types of unions, determined by discriminator.
 749
 750     # With no discriminator it is a simple union.
 751     if discriminator is None:
 752         enum_define = None
 753         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 754         if base is not None:
 755             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 756                                name)
 757
 758     # Else, it's a flat union.
 759     else:
 760         # The object must have a string or dictionary 'base'.
 761         check_type(info, "'base' for union '%s'" % name,
 762                    base, allow_dict=True, allow_optional=True,
 763                    allow_metas=['struct'])
 764         if not base:
 765             raise QAPISemError(info, "Flat union '%s' must have a base"
 766                                % name)
 767         base_members = find_base_members(base)
 768         assert base_members is not None
 769
 770         # The value of member 'discriminator' must name a non-optional
 771         # member of the base struct.
 772         check_name(info, "Discriminator of flat union '%s'" % name,
 773                    discriminator)
 774         discriminator_type = base_members.get(discriminator)
 775         if not discriminator_type:
 776             raise QAPISemError(info,
 777                                "Discriminator '%s' is not a member of base "
 778                                "struct '%s'"
 779                                % (discriminator, base))
 780         enum_define = find_enum(discriminator_type)
 781         allow_metas = ['struct']
 782         # Do not allow string discriminator
 783         if not enum_define:
 784             raise QAPISemError(info,
 785                                "Discriminator '%s' must be of enumeration "
 786                                "type" % discriminator)
 787
 788     # Check every branch; don't allow an empty union
 789     if len(members) == 0:
 790         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 791     for (key, value) in members.items():
 792         check_name(info, "Member of union '%s'" % name, key)
 793
 794         # Each value must name a known type
 795         check_type(info, "Member '%s' of union '%s'" % (key, name),
 796                    value, allow_array=not base, allow_metas=allow_metas)
 797
 798         # If the discriminator names an enum type, then all members
 799         # of 'data' must also be members of the enum type.
 800         if enum_define:
 801             if key not in enum_define['enum_values']:
 802                 raise QAPISemError(info,
 803                                    "Discriminator value '%s' is not found in "
 804                                    "enum '%s'"
 805                                    % (key, enum_define['enum_name']))
 806
 807     # If discriminator is user-defined, ensure all values are covered
 808     if enum_define:
 809         for value in enum_define['enum_values']:
 810             if value not in members.keys():
 811                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 812                                    % (name, value))
 813
 814
 815 def check_alternate(expr, info):
 816     name = expr['alternate']
 817     members = expr['data']
 818     types_seen = {}
 819
 820     # Check every branch; require at least two branches
 821     if len(members) < 2:
 822         raise QAPISemError(info,
 823                            "Alternate '%s' should have at least two branches "
 824                            "in 'data'" % name)
 825     for (key, value) in members.items():
 826         check_name(info, "Member of alternate '%s'" % name, key)
 827
 828         # Ensure alternates have no type conflicts.
 829         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 830                    value,
 831                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 832         qtype = find_alternate_member_qtype(value)
 833         if not qtype:
 834             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 835                                "type '%s'" % (name, key, value))
 836         if qtype in types_seen:
 837             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 838                                "be distinguished from member '%s'"
 839                                % (name, key, types_seen[qtype]))
 840         types_seen[qtype] = key
 841
 842
 843 def check_enum(expr, info):
 844     name = expr['enum']
 845     members = expr.get('data')
 846     prefix = expr.get('prefix')
 847
 848     if not isinstance(members, list):
 849         raise QAPISemError(info,
 850                            "Enum '%s' requires an array for 'data'" % name)
 851     if prefix is not None and not isinstance(prefix, str):
 852         raise QAPISemError(info,
 853                            "Enum '%s' requires a string for 'prefix'" % name)
 854     for member in members:
 855         check_name(info, "Member of enum '%s'" % name, member,
 856                    enum_member=True)
 857
 858
 859 def check_struct(expr, info):
 860     name = expr['struct']
 861     members = expr['data']
 862
 863     check_type(info, "'data' for struct '%s'" % name, members,
 864                allow_dict=True, allow_optional=True)
 865     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 866                allow_metas=['struct'])
 867
 868
 869 def check_keys(expr_elem, meta, required, optional=[]):
 870     expr = expr_elem['expr']
 871     info = expr_elem['info']
 872     name = expr[meta]
 873     if not isinstance(name, str):
 874         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 875     required = required + [meta]
 876     for (key, value) in expr.items():
 877         if key not in required and key not in optional:
 878             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 879                                % (key, meta, name))
 880         if (key == 'gen' or key == 'success-response') and value is not False:
 881             raise QAPISemError(info,
 882                                "'%s' of %s '%s' should only use false value"
 883                                % (key, meta, name))
 884         if key == 'boxed' and value is not True:
 885             raise QAPISemError(info,
 886                                "'%s' of %s '%s' should only use true value"
 887                                % (key, meta, name))
 888     for key in required:
 889         if key not in expr:
 890             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 891                                % (key, meta, name))
 892
 893
 894 def check_exprs(exprs):
 895     global all_names
 896
 897     # Learn the types and check for valid expression keys
 898     for builtin in builtin_types.keys():
 899         all_names[builtin] = 'built-in'
 900     for expr_elem in exprs:
 901         expr = expr_elem['expr']
 902         info = expr_elem['info']
 903
 904         if 'doc' not in expr_elem and doc_required:
 905             raise QAPISemError(info,
 906                                "Expression missing documentation comment")
 907
 908         if 'enum' in expr:
 909             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 910             add_enum(expr['enum'], info, expr['data'])
 911         elif 'union' in expr:
 912             check_keys(expr_elem, 'union', ['data'],
 913                        ['base', 'discriminator'])
 914             add_union(expr, info)
 915         elif 'alternate' in expr:
 916             check_keys(expr_elem, 'alternate', ['data'])
 917             add_name(expr['alternate'], info, 'alternate')
 918         elif 'struct' in expr:
 919             check_keys(expr_elem, 'struct', ['data'], ['base'])
 920             add_struct(expr, info)
 921         elif 'command' in expr:
 922             check_keys(expr_elem, 'command', [],
 923                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 924             add_name(expr['command'], info, 'command')
 925         elif 'event' in expr:
 926             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 927             add_name(expr['event'], info, 'event')
 928         else:
 929             raise QAPISemError(expr_elem['info'],
 930                                "Expression is missing metatype")
 931
 932     # Try again for hidden UnionKind enum
 933     for expr_elem in exprs:
 934         expr = expr_elem['expr']
 935         if 'union' in expr:
 936             if not discriminator_find_enum_define(expr):
 937                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 938                          implicit=True)
 939         elif 'alternate' in expr:
 940             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 941                      implicit=True)
 942
 943     # Validate that exprs make sense
 944     for expr_elem in exprs:
 945         expr = expr_elem['expr']
 946         info = expr_elem['info']
 947
 948         if 'enum' in expr:
 949             check_enum(expr, info)
 950         elif 'union' in expr:
 951             check_union(expr, info)
 952         elif 'alternate' in expr:
 953             check_alternate(expr, info)
 954         elif 'struct' in expr:
 955             check_struct(expr, info)
 956         elif 'command' in expr:
 957             check_command(expr, info)
 958         elif 'event' in expr:
 959             check_event(expr, info)
 960         else:
 961             assert False, 'unexpected meta type'
 962
 963     return exprs
 964
 965
 966 def check_freeform_doc(doc):
 967     body = str(doc.body)
 968     if re.search(r'@\S+:', body, re.MULTILINE):
 969         raise QAPISemError(doc.info,
 970                            "Free-form documentation block must not contain"
 971                            " @NAME: sections")
 972
 973
 974 def check_definition_doc(doc, expr, info):
 975     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 976         if i in expr:
 977             meta = i
 978             break
 979
 980     name = expr[meta]
 981     if doc.symbol != name:
 982         raise QAPISemError(info, "Definition of '%s' follows documentation"
 983                            " for '%s'" % (name, doc.symbol))
 984     if doc.has_section('Returns') and 'command' not in expr:
 985         raise QAPISemError(info, "'Returns:' is only valid for commands")
 986
 987     if meta == 'union':
 988         args = expr.get('base', [])
 989     else:
 990         args = expr.get('data', [])
 991     if isinstance(args, str):
 992         return
 993     if isinstance(args, dict):
 994         args = args.keys()
 995     assert isinstance(args, list)
 996
 997     if (meta == 'alternate'
 998             or (meta == 'union' and not expr.get('discriminator'))):
 999         args.append('type')
1000
1001     doc_args = set(doc.args.keys())
1002     args = set([name.strip('*') for name in args])
1003     if not doc_args.issubset(args):
1004         raise QAPISemError(info, "The following documented members are not in "
1005                            "the declaration: %s" % ', '.join(doc_args - args))
1006
1007
1008 def check_docs(docs):
1009     for doc in docs:
1010         for section in doc.args.values() + doc.sections:
1011             content = str(section)
1012             if not content or content.isspace():
1013                 raise QAPISemError(doc.info,
1014                                    "Empty doc section '%s'" % section.name)
1015
1016         if not doc.expr:
1017             check_freeform_doc(doc)
1018         else:
1019             check_definition_doc(doc, doc.expr, doc.info)
1020
1021     return docs
1022
1023
1024 #
1025 # Schema compiler frontend
1026 #
1027
1028 class QAPISchemaEntity(object):
1029     def __init__(self, name, info, doc):
1030         assert isinstance(name, str)
1031         self.name = name
1032         # For explicitly defined entities, info points to the (explicit)
1033         # definition.  For builtins (and their arrays), info is None.
1034         # For implicitly defined entities, info points to a place that
1035         # triggered the implicit definition (there may be more than one
1036         # such place).
1037         self.info = info
1038         self.doc = doc
1039
1040     def c_name(self):
1041         return c_name(self.name)
1042
1043     def check(self, schema):
1044         pass
1045
1046     def is_implicit(self):
1047         return not self.info
1048
1049     def visit(self, visitor):
1050         pass
1051
1052
1053 class QAPISchemaVisitor(object):
1054     def visit_begin(self, schema):
1055         pass
1056
1057     def visit_end(self):
1058         pass
1059
1060     def visit_needed(self, entity):
1061         # Default to visiting everything
1062         return True
1063
1064     def visit_builtin_type(self, name, info, json_type):
1065         pass
1066
1067     def visit_enum_type(self, name, info, values, prefix):
1068         pass
1069
1070     def visit_array_type(self, name, info, element_type):
1071         pass
1072
1073     def visit_object_type(self, name, info, base, members, variants):
1074         pass
1075
1076     def visit_object_type_flat(self, name, info, members, variants):
1077         pass
1078
1079     def visit_alternate_type(self, name, info, variants):
1080         pass
1081
1082     def visit_command(self, name, info, arg_type, ret_type,
1083                       gen, success_response, boxed):
1084         pass
1085
1086     def visit_event(self, name, info, arg_type, boxed):
1087         pass
1088
1089
1090 class QAPISchemaType(QAPISchemaEntity):
1091     # Return the C type for common use.
1092     # For the types we commonly box, this is a pointer type.
1093     def c_type(self):
1094         pass
1095
1096     # Return the C type to be used in a parameter list.
1097     def c_param_type(self):
1098         return self.c_type()
1099
1100     # Return the C type to be used where we suppress boxing.
1101     def c_unboxed_type(self):
1102         return self.c_type()
1103
1104     def json_type(self):
1105         pass
1106
1107     def alternate_qtype(self):
1108         json2qtype = {
1109             'string':  'QTYPE_QSTRING',
1110             'number':  'QTYPE_QFLOAT',
1111             'int':     'QTYPE_QINT',
1112             'boolean': 'QTYPE_QBOOL',
1113             'object':  'QTYPE_QDICT'
1114         }
1115         return json2qtype.get(self.json_type())
1116
1117     def doc_type(self):
1118         if self.is_implicit():
1119             return None
1120         return self.name
1121
1122
1123 class QAPISchemaBuiltinType(QAPISchemaType):
1124     def __init__(self, name, json_type, c_type):
1125         QAPISchemaType.__init__(self, name, None, None)
1126         assert not c_type or isinstance(c_type, str)
1127         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1128                              'value')
1129         self._json_type_name = json_type
1130         self._c_type_name = c_type
1131
1132     def c_name(self):
1133         return self.name
1134
1135     def c_type(self):
1136         return self._c_type_name
1137
1138     def c_param_type(self):
1139         if self.name == 'str':
1140             return 'const ' + self._c_type_name
1141         return self._c_type_name
1142
1143     def json_type(self):
1144         return self._json_type_name
1145
1146     def doc_type(self):
1147         return self.json_type()
1148
1149     def visit(self, visitor):
1150         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1151
1152
1153 class QAPISchemaEnumType(QAPISchemaType):
1154     def __init__(self, name, info, doc, values, prefix):
1155         QAPISchemaType.__init__(self, name, info, doc)
1156         for v in values:
1157             assert isinstance(v, QAPISchemaMember)
1158             v.set_owner(name)
1159         assert prefix is None or isinstance(prefix, str)
1160         self.values = values
1161         self.prefix = prefix
1162
1163     def check(self, schema):
1164         seen = {}
1165         for v in self.values:
1166             v.check_clash(self.info, seen)
1167             if self.doc:
1168                 self.doc.connect_member(v)
1169
1170     def is_implicit(self):
1171         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1172         return self.name.endswith('Kind') or self.name == 'QType'
1173
1174     def c_type(self):
1175         return c_name(self.name)
1176
1177     def member_names(self):
1178         return [v.name for v in self.values]
1179
1180     def json_type(self):
1181         return 'string'
1182
1183     def visit(self, visitor):
1184         visitor.visit_enum_type(self.name, self.info,
1185                                 self.member_names(), self.prefix)
1186
1187
1188 class QAPISchemaArrayType(QAPISchemaType):
1189     def __init__(self, name, info, element_type):
1190         QAPISchemaType.__init__(self, name, info, None)
1191         assert isinstance(element_type, str)
1192         self._element_type_name = element_type
1193         self.element_type = None
1194
1195     def check(self, schema):
1196         self.element_type = schema.lookup_type(self._element_type_name)
1197         assert self.element_type
1198
1199     def is_implicit(self):
1200         return True
1201
1202     def c_type(self):
1203         return c_name(self.name) + pointer_suffix
1204
1205     def json_type(self):
1206         return 'array'
1207
1208     def doc_type(self):
1209         elt_doc_type = self.element_type.doc_type()
1210         if not elt_doc_type:
1211             return None
1212         return 'array of ' + elt_doc_type
1213
1214     def visit(self, visitor):
1215         visitor.visit_array_type(self.name, self.info, self.element_type)
1216
1217
1218 class QAPISchemaObjectType(QAPISchemaType):
1219     def __init__(self, name, info, doc, base, local_members, variants):
1220         # struct has local_members, optional base, and no variants
1221         # flat union has base, variants, and no local_members
1222         # simple union has local_members, variants, and no base
1223         QAPISchemaType.__init__(self, name, info, doc)
1224         assert base is None or isinstance(base, str)
1225         for m in local_members:
1226             assert isinstance(m, QAPISchemaObjectTypeMember)
1227             m.set_owner(name)
1228         if variants is not None:
1229             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1230             variants.set_owner(name)
1231         self._base_name = base
1232         self.base = None
1233         self.local_members = local_members
1234         self.variants = variants
1235         self.members = None
1236
1237     def check(self, schema):
1238         if self.members is False:               # check for cycles
1239             raise QAPISemError(self.info,
1240                                "Object %s contains itself" % self.name)
1241         if self.members:
1242             return
1243         self.members = False                    # mark as being checked
1244         seen = OrderedDict()
1245         if self._base_name:
1246             self.base = schema.lookup_type(self._base_name)
1247             assert isinstance(self.base, QAPISchemaObjectType)
1248             self.base.check(schema)
1249             self.base.check_clash(schema, self.info, seen)
1250         for m in self.local_members:
1251             m.check(schema)
1252             m.check_clash(self.info, seen)
1253             if self.doc:
1254                 self.doc.connect_member(m)
1255         self.members = seen.values()
1256         if self.variants:
1257             self.variants.check(schema, seen)
1258             assert self.variants.tag_member in self.members
1259             self.variants.check_clash(schema, self.info, seen)
1260
1261     # Check that the members of this type do not cause duplicate JSON members,
1262     # and update seen to track the members seen so far. Report any errors
1263     # on behalf of info, which is not necessarily self.info
1264     def check_clash(self, schema, info, seen):
1265         assert not self.variants       # not implemented
1266         for m in self.members:
1267             m.check_clash(info, seen)
1268
1269     def is_implicit(self):
1270         # See QAPISchema._make_implicit_object_type(), as well as
1271         # _def_predefineds()
1272         return self.name.startswith('q_')
1273
1274     def is_empty(self):
1275         assert self.members is not None
1276         return not self.members and not self.variants
1277
1278     def c_name(self):
1279         assert self.name != 'q_empty'
1280         return QAPISchemaType.c_name(self)
1281
1282     def c_type(self):
1283         assert not self.is_implicit()
1284         return c_name(self.name) + pointer_suffix
1285
1286     def c_unboxed_type(self):
1287         return c_name(self.name)
1288
1289     def json_type(self):
1290         return 'object'
1291
1292     def visit(self, visitor):
1293         visitor.visit_object_type(self.name, self.info,
1294                                   self.base, self.local_members, self.variants)
1295         visitor.visit_object_type_flat(self.name, self.info,
1296                                        self.members, self.variants)
1297
1298
1299 class QAPISchemaMember(object):
1300     role = 'member'
1301
1302     def __init__(self, name):
1303         assert isinstance(name, str)
1304         self.name = name
1305         self.owner = None
1306
1307     def set_owner(self, name):
1308         assert not self.owner
1309         self.owner = name
1310
1311     def check_clash(self, info, seen):
1312         cname = c_name(self.name)
1313         if cname.lower() != cname and self.owner not in name_case_whitelist:
1314             raise QAPISemError(info,
1315                                "%s should not use uppercase" % self.describe())
1316         if cname in seen:
1317             raise QAPISemError(info, "%s collides with %s" %
1318                                (self.describe(), seen[cname].describe()))
1319         seen[cname] = self
1320
1321     def _pretty_owner(self):
1322         owner = self.owner
1323         if owner.startswith('q_obj_'):
1324             # See QAPISchema._make_implicit_object_type() - reverse the
1325             # mapping there to create a nice human-readable description
1326             owner = owner[6:]
1327             if owner.endswith('-arg'):
1328                 return '(parameter of %s)' % owner[:-4]
1329             elif owner.endswith('-base'):
1330                 return '(base of %s)' % owner[:-5]
1331             else:
1332                 assert owner.endswith('-wrapper')
1333                 # Unreachable and not implemented
1334                 assert False
1335         if owner.endswith('Kind'):
1336             # See QAPISchema._make_implicit_enum_type()
1337             return '(branch of %s)' % owner[:-4]
1338         return '(%s of %s)' % (self.role, owner)
1339
1340     def describe(self):
1341         return "'%s' %s" % (self.name, self._pretty_owner())
1342
1343
1344 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1345     def __init__(self, name, typ, optional):
1346         QAPISchemaMember.__init__(self, name)
1347         assert isinstance(typ, str)
1348         assert isinstance(optional, bool)
1349         self._type_name = typ
1350         self.type = None
1351         self.optional = optional
1352
1353     def check(self, schema):
1354         assert self.owner
1355         self.type = schema.lookup_type(self._type_name)
1356         assert self.type
1357
1358
1359 class QAPISchemaObjectTypeVariants(object):
1360     def __init__(self, tag_name, tag_member, variants):
1361         # Flat unions pass tag_name but not tag_member.
1362         # Simple unions and alternates pass tag_member but not tag_name.
1363         # After check(), tag_member is always set, and tag_name remains
1364         # a reliable witness of being used by a flat union.
1365         assert bool(tag_member) != bool(tag_name)
1366         assert (isinstance(tag_name, str) or
1367                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1368         assert len(variants) > 0
1369         for v in variants:
1370             assert isinstance(v, QAPISchemaObjectTypeVariant)
1371         self._tag_name = tag_name
1372         self.tag_member = tag_member
1373         self.variants = variants
1374
1375     def set_owner(self, name):
1376         for v in self.variants:
1377             v.set_owner(name)
1378
1379     def check(self, schema, seen):
1380         if not self.tag_member:    # flat union
1381             self.tag_member = seen[c_name(self._tag_name)]
1382             assert self._tag_name == self.tag_member.name
1383         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1384         for v in self.variants:
1385             v.check(schema)
1386             # Union names must match enum values; alternate names are
1387             # checked separately. Use 'seen' to tell the two apart.
1388             if seen:
1389                 assert v.name in self.tag_member.type.member_names()
1390                 assert isinstance(v.type, QAPISchemaObjectType)
1391                 v.type.check(schema)
1392
1393     def check_clash(self, schema, info, seen):
1394         for v in self.variants:
1395             # Reset seen map for each variant, since qapi names from one
1396             # branch do not affect another branch
1397             assert isinstance(v.type, QAPISchemaObjectType)
1398             v.type.check_clash(schema, info, dict(seen))
1399
1400
1401 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1402     role = 'branch'
1403
1404     def __init__(self, name, typ):
1405         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1406
1407
1408 class QAPISchemaAlternateType(QAPISchemaType):
1409     def __init__(self, name, info, doc, variants):
1410         QAPISchemaType.__init__(self, name, info, doc)
1411         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1412         assert variants.tag_member
1413         variants.set_owner(name)
1414         variants.tag_member.set_owner(self.name)
1415         self.variants = variants
1416
1417     def check(self, schema):
1418         self.variants.tag_member.check(schema)
1419         # Not calling self.variants.check_clash(), because there's nothing
1420         # to clash with
1421         self.variants.check(schema, {})
1422         # Alternate branch names have no relation to the tag enum values;
1423         # so we have to check for potential name collisions ourselves.
1424         seen = {}
1425         for v in self.variants.variants:
1426             v.check_clash(self.info, seen)
1427             if self.doc:
1428                 self.doc.connect_member(v)
1429
1430     def c_type(self):
1431         return c_name(self.name) + pointer_suffix
1432
1433     def json_type(self):
1434         return 'value'
1435
1436     def visit(self, visitor):
1437         visitor.visit_alternate_type(self.name, self.info, self.variants)
1438
1439     def is_empty(self):
1440         return False
1441
1442
1443 class QAPISchemaCommand(QAPISchemaEntity):
1444     def __init__(self, name, info, doc, arg_type, ret_type,
1445                  gen, success_response, boxed):
1446         QAPISchemaEntity.__init__(self, name, info, doc)
1447         assert not arg_type or isinstance(arg_type, str)
1448         assert not ret_type or isinstance(ret_type, str)
1449         self._arg_type_name = arg_type
1450         self.arg_type = None
1451         self._ret_type_name = ret_type
1452         self.ret_type = None
1453         self.gen = gen
1454         self.success_response = success_response
1455         self.boxed = boxed
1456
1457     def check(self, schema):
1458         if self._arg_type_name:
1459             self.arg_type = schema.lookup_type(self._arg_type_name)
1460             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1461                     isinstance(self.arg_type, QAPISchemaAlternateType))
1462             self.arg_type.check(schema)
1463             if self.boxed:
1464                 if self.arg_type.is_empty():
1465                     raise QAPISemError(self.info,
1466                                        "Cannot use 'boxed' with empty type")
1467             else:
1468                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1469                 assert not self.arg_type.variants
1470         elif self.boxed:
1471             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1472         if self._ret_type_name:
1473             self.ret_type = schema.lookup_type(self._ret_type_name)
1474             assert isinstance(self.ret_type, QAPISchemaType)
1475
1476     def visit(self, visitor):
1477         visitor.visit_command(self.name, self.info,
1478                               self.arg_type, self.ret_type,
1479                               self.gen, self.success_response, self.boxed)
1480
1481
1482 class QAPISchemaEvent(QAPISchemaEntity):
1483     def __init__(self, name, info, doc, arg_type, boxed):
1484         QAPISchemaEntity.__init__(self, name, info, doc)
1485         assert not arg_type or isinstance(arg_type, str)
1486         self._arg_type_name = arg_type
1487         self.arg_type = None
1488         self.boxed = boxed
1489
1490     def check(self, schema):
1491         if self._arg_type_name:
1492             self.arg_type = schema.lookup_type(self._arg_type_name)
1493             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1494                     isinstance(self.arg_type, QAPISchemaAlternateType))
1495             self.arg_type.check(schema)
1496             if self.boxed:
1497                 if self.arg_type.is_empty():
1498                     raise QAPISemError(self.info,
1499                                        "Cannot use 'boxed' with empty type")
1500             else:
1501                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1502                 assert not self.arg_type.variants
1503         elif self.boxed:
1504             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1505
1506     def visit(self, visitor):
1507         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1508
1509
1510 class QAPISchema(object):
1511     def __init__(self, fname):
1512         try:
1513             parser = QAPISchemaParser(open(fname, 'r'))
1514             self.exprs = check_exprs(parser.exprs)
1515             self.docs = check_docs(parser.docs)
1516             self._entity_dict = {}
1517             self._predefining = True
1518             self._def_predefineds()
1519             self._predefining = False
1520             self._def_exprs()
1521             self.check()
1522         except QAPIError as err:
1523             print >>sys.stderr, err
1524             exit(1)
1525
1526     def _def_entity(self, ent):
1527         # Only the predefined types are allowed to not have info
1528         assert ent.info or self._predefining
1529         assert ent.name not in self._entity_dict
1530         self._entity_dict[ent.name] = ent
1531
1532     def lookup_entity(self, name, typ=None):
1533         ent = self._entity_dict.get(name)
1534         if typ and not isinstance(ent, typ):
1535             return None
1536         return ent
1537
1538     def lookup_type(self, name):
1539         return self.lookup_entity(name, QAPISchemaType)
1540
1541     def _def_builtin_type(self, name, json_type, c_type):
1542         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1543         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1544         # qapi-types.h from a single .c, all arrays of builtins must be
1545         # declared in the first file whether or not they are used.  Nicer
1546         # would be to use lazy instantiation, while figuring out how to
1547         # avoid compilation issues with multiple qapi-types.h.
1548         self._make_array_type(name, None)
1549
1550     def _def_predefineds(self):
1551         for t in [('str',    'string',  'char' + pointer_suffix),
1552                   ('number', 'number',  'double'),
1553                   ('int',    'int',     'int64_t'),
1554                   ('int8',   'int',     'int8_t'),
1555                   ('int16',  'int',     'int16_t'),
1556                   ('int32',  'int',     'int32_t'),
1557                   ('int64',  'int',     'int64_t'),
1558                   ('uint8',  'int',     'uint8_t'),
1559                   ('uint16', 'int',     'uint16_t'),
1560                   ('uint32', 'int',     'uint32_t'),
1561                   ('uint64', 'int',     'uint64_t'),
1562                   ('size',   'int',     'uint64_t'),
1563                   ('bool',   'boolean', 'bool'),
1564                   ('any',    'value',   'QObject' + pointer_suffix)]:
1565             self._def_builtin_type(*t)
1566         self.the_empty_object_type = QAPISchemaObjectType(
1567             'q_empty', None, None, None, [], None)
1568         self._def_entity(self.the_empty_object_type)
1569         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1570                                                 'qstring', 'qdict', 'qlist',
1571                                                 'qfloat', 'qbool'])
1572         self._def_entity(QAPISchemaEnumType('QType', None, None,
1573                                             qtype_values, 'QTYPE'))
1574
1575     def _make_enum_members(self, values):
1576         return [QAPISchemaMember(v) for v in values]
1577
1578     def _make_implicit_enum_type(self, name, info, values):
1579         # See also QAPISchemaObjectTypeMember._pretty_owner()
1580         name = name + 'Kind'   # Use namespace reserved by add_name()
1581         self._def_entity(QAPISchemaEnumType(
1582             name, info, None, self._make_enum_members(values), None))
1583         return name
1584
1585     def _make_array_type(self, element_type, info):
1586         name = element_type + 'List'   # Use namespace reserved by add_name()
1587         if not self.lookup_type(name):
1588             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1589         return name
1590
1591     def _make_implicit_object_type(self, name, info, doc, role, members):
1592         if not members:
1593             return None
1594         # See also QAPISchemaObjectTypeMember._pretty_owner()
1595         name = 'q_obj_%s-%s' % (name, role)
1596         if not self.lookup_entity(name, QAPISchemaObjectType):
1597             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1598                                                   members, None))
1599         return name
1600
1601     def _def_enum_type(self, expr, info, doc):
1602         name = expr['enum']
1603         data = expr['data']
1604         prefix = expr.get('prefix')
1605         self._def_entity(QAPISchemaEnumType(
1606             name, info, doc, self._make_enum_members(data), prefix))
1607
1608     def _make_member(self, name, typ, info):
1609         optional = False
1610         if name.startswith('*'):
1611             name = name[1:]
1612             optional = True
1613         if isinstance(typ, list):
1614             assert len(typ) == 1
1615             typ = self._make_array_type(typ[0], info)
1616         return QAPISchemaObjectTypeMember(name, typ, optional)
1617
1618     def _make_members(self, data, info):
1619         return [self._make_member(key, value, info)
1620                 for (key, value) in data.iteritems()]
1621
1622     def _def_struct_type(self, expr, info, doc):
1623         name = expr['struct']
1624         base = expr.get('base')
1625         data = expr['data']
1626         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1627                                               self._make_members(data, info),
1628                                               None))
1629
1630     def _make_variant(self, case, typ):
1631         return QAPISchemaObjectTypeVariant(case, typ)
1632
1633     def _make_simple_variant(self, case, typ, info):
1634         if isinstance(typ, list):
1635             assert len(typ) == 1
1636             typ = self._make_array_type(typ[0], info)
1637         typ = self._make_implicit_object_type(
1638             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1639         return QAPISchemaObjectTypeVariant(case, typ)
1640
1641     def _def_union_type(self, expr, info, doc):
1642         name = expr['union']
1643         data = expr['data']
1644         base = expr.get('base')
1645         tag_name = expr.get('discriminator')
1646         tag_member = None
1647         if isinstance(base, dict):
1648             base = (self._make_implicit_object_type(
1649                     name, info, doc, 'base', self._make_members(base, info)))
1650         if tag_name:
1651             variants = [self._make_variant(key, value)
1652                         for (key, value) in data.iteritems()]
1653             members = []
1654         else:
1655             variants = [self._make_simple_variant(key, value, info)
1656                         for (key, value) in data.iteritems()]
1657             typ = self._make_implicit_enum_type(name, info,
1658                                                 [v.name for v in variants])
1659             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1660             members = [tag_member]
1661         self._def_entity(
1662             QAPISchemaObjectType(name, info, doc, base, members,
1663                                  QAPISchemaObjectTypeVariants(tag_name,
1664                                                               tag_member,
1665                                                               variants)))
1666
1667     def _def_alternate_type(self, expr, info, doc):
1668         name = expr['alternate']
1669         data = expr['data']
1670         variants = [self._make_variant(key, value)
1671                     for (key, value) in data.iteritems()]
1672         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1673         self._def_entity(
1674             QAPISchemaAlternateType(name, info, doc,
1675                                     QAPISchemaObjectTypeVariants(None,
1676                                                                  tag_member,
1677                                                                  variants)))
1678
1679     def _def_command(self, expr, info, doc):
1680         name = expr['command']
1681         data = expr.get('data')
1682         rets = expr.get('returns')
1683         gen = expr.get('gen', True)
1684         success_response = expr.get('success-response', True)
1685         boxed = expr.get('boxed', False)
1686         if isinstance(data, OrderedDict):
1687             data = self._make_implicit_object_type(
1688                 name, info, doc, 'arg', self._make_members(data, info))
1689         if isinstance(rets, list):
1690             assert len(rets) == 1
1691             rets = self._make_array_type(rets[0], info)
1692         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1693                                            gen, success_response, boxed))
1694
1695     def _def_event(self, expr, info, doc):
1696         name = expr['event']
1697         data = expr.get('data')
1698         boxed = expr.get('boxed', False)
1699         if isinstance(data, OrderedDict):
1700             data = self._make_implicit_object_type(
1701                 name, info, doc, 'arg', self._make_members(data, info))
1702         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1703
1704     def _def_exprs(self):
1705         for expr_elem in self.exprs:
1706             expr = expr_elem['expr']
1707             info = expr_elem['info']
1708             doc = expr_elem.get('doc')
1709             if 'enum' in expr:
1710                 self._def_enum_type(expr, info, doc)
1711             elif 'struct' in expr:
1712                 self._def_struct_type(expr, info, doc)
1713             elif 'union' in expr:
1714                 self._def_union_type(expr, info, doc)
1715             elif 'alternate' in expr:
1716                 self._def_alternate_type(expr, info, doc)
1717             elif 'command' in expr:
1718                 self._def_command(expr, info, doc)
1719             elif 'event' in expr:
1720                 self._def_event(expr, info, doc)
1721             else:
1722                 assert False
1723
1724     def check(self):
1725         for ent in self._entity_dict.values():
1726             ent.check(self)
1727
1728     def visit(self, visitor):
1729         visitor.visit_begin(self)
1730         for (name, entity) in sorted(self._entity_dict.items()):
1731             if visitor.visit_needed(entity):
1732                 entity.visit(visitor)
1733         visitor.visit_end()
1734
1735
1736 #
1737 # Code generation helpers
1738 #
1739
1740 def camel_case(name):
1741     new_name = ''
1742     first = True
1743     for ch in name:
1744         if ch in ['_', '-']:
1745             first = True
1746         elif first:
1747             new_name += ch.upper()
1748             first = False
1749         else:
1750             new_name += ch.lower()
1751     return new_name
1752
1753
1754 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1755 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1756 # ENUM24_Name -> ENUM24_NAME
1757 def camel_to_upper(value):
1758     c_fun_str = c_name(value, False)
1759     if value.isupper():
1760         return c_fun_str
1761
1762     new_name = ''
1763     l = len(c_fun_str)
1764     for i in range(l):
1765         c = c_fun_str[i]
1766         # When c is upper and no '_' appears before, do more checks
1767         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1768             if i < l - 1 and c_fun_str[i + 1].islower():
1769                 new_name += '_'
1770             elif c_fun_str[i - 1].isdigit():
1771                 new_name += '_'
1772         new_name += c
1773     return new_name.lstrip('_').upper()
1774
1775
1776 def c_enum_const(type_name, const_name, prefix=None):
1777     if prefix is not None:
1778         type_name = prefix
1779     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1780
1781 c_name_trans = string.maketrans('.-', '__')
1782
1783
1784 # Map @name to a valid C identifier.
1785 # If @protect, avoid returning certain ticklish identifiers (like
1786 # C keywords) by prepending 'q_'.
1787 #
1788 # Used for converting 'name' from a 'name':'type' qapi definition
1789 # into a generated struct member, as well as converting type names
1790 # into substrings of a generated C function name.
1791 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1792 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1793 def c_name(name, protect=True):
1794     # ANSI X3J11/88-090, 3.1.1
1795     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1796                      'default', 'do', 'double', 'else', 'enum', 'extern',
1797                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1798                      'return', 'short', 'signed', 'sizeof', 'static',
1799                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1800                      'void', 'volatile', 'while'])
1801     # ISO/IEC 9899:1999, 6.4.1
1802     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1803     # ISO/IEC 9899:2011, 6.4.1
1804     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1805                      '_Noreturn', '_Static_assert', '_Thread_local'])
1806     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1807     # excluding _.*
1808     gcc_words = set(['asm', 'typeof'])
1809     # C++ ISO/IEC 14882:2003 2.11
1810     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1811                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1812                      'namespace', 'new', 'operator', 'private', 'protected',
1813                      'public', 'reinterpret_cast', 'static_cast', 'template',
1814                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1815                      'using', 'virtual', 'wchar_t',
1816                      # alternative representations
1817                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1818                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1819     # namespace pollution:
1820     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1821     name = name.translate(c_name_trans)
1822     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1823                     | cpp_words | polluted_words):
1824         return 'q_' + name
1825     return name
1826
1827 eatspace = '\033EATSPACE.'
1828 pointer_suffix = ' *' + eatspace
1829
1830
1831 def genindent(count):
1832     ret = ''
1833     for _ in range(count):
1834         ret += ' '
1835     return ret
1836
1837 indent_level = 0
1838
1839
1840 def push_indent(indent_amount=4):
1841     global indent_level
1842     indent_level += indent_amount
1843
1844
1845 def pop_indent(indent_amount=4):
1846     global indent_level
1847     indent_level -= indent_amount
1848
1849
1850 # Generate @code with @kwds interpolated.
1851 # Obey indent_level, and strip eatspace.
1852 def cgen(code, **kwds):
1853     raw = code % kwds
1854     if indent_level:
1855         indent = genindent(indent_level)
1856         # re.subn() lacks flags support before Python 2.7, use re.compile()
1857         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1858                       indent + r'\g<0>', raw)
1859         raw = raw[0]
1860     return re.sub(re.escape(eatspace) + r' *', '', raw)
1861
1862
1863 def mcgen(code, **kwds):
1864     if code[0] == '\n':
1865         code = code[1:]
1866     return cgen(code, **kwds)
1867
1868
1869 def guardname(filename):
1870     return c_name(filename, protect=False).upper()
1871
1872
1873 def guardstart(name):
1874     return mcgen('''
1875
1876 #ifndef %(name)s
1877 #define %(name)s
1878
1879 ''',
1880                  name=guardname(name))
1881
1882
1883 def guardend(name):
1884     return mcgen('''
1885
1886 #endif /* %(name)s */
1887
1888 ''',
1889                  name=guardname(name))
1890
1891
1892 def gen_enum_lookup(name, values, prefix=None):
1893     ret = mcgen('''
1894
1895 const char *const %(c_name)s_lookup[] = {
1896 ''',
1897                 c_name=c_name(name))
1898     for value in values:
1899         index = c_enum_const(name, value, prefix)
1900         ret += mcgen('''
1901     [%(index)s] = "%(value)s",
1902 ''',
1903                      index=index, value=value)
1904
1905     max_index = c_enum_const(name, '_MAX', prefix)
1906     ret += mcgen('''
1907     [%(max_index)s] = NULL,
1908 };
1909 ''',
1910                  max_index=max_index)
1911     return ret
1912
1913
1914 def gen_enum(name, values, prefix=None):
1915     # append automatically generated _MAX value
1916     enum_values = values + ['_MAX']
1917
1918     ret = mcgen('''
1919
1920 typedef enum %(c_name)s {
1921 ''',
1922                 c_name=c_name(name))
1923
1924     i = 0
1925     for value in enum_values:
1926         ret += mcgen('''
1927     %(c_enum)s = %(i)d,
1928 ''',
1929                      c_enum=c_enum_const(name, value, prefix),
1930                      i=i)
1931         i += 1
1932
1933     ret += mcgen('''
1934 } %(c_name)s;
1935 ''',
1936                  c_name=c_name(name))
1937
1938     ret += mcgen('''
1939
1940 extern const char *const %(c_name)s_lookup[];
1941 ''',
1942                  c_name=c_name(name))
1943     return ret
1944
1945
1946 def gen_params(arg_type, boxed, extra):
1947     if not arg_type:
1948         assert not boxed
1949         return extra
1950     ret = ''
1951     sep = ''
1952     if boxed:
1953         ret += '%s arg' % arg_type.c_param_type()
1954         sep = ', '
1955     else:
1956         assert not arg_type.variants
1957         for memb in arg_type.members:
1958             ret += sep
1959             sep = ', '
1960             if memb.optional:
1961                 ret += 'bool has_%s, ' % c_name(memb.name)
1962             ret += '%s %s' % (memb.type.c_param_type(),
1963                               c_name(memb.name))
1964     if extra:
1965         ret += sep + extra
1966     return ret
1967
1968
1969 #
1970 # Common command line parsing
1971 #
1972
1973
1974 def parse_command_line(extra_options='', extra_long_options=[]):
1975
1976     try:
1977         opts, args = getopt.gnu_getopt(sys.argv[1:],
1978                                        'chp:o:' + extra_options,
1979                                        ['source', 'header', 'prefix=',
1980                                         'output-dir='] + extra_long_options)
1981     except getopt.GetoptError as err:
1982         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1983         sys.exit(1)
1984
1985     output_dir = ''
1986     prefix = ''
1987     do_c = False
1988     do_h = False
1989     extra_opts = []
1990
1991     for oa in opts:
1992         o, a = oa
1993         if o in ('-p', '--prefix'):
1994             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1995             if match.end() != len(a):
1996                 print >>sys.stderr, \
1997                     "%s: 'funny character '%s' in argument of --prefix" \
1998                     % (sys.argv[0], a[match.end()])
1999                 sys.exit(1)
2000             prefix = a
2001         elif o in ('-o', '--output-dir'):
2002             output_dir = a + '/'
2003         elif o in ('-c', '--source'):
2004             do_c = True
2005         elif o in ('-h', '--header'):
2006             do_h = True
2007         else:
2008             extra_opts.append(oa)
2009
2010     if not do_c and not do_h:
2011         do_c = True
2012         do_h = True
2013
2014     if len(args) != 1:
2015         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2016         sys.exit(1)
2017     fname = args[0]
2018
2019     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2020
2021 #
2022 # Generate output files with boilerplate
2023 #
2024
2025
2026 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2027                 c_comment, h_comment):
2028     guard = guardname(prefix + h_file)
2029     c_file = output_dir + prefix + c_file
2030     h_file = output_dir + prefix + h_file
2031
2032     if output_dir:
2033         try:
2034             os.makedirs(output_dir)
2035         except os.error as e:
2036             if e.errno != errno.EEXIST:
2037                 raise
2038
2039     def maybe_open(really, name, opt):
2040         if really:
2041             return open(name, opt)
2042         else:
2043             import StringIO
2044             return StringIO.StringIO()
2045
2046     fdef = maybe_open(do_c, c_file, 'w')
2047     fdecl = maybe_open(do_h, h_file, 'w')
2048
2049     fdef.write(mcgen('''
2050 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2051 %(comment)s
2052 ''',
2053                      comment=c_comment))
2054
2055     fdecl.write(mcgen('''
2056 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2057 %(comment)s
2058 #ifndef %(guard)s
2059 #define %(guard)s
2060
2061 ''',
2062                       comment=h_comment, guard=guard))
2063
2064     return (fdef, fdecl)
2065
2066
2067 def close_output(fdef, fdecl):
2068     fdecl.write('''
2069 #endif
2070 ''')
2071     fdecl.close()
2072     fdef.close()