scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 events = []
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ""
  62     while parent:
  63         res = ("In file included from %s:%d:\n" % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = "%s:%d" % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ":%s" % self.col
  82         return error_path(self.info) + "%s: %s" % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110             self.optional = False
 111
 112         def append(self, line):
 113             self.content.append(line)
 114
 115         def __repr__(self):
 116             return "\n".join(self.content).strip()
 117
 118     class ArgSection(Section):
 119         def __init__(self, name):
 120             QAPIDoc.Section.__init__(self, name)
 121             self.member = None
 122
 123         def connect(self, member):
 124             self.member = member
 125
 126     def __init__(self, parser, info):
 127         # self.parser is used to report errors with QAPIParseError.  The
 128         # resulting error position depends on the state of the parser.
 129         # It happens to be the beginning of the comment.  More or less
 130         # servicable, but action at a distance.
 131         self.parser = parser
 132         self.info = info
 133         self.symbol = None
 134         self.body = QAPIDoc.Section()
 135         # dict mapping parameter name to ArgSection
 136         self.args = OrderedDict()
 137         # a list of Section
 138         self.sections = []
 139         # the current section
 140         self.section = self.body
 141         # associated expression (to be set by expression parser)
 142         self.expr = None
 143
 144     def has_section(self, name):
 145         """Return True if we have a section with this name."""
 146         for i in self.sections:
 147             if i.name == name:
 148                 return True
 149         return False
 150
 151     def append(self, line):
 152         """Parse a comment line and add it to the documentation."""
 153         line = line[1:]
 154         if not line:
 155             self._append_freeform(line)
 156             return
 157
 158         if line[0] != ' ':
 159             raise QAPIParseError(self.parser, "Missing space after #")
 160         line = line[1:]
 161
 162         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 163         # recognized, and get silently treated as ordinary text
 164         if self.symbol:
 165             self._append_symbol_line(line)
 166         elif not self.body.content and line.startswith("@"):
 167             if not line.endswith(":"):
 168                 raise QAPIParseError(self.parser, "Line should end with :")
 169             self.symbol = line[1:-1]
 170             # FIXME invalid names other than the empty string aren't flagged
 171             if not self.symbol:
 172                 raise QAPIParseError(self.parser, "Invalid name")
 173         else:
 174             self._append_freeform(line)
 175
 176     def _append_symbol_line(self, line):
 177         name = line.split(' ', 1)[0]
 178
 179         if name.startswith("@") and name.endswith(":"):
 180             line = line[len(name)+1:]
 181             self._start_args_section(name[1:-1])
 182         elif name in ("Returns:", "Since:",
 183                       # those are often singular or plural
 184                       "Note:", "Notes:",
 185                       "Example:", "Examples:",
 186                       "TODO:"):
 187             line = line[len(name)+1:]
 188             self._start_section(name[:-1])
 189
 190         self._append_freeform(line)
 191
 192     def _start_args_section(self, name):
 193         # FIXME invalid names other than the empty string aren't flagged
 194         if not name:
 195             raise QAPIParseError(self.parser, "Invalid parameter name")
 196         if name in self.args:
 197             raise QAPIParseError(self.parser,
 198                                  "'%s' parameter name duplicated" % name)
 199         if self.sections:
 200             raise QAPIParseError(self.parser,
 201                                  "'@%s:' can't follow '%s' section"
 202                                  % (name, self.sections[0].name))
 203         self.section = QAPIDoc.ArgSection(name)
 204         self.args[name] = self.section
 205
 206     def _start_section(self, name=""):
 207         if name in ("Returns", "Since") and self.has_section(name):
 208             raise QAPIParseError(self.parser,
 209                                  "Duplicated '%s' section" % name)
 210         self.section = QAPIDoc.Section(name)
 211         self.sections.append(self.section)
 212
 213     def _append_freeform(self, line):
 214         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 215         if (in_arg and self.section.content
 216                 and not self.section.content[-1]
 217                 and line and not line[0].isspace()):
 218             self._start_section()
 219         if (in_arg or not self.section.name
 220                 or not self.section.name.startswith("Example")):
 221             line = line.strip()
 222         # TODO Drop this once the dust has settled
 223         if (isinstance(self.section, QAPIDoc.ArgSection)
 224                 and '#optional' in line):
 225             raise QAPISemError(self.info, "Please drop the #optional tag")
 226         self.section.append(line)
 227
 228     def connect_member(self, member):
 229         if member.name not in self.args:
 230             # Undocumented TODO outlaw
 231             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 232         self.args[member.name].connect(member)
 233
 234
 235 class QAPISchemaParser(object):
 236
 237     def __init__(self, fp, previously_included=[], incl_info=None):
 238         abs_fname = os.path.abspath(fp.name)
 239         fname = fp.name
 240         self.fname = fname
 241         previously_included.append(abs_fname)
 242         self.incl_info = incl_info
 243         self.src = fp.read()
 244         if self.src == '' or self.src[-1] != '\n':
 245             self.src += '\n'
 246         self.cursor = 0
 247         self.line = 1
 248         self.line_pos = 0
 249         self.exprs = []
 250         self.docs = []
 251         self.accept()
 252
 253         while self.tok is not None:
 254             info = {'file': fname, 'line': self.line,
 255                     'parent': self.incl_info}
 256             if self.tok == '#':
 257                 doc = self.get_doc(info)
 258                 self.docs.append(doc)
 259                 continue
 260
 261             expr = self.get_expr(False)
 262             if 'include' in expr:
 263                 if len(expr) != 1:
 264                     raise QAPISemError(info, "Invalid 'include' directive")
 265                 include = expr["include"]
 266                 if not isinstance(include, str):
 267                     raise QAPISemError(info,
 268                                        "Value of 'include' must be a string")
 269                 self._include(include, info, os.path.dirname(abs_fname),
 270                               previously_included)
 271             elif "pragma" in expr:
 272                 if len(expr) != 1:
 273                     raise QAPISemError(info, "Invalid 'pragma' directive")
 274                 pragma = expr['pragma']
 275                 if not isinstance(pragma, dict):
 276                     raise QAPISemError(
 277                         info, "Value of 'pragma' must be a dictionary")
 278                 for name, value in pragma.iteritems():
 279                     self._pragma(name, value, info)
 280             else:
 281                 expr_elem = {'expr': expr,
 282                              'info': info}
 283                 if (self.docs
 284                         and self.docs[-1].info['file'] == fname
 285                         and not self.docs[-1].expr):
 286                     self.docs[-1].expr = expr
 287                     expr_elem['doc'] = self.docs[-1]
 288
 289                 self.exprs.append(expr_elem)
 290
 291     def _include(self, include, info, base_dir, previously_included):
 292         incl_abs_fname = os.path.join(base_dir, include)
 293         # catch inclusion cycle
 294         inf = info
 295         while inf:
 296             if incl_abs_fname == os.path.abspath(inf['file']):
 297                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 298             inf = inf['parent']
 299
 300         # skip multiple include of the same file
 301         if incl_abs_fname in previously_included:
 302             return
 303         try:
 304             fobj = open(incl_abs_fname, 'r')
 305         except IOError as e:
 306             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 307         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 308         self.exprs.extend(exprs_include.exprs)
 309         self.docs.extend(exprs_include.docs)
 310
 311     def _pragma(self, name, value, info):
 312         global doc_required, returns_whitelist, name_case_whitelist
 313         if name == 'doc-required':
 314             if not isinstance(value, bool):
 315                 raise QAPISemError(info,
 316                                    "Pragma 'doc-required' must be boolean")
 317             doc_required = value
 318         elif name == 'returns-whitelist':
 319             if (not isinstance(value, list)
 320                     or any([not isinstance(elt, str) for elt in value])):
 321                 raise QAPISemError(info,
 322                                    "Pragma returns-whitelist must be"
 323                                    " a list of strings")
 324             returns_whitelist = value
 325         elif name == 'name-case-whitelist':
 326             if (not isinstance(value, list)
 327                     or any([not isinstance(elt, str) for elt in value])):
 328                 raise QAPISemError(info,
 329                                    "Pragma name-case-whitelist must be"
 330                                    " a list of strings")
 331             name_case_whitelist = value
 332         else:
 333             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 334
 335     def accept(self, skip_comment=True):
 336         while True:
 337             self.tok = self.src[self.cursor]
 338             self.pos = self.cursor
 339             self.cursor += 1
 340             self.val = None
 341
 342             if self.tok == '#':
 343                 if self.src[self.cursor] == '#':
 344                     # Start of doc comment
 345                     skip_comment = False
 346                 self.cursor = self.src.find('\n', self.cursor)
 347                 if not skip_comment:
 348                     self.val = self.src[self.pos:self.cursor]
 349                     return
 350             elif self.tok in "{}:,[]":
 351                 return
 352             elif self.tok == "'":
 353                 string = ''
 354                 esc = False
 355                 while True:
 356                     ch = self.src[self.cursor]
 357                     self.cursor += 1
 358                     if ch == '\n':
 359                         raise QAPIParseError(self, 'Missing terminating "\'"')
 360                     if esc:
 361                         if ch == 'b':
 362                             string += '\b'
 363                         elif ch == 'f':
 364                             string += '\f'
 365                         elif ch == 'n':
 366                             string += '\n'
 367                         elif ch == 'r':
 368                             string += '\r'
 369                         elif ch == 't':
 370                             string += '\t'
 371                         elif ch == 'u':
 372                             value = 0
 373                             for _ in range(0, 4):
 374                                 ch = self.src[self.cursor]
 375                                 self.cursor += 1
 376                                 if ch not in "0123456789abcdefABCDEF":
 377                                     raise QAPIParseError(self,
 378                                                          '\\u escape needs 4 '
 379                                                          'hex digits')
 380                                 value = (value << 4) + int(ch, 16)
 381                             # If Python 2 and 3 didn't disagree so much on
 382                             # how to handle Unicode, then we could allow
 383                             # Unicode string defaults.  But most of QAPI is
 384                             # ASCII-only, so we aren't losing much for now.
 385                             if not value or value > 0x7f:
 386                                 raise QAPIParseError(self,
 387                                                      'For now, \\u escape '
 388                                                      'only supports non-zero '
 389                                                      'values up to \\u007f')
 390                             string += chr(value)
 391                         elif ch in "\\/'\"":
 392                             string += ch
 393                         else:
 394                             raise QAPIParseError(self,
 395                                                  "Unknown escape \\%s" % ch)
 396                         esc = False
 397                     elif ch == "\\":
 398                         esc = True
 399                     elif ch == "'":
 400                         self.val = string
 401                         return
 402                     else:
 403                         string += ch
 404             elif self.src.startswith("true", self.pos):
 405                 self.val = True
 406                 self.cursor += 3
 407                 return
 408             elif self.src.startswith("false", self.pos):
 409                 self.val = False
 410                 self.cursor += 4
 411                 return
 412             elif self.src.startswith("null", self.pos):
 413                 self.val = None
 414                 self.cursor += 3
 415                 return
 416             elif self.tok == '\n':
 417                 if self.cursor == len(self.src):
 418                     self.tok = None
 419                     return
 420                 self.line += 1
 421                 self.line_pos = self.cursor
 422             elif not self.tok.isspace():
 423                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 424
 425     def get_members(self):
 426         expr = OrderedDict()
 427         if self.tok == '}':
 428             self.accept()
 429             return expr
 430         if self.tok != "'":
 431             raise QAPIParseError(self, 'Expected string or "}"')
 432         while True:
 433             key = self.val
 434             self.accept()
 435             if self.tok != ':':
 436                 raise QAPIParseError(self, 'Expected ":"')
 437             self.accept()
 438             if key in expr:
 439                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 440             expr[key] = self.get_expr(True)
 441             if self.tok == '}':
 442                 self.accept()
 443                 return expr
 444             if self.tok != ',':
 445                 raise QAPIParseError(self, 'Expected "," or "}"')
 446             self.accept()
 447             if self.tok != "'":
 448                 raise QAPIParseError(self, 'Expected string')
 449
 450     def get_values(self):
 451         expr = []
 452         if self.tok == ']':
 453             self.accept()
 454             return expr
 455         if self.tok not in "{['tfn":
 456             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 457                                  'boolean or "null"')
 458         while True:
 459             expr.append(self.get_expr(True))
 460             if self.tok == ']':
 461                 self.accept()
 462                 return expr
 463             if self.tok != ',':
 464                 raise QAPIParseError(self, 'Expected "," or "]"')
 465             self.accept()
 466
 467     def get_expr(self, nested):
 468         if self.tok != '{' and not nested:
 469             raise QAPIParseError(self, 'Expected "{"')
 470         if self.tok == '{':
 471             self.accept()
 472             expr = self.get_members()
 473         elif self.tok == '[':
 474             self.accept()
 475             expr = self.get_values()
 476         elif self.tok in "'tfn":
 477             expr = self.val
 478             self.accept()
 479         else:
 480             raise QAPIParseError(self, 'Expected "{", "[" or string')
 481         return expr
 482
 483     def get_doc(self, info):
 484         if self.val != '##':
 485             raise QAPIParseError(self, "Junk after '##' at start of "
 486                                  "documentation comment")
 487
 488         doc = QAPIDoc(self, info)
 489         self.accept(False)
 490         while self.tok == '#':
 491             if self.val.startswith('##'):
 492                 # End of doc comment
 493                 if self.val != '##':
 494                     raise QAPIParseError(self, "Junk after '##' at end of "
 495                                          "documentation comment")
 496                 self.accept()
 497                 return doc
 498             else:
 499                 doc.append(self.val)
 500             self.accept(False)
 501
 502         raise QAPIParseError(self, "Documentation comment must end with '##'")
 503
 504
 505 #
 506 # Semantic analysis of schema expressions
 507 # TODO fold into QAPISchema
 508 # TODO catching name collisions in generated code would be nice
 509 #
 510
 511
 512 def find_base_members(base):
 513     if isinstance(base, dict):
 514         return base
 515     base_struct_define = find_struct(base)
 516     if not base_struct_define:
 517         return None
 518     return base_struct_define['data']
 519
 520
 521 # Return the qtype of an alternate branch, or None on error.
 522 def find_alternate_member_qtype(qapi_type):
 523     if qapi_type in builtin_types:
 524         return builtin_types[qapi_type]
 525     elif find_struct(qapi_type):
 526         return "QTYPE_QDICT"
 527     elif find_enum(qapi_type):
 528         return "QTYPE_QSTRING"
 529     elif find_union(qapi_type):
 530         return "QTYPE_QDICT"
 531     return None
 532
 533
 534 # Return the discriminator enum define if discriminator is specified as an
 535 # enum type, otherwise return None.
 536 def discriminator_find_enum_define(expr):
 537     base = expr.get('base')
 538     discriminator = expr.get('discriminator')
 539
 540     if not (discriminator and base):
 541         return None
 542
 543     base_members = find_base_members(base)
 544     if not base_members:
 545         return None
 546
 547     discriminator_type = base_members.get(discriminator)
 548     if not discriminator_type:
 549         return None
 550
 551     return find_enum(discriminator_type)
 552
 553
 554 # Names must be letters, numbers, -, and _.  They must start with letter,
 555 # except for downstream extensions which must start with __RFQDN_.
 556 # Dots are only valid in the downstream extension prefix.
 557 valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
 558                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 559
 560
 561 def check_name(info, source, name, allow_optional=False,
 562                enum_member=False):
 563     global valid_name
 564     membername = name
 565
 566     if not isinstance(name, str):
 567         raise QAPISemError(info, "%s requires a string name" % source)
 568     if name.startswith('*'):
 569         membername = name[1:]
 570         if not allow_optional:
 571             raise QAPISemError(info, "%s does not allow optional name '%s'"
 572                                % (source, name))
 573     # Enum members can start with a digit, because the generated C
 574     # code always prefixes it with the enum name
 575     if enum_member and membername[0].isdigit():
 576         membername = 'D' + membername
 577     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 578     # and 'q_obj_*' implicit type names.
 579     if not valid_name.match(membername) or \
 580        c_name(membername, False).startswith('q_'):
 581         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 582
 583
 584 def add_name(name, info, meta, implicit=False):
 585     global all_names
 586     check_name(info, "'%s'" % meta, name)
 587     # FIXME should reject names that differ only in '_' vs. '.'
 588     # vs. '-', because they're liable to clash in generated C.
 589     if name in all_names:
 590         raise QAPISemError(info, "%s '%s' is already defined"
 591                            % (all_names[name], name))
 592     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 593         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 594                            % (meta, name, name[-4:]))
 595     all_names[name] = meta
 596
 597
 598 def add_struct(definition, info):
 599     global struct_types
 600     name = definition['struct']
 601     add_name(name, info, 'struct')
 602     struct_types.append(definition)
 603
 604
 605 def find_struct(name):
 606     global struct_types
 607     for struct in struct_types:
 608         if struct['struct'] == name:
 609             return struct
 610     return None
 611
 612
 613 def add_union(definition, info):
 614     global union_types
 615     name = definition['union']
 616     add_name(name, info, 'union')
 617     union_types.append(definition)
 618
 619
 620 def find_union(name):
 621     global union_types
 622     for union in union_types:
 623         if union['union'] == name:
 624             return union
 625     return None
 626
 627
 628 def add_enum(name, info, enum_values=None, implicit=False):
 629     global enum_types
 630     add_name(name, info, 'enum', implicit)
 631     enum_types.append({"enum_name": name, "enum_values": enum_values})
 632
 633
 634 def find_enum(name):
 635     global enum_types
 636     for enum in enum_types:
 637         if enum['enum_name'] == name:
 638             return enum
 639     return None
 640
 641
 642 def is_enum(name):
 643     return find_enum(name) is not None
 644
 645
 646 def check_type(info, source, value, allow_array=False,
 647                allow_dict=False, allow_optional=False,
 648                allow_metas=[]):
 649     global all_names
 650
 651     if value is None:
 652         return
 653
 654     # Check if array type for value is okay
 655     if isinstance(value, list):
 656         if not allow_array:
 657             raise QAPISemError(info, "%s cannot be an array" % source)
 658         if len(value) != 1 or not isinstance(value[0], str):
 659             raise QAPISemError(info,
 660                                "%s: array type must contain single type name" %
 661                                source)
 662         value = value[0]
 663
 664     # Check if type name for value is okay
 665     if isinstance(value, str):
 666         if value not in all_names:
 667             raise QAPISemError(info, "%s uses unknown type '%s'"
 668                                % (source, value))
 669         if not all_names[value] in allow_metas:
 670             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 671                                (source, all_names[value], value))
 672         return
 673
 674     if not allow_dict:
 675         raise QAPISemError(info, "%s should be a type name" % source)
 676
 677     if not isinstance(value, OrderedDict):
 678         raise QAPISemError(info,
 679                            "%s should be a dictionary or type name" % source)
 680
 681     # value is a dictionary, check that each member is okay
 682     for (key, arg) in value.items():
 683         check_name(info, "Member of %s" % source, key,
 684                    allow_optional=allow_optional)
 685         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 686             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 687                                % (source, key))
 688         # Todo: allow dictionaries to represent default values of
 689         # an optional argument.
 690         check_type(info, "Member '%s' of %s" % (key, source), arg,
 691                    allow_array=True,
 692                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 693                                 'enum'])
 694
 695
 696 def check_command(expr, info):
 697     name = expr['command']
 698     boxed = expr.get('boxed', False)
 699
 700     args_meta = ['struct']
 701     if boxed:
 702         args_meta += ['union', 'alternate']
 703     check_type(info, "'data' for command '%s'" % name,
 704                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 705                allow_metas=args_meta)
 706     returns_meta = ['union', 'struct']
 707     if name in returns_whitelist:
 708         returns_meta += ['built-in', 'alternate', 'enum']
 709     check_type(info, "'returns' for command '%s'" % name,
 710                expr.get('returns'), allow_array=True,
 711                allow_optional=True, allow_metas=returns_meta)
 712
 713
 714 def check_event(expr, info):
 715     global events
 716     name = expr['event']
 717     boxed = expr.get('boxed', False)
 718
 719     meta = ['struct']
 720     if boxed:
 721         meta += ['union', 'alternate']
 722     events.append(name)
 723     check_type(info, "'data' for event '%s'" % name,
 724                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 725                allow_metas=meta)
 726
 727
 728 def check_union(expr, info):
 729     name = expr['union']
 730     base = expr.get('base')
 731     discriminator = expr.get('discriminator')
 732     members = expr['data']
 733
 734     # Two types of unions, determined by discriminator.
 735
 736     # With no discriminator it is a simple union.
 737     if discriminator is None:
 738         enum_define = None
 739         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 740         if base is not None:
 741             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 742                                name)
 743
 744     # Else, it's a flat union.
 745     else:
 746         # The object must have a string or dictionary 'base'.
 747         check_type(info, "'base' for union '%s'" % name,
 748                    base, allow_dict=True, allow_optional=True,
 749                    allow_metas=['struct'])
 750         if not base:
 751             raise QAPISemError(info, "Flat union '%s' must have a base"
 752                                % name)
 753         base_members = find_base_members(base)
 754         assert base_members is not None
 755
 756         # The value of member 'discriminator' must name a non-optional
 757         # member of the base struct.
 758         check_name(info, "Discriminator of flat union '%s'" % name,
 759                    discriminator)
 760         discriminator_type = base_members.get(discriminator)
 761         if not discriminator_type:
 762             raise QAPISemError(info,
 763                                "Discriminator '%s' is not a member of base "
 764                                "struct '%s'"
 765                                % (discriminator, base))
 766         enum_define = find_enum(discriminator_type)
 767         allow_metas = ['struct']
 768         # Do not allow string discriminator
 769         if not enum_define:
 770             raise QAPISemError(info,
 771                                "Discriminator '%s' must be of enumeration "
 772                                "type" % discriminator)
 773
 774     # Check every branch; don't allow an empty union
 775     if len(members) == 0:
 776         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 777     for (key, value) in members.items():
 778         check_name(info, "Member of union '%s'" % name, key)
 779
 780         # Each value must name a known type
 781         check_type(info, "Member '%s' of union '%s'" % (key, name),
 782                    value, allow_array=not base, allow_metas=allow_metas)
 783
 784         # If the discriminator names an enum type, then all members
 785         # of 'data' must also be members of the enum type.
 786         if enum_define:
 787             if key not in enum_define['enum_values']:
 788                 raise QAPISemError(info,
 789                                    "Discriminator value '%s' is not found in "
 790                                    "enum '%s'"
 791                                    % (key, enum_define["enum_name"]))
 792
 793     # If discriminator is user-defined, ensure all values are covered
 794     if enum_define:
 795         for value in enum_define['enum_values']:
 796             if value not in members.keys():
 797                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 798                                    % (name, value))
 799
 800
 801 def check_alternate(expr, info):
 802     name = expr['alternate']
 803     members = expr['data']
 804     types_seen = {}
 805
 806     # Check every branch; require at least two branches
 807     if len(members) < 2:
 808         raise QAPISemError(info,
 809                            "Alternate '%s' should have at least two branches "
 810                            "in 'data'" % name)
 811     for (key, value) in members.items():
 812         check_name(info, "Member of alternate '%s'" % name, key)
 813
 814         # Ensure alternates have no type conflicts.
 815         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 816                    value,
 817                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 818         qtype = find_alternate_member_qtype(value)
 819         if not qtype:
 820             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 821                                "type '%s'" % (name, key, value))
 822         if qtype in types_seen:
 823             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 824                                "be distinguished from member '%s'"
 825                                % (name, key, types_seen[qtype]))
 826         types_seen[qtype] = key
 827
 828
 829 def check_enum(expr, info):
 830     name = expr['enum']
 831     members = expr.get('data')
 832     prefix = expr.get('prefix')
 833
 834     if not isinstance(members, list):
 835         raise QAPISemError(info,
 836                            "Enum '%s' requires an array for 'data'" % name)
 837     if prefix is not None and not isinstance(prefix, str):
 838         raise QAPISemError(info,
 839                            "Enum '%s' requires a string for 'prefix'" % name)
 840     for member in members:
 841         check_name(info, "Member of enum '%s'" % name, member,
 842                    enum_member=True)
 843
 844
 845 def check_struct(expr, info):
 846     name = expr['struct']
 847     members = expr['data']
 848
 849     check_type(info, "'data' for struct '%s'" % name, members,
 850                allow_dict=True, allow_optional=True)
 851     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 852                allow_metas=['struct'])
 853
 854
 855 def check_keys(expr_elem, meta, required, optional=[]):
 856     expr = expr_elem['expr']
 857     info = expr_elem['info']
 858     name = expr[meta]
 859     if not isinstance(name, str):
 860         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 861     required = required + [meta]
 862     for (key, value) in expr.items():
 863         if key not in required and key not in optional:
 864             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 865                                % (key, meta, name))
 866         if (key == 'gen' or key == 'success-response') and value is not False:
 867             raise QAPISemError(info,
 868                                "'%s' of %s '%s' should only use false value"
 869                                % (key, meta, name))
 870         if key == 'boxed' and value is not True:
 871             raise QAPISemError(info,
 872                                "'%s' of %s '%s' should only use true value"
 873                                % (key, meta, name))
 874     for key in required:
 875         if key not in expr:
 876             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 877                                % (key, meta, name))
 878
 879
 880 def check_exprs(exprs):
 881     global all_names
 882
 883     # Learn the types and check for valid expression keys
 884     for builtin in builtin_types.keys():
 885         all_names[builtin] = 'built-in'
 886     for expr_elem in exprs:
 887         expr = expr_elem['expr']
 888         info = expr_elem['info']
 889
 890         if 'doc' not in expr_elem and doc_required:
 891             raise QAPISemError(info,
 892                                "Expression missing documentation comment")
 893
 894         if 'enum' in expr:
 895             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 896             add_enum(expr['enum'], info, expr['data'])
 897         elif 'union' in expr:
 898             check_keys(expr_elem, 'union', ['data'],
 899                        ['base', 'discriminator'])
 900             add_union(expr, info)
 901         elif 'alternate' in expr:
 902             check_keys(expr_elem, 'alternate', ['data'])
 903             add_name(expr['alternate'], info, 'alternate')
 904         elif 'struct' in expr:
 905             check_keys(expr_elem, 'struct', ['data'], ['base'])
 906             add_struct(expr, info)
 907         elif 'command' in expr:
 908             check_keys(expr_elem, 'command', [],
 909                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 910             add_name(expr['command'], info, 'command')
 911         elif 'event' in expr:
 912             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 913             add_name(expr['event'], info, 'event')
 914         else:
 915             raise QAPISemError(expr_elem['info'],
 916                                "Expression is missing metatype")
 917
 918     # Try again for hidden UnionKind enum
 919     for expr_elem in exprs:
 920         expr = expr_elem['expr']
 921         if 'union' in expr:
 922             if not discriminator_find_enum_define(expr):
 923                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 924                          implicit=True)
 925         elif 'alternate' in expr:
 926             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 927                      implicit=True)
 928
 929     # Validate that exprs make sense
 930     for expr_elem in exprs:
 931         expr = expr_elem['expr']
 932         info = expr_elem['info']
 933
 934         if 'enum' in expr:
 935             check_enum(expr, info)
 936         elif 'union' in expr:
 937             check_union(expr, info)
 938         elif 'alternate' in expr:
 939             check_alternate(expr, info)
 940         elif 'struct' in expr:
 941             check_struct(expr, info)
 942         elif 'command' in expr:
 943             check_command(expr, info)
 944         elif 'event' in expr:
 945             check_event(expr, info)
 946         else:
 947             assert False, 'unexpected meta type'
 948
 949     return exprs
 950
 951
 952 def check_freeform_doc(doc):
 953     if doc.symbol:
 954         raise QAPISemError(doc.info,
 955                            "Documention for '%s' is not followed"
 956                            " by the definition" % doc.symbol)
 957
 958     body = str(doc.body)
 959     if re.search(r'@\S+:', body, re.MULTILINE):
 960         raise QAPISemError(doc.info,
 961                            "Free-form documentation block must not contain"
 962                            " @NAME: sections")
 963
 964
 965 def check_definition_doc(doc, expr, info):
 966     for i in ('enum', 'union', 'alternate', 'struct', 'command', 'event'):
 967         if i in expr:
 968             meta = i
 969             break
 970
 971     name = expr[meta]
 972     if doc.symbol != name:
 973         raise QAPISemError(info, "Definition of '%s' follows documentation"
 974                            " for '%s'" % (name, doc.symbol))
 975     if doc.has_section('Returns') and 'command' not in expr:
 976         raise QAPISemError(info, "'Returns:' is only valid for commands")
 977
 978     if meta == 'union':
 979         args = expr.get('base', [])
 980     else:
 981         args = expr.get('data', [])
 982     if isinstance(args, str):
 983         return
 984     if isinstance(args, dict):
 985         args = args.keys()
 986     assert isinstance(args, list)
 987
 988     if (meta == 'alternate'
 989             or (meta == 'union' and not expr.get('discriminator'))):
 990         args.append('type')
 991
 992     doc_args = set(doc.args.keys())
 993     args = set([name.strip('*') for name in args])
 994     if not doc_args.issubset(args):
 995         raise QAPISemError(info, "The following documented members are not in "
 996                            "the declaration: %s" % ", ".join(doc_args - args))
 997
 998
 999 def check_docs(docs):
1000     for doc in docs:
1001         for section in doc.args.values() + doc.sections:
1002             content = str(section)
1003             if not content or content.isspace():
1004                 raise QAPISemError(doc.info,
1005                                    "Empty doc section '%s'" % section.name)
1006
1007         if not doc.expr:
1008             check_freeform_doc(doc)
1009         else:
1010             check_definition_doc(doc, doc.expr, doc.info)
1011
1012     return docs
1013
1014
1015 #
1016 # Schema compiler frontend
1017 #
1018
1019 class QAPISchemaEntity(object):
1020     def __init__(self, name, info, doc):
1021         assert isinstance(name, str)
1022         self.name = name
1023         # For explicitly defined entities, info points to the (explicit)
1024         # definition.  For builtins (and their arrays), info is None.
1025         # For implicitly defined entities, info points to a place that
1026         # triggered the implicit definition (there may be more than one
1027         # such place).
1028         self.info = info
1029         self.doc = doc
1030
1031     def c_name(self):
1032         return c_name(self.name)
1033
1034     def check(self, schema):
1035         pass
1036
1037     def is_implicit(self):
1038         return not self.info
1039
1040     def visit(self, visitor):
1041         pass
1042
1043
1044 class QAPISchemaVisitor(object):
1045     def visit_begin(self, schema):
1046         pass
1047
1048     def visit_end(self):
1049         pass
1050
1051     def visit_needed(self, entity):
1052         # Default to visiting everything
1053         return True
1054
1055     def visit_builtin_type(self, name, info, json_type):
1056         pass
1057
1058     def visit_enum_type(self, name, info, values, prefix):
1059         pass
1060
1061     def visit_array_type(self, name, info, element_type):
1062         pass
1063
1064     def visit_object_type(self, name, info, base, members, variants):
1065         pass
1066
1067     def visit_object_type_flat(self, name, info, members, variants):
1068         pass
1069
1070     def visit_alternate_type(self, name, info, variants):
1071         pass
1072
1073     def visit_command(self, name, info, arg_type, ret_type,
1074                       gen, success_response, boxed):
1075         pass
1076
1077     def visit_event(self, name, info, arg_type, boxed):
1078         pass
1079
1080
1081 class QAPISchemaType(QAPISchemaEntity):
1082     # Return the C type for common use.
1083     # For the types we commonly box, this is a pointer type.
1084     def c_type(self):
1085         pass
1086
1087     # Return the C type to be used in a parameter list.
1088     def c_param_type(self):
1089         return self.c_type()
1090
1091     # Return the C type to be used where we suppress boxing.
1092     def c_unboxed_type(self):
1093         return self.c_type()
1094
1095     def json_type(self):
1096         pass
1097
1098     def alternate_qtype(self):
1099         json2qtype = {
1100             'string':  'QTYPE_QSTRING',
1101             'number':  'QTYPE_QFLOAT',
1102             'int':     'QTYPE_QINT',
1103             'boolean': 'QTYPE_QBOOL',
1104             'object':  'QTYPE_QDICT'
1105         }
1106         return json2qtype.get(self.json_type())
1107
1108
1109 class QAPISchemaBuiltinType(QAPISchemaType):
1110     def __init__(self, name, json_type, c_type):
1111         QAPISchemaType.__init__(self, name, None, None)
1112         assert not c_type or isinstance(c_type, str)
1113         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1114                              'value')
1115         self._json_type_name = json_type
1116         self._c_type_name = c_type
1117
1118     def c_name(self):
1119         return self.name
1120
1121     def c_type(self):
1122         return self._c_type_name
1123
1124     def c_param_type(self):
1125         if self.name == 'str':
1126             return 'const ' + self._c_type_name
1127         return self._c_type_name
1128
1129     def json_type(self):
1130         return self._json_type_name
1131
1132     def visit(self, visitor):
1133         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1134
1135
1136 class QAPISchemaEnumType(QAPISchemaType):
1137     def __init__(self, name, info, doc, values, prefix):
1138         QAPISchemaType.__init__(self, name, info, doc)
1139         for v in values:
1140             assert isinstance(v, QAPISchemaMember)
1141             v.set_owner(name)
1142         assert prefix is None or isinstance(prefix, str)
1143         self.values = values
1144         self.prefix = prefix
1145
1146     def check(self, schema):
1147         seen = {}
1148         for v in self.values:
1149             v.check_clash(self.info, seen)
1150             if self.doc:
1151                 self.doc.connect_member(v)
1152
1153     def is_implicit(self):
1154         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1155         return self.name.endswith('Kind') or self.name == 'QType'
1156
1157     def c_type(self):
1158         return c_name(self.name)
1159
1160     def member_names(self):
1161         return [v.name for v in self.values]
1162
1163     def json_type(self):
1164         return 'string'
1165
1166     def visit(self, visitor):
1167         visitor.visit_enum_type(self.name, self.info,
1168                                 self.member_names(), self.prefix)
1169
1170
1171 class QAPISchemaArrayType(QAPISchemaType):
1172     def __init__(self, name, info, element_type):
1173         QAPISchemaType.__init__(self, name, info, None)
1174         assert isinstance(element_type, str)
1175         self._element_type_name = element_type
1176         self.element_type = None
1177
1178     def check(self, schema):
1179         self.element_type = schema.lookup_type(self._element_type_name)
1180         assert self.element_type
1181
1182     def is_implicit(self):
1183         return True
1184
1185     def c_type(self):
1186         return c_name(self.name) + pointer_suffix
1187
1188     def json_type(self):
1189         return 'array'
1190
1191     def visit(self, visitor):
1192         visitor.visit_array_type(self.name, self.info, self.element_type)
1193
1194
1195 class QAPISchemaObjectType(QAPISchemaType):
1196     def __init__(self, name, info, doc, base, local_members, variants):
1197         # struct has local_members, optional base, and no variants
1198         # flat union has base, variants, and no local_members
1199         # simple union has local_members, variants, and no base
1200         QAPISchemaType.__init__(self, name, info, doc)
1201         assert base is None or isinstance(base, str)
1202         for m in local_members:
1203             assert isinstance(m, QAPISchemaObjectTypeMember)
1204             m.set_owner(name)
1205         if variants is not None:
1206             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1207             variants.set_owner(name)
1208         self._base_name = base
1209         self.base = None
1210         self.local_members = local_members
1211         self.variants = variants
1212         self.members = None
1213
1214     def check(self, schema):
1215         if self.members is False:               # check for cycles
1216             raise QAPISemError(self.info,
1217                                "Object %s contains itself" % self.name)
1218         if self.members:
1219             return
1220         self.members = False                    # mark as being checked
1221         seen = OrderedDict()
1222         if self._base_name:
1223             self.base = schema.lookup_type(self._base_name)
1224             assert isinstance(self.base, QAPISchemaObjectType)
1225             self.base.check(schema)
1226             self.base.check_clash(schema, self.info, seen)
1227         for m in self.local_members:
1228             m.check(schema)
1229             m.check_clash(self.info, seen)
1230             if self.doc:
1231                 self.doc.connect_member(m)
1232         self.members = seen.values()
1233         if self.variants:
1234             self.variants.check(schema, seen)
1235             assert self.variants.tag_member in self.members
1236             self.variants.check_clash(schema, self.info, seen)
1237
1238     # Check that the members of this type do not cause duplicate JSON members,
1239     # and update seen to track the members seen so far. Report any errors
1240     # on behalf of info, which is not necessarily self.info
1241     def check_clash(self, schema, info, seen):
1242         assert not self.variants       # not implemented
1243         for m in self.members:
1244             m.check_clash(info, seen)
1245
1246     def is_implicit(self):
1247         # See QAPISchema._make_implicit_object_type(), as well as
1248         # _def_predefineds()
1249         return self.name.startswith('q_')
1250
1251     def is_empty(self):
1252         assert self.members is not None
1253         return not self.members and not self.variants
1254
1255     def c_name(self):
1256         assert self.name != 'q_empty'
1257         return QAPISchemaType.c_name(self)
1258
1259     def c_type(self):
1260         assert not self.is_implicit()
1261         return c_name(self.name) + pointer_suffix
1262
1263     def c_unboxed_type(self):
1264         return c_name(self.name)
1265
1266     def json_type(self):
1267         return 'object'
1268
1269     def visit(self, visitor):
1270         visitor.visit_object_type(self.name, self.info,
1271                                   self.base, self.local_members, self.variants)
1272         visitor.visit_object_type_flat(self.name, self.info,
1273                                        self.members, self.variants)
1274
1275
1276 class QAPISchemaMember(object):
1277     role = 'member'
1278
1279     def __init__(self, name):
1280         assert isinstance(name, str)
1281         self.name = name
1282         self.owner = None
1283
1284     def set_owner(self, name):
1285         assert not self.owner
1286         self.owner = name
1287
1288     def check_clash(self, info, seen):
1289         cname = c_name(self.name)
1290         if cname.lower() != cname and self.owner not in name_case_whitelist:
1291             raise QAPISemError(info,
1292                                "%s should not use uppercase" % self.describe())
1293         if cname in seen:
1294             raise QAPISemError(info, "%s collides with %s" %
1295                                (self.describe(), seen[cname].describe()))
1296         seen[cname] = self
1297
1298     def _pretty_owner(self):
1299         owner = self.owner
1300         if owner.startswith('q_obj_'):
1301             # See QAPISchema._make_implicit_object_type() - reverse the
1302             # mapping there to create a nice human-readable description
1303             owner = owner[6:]
1304             if owner.endswith('-arg'):
1305                 return '(parameter of %s)' % owner[:-4]
1306             elif owner.endswith('-base'):
1307                 return '(base of %s)' % owner[:-5]
1308             else:
1309                 assert owner.endswith('-wrapper')
1310                 # Unreachable and not implemented
1311                 assert False
1312         if owner.endswith('Kind'):
1313             # See QAPISchema._make_implicit_enum_type()
1314             return '(branch of %s)' % owner[:-4]
1315         return '(%s of %s)' % (self.role, owner)
1316
1317     def describe(self):
1318         return "'%s' %s" % (self.name, self._pretty_owner())
1319
1320
1321 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1322     def __init__(self, name, typ, optional):
1323         QAPISchemaMember.__init__(self, name)
1324         assert isinstance(typ, str)
1325         assert isinstance(optional, bool)
1326         self._type_name = typ
1327         self.type = None
1328         self.optional = optional
1329
1330     def check(self, schema):
1331         assert self.owner
1332         self.type = schema.lookup_type(self._type_name)
1333         assert self.type
1334
1335
1336 class QAPISchemaObjectTypeVariants(object):
1337     def __init__(self, tag_name, tag_member, variants):
1338         # Flat unions pass tag_name but not tag_member.
1339         # Simple unions and alternates pass tag_member but not tag_name.
1340         # After check(), tag_member is always set, and tag_name remains
1341         # a reliable witness of being used by a flat union.
1342         assert bool(tag_member) != bool(tag_name)
1343         assert (isinstance(tag_name, str) or
1344                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1345         assert len(variants) > 0
1346         for v in variants:
1347             assert isinstance(v, QAPISchemaObjectTypeVariant)
1348         self._tag_name = tag_name
1349         self.tag_member = tag_member
1350         self.variants = variants
1351
1352     def set_owner(self, name):
1353         for v in self.variants:
1354             v.set_owner(name)
1355
1356     def check(self, schema, seen):
1357         if not self.tag_member:    # flat union
1358             self.tag_member = seen[c_name(self._tag_name)]
1359             assert self._tag_name == self.tag_member.name
1360         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1361         for v in self.variants:
1362             v.check(schema)
1363             # Union names must match enum values; alternate names are
1364             # checked separately. Use 'seen' to tell the two apart.
1365             if seen:
1366                 assert v.name in self.tag_member.type.member_names()
1367                 assert isinstance(v.type, QAPISchemaObjectType)
1368                 v.type.check(schema)
1369
1370     def check_clash(self, schema, info, seen):
1371         for v in self.variants:
1372             # Reset seen map for each variant, since qapi names from one
1373             # branch do not affect another branch
1374             assert isinstance(v.type, QAPISchemaObjectType)
1375             v.type.check_clash(schema, info, dict(seen))
1376
1377
1378 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1379     role = 'branch'
1380
1381     def __init__(self, name, typ):
1382         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1383
1384
1385 class QAPISchemaAlternateType(QAPISchemaType):
1386     def __init__(self, name, info, doc, variants):
1387         QAPISchemaType.__init__(self, name, info, doc)
1388         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1389         assert variants.tag_member
1390         variants.set_owner(name)
1391         variants.tag_member.set_owner(self.name)
1392         self.variants = variants
1393
1394     def check(self, schema):
1395         self.variants.tag_member.check(schema)
1396         # Not calling self.variants.check_clash(), because there's nothing
1397         # to clash with
1398         self.variants.check(schema, {})
1399         # Alternate branch names have no relation to the tag enum values;
1400         # so we have to check for potential name collisions ourselves.
1401         seen = {}
1402         for v in self.variants.variants:
1403             v.check_clash(self.info, seen)
1404             if self.doc:
1405                 self.doc.connect_member(v)
1406
1407     def c_type(self):
1408         return c_name(self.name) + pointer_suffix
1409
1410     def json_type(self):
1411         return 'value'
1412
1413     def visit(self, visitor):
1414         visitor.visit_alternate_type(self.name, self.info, self.variants)
1415
1416     def is_empty(self):
1417         return False
1418
1419
1420 class QAPISchemaCommand(QAPISchemaEntity):
1421     def __init__(self, name, info, doc, arg_type, ret_type,
1422                  gen, success_response, boxed):
1423         QAPISchemaEntity.__init__(self, name, info, doc)
1424         assert not arg_type or isinstance(arg_type, str)
1425         assert not ret_type or isinstance(ret_type, str)
1426         self._arg_type_name = arg_type
1427         self.arg_type = None
1428         self._ret_type_name = ret_type
1429         self.ret_type = None
1430         self.gen = gen
1431         self.success_response = success_response
1432         self.boxed = boxed
1433
1434     def check(self, schema):
1435         if self._arg_type_name:
1436             self.arg_type = schema.lookup_type(self._arg_type_name)
1437             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1438                     isinstance(self.arg_type, QAPISchemaAlternateType))
1439             self.arg_type.check(schema)
1440             if self.boxed:
1441                 if self.arg_type.is_empty():
1442                     raise QAPISemError(self.info,
1443                                        "Cannot use 'boxed' with empty type")
1444             else:
1445                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1446                 assert not self.arg_type.variants
1447         elif self.boxed:
1448             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1449         if self._ret_type_name:
1450             self.ret_type = schema.lookup_type(self._ret_type_name)
1451             assert isinstance(self.ret_type, QAPISchemaType)
1452
1453     def visit(self, visitor):
1454         visitor.visit_command(self.name, self.info,
1455                               self.arg_type, self.ret_type,
1456                               self.gen, self.success_response, self.boxed)
1457
1458
1459 class QAPISchemaEvent(QAPISchemaEntity):
1460     def __init__(self, name, info, doc, arg_type, boxed):
1461         QAPISchemaEntity.__init__(self, name, info, doc)
1462         assert not arg_type or isinstance(arg_type, str)
1463         self._arg_type_name = arg_type
1464         self.arg_type = None
1465         self.boxed = boxed
1466
1467     def check(self, schema):
1468         if self._arg_type_name:
1469             self.arg_type = schema.lookup_type(self._arg_type_name)
1470             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1471                     isinstance(self.arg_type, QAPISchemaAlternateType))
1472             self.arg_type.check(schema)
1473             if self.boxed:
1474                 if self.arg_type.is_empty():
1475                     raise QAPISemError(self.info,
1476                                        "Cannot use 'boxed' with empty type")
1477             else:
1478                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1479                 assert not self.arg_type.variants
1480         elif self.boxed:
1481             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1482
1483     def visit(self, visitor):
1484         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1485
1486
1487 class QAPISchema(object):
1488     def __init__(self, fname):
1489         try:
1490             parser = QAPISchemaParser(open(fname, "r"))
1491             self.exprs = check_exprs(parser.exprs)
1492             self.docs = check_docs(parser.docs)
1493             self._entity_dict = {}
1494             self._predefining = True
1495             self._def_predefineds()
1496             self._predefining = False
1497             self._def_exprs()
1498             self.check()
1499         except QAPIError as err:
1500             print >>sys.stderr, err
1501             exit(1)
1502
1503     def _def_entity(self, ent):
1504         # Only the predefined types are allowed to not have info
1505         assert ent.info or self._predefining
1506         assert ent.name not in self._entity_dict
1507         self._entity_dict[ent.name] = ent
1508
1509     def lookup_entity(self, name, typ=None):
1510         ent = self._entity_dict.get(name)
1511         if typ and not isinstance(ent, typ):
1512             return None
1513         return ent
1514
1515     def lookup_type(self, name):
1516         return self.lookup_entity(name, QAPISchemaType)
1517
1518     def _def_builtin_type(self, name, json_type, c_type):
1519         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1520         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1521         # qapi-types.h from a single .c, all arrays of builtins must be
1522         # declared in the first file whether or not they are used.  Nicer
1523         # would be to use lazy instantiation, while figuring out how to
1524         # avoid compilation issues with multiple qapi-types.h.
1525         self._make_array_type(name, None)
1526
1527     def _def_predefineds(self):
1528         for t in [('str',    'string',  'char' + pointer_suffix),
1529                   ('number', 'number',  'double'),
1530                   ('int',    'int',     'int64_t'),
1531                   ('int8',   'int',     'int8_t'),
1532                   ('int16',  'int',     'int16_t'),
1533                   ('int32',  'int',     'int32_t'),
1534                   ('int64',  'int',     'int64_t'),
1535                   ('uint8',  'int',     'uint8_t'),
1536                   ('uint16', 'int',     'uint16_t'),
1537                   ('uint32', 'int',     'uint32_t'),
1538                   ('uint64', 'int',     'uint64_t'),
1539                   ('size',   'int',     'uint64_t'),
1540                   ('bool',   'boolean', 'bool'),
1541                   ('any',    'value',   'QObject' + pointer_suffix)]:
1542             self._def_builtin_type(*t)
1543         self.the_empty_object_type = QAPISchemaObjectType(
1544             'q_empty', None, None, None, [], None)
1545         self._def_entity(self.the_empty_object_type)
1546         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1547                                                 'qstring', 'qdict', 'qlist',
1548                                                 'qfloat', 'qbool'])
1549         self._def_entity(QAPISchemaEnumType('QType', None, None,
1550                                             qtype_values, 'QTYPE'))
1551
1552     def _make_enum_members(self, values):
1553         return [QAPISchemaMember(v) for v in values]
1554
1555     def _make_implicit_enum_type(self, name, info, values):
1556         # See also QAPISchemaObjectTypeMember._pretty_owner()
1557         name = name + 'Kind'   # Use namespace reserved by add_name()
1558         self._def_entity(QAPISchemaEnumType(
1559             name, info, None, self._make_enum_members(values), None))
1560         return name
1561
1562     def _make_array_type(self, element_type, info):
1563         name = element_type + 'List'   # Use namespace reserved by add_name()
1564         if not self.lookup_type(name):
1565             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1566         return name
1567
1568     def _make_implicit_object_type(self, name, info, doc, role, members):
1569         if not members:
1570             return None
1571         # See also QAPISchemaObjectTypeMember._pretty_owner()
1572         name = 'q_obj_%s-%s' % (name, role)
1573         if not self.lookup_entity(name, QAPISchemaObjectType):
1574             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1575                                                   members, None))
1576         return name
1577
1578     def _def_enum_type(self, expr, info, doc):
1579         name = expr['enum']
1580         data = expr['data']
1581         prefix = expr.get('prefix')
1582         self._def_entity(QAPISchemaEnumType(
1583             name, info, doc, self._make_enum_members(data), prefix))
1584
1585     def _make_member(self, name, typ, info):
1586         optional = False
1587         if name.startswith('*'):
1588             name = name[1:]
1589             optional = True
1590         if isinstance(typ, list):
1591             assert len(typ) == 1
1592             typ = self._make_array_type(typ[0], info)
1593         return QAPISchemaObjectTypeMember(name, typ, optional)
1594
1595     def _make_members(self, data, info):
1596         return [self._make_member(key, value, info)
1597                 for (key, value) in data.iteritems()]
1598
1599     def _def_struct_type(self, expr, info, doc):
1600         name = expr['struct']
1601         base = expr.get('base')
1602         data = expr['data']
1603         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1604                                               self._make_members(data, info),
1605                                               None))
1606
1607     def _make_variant(self, case, typ):
1608         return QAPISchemaObjectTypeVariant(case, typ)
1609
1610     def _make_simple_variant(self, case, typ, info):
1611         if isinstance(typ, list):
1612             assert len(typ) == 1
1613             typ = self._make_array_type(typ[0], info)
1614         typ = self._make_implicit_object_type(
1615             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1616         return QAPISchemaObjectTypeVariant(case, typ)
1617
1618     def _def_union_type(self, expr, info, doc):
1619         name = expr['union']
1620         data = expr['data']
1621         base = expr.get('base')
1622         tag_name = expr.get('discriminator')
1623         tag_member = None
1624         if isinstance(base, dict):
1625             base = (self._make_implicit_object_type(
1626                     name, info, doc, 'base', self._make_members(base, info)))
1627         if tag_name:
1628             variants = [self._make_variant(key, value)
1629                         for (key, value) in data.iteritems()]
1630             members = []
1631         else:
1632             variants = [self._make_simple_variant(key, value, info)
1633                         for (key, value) in data.iteritems()]
1634             typ = self._make_implicit_enum_type(name, info,
1635                                                 [v.name for v in variants])
1636             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1637             members = [tag_member]
1638         self._def_entity(
1639             QAPISchemaObjectType(name, info, doc, base, members,
1640                                  QAPISchemaObjectTypeVariants(tag_name,
1641                                                               tag_member,
1642                                                               variants)))
1643
1644     def _def_alternate_type(self, expr, info, doc):
1645         name = expr['alternate']
1646         data = expr['data']
1647         variants = [self._make_variant(key, value)
1648                     for (key, value) in data.iteritems()]
1649         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1650         self._def_entity(
1651             QAPISchemaAlternateType(name, info, doc,
1652                                     QAPISchemaObjectTypeVariants(None,
1653                                                                  tag_member,
1654                                                                  variants)))
1655
1656     def _def_command(self, expr, info, doc):
1657         name = expr['command']
1658         data = expr.get('data')
1659         rets = expr.get('returns')
1660         gen = expr.get('gen', True)
1661         success_response = expr.get('success-response', True)
1662         boxed = expr.get('boxed', False)
1663         if isinstance(data, OrderedDict):
1664             data = self._make_implicit_object_type(
1665                 name, info, doc, 'arg', self._make_members(data, info))
1666         if isinstance(rets, list):
1667             assert len(rets) == 1
1668             rets = self._make_array_type(rets[0], info)
1669         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1670                                            gen, success_response, boxed))
1671
1672     def _def_event(self, expr, info, doc):
1673         name = expr['event']
1674         data = expr.get('data')
1675         boxed = expr.get('boxed', False)
1676         if isinstance(data, OrderedDict):
1677             data = self._make_implicit_object_type(
1678                 name, info, doc, 'arg', self._make_members(data, info))
1679         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1680
1681     def _def_exprs(self):
1682         for expr_elem in self.exprs:
1683             expr = expr_elem['expr']
1684             info = expr_elem['info']
1685             doc = expr_elem.get('doc')
1686             if 'enum' in expr:
1687                 self._def_enum_type(expr, info, doc)
1688             elif 'struct' in expr:
1689                 self._def_struct_type(expr, info, doc)
1690             elif 'union' in expr:
1691                 self._def_union_type(expr, info, doc)
1692             elif 'alternate' in expr:
1693                 self._def_alternate_type(expr, info, doc)
1694             elif 'command' in expr:
1695                 self._def_command(expr, info, doc)
1696             elif 'event' in expr:
1697                 self._def_event(expr, info, doc)
1698             else:
1699                 assert False
1700
1701     def check(self):
1702         for ent in self._entity_dict.values():
1703             ent.check(self)
1704
1705     def visit(self, visitor):
1706         visitor.visit_begin(self)
1707         for (name, entity) in sorted(self._entity_dict.items()):
1708             if visitor.visit_needed(entity):
1709                 entity.visit(visitor)
1710         visitor.visit_end()
1711
1712
1713 #
1714 # Code generation helpers
1715 #
1716
1717 def camel_case(name):
1718     new_name = ''
1719     first = True
1720     for ch in name:
1721         if ch in ['_', '-']:
1722             first = True
1723         elif first:
1724             new_name += ch.upper()
1725             first = False
1726         else:
1727             new_name += ch.lower()
1728     return new_name
1729
1730
1731 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1732 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1733 # ENUM24_Name -> ENUM24_NAME
1734 def camel_to_upper(value):
1735     c_fun_str = c_name(value, False)
1736     if value.isupper():
1737         return c_fun_str
1738
1739     new_name = ''
1740     l = len(c_fun_str)
1741     for i in range(l):
1742         c = c_fun_str[i]
1743         # When c is upper and no "_" appears before, do more checks
1744         if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
1745             if i < l - 1 and c_fun_str[i + 1].islower():
1746                 new_name += '_'
1747             elif c_fun_str[i - 1].isdigit():
1748                 new_name += '_'
1749         new_name += c
1750     return new_name.lstrip('_').upper()
1751
1752
1753 def c_enum_const(type_name, const_name, prefix=None):
1754     if prefix is not None:
1755         type_name = prefix
1756     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1757
1758 c_name_trans = string.maketrans('.-', '__')
1759
1760
1761 # Map @name to a valid C identifier.
1762 # If @protect, avoid returning certain ticklish identifiers (like
1763 # C keywords) by prepending "q_".
1764 #
1765 # Used for converting 'name' from a 'name':'type' qapi definition
1766 # into a generated struct member, as well as converting type names
1767 # into substrings of a generated C function name.
1768 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1769 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1770 def c_name(name, protect=True):
1771     # ANSI X3J11/88-090, 3.1.1
1772     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1773                      'default', 'do', 'double', 'else', 'enum', 'extern',
1774                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1775                      'return', 'short', 'signed', 'sizeof', 'static',
1776                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1777                      'void', 'volatile', 'while'])
1778     # ISO/IEC 9899:1999, 6.4.1
1779     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1780     # ISO/IEC 9899:2011, 6.4.1
1781     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1782                      '_Noreturn', '_Static_assert', '_Thread_local'])
1783     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1784     # excluding _.*
1785     gcc_words = set(['asm', 'typeof'])
1786     # C++ ISO/IEC 14882:2003 2.11
1787     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1788                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1789                      'namespace', 'new', 'operator', 'private', 'protected',
1790                      'public', 'reinterpret_cast', 'static_cast', 'template',
1791                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1792                      'using', 'virtual', 'wchar_t',
1793                      # alternative representations
1794                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1795                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1796     # namespace pollution:
1797     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1798     name = name.translate(c_name_trans)
1799     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1800                     | cpp_words | polluted_words):
1801         return "q_" + name
1802     return name
1803
1804 eatspace = '\033EATSPACE.'
1805 pointer_suffix = ' *' + eatspace
1806
1807
1808 def genindent(count):
1809     ret = ""
1810     for _ in range(count):
1811         ret += " "
1812     return ret
1813
1814 indent_level = 0
1815
1816
1817 def push_indent(indent_amount=4):
1818     global indent_level
1819     indent_level += indent_amount
1820
1821
1822 def pop_indent(indent_amount=4):
1823     global indent_level
1824     indent_level -= indent_amount
1825
1826
1827 # Generate @code with @kwds interpolated.
1828 # Obey indent_level, and strip eatspace.
1829 def cgen(code, **kwds):
1830     raw = code % kwds
1831     if indent_level:
1832         indent = genindent(indent_level)
1833         # re.subn() lacks flags support before Python 2.7, use re.compile()
1834         raw = re.subn(re.compile("^.", re.MULTILINE),
1835                       indent + r'\g<0>', raw)
1836         raw = raw[0]
1837     return re.sub(re.escape(eatspace) + ' *', '', raw)
1838
1839
1840 def mcgen(code, **kwds):
1841     if code[0] == '\n':
1842         code = code[1:]
1843     return cgen(code, **kwds)
1844
1845
1846 def guardname(filename):
1847     return c_name(filename, protect=False).upper()
1848
1849
1850 def guardstart(name):
1851     return mcgen('''
1852
1853 #ifndef %(name)s
1854 #define %(name)s
1855
1856 ''',
1857                  name=guardname(name))
1858
1859
1860 def guardend(name):
1861     return mcgen('''
1862
1863 #endif /* %(name)s */
1864
1865 ''',
1866                  name=guardname(name))
1867
1868
1869 def gen_enum_lookup(name, values, prefix=None):
1870     ret = mcgen('''
1871
1872 const char *const %(c_name)s_lookup[] = {
1873 ''',
1874                 c_name=c_name(name))
1875     for value in values:
1876         index = c_enum_const(name, value, prefix)
1877         ret += mcgen('''
1878     [%(index)s] = "%(value)s",
1879 ''',
1880                      index=index, value=value)
1881
1882     max_index = c_enum_const(name, '_MAX', prefix)
1883     ret += mcgen('''
1884     [%(max_index)s] = NULL,
1885 };
1886 ''',
1887                  max_index=max_index)
1888     return ret
1889
1890
1891 def gen_enum(name, values, prefix=None):
1892     # append automatically generated _MAX value
1893     enum_values = values + ['_MAX']
1894
1895     ret = mcgen('''
1896
1897 typedef enum %(c_name)s {
1898 ''',
1899                 c_name=c_name(name))
1900
1901     i = 0
1902     for value in enum_values:
1903         ret += mcgen('''
1904     %(c_enum)s = %(i)d,
1905 ''',
1906                      c_enum=c_enum_const(name, value, prefix),
1907                      i=i)
1908         i += 1
1909
1910     ret += mcgen('''
1911 } %(c_name)s;
1912 ''',
1913                  c_name=c_name(name))
1914
1915     ret += mcgen('''
1916
1917 extern const char *const %(c_name)s_lookup[];
1918 ''',
1919                  c_name=c_name(name))
1920     return ret
1921
1922
1923 def gen_params(arg_type, boxed, extra):
1924     if not arg_type:
1925         assert not boxed
1926         return extra
1927     ret = ''
1928     sep = ''
1929     if boxed:
1930         ret += '%s arg' % arg_type.c_param_type()
1931         sep = ', '
1932     else:
1933         assert not arg_type.variants
1934         for memb in arg_type.members:
1935             ret += sep
1936             sep = ', '
1937             if memb.optional:
1938                 ret += 'bool has_%s, ' % c_name(memb.name)
1939             ret += '%s %s' % (memb.type.c_param_type(),
1940                               c_name(memb.name))
1941     if extra:
1942         ret += sep + extra
1943     return ret
1944
1945
1946 #
1947 # Common command line parsing
1948 #
1949
1950
1951 def parse_command_line(extra_options="", extra_long_options=[]):
1952
1953     try:
1954         opts, args = getopt.gnu_getopt(sys.argv[1:],
1955                                        "chp:o:" + extra_options,
1956                                        ["source", "header", "prefix=",
1957                                         "output-dir="] + extra_long_options)
1958     except getopt.GetoptError as err:
1959         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1960         sys.exit(1)
1961
1962     output_dir = ""
1963     prefix = ""
1964     do_c = False
1965     do_h = False
1966     extra_opts = []
1967
1968     for oa in opts:
1969         o, a = oa
1970         if o in ("-p", "--prefix"):
1971             match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1972             if match.end() != len(a):
1973                 print >>sys.stderr, \
1974                     "%s: 'funny character '%s' in argument of --prefix" \
1975                     % (sys.argv[0], a[match.end()])
1976                 sys.exit(1)
1977             prefix = a
1978         elif o in ("-o", "--output-dir"):
1979             output_dir = a + "/"
1980         elif o in ("-c", "--source"):
1981             do_c = True
1982         elif o in ("-h", "--header"):
1983             do_h = True
1984         else:
1985             extra_opts.append(oa)
1986
1987     if not do_c and not do_h:
1988         do_c = True
1989         do_h = True
1990
1991     if len(args) != 1:
1992         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
1993         sys.exit(1)
1994     fname = args[0]
1995
1996     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
1997
1998 #
1999 # Generate output files with boilerplate
2000 #
2001
2002
2003 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2004                 c_comment, h_comment):
2005     guard = guardname(prefix + h_file)
2006     c_file = output_dir + prefix + c_file
2007     h_file = output_dir + prefix + h_file
2008
2009     if output_dir:
2010         try:
2011             os.makedirs(output_dir)
2012         except os.error as e:
2013             if e.errno != errno.EEXIST:
2014                 raise
2015
2016     def maybe_open(really, name, opt):
2017         if really:
2018             return open(name, opt)
2019         else:
2020             import StringIO
2021             return StringIO.StringIO()
2022
2023     fdef = maybe_open(do_c, c_file, 'w')
2024     fdecl = maybe_open(do_h, h_file, 'w')
2025
2026     fdef.write(mcgen('''
2027 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2028 %(comment)s
2029 ''',
2030                      comment=c_comment))
2031
2032     fdecl.write(mcgen('''
2033 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2034 %(comment)s
2035 #ifndef %(guard)s
2036 #define %(guard)s
2037
2038 ''',
2039                       comment=h_comment, guard=guard))
2040
2041     return (fdef, fdecl)
2042
2043
2044 def close_output(fdef, fdecl):
2045     fdecl.write('''
2046 #endif
2047 ''')
2048     fdecl.close()
2049     fdef.close()