scripts/qapi.py

   1 #
   2 # QAPI helper library
   3 #
   4 # Copyright IBM, Corp. 2011
   5 # Copyright (c) 2013-2016 Red Hat Inc.
   6 #
   7 # Authors:
   8 #  Anthony Liguori <aliguori@us.ibm.com>
   9 #  Markus Armbruster <armbru@redhat.com>
  10 #
  11 # This work is licensed under the terms of the GNU GPL, version 2.
  12 # See the COPYING file in the top-level directory.
  13
  14 import re
  15 from ordereddict import OrderedDict
  16 import errno
  17 import getopt
  18 import os
  19 import sys
  20 import string
  21
  22 builtin_types = {
  23     'str':      'QTYPE_QSTRING',
  24     'int':      'QTYPE_QINT',
  25     'number':   'QTYPE_QFLOAT',
  26     'bool':     'QTYPE_QBOOL',
  27     'int8':     'QTYPE_QINT',
  28     'int16':    'QTYPE_QINT',
  29     'int32':    'QTYPE_QINT',
  30     'int64':    'QTYPE_QINT',
  31     'uint8':    'QTYPE_QINT',
  32     'uint16':   'QTYPE_QINT',
  33     'uint32':   'QTYPE_QINT',
  34     'uint64':   'QTYPE_QINT',
  35     'size':     'QTYPE_QINT',
  36     'any':      None,           # any QType possible, actually
  37     'QType':    'QTYPE_QSTRING',
  38 }
  39
  40 # Are documentation comments required?
  41 doc_required = False
  42
  43 # Whitelist of commands allowed to return a non-dictionary
  44 returns_whitelist = []
  45
  46 # Whitelist of entities allowed to violate case conventions
  47 name_case_whitelist = []
  48
  49 enum_types = []
  50 struct_types = []
  51 union_types = []
  52 events = []
  53 all_names = {}
  54
  55 #
  56 # Parsing the schema into expressions
  57 #
  58
  59
  60 def error_path(parent):
  61     res = ''
  62     while parent:
  63         res = ('In file included from %s:%d:\n' % (parent['file'],
  64                                                    parent['line'])) + res
  65         parent = parent['parent']
  66     return res
  67
  68
  69 class QAPIError(Exception):
  70     def __init__(self, fname, line, col, incl_info, msg):
  71         Exception.__init__(self)
  72         self.fname = fname
  73         self.line = line
  74         self.col = col
  75         self.info = incl_info
  76         self.msg = msg
  77
  78     def __str__(self):
  79         loc = '%s:%d' % (self.fname, self.line)
  80         if self.col is not None:
  81             loc += ':%s' % self.col
  82         return error_path(self.info) + '%s: %s' % (loc, self.msg)
  83
  84
  85 class QAPIParseError(QAPIError):
  86     def __init__(self, parser, msg):
  87         col = 1
  88         for ch in parser.src[parser.line_pos:parser.pos]:
  89             if ch == '\t':
  90                 col = (col + 7) % 8 + 1
  91             else:
  92                 col += 1
  93         QAPIError.__init__(self, parser.fname, parser.line, col,
  94                            parser.incl_info, msg)
  95
  96
  97 class QAPISemError(QAPIError):
  98     def __init__(self, info, msg):
  99         QAPIError.__init__(self, info['file'], info['line'], None,
 100                            info['parent'], msg)
 101
 102
 103 class QAPIDoc(object):
 104     class Section(object):
 105         def __init__(self, name=None):
 106             # optional section name (argument/member or section name)
 107             self.name = name
 108             # the list of lines for this section
 109             self.content = []
 110             self.optional = False
 111
 112         def append(self, line):
 113             self.content.append(line)
 114
 115         def __repr__(self):
 116             return '\n'.join(self.content).strip()
 117
 118     class ArgSection(Section):
 119         def __init__(self, name):
 120             QAPIDoc.Section.__init__(self, name)
 121             self.member = None
 122
 123         def connect(self, member):
 124             self.member = member
 125
 126     def __init__(self, parser, info):
 127         # self.parser is used to report errors with QAPIParseError.  The
 128         # resulting error position depends on the state of the parser.
 129         # It happens to be the beginning of the comment.  More or less
 130         # servicable, but action at a distance.
 131         self.parser = parser
 132         self.info = info
 133         self.symbol = None
 134         self.body = QAPIDoc.Section()
 135         # dict mapping parameter name to ArgSection
 136         self.args = OrderedDict()
 137         # a list of Section
 138         self.sections = []
 139         # the current section
 140         self.section = self.body
 141
 142     def has_section(self, name):
 143         """Return True if we have a section with this name."""
 144         for i in self.sections:
 145             if i.name == name:
 146                 return True
 147         return False
 148
 149     def append(self, line):
 150         """Parse a comment line and add it to the documentation."""
 151         line = line[1:]
 152         if not line:
 153             self._append_freeform(line)
 154             return
 155
 156         if line[0] != ' ':
 157             raise QAPIParseError(self.parser, "Missing space after #")
 158         line = line[1:]
 159
 160         # FIXME not nice: things like '#  @foo:' and '# @foo: ' aren't
 161         # recognized, and get silently treated as ordinary text
 162         if self.symbol:
 163             self._append_symbol_line(line)
 164         elif not self.body.content and line.startswith('@'):
 165             if not line.endswith(':'):
 166                 raise QAPIParseError(self.parser, "Line should end with :")
 167             self.symbol = line[1:-1]
 168             # FIXME invalid names other than the empty string aren't flagged
 169             if not self.symbol:
 170                 raise QAPIParseError(self.parser, "Invalid name")
 171         else:
 172             self._append_freeform(line)
 173
 174     def end_comment(self):
 175         self._end_section()
 176
 177     def _append_symbol_line(self, line):
 178         name = line.split(' ', 1)[0]
 179
 180         if name.startswith('@') and name.endswith(':'):
 181             line = line[len(name)+1:]
 182             self._start_args_section(name[1:-1])
 183         elif name in ('Returns:', 'Since:',
 184                       # those are often singular or plural
 185                       'Note:', 'Notes:',
 186                       'Example:', 'Examples:',
 187                       'TODO:'):
 188             line = line[len(name)+1:]
 189             self._start_section(name[:-1])
 190
 191         self._append_freeform(line)
 192
 193     def _start_args_section(self, name):
 194         # FIXME invalid names other than the empty string aren't flagged
 195         if not name:
 196             raise QAPIParseError(self.parser, "Invalid parameter name")
 197         if name in self.args:
 198             raise QAPIParseError(self.parser,
 199                                  "'%s' parameter name duplicated" % name)
 200         if self.sections:
 201             raise QAPIParseError(self.parser,
 202                                  "'@%s:' can't follow '%s' section"
 203                                  % (name, self.sections[0].name))
 204         self._end_section()
 205         self.section = QAPIDoc.ArgSection(name)
 206         self.args[name] = self.section
 207
 208     def _start_section(self, name=''):
 209         if name in ('Returns', 'Since') and self.has_section(name):
 210             raise QAPIParseError(self.parser,
 211                                  "Duplicated '%s' section" % name)
 212         self._end_section()
 213         self.section = QAPIDoc.Section(name)
 214         self.sections.append(self.section)
 215
 216     def _end_section(self):
 217         if self.section:
 218             contents = str(self.section)
 219             if self.section.name and (not contents or contents.isspace()):
 220                 raise QAPIParseError(self.parser, "Empty doc section '%s'"
 221                                      % self.section.name)
 222             self.section = None
 223
 224     def _append_freeform(self, line):
 225         in_arg = isinstance(self.section, QAPIDoc.ArgSection)
 226         if (in_arg and self.section.content
 227                 and not self.section.content[-1]
 228                 and line and not line[0].isspace()):
 229             self._start_section()
 230         if (in_arg or not self.section.name
 231                 or not self.section.name.startswith('Example')):
 232             line = line.strip()
 233         match = re.match(r'(@\S+:)', line)
 234         if match:
 235             raise QAPIParseError(self.parser,
 236                                  "'%s' not allowed in free-form documentation"
 237                                  % match.group(1))
 238         # TODO Drop this once the dust has settled
 239         if (isinstance(self.section, QAPIDoc.ArgSection)
 240                 and '#optional' in line):
 241             raise QAPISemError(self.info, "Please drop the #optional tag")
 242         self.section.append(line)
 243
 244     def connect_member(self, member):
 245         if member.name not in self.args:
 246             # Undocumented TODO outlaw
 247             self.args[member.name] = QAPIDoc.ArgSection(member.name)
 248         self.args[member.name].connect(member)
 249
 250     def check_expr(self, expr):
 251         if self.has_section('Returns') and 'command' not in expr:
 252             raise QAPISemError(self.info,
 253                                "'Returns:' is only valid for commands")
 254
 255     def check(self):
 256         bogus = [name for name, section in self.args.iteritems()
 257                  if not section.member]
 258         if bogus:
 259             raise QAPISemError(
 260                 self.info,
 261                 "The following documented members are not in "
 262                 "the declaration: %s" % ", ".join(bogus))
 263
 264
 265 class QAPISchemaParser(object):
 266
 267     def __init__(self, fp, previously_included=[], incl_info=None):
 268         abs_fname = os.path.abspath(fp.name)
 269         fname = fp.name
 270         self.fname = fname
 271         previously_included.append(abs_fname)
 272         self.incl_info = incl_info
 273         self.src = fp.read()
 274         if self.src == '' or self.src[-1] != '\n':
 275             self.src += '\n'
 276         self.cursor = 0
 277         self.line = 1
 278         self.line_pos = 0
 279         self.exprs = []
 280         self.docs = []
 281         self.cur_doc = None
 282         self.accept()
 283
 284         while self.tok is not None:
 285             info = {'file': fname, 'line': self.line,
 286                     'parent': self.incl_info}
 287             if self.tok == '#':
 288                 self.reject_expr_doc()
 289                 self.cur_doc = self.get_doc(info)
 290                 self.docs.append(self.cur_doc)
 291                 continue
 292
 293             expr = self.get_expr(False)
 294             if 'include' in expr:
 295                 self.reject_expr_doc()
 296                 if len(expr) != 1:
 297                     raise QAPISemError(info, "Invalid 'include' directive")
 298                 include = expr['include']
 299                 if not isinstance(include, str):
 300                     raise QAPISemError(info,
 301                                        "Value of 'include' must be a string")
 302                 self._include(include, info, os.path.dirname(abs_fname),
 303                               previously_included)
 304             elif "pragma" in expr:
 305                 self.reject_expr_doc()
 306                 if len(expr) != 1:
 307                     raise QAPISemError(info, "Invalid 'pragma' directive")
 308                 pragma = expr['pragma']
 309                 if not isinstance(pragma, dict):
 310                     raise QAPISemError(
 311                         info, "Value of 'pragma' must be a dictionary")
 312                 for name, value in pragma.iteritems():
 313                     self._pragma(name, value, info)
 314             else:
 315                 expr_elem = {'expr': expr,
 316                              'info': info}
 317                 if self.cur_doc:
 318                     if not self.cur_doc.symbol:
 319                         raise QAPISemError(
 320                             self.cur_doc.info,
 321                             "Expression documentation required")
 322                     expr_elem['doc'] = self.cur_doc
 323                 self.exprs.append(expr_elem)
 324             self.cur_doc = None
 325         self.reject_expr_doc()
 326
 327     def reject_expr_doc(self):
 328         if self.cur_doc and self.cur_doc.symbol:
 329             raise QAPISemError(
 330                 self.cur_doc.info,
 331                 "Documentation for '%s' is not followed by the definition"
 332                 % self.cur_doc.symbol)
 333
 334     def _include(self, include, info, base_dir, previously_included):
 335         incl_abs_fname = os.path.join(base_dir, include)
 336         # catch inclusion cycle
 337         inf = info
 338         while inf:
 339             if incl_abs_fname == os.path.abspath(inf['file']):
 340                 raise QAPISemError(info, "Inclusion loop for %s" % include)
 341             inf = inf['parent']
 342
 343         # skip multiple include of the same file
 344         if incl_abs_fname in previously_included:
 345             return
 346         try:
 347             fobj = open(incl_abs_fname, 'r')
 348         except IOError as e:
 349             raise QAPISemError(info, '%s: %s' % (e.strerror, include))
 350         exprs_include = QAPISchemaParser(fobj, previously_included, info)
 351         self.exprs.extend(exprs_include.exprs)
 352         self.docs.extend(exprs_include.docs)
 353
 354     def _pragma(self, name, value, info):
 355         global doc_required, returns_whitelist, name_case_whitelist
 356         if name == 'doc-required':
 357             if not isinstance(value, bool):
 358                 raise QAPISemError(info,
 359                                    "Pragma 'doc-required' must be boolean")
 360             doc_required = value
 361         elif name == 'returns-whitelist':
 362             if (not isinstance(value, list)
 363                     or any([not isinstance(elt, str) for elt in value])):
 364                 raise QAPISemError(info,
 365                                    "Pragma returns-whitelist must be"
 366                                    " a list of strings")
 367             returns_whitelist = value
 368         elif name == 'name-case-whitelist':
 369             if (not isinstance(value, list)
 370                     or any([not isinstance(elt, str) for elt in value])):
 371                 raise QAPISemError(info,
 372                                    "Pragma name-case-whitelist must be"
 373                                    " a list of strings")
 374             name_case_whitelist = value
 375         else:
 376             raise QAPISemError(info, "Unknown pragma '%s'" % name)
 377
 378     def accept(self, skip_comment=True):
 379         while True:
 380             self.tok = self.src[self.cursor]
 381             self.pos = self.cursor
 382             self.cursor += 1
 383             self.val = None
 384
 385             if self.tok == '#':
 386                 if self.src[self.cursor] == '#':
 387                     # Start of doc comment
 388                     skip_comment = False
 389                 self.cursor = self.src.find('\n', self.cursor)
 390                 if not skip_comment:
 391                     self.val = self.src[self.pos:self.cursor]
 392                     return
 393             elif self.tok in '{}:,[]':
 394                 return
 395             elif self.tok == "'":
 396                 string = ''
 397                 esc = False
 398                 while True:
 399                     ch = self.src[self.cursor]
 400                     self.cursor += 1
 401                     if ch == '\n':
 402                         raise QAPIParseError(self, 'Missing terminating "\'"')
 403                     if esc:
 404                         if ch == 'b':
 405                             string += '\b'
 406                         elif ch == 'f':
 407                             string += '\f'
 408                         elif ch == 'n':
 409                             string += '\n'
 410                         elif ch == 'r':
 411                             string += '\r'
 412                         elif ch == 't':
 413                             string += '\t'
 414                         elif ch == 'u':
 415                             value = 0
 416                             for _ in range(0, 4):
 417                                 ch = self.src[self.cursor]
 418                                 self.cursor += 1
 419                                 if ch not in '0123456789abcdefABCDEF':
 420                                     raise QAPIParseError(self,
 421                                                          '\\u escape needs 4 '
 422                                                          'hex digits')
 423                                 value = (value << 4) + int(ch, 16)
 424                             # If Python 2 and 3 didn't disagree so much on
 425                             # how to handle Unicode, then we could allow
 426                             # Unicode string defaults.  But most of QAPI is
 427                             # ASCII-only, so we aren't losing much for now.
 428                             if not value or value > 0x7f:
 429                                 raise QAPIParseError(self,
 430                                                      'For now, \\u escape '
 431                                                      'only supports non-zero '
 432                                                      'values up to \\u007f')
 433                             string += chr(value)
 434                         elif ch in '\\/\'"':
 435                             string += ch
 436                         else:
 437                             raise QAPIParseError(self,
 438                                                  "Unknown escape \\%s" % ch)
 439                         esc = False
 440                     elif ch == '\\':
 441                         esc = True
 442                     elif ch == "'":
 443                         self.val = string
 444                         return
 445                     else:
 446                         string += ch
 447             elif self.src.startswith('true', self.pos):
 448                 self.val = True
 449                 self.cursor += 3
 450                 return
 451             elif self.src.startswith('false', self.pos):
 452                 self.val = False
 453                 self.cursor += 4
 454                 return
 455             elif self.src.startswith('null', self.pos):
 456                 self.val = None
 457                 self.cursor += 3
 458                 return
 459             elif self.tok == '\n':
 460                 if self.cursor == len(self.src):
 461                     self.tok = None
 462                     return
 463                 self.line += 1
 464                 self.line_pos = self.cursor
 465             elif not self.tok.isspace():
 466                 raise QAPIParseError(self, 'Stray "%s"' % self.tok)
 467
 468     def get_members(self):
 469         expr = OrderedDict()
 470         if self.tok == '}':
 471             self.accept()
 472             return expr
 473         if self.tok != "'":
 474             raise QAPIParseError(self, 'Expected string or "}"')
 475         while True:
 476             key = self.val
 477             self.accept()
 478             if self.tok != ':':
 479                 raise QAPIParseError(self, 'Expected ":"')
 480             self.accept()
 481             if key in expr:
 482                 raise QAPIParseError(self, 'Duplicate key "%s"' % key)
 483             expr[key] = self.get_expr(True)
 484             if self.tok == '}':
 485                 self.accept()
 486                 return expr
 487             if self.tok != ',':
 488                 raise QAPIParseError(self, 'Expected "," or "}"')
 489             self.accept()
 490             if self.tok != "'":
 491                 raise QAPIParseError(self, 'Expected string')
 492
 493     def get_values(self):
 494         expr = []
 495         if self.tok == ']':
 496             self.accept()
 497             return expr
 498         if self.tok not in "{['tfn":
 499             raise QAPIParseError(self, 'Expected "{", "[", "]", string, '
 500                                  'boolean or "null"')
 501         while True:
 502             expr.append(self.get_expr(True))
 503             if self.tok == ']':
 504                 self.accept()
 505                 return expr
 506             if self.tok != ',':
 507                 raise QAPIParseError(self, 'Expected "," or "]"')
 508             self.accept()
 509
 510     def get_expr(self, nested):
 511         if self.tok != '{' and not nested:
 512             raise QAPIParseError(self, 'Expected "{"')
 513         if self.tok == '{':
 514             self.accept()
 515             expr = self.get_members()
 516         elif self.tok == '[':
 517             self.accept()
 518             expr = self.get_values()
 519         elif self.tok in "'tfn":
 520             expr = self.val
 521             self.accept()
 522         else:
 523             raise QAPIParseError(self, 'Expected "{", "[" or string')
 524         return expr
 525
 526     def get_doc(self, info):
 527         if self.val != '##':
 528             raise QAPIParseError(self, "Junk after '##' at start of "
 529                                  "documentation comment")
 530
 531         doc = QAPIDoc(self, info)
 532         self.accept(False)
 533         while self.tok == '#':
 534             if self.val.startswith('##'):
 535                 # End of doc comment
 536                 if self.val != '##':
 537                     raise QAPIParseError(self, "Junk after '##' at end of "
 538                                          "documentation comment")
 539                 doc.end_comment()
 540                 self.accept()
 541                 return doc
 542             else:
 543                 doc.append(self.val)
 544             self.accept(False)
 545
 546         raise QAPIParseError(self, "Documentation comment must end with '##'")
 547
 548
 549 #
 550 # Semantic analysis of schema expressions
 551 # TODO fold into QAPISchema
 552 # TODO catching name collisions in generated code would be nice
 553 #
 554
 555
 556 def find_base_members(base):
 557     if isinstance(base, dict):
 558         return base
 559     base_struct_define = find_struct(base)
 560     if not base_struct_define:
 561         return None
 562     return base_struct_define['data']
 563
 564
 565 # Return the qtype of an alternate branch, or None on error.
 566 def find_alternate_member_qtype(qapi_type):
 567     if qapi_type in builtin_types:
 568         return builtin_types[qapi_type]
 569     elif find_struct(qapi_type):
 570         return 'QTYPE_QDICT'
 571     elif find_enum(qapi_type):
 572         return 'QTYPE_QSTRING'
 573     elif find_union(qapi_type):
 574         return 'QTYPE_QDICT'
 575     return None
 576
 577
 578 # Return the discriminator enum define if discriminator is specified as an
 579 # enum type, otherwise return None.
 580 def discriminator_find_enum_define(expr):
 581     base = expr.get('base')
 582     discriminator = expr.get('discriminator')
 583
 584     if not (discriminator and base):
 585         return None
 586
 587     base_members = find_base_members(base)
 588     if not base_members:
 589         return None
 590
 591     discriminator_type = base_members.get(discriminator)
 592     if not discriminator_type:
 593         return None
 594
 595     return find_enum(discriminator_type)
 596
 597
 598 # Names must be letters, numbers, -, and _.  They must start with letter,
 599 # except for downstream extensions which must start with __RFQDN_.
 600 # Dots are only valid in the downstream extension prefix.
 601 valid_name = re.compile(r'^(__[a-zA-Z0-9.-]+_)?'
 602                         '[a-zA-Z][a-zA-Z0-9_-]*$')
 603
 604
 605 def check_name(info, source, name, allow_optional=False,
 606                enum_member=False):
 607     global valid_name
 608     membername = name
 609
 610     if not isinstance(name, str):
 611         raise QAPISemError(info, "%s requires a string name" % source)
 612     if name.startswith('*'):
 613         membername = name[1:]
 614         if not allow_optional:
 615             raise QAPISemError(info, "%s does not allow optional name '%s'"
 616                                % (source, name))
 617     # Enum members can start with a digit, because the generated C
 618     # code always prefixes it with the enum name
 619     if enum_member and membername[0].isdigit():
 620         membername = 'D' + membername
 621     # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
 622     # and 'q_obj_*' implicit type names.
 623     if not valid_name.match(membername) or \
 624        c_name(membername, False).startswith('q_'):
 625         raise QAPISemError(info, "%s uses invalid name '%s'" % (source, name))
 626
 627
 628 def add_name(name, info, meta, implicit=False):
 629     global all_names
 630     check_name(info, "'%s'" % meta, name)
 631     # FIXME should reject names that differ only in '_' vs. '.'
 632     # vs. '-', because they're liable to clash in generated C.
 633     if name in all_names:
 634         raise QAPISemError(info, "%s '%s' is already defined"
 635                            % (all_names[name], name))
 636     if not implicit and (name.endswith('Kind') or name.endswith('List')):
 637         raise QAPISemError(info, "%s '%s' should not end in '%s'"
 638                            % (meta, name, name[-4:]))
 639     all_names[name] = meta
 640
 641
 642 def add_struct(definition, info):
 643     global struct_types
 644     name = definition['struct']
 645     add_name(name, info, 'struct')
 646     struct_types.append(definition)
 647
 648
 649 def find_struct(name):
 650     global struct_types
 651     for struct in struct_types:
 652         if struct['struct'] == name:
 653             return struct
 654     return None
 655
 656
 657 def add_union(definition, info):
 658     global union_types
 659     name = definition['union']
 660     add_name(name, info, 'union')
 661     union_types.append(definition)
 662
 663
 664 def find_union(name):
 665     global union_types
 666     for union in union_types:
 667         if union['union'] == name:
 668             return union
 669     return None
 670
 671
 672 def add_enum(name, info, enum_values=None, implicit=False):
 673     global enum_types
 674     add_name(name, info, 'enum', implicit)
 675     enum_types.append({'enum_name': name, 'enum_values': enum_values})
 676
 677
 678 def find_enum(name):
 679     global enum_types
 680     for enum in enum_types:
 681         if enum['enum_name'] == name:
 682             return enum
 683     return None
 684
 685
 686 def is_enum(name):
 687     return find_enum(name) is not None
 688
 689
 690 def check_type(info, source, value, allow_array=False,
 691                allow_dict=False, allow_optional=False,
 692                allow_metas=[]):
 693     global all_names
 694
 695     if value is None:
 696         return
 697
 698     # Check if array type for value is okay
 699     if isinstance(value, list):
 700         if not allow_array:
 701             raise QAPISemError(info, "%s cannot be an array" % source)
 702         if len(value) != 1 or not isinstance(value[0], str):
 703             raise QAPISemError(info,
 704                                "%s: array type must contain single type name" %
 705                                source)
 706         value = value[0]
 707
 708     # Check if type name for value is okay
 709     if isinstance(value, str):
 710         if value not in all_names:
 711             raise QAPISemError(info, "%s uses unknown type '%s'"
 712                                % (source, value))
 713         if not all_names[value] in allow_metas:
 714             raise QAPISemError(info, "%s cannot use %s type '%s'" %
 715                                (source, all_names[value], value))
 716         return
 717
 718     if not allow_dict:
 719         raise QAPISemError(info, "%s should be a type name" % source)
 720
 721     if not isinstance(value, OrderedDict):
 722         raise QAPISemError(info,
 723                            "%s should be a dictionary or type name" % source)
 724
 725     # value is a dictionary, check that each member is okay
 726     for (key, arg) in value.items():
 727         check_name(info, "Member of %s" % source, key,
 728                    allow_optional=allow_optional)
 729         if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
 730             raise QAPISemError(info, "Member of %s uses reserved name '%s'"
 731                                % (source, key))
 732         # Todo: allow dictionaries to represent default values of
 733         # an optional argument.
 734         check_type(info, "Member '%s' of %s" % (key, source), arg,
 735                    allow_array=True,
 736                    allow_metas=['built-in', 'union', 'alternate', 'struct',
 737                                 'enum'])
 738
 739
 740 def check_command(expr, info):
 741     name = expr['command']
 742     boxed = expr.get('boxed', False)
 743
 744     args_meta = ['struct']
 745     if boxed:
 746         args_meta += ['union', 'alternate']
 747     check_type(info, "'data' for command '%s'" % name,
 748                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 749                allow_metas=args_meta)
 750     returns_meta = ['union', 'struct']
 751     if name in returns_whitelist:
 752         returns_meta += ['built-in', 'alternate', 'enum']
 753     check_type(info, "'returns' for command '%s'" % name,
 754                expr.get('returns'), allow_array=True,
 755                allow_optional=True, allow_metas=returns_meta)
 756
 757
 758 def check_event(expr, info):
 759     global events
 760     name = expr['event']
 761     boxed = expr.get('boxed', False)
 762
 763     meta = ['struct']
 764     if boxed:
 765         meta += ['union', 'alternate']
 766     events.append(name)
 767     check_type(info, "'data' for event '%s'" % name,
 768                expr.get('data'), allow_dict=not boxed, allow_optional=True,
 769                allow_metas=meta)
 770
 771
 772 def check_union(expr, info):
 773     name = expr['union']
 774     base = expr.get('base')
 775     discriminator = expr.get('discriminator')
 776     members = expr['data']
 777
 778     # Two types of unions, determined by discriminator.
 779
 780     # With no discriminator it is a simple union.
 781     if discriminator is None:
 782         enum_define = None
 783         allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
 784         if base is not None:
 785             raise QAPISemError(info, "Simple union '%s' must not have a base" %
 786                                name)
 787
 788     # Else, it's a flat union.
 789     else:
 790         # The object must have a string or dictionary 'base'.
 791         check_type(info, "'base' for union '%s'" % name,
 792                    base, allow_dict=True, allow_optional=True,
 793                    allow_metas=['struct'])
 794         if not base:
 795             raise QAPISemError(info, "Flat union '%s' must have a base"
 796                                % name)
 797         base_members = find_base_members(base)
 798         assert base_members is not None
 799
 800         # The value of member 'discriminator' must name a non-optional
 801         # member of the base struct.
 802         check_name(info, "Discriminator of flat union '%s'" % name,
 803                    discriminator)
 804         discriminator_type = base_members.get(discriminator)
 805         if not discriminator_type:
 806             raise QAPISemError(info,
 807                                "Discriminator '%s' is not a member of base "
 808                                "struct '%s'"
 809                                % (discriminator, base))
 810         enum_define = find_enum(discriminator_type)
 811         allow_metas = ['struct']
 812         # Do not allow string discriminator
 813         if not enum_define:
 814             raise QAPISemError(info,
 815                                "Discriminator '%s' must be of enumeration "
 816                                "type" % discriminator)
 817
 818     # Check every branch; don't allow an empty union
 819     if len(members) == 0:
 820         raise QAPISemError(info, "Union '%s' cannot have empty 'data'" % name)
 821     for (key, value) in members.items():
 822         check_name(info, "Member of union '%s'" % name, key)
 823
 824         # Each value must name a known type
 825         check_type(info, "Member '%s' of union '%s'" % (key, name),
 826                    value, allow_array=not base, allow_metas=allow_metas)
 827
 828         # If the discriminator names an enum type, then all members
 829         # of 'data' must also be members of the enum type.
 830         if enum_define:
 831             if key not in enum_define['enum_values']:
 832                 raise QAPISemError(info,
 833                                    "Discriminator value '%s' is not found in "
 834                                    "enum '%s'"
 835                                    % (key, enum_define['enum_name']))
 836
 837     # If discriminator is user-defined, ensure all values are covered
 838     if enum_define:
 839         for value in enum_define['enum_values']:
 840             if value not in members.keys():
 841                 raise QAPISemError(info, "Union '%s' data missing '%s' branch"
 842                                    % (name, value))
 843
 844
 845 def check_alternate(expr, info):
 846     name = expr['alternate']
 847     members = expr['data']
 848     types_seen = {}
 849
 850     # Check every branch; require at least two branches
 851     if len(members) < 2:
 852         raise QAPISemError(info,
 853                            "Alternate '%s' should have at least two branches "
 854                            "in 'data'" % name)
 855     for (key, value) in members.items():
 856         check_name(info, "Member of alternate '%s'" % name, key)
 857
 858         # Ensure alternates have no type conflicts.
 859         check_type(info, "Member '%s' of alternate '%s'" % (key, name),
 860                    value,
 861                    allow_metas=['built-in', 'union', 'struct', 'enum'])
 862         qtype = find_alternate_member_qtype(value)
 863         if not qtype:
 864             raise QAPISemError(info, "Alternate '%s' member '%s' cannot use "
 865                                "type '%s'" % (name, key, value))
 866         if qtype in types_seen:
 867             raise QAPISemError(info, "Alternate '%s' member '%s' can't "
 868                                "be distinguished from member '%s'"
 869                                % (name, key, types_seen[qtype]))
 870         types_seen[qtype] = key
 871
 872
 873 def check_enum(expr, info):
 874     name = expr['enum']
 875     members = expr.get('data')
 876     prefix = expr.get('prefix')
 877
 878     if not isinstance(members, list):
 879         raise QAPISemError(info,
 880                            "Enum '%s' requires an array for 'data'" % name)
 881     if prefix is not None and not isinstance(prefix, str):
 882         raise QAPISemError(info,
 883                            "Enum '%s' requires a string for 'prefix'" % name)
 884     for member in members:
 885         check_name(info, "Member of enum '%s'" % name, member,
 886                    enum_member=True)
 887
 888
 889 def check_struct(expr, info):
 890     name = expr['struct']
 891     members = expr['data']
 892
 893     check_type(info, "'data' for struct '%s'" % name, members,
 894                allow_dict=True, allow_optional=True)
 895     check_type(info, "'base' for struct '%s'" % name, expr.get('base'),
 896                allow_metas=['struct'])
 897
 898
 899 def check_keys(expr_elem, meta, required, optional=[]):
 900     expr = expr_elem['expr']
 901     info = expr_elem['info']
 902     name = expr[meta]
 903     if not isinstance(name, str):
 904         raise QAPISemError(info, "'%s' key must have a string value" % meta)
 905     required = required + [meta]
 906     for (key, value) in expr.items():
 907         if key not in required and key not in optional:
 908             raise QAPISemError(info, "Unknown key '%s' in %s '%s'"
 909                                % (key, meta, name))
 910         if (key == 'gen' or key == 'success-response') and value is not False:
 911             raise QAPISemError(info,
 912                                "'%s' of %s '%s' should only use false value"
 913                                % (key, meta, name))
 914         if key == 'boxed' and value is not True:
 915             raise QAPISemError(info,
 916                                "'%s' of %s '%s' should only use true value"
 917                                % (key, meta, name))
 918     for key in required:
 919         if key not in expr:
 920             raise QAPISemError(info, "Key '%s' is missing from %s '%s'"
 921                                % (key, meta, name))
 922
 923
 924 def check_exprs(exprs):
 925     global all_names
 926
 927     # Populate name table with names of built-in types
 928     for builtin in builtin_types.keys():
 929         all_names[builtin] = 'built-in'
 930
 931     # Learn the types and check for valid expression keys
 932     for expr_elem in exprs:
 933         expr = expr_elem['expr']
 934         info = expr_elem['info']
 935         doc = expr_elem.get('doc')
 936
 937         if not doc and doc_required:
 938             raise QAPISemError(info,
 939                                "Expression missing documentation comment")
 940
 941         if 'enum' in expr:
 942             name = expr['enum']
 943             check_keys(expr_elem, 'enum', ['data'], ['prefix'])
 944             add_enum(name, info, expr['data'])
 945         elif 'union' in expr:
 946             name = expr['union']
 947             check_keys(expr_elem, 'union', ['data'],
 948                        ['base', 'discriminator'])
 949             add_union(expr, info)
 950         elif 'alternate' in expr:
 951             name = expr['alternate']
 952             check_keys(expr_elem, 'alternate', ['data'])
 953             add_name(name, info, 'alternate')
 954         elif 'struct' in expr:
 955             name = expr['struct']
 956             check_keys(expr_elem, 'struct', ['data'], ['base'])
 957             add_struct(expr, info)
 958         elif 'command' in expr:
 959             name = expr['command']
 960             check_keys(expr_elem, 'command', [],
 961                        ['data', 'returns', 'gen', 'success-response', 'boxed'])
 962             add_name(name, info, 'command')
 963         elif 'event' in expr:
 964             name = expr['event']
 965             check_keys(expr_elem, 'event', [], ['data', 'boxed'])
 966             add_name(name, info, 'event')
 967         else:
 968             raise QAPISemError(expr_elem['info'],
 969                                "Expression is missing metatype")
 970         if doc and doc.symbol != name:
 971             raise QAPISemError(info, "Definition of '%s' follows documentation"
 972                                " for '%s'" % (name, doc.symbol))
 973
 974     # Try again for hidden UnionKind enum
 975     for expr_elem in exprs:
 976         expr = expr_elem['expr']
 977         if 'union' in expr:
 978             if not discriminator_find_enum_define(expr):
 979                 add_enum('%sKind' % expr['union'], expr_elem['info'],
 980                          implicit=True)
 981         elif 'alternate' in expr:
 982             add_enum('%sKind' % expr['alternate'], expr_elem['info'],
 983                      implicit=True)
 984
 985     # Validate that exprs make sense
 986     for expr_elem in exprs:
 987         expr = expr_elem['expr']
 988         info = expr_elem['info']
 989         doc = expr_elem.get('doc')
 990
 991         if 'enum' in expr:
 992             check_enum(expr, info)
 993         elif 'union' in expr:
 994             check_union(expr, info)
 995         elif 'alternate' in expr:
 996             check_alternate(expr, info)
 997         elif 'struct' in expr:
 998             check_struct(expr, info)
 999         elif 'command' in expr:
1000             check_command(expr, info)
1001         elif 'event' in expr:
1002             check_event(expr, info)
1003         else:
1004             assert False, 'unexpected meta type'
1005
1006         if doc:
1007             doc.check_expr(expr)
1008
1009     return exprs
1010
1011
1012 #
1013 # Schema compiler frontend
1014 #
1015
1016 class QAPISchemaEntity(object):
1017     def __init__(self, name, info, doc):
1018         assert isinstance(name, str)
1019         self.name = name
1020         # For explicitly defined entities, info points to the (explicit)
1021         # definition.  For builtins (and their arrays), info is None.
1022         # For implicitly defined entities, info points to a place that
1023         # triggered the implicit definition (there may be more than one
1024         # such place).
1025         self.info = info
1026         self.doc = doc
1027
1028     def c_name(self):
1029         return c_name(self.name)
1030
1031     def check(self, schema):
1032         pass
1033
1034     def is_implicit(self):
1035         return not self.info
1036
1037     def visit(self, visitor):
1038         pass
1039
1040
1041 class QAPISchemaVisitor(object):
1042     def visit_begin(self, schema):
1043         pass
1044
1045     def visit_end(self):
1046         pass
1047
1048     def visit_needed(self, entity):
1049         # Default to visiting everything
1050         return True
1051
1052     def visit_builtin_type(self, name, info, json_type):
1053         pass
1054
1055     def visit_enum_type(self, name, info, values, prefix):
1056         pass
1057
1058     def visit_array_type(self, name, info, element_type):
1059         pass
1060
1061     def visit_object_type(self, name, info, base, members, variants):
1062         pass
1063
1064     def visit_object_type_flat(self, name, info, members, variants):
1065         pass
1066
1067     def visit_alternate_type(self, name, info, variants):
1068         pass
1069
1070     def visit_command(self, name, info, arg_type, ret_type,
1071                       gen, success_response, boxed):
1072         pass
1073
1074     def visit_event(self, name, info, arg_type, boxed):
1075         pass
1076
1077
1078 class QAPISchemaType(QAPISchemaEntity):
1079     # Return the C type for common use.
1080     # For the types we commonly box, this is a pointer type.
1081     def c_type(self):
1082         pass
1083
1084     # Return the C type to be used in a parameter list.
1085     def c_param_type(self):
1086         return self.c_type()
1087
1088     # Return the C type to be used where we suppress boxing.
1089     def c_unboxed_type(self):
1090         return self.c_type()
1091
1092     def json_type(self):
1093         pass
1094
1095     def alternate_qtype(self):
1096         json2qtype = {
1097             'string':  'QTYPE_QSTRING',
1098             'number':  'QTYPE_QFLOAT',
1099             'int':     'QTYPE_QINT',
1100             'boolean': 'QTYPE_QBOOL',
1101             'object':  'QTYPE_QDICT'
1102         }
1103         return json2qtype.get(self.json_type())
1104
1105     def doc_type(self):
1106         if self.is_implicit():
1107             return None
1108         return self.name
1109
1110
1111 class QAPISchemaBuiltinType(QAPISchemaType):
1112     def __init__(self, name, json_type, c_type):
1113         QAPISchemaType.__init__(self, name, None, None)
1114         assert not c_type or isinstance(c_type, str)
1115         assert json_type in ('string', 'number', 'int', 'boolean', 'null',
1116                              'value')
1117         self._json_type_name = json_type
1118         self._c_type_name = c_type
1119
1120     def c_name(self):
1121         return self.name
1122
1123     def c_type(self):
1124         return self._c_type_name
1125
1126     def c_param_type(self):
1127         if self.name == 'str':
1128             return 'const ' + self._c_type_name
1129         return self._c_type_name
1130
1131     def json_type(self):
1132         return self._json_type_name
1133
1134     def doc_type(self):
1135         return self.json_type()
1136
1137     def visit(self, visitor):
1138         visitor.visit_builtin_type(self.name, self.info, self.json_type())
1139
1140
1141 class QAPISchemaEnumType(QAPISchemaType):
1142     def __init__(self, name, info, doc, values, prefix):
1143         QAPISchemaType.__init__(self, name, info, doc)
1144         for v in values:
1145             assert isinstance(v, QAPISchemaMember)
1146             v.set_owner(name)
1147         assert prefix is None or isinstance(prefix, str)
1148         self.values = values
1149         self.prefix = prefix
1150
1151     def check(self, schema):
1152         seen = {}
1153         for v in self.values:
1154             v.check_clash(self.info, seen)
1155             if self.doc:
1156                 self.doc.connect_member(v)
1157
1158     def is_implicit(self):
1159         # See QAPISchema._make_implicit_enum_type() and ._def_predefineds()
1160         return self.name.endswith('Kind') or self.name == 'QType'
1161
1162     def c_type(self):
1163         return c_name(self.name)
1164
1165     def member_names(self):
1166         return [v.name for v in self.values]
1167
1168     def json_type(self):
1169         return 'string'
1170
1171     def visit(self, visitor):
1172         visitor.visit_enum_type(self.name, self.info,
1173                                 self.member_names(), self.prefix)
1174
1175
1176 class QAPISchemaArrayType(QAPISchemaType):
1177     def __init__(self, name, info, element_type):
1178         QAPISchemaType.__init__(self, name, info, None)
1179         assert isinstance(element_type, str)
1180         self._element_type_name = element_type
1181         self.element_type = None
1182
1183     def check(self, schema):
1184         self.element_type = schema.lookup_type(self._element_type_name)
1185         assert self.element_type
1186
1187     def is_implicit(self):
1188         return True
1189
1190     def c_type(self):
1191         return c_name(self.name) + pointer_suffix
1192
1193     def json_type(self):
1194         return 'array'
1195
1196     def doc_type(self):
1197         elt_doc_type = self.element_type.doc_type()
1198         if not elt_doc_type:
1199             return None
1200         return 'array of ' + elt_doc_type
1201
1202     def visit(self, visitor):
1203         visitor.visit_array_type(self.name, self.info, self.element_type)
1204
1205
1206 class QAPISchemaObjectType(QAPISchemaType):
1207     def __init__(self, name, info, doc, base, local_members, variants):
1208         # struct has local_members, optional base, and no variants
1209         # flat union has base, variants, and no local_members
1210         # simple union has local_members, variants, and no base
1211         QAPISchemaType.__init__(self, name, info, doc)
1212         assert base is None or isinstance(base, str)
1213         for m in local_members:
1214             assert isinstance(m, QAPISchemaObjectTypeMember)
1215             m.set_owner(name)
1216         if variants is not None:
1217             assert isinstance(variants, QAPISchemaObjectTypeVariants)
1218             variants.set_owner(name)
1219         self._base_name = base
1220         self.base = None
1221         self.local_members = local_members
1222         self.variants = variants
1223         self.members = None
1224
1225     def check(self, schema):
1226         if self.members is False:               # check for cycles
1227             raise QAPISemError(self.info,
1228                                "Object %s contains itself" % self.name)
1229         if self.members:
1230             return
1231         self.members = False                    # mark as being checked
1232         seen = OrderedDict()
1233         if self._base_name:
1234             self.base = schema.lookup_type(self._base_name)
1235             assert isinstance(self.base, QAPISchemaObjectType)
1236             self.base.check(schema)
1237             self.base.check_clash(schema, self.info, seen)
1238         for m in self.local_members:
1239             m.check(schema)
1240             m.check_clash(self.info, seen)
1241             if self.doc:
1242                 self.doc.connect_member(m)
1243         self.members = seen.values()
1244         if self.variants:
1245             self.variants.check(schema, seen)
1246             assert self.variants.tag_member in self.members
1247             self.variants.check_clash(schema, self.info, seen)
1248         if self.doc:
1249             self.doc.check()
1250
1251     # Check that the members of this type do not cause duplicate JSON members,
1252     # and update seen to track the members seen so far. Report any errors
1253     # on behalf of info, which is not necessarily self.info
1254     def check_clash(self, schema, info, seen):
1255         assert not self.variants       # not implemented
1256         for m in self.members:
1257             m.check_clash(info, seen)
1258
1259     def is_implicit(self):
1260         # See QAPISchema._make_implicit_object_type(), as well as
1261         # _def_predefineds()
1262         return self.name.startswith('q_')
1263
1264     def is_empty(self):
1265         assert self.members is not None
1266         return not self.members and not self.variants
1267
1268     def c_name(self):
1269         assert self.name != 'q_empty'
1270         return QAPISchemaType.c_name(self)
1271
1272     def c_type(self):
1273         assert not self.is_implicit()
1274         return c_name(self.name) + pointer_suffix
1275
1276     def c_unboxed_type(self):
1277         return c_name(self.name)
1278
1279     def json_type(self):
1280         return 'object'
1281
1282     def visit(self, visitor):
1283         visitor.visit_object_type(self.name, self.info,
1284                                   self.base, self.local_members, self.variants)
1285         visitor.visit_object_type_flat(self.name, self.info,
1286                                        self.members, self.variants)
1287
1288
1289 class QAPISchemaMember(object):
1290     role = 'member'
1291
1292     def __init__(self, name):
1293         assert isinstance(name, str)
1294         self.name = name
1295         self.owner = None
1296
1297     def set_owner(self, name):
1298         assert not self.owner
1299         self.owner = name
1300
1301     def check_clash(self, info, seen):
1302         cname = c_name(self.name)
1303         if cname.lower() != cname and self.owner not in name_case_whitelist:
1304             raise QAPISemError(info,
1305                                "%s should not use uppercase" % self.describe())
1306         if cname in seen:
1307             raise QAPISemError(info, "%s collides with %s" %
1308                                (self.describe(), seen[cname].describe()))
1309         seen[cname] = self
1310
1311     def _pretty_owner(self):
1312         owner = self.owner
1313         if owner.startswith('q_obj_'):
1314             # See QAPISchema._make_implicit_object_type() - reverse the
1315             # mapping there to create a nice human-readable description
1316             owner = owner[6:]
1317             if owner.endswith('-arg'):
1318                 return '(parameter of %s)' % owner[:-4]
1319             elif owner.endswith('-base'):
1320                 return '(base of %s)' % owner[:-5]
1321             else:
1322                 assert owner.endswith('-wrapper')
1323                 # Unreachable and not implemented
1324                 assert False
1325         if owner.endswith('Kind'):
1326             # See QAPISchema._make_implicit_enum_type()
1327             return '(branch of %s)' % owner[:-4]
1328         return '(%s of %s)' % (self.role, owner)
1329
1330     def describe(self):
1331         return "'%s' %s" % (self.name, self._pretty_owner())
1332
1333
1334 class QAPISchemaObjectTypeMember(QAPISchemaMember):
1335     def __init__(self, name, typ, optional):
1336         QAPISchemaMember.__init__(self, name)
1337         assert isinstance(typ, str)
1338         assert isinstance(optional, bool)
1339         self._type_name = typ
1340         self.type = None
1341         self.optional = optional
1342
1343     def check(self, schema):
1344         assert self.owner
1345         self.type = schema.lookup_type(self._type_name)
1346         assert self.type
1347
1348
1349 class QAPISchemaObjectTypeVariants(object):
1350     def __init__(self, tag_name, tag_member, variants):
1351         # Flat unions pass tag_name but not tag_member.
1352         # Simple unions and alternates pass tag_member but not tag_name.
1353         # After check(), tag_member is always set, and tag_name remains
1354         # a reliable witness of being used by a flat union.
1355         assert bool(tag_member) != bool(tag_name)
1356         assert (isinstance(tag_name, str) or
1357                 isinstance(tag_member, QAPISchemaObjectTypeMember))
1358         assert len(variants) > 0
1359         for v in variants:
1360             assert isinstance(v, QAPISchemaObjectTypeVariant)
1361         self._tag_name = tag_name
1362         self.tag_member = tag_member
1363         self.variants = variants
1364
1365     def set_owner(self, name):
1366         for v in self.variants:
1367             v.set_owner(name)
1368
1369     def check(self, schema, seen):
1370         if not self.tag_member:    # flat union
1371             self.tag_member = seen[c_name(self._tag_name)]
1372             assert self._tag_name == self.tag_member.name
1373         assert isinstance(self.tag_member.type, QAPISchemaEnumType)
1374         for v in self.variants:
1375             v.check(schema)
1376             # Union names must match enum values; alternate names are
1377             # checked separately. Use 'seen' to tell the two apart.
1378             if seen:
1379                 assert v.name in self.tag_member.type.member_names()
1380                 assert isinstance(v.type, QAPISchemaObjectType)
1381                 v.type.check(schema)
1382
1383     def check_clash(self, schema, info, seen):
1384         for v in self.variants:
1385             # Reset seen map for each variant, since qapi names from one
1386             # branch do not affect another branch
1387             assert isinstance(v.type, QAPISchemaObjectType)
1388             v.type.check_clash(schema, info, dict(seen))
1389
1390
1391 class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
1392     role = 'branch'
1393
1394     def __init__(self, name, typ):
1395         QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
1396
1397
1398 class QAPISchemaAlternateType(QAPISchemaType):
1399     def __init__(self, name, info, doc, variants):
1400         QAPISchemaType.__init__(self, name, info, doc)
1401         assert isinstance(variants, QAPISchemaObjectTypeVariants)
1402         assert variants.tag_member
1403         variants.set_owner(name)
1404         variants.tag_member.set_owner(self.name)
1405         self.variants = variants
1406
1407     def check(self, schema):
1408         self.variants.tag_member.check(schema)
1409         # Not calling self.variants.check_clash(), because there's nothing
1410         # to clash with
1411         self.variants.check(schema, {})
1412         # Alternate branch names have no relation to the tag enum values;
1413         # so we have to check for potential name collisions ourselves.
1414         seen = {}
1415         for v in self.variants.variants:
1416             v.check_clash(self.info, seen)
1417             if self.doc:
1418                 self.doc.connect_member(v)
1419         if self.doc:
1420             self.doc.check()
1421
1422     def c_type(self):
1423         return c_name(self.name) + pointer_suffix
1424
1425     def json_type(self):
1426         return 'value'
1427
1428     def visit(self, visitor):
1429         visitor.visit_alternate_type(self.name, self.info, self.variants)
1430
1431     def is_empty(self):
1432         return False
1433
1434
1435 class QAPISchemaCommand(QAPISchemaEntity):
1436     def __init__(self, name, info, doc, arg_type, ret_type,
1437                  gen, success_response, boxed):
1438         QAPISchemaEntity.__init__(self, name, info, doc)
1439         assert not arg_type or isinstance(arg_type, str)
1440         assert not ret_type or isinstance(ret_type, str)
1441         self._arg_type_name = arg_type
1442         self.arg_type = None
1443         self._ret_type_name = ret_type
1444         self.ret_type = None
1445         self.gen = gen
1446         self.success_response = success_response
1447         self.boxed = boxed
1448
1449     def check(self, schema):
1450         if self._arg_type_name:
1451             self.arg_type = schema.lookup_type(self._arg_type_name)
1452             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1453                     isinstance(self.arg_type, QAPISchemaAlternateType))
1454             self.arg_type.check(schema)
1455             if self.boxed:
1456                 if self.arg_type.is_empty():
1457                     raise QAPISemError(self.info,
1458                                        "Cannot use 'boxed' with empty type")
1459             else:
1460                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1461                 assert not self.arg_type.variants
1462         elif self.boxed:
1463             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1464         if self._ret_type_name:
1465             self.ret_type = schema.lookup_type(self._ret_type_name)
1466             assert isinstance(self.ret_type, QAPISchemaType)
1467
1468     def visit(self, visitor):
1469         visitor.visit_command(self.name, self.info,
1470                               self.arg_type, self.ret_type,
1471                               self.gen, self.success_response, self.boxed)
1472
1473
1474 class QAPISchemaEvent(QAPISchemaEntity):
1475     def __init__(self, name, info, doc, arg_type, boxed):
1476         QAPISchemaEntity.__init__(self, name, info, doc)
1477         assert not arg_type or isinstance(arg_type, str)
1478         self._arg_type_name = arg_type
1479         self.arg_type = None
1480         self.boxed = boxed
1481
1482     def check(self, schema):
1483         if self._arg_type_name:
1484             self.arg_type = schema.lookup_type(self._arg_type_name)
1485             assert (isinstance(self.arg_type, QAPISchemaObjectType) or
1486                     isinstance(self.arg_type, QAPISchemaAlternateType))
1487             self.arg_type.check(schema)
1488             if self.boxed:
1489                 if self.arg_type.is_empty():
1490                     raise QAPISemError(self.info,
1491                                        "Cannot use 'boxed' with empty type")
1492             else:
1493                 assert not isinstance(self.arg_type, QAPISchemaAlternateType)
1494                 assert not self.arg_type.variants
1495         elif self.boxed:
1496             raise QAPISemError(self.info, "Use of 'boxed' requires 'data'")
1497
1498     def visit(self, visitor):
1499         visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
1500
1501
1502 class QAPISchema(object):
1503     def __init__(self, fname):
1504         try:
1505             parser = QAPISchemaParser(open(fname, 'r'))
1506             self.exprs = check_exprs(parser.exprs)
1507             self.docs = parser.docs
1508             self._entity_dict = {}
1509             self._predefining = True
1510             self._def_predefineds()
1511             self._predefining = False
1512             self._def_exprs()
1513             self.check()
1514         except QAPIError as err:
1515             print >>sys.stderr, err
1516             exit(1)
1517
1518     def _def_entity(self, ent):
1519         # Only the predefined types are allowed to not have info
1520         assert ent.info or self._predefining
1521         assert ent.name not in self._entity_dict
1522         self._entity_dict[ent.name] = ent
1523
1524     def lookup_entity(self, name, typ=None):
1525         ent = self._entity_dict.get(name)
1526         if typ and not isinstance(ent, typ):
1527             return None
1528         return ent
1529
1530     def lookup_type(self, name):
1531         return self.lookup_entity(name, QAPISchemaType)
1532
1533     def _def_builtin_type(self, name, json_type, c_type):
1534         self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
1535         # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
1536         # qapi-types.h from a single .c, all arrays of builtins must be
1537         # declared in the first file whether or not they are used.  Nicer
1538         # would be to use lazy instantiation, while figuring out how to
1539         # avoid compilation issues with multiple qapi-types.h.
1540         self._make_array_type(name, None)
1541
1542     def _def_predefineds(self):
1543         for t in [('str',    'string',  'char' + pointer_suffix),
1544                   ('number', 'number',  'double'),
1545                   ('int',    'int',     'int64_t'),
1546                   ('int8',   'int',     'int8_t'),
1547                   ('int16',  'int',     'int16_t'),
1548                   ('int32',  'int',     'int32_t'),
1549                   ('int64',  'int',     'int64_t'),
1550                   ('uint8',  'int',     'uint8_t'),
1551                   ('uint16', 'int',     'uint16_t'),
1552                   ('uint32', 'int',     'uint32_t'),
1553                   ('uint64', 'int',     'uint64_t'),
1554                   ('size',   'int',     'uint64_t'),
1555                   ('bool',   'boolean', 'bool'),
1556                   ('any',    'value',   'QObject' + pointer_suffix)]:
1557             self._def_builtin_type(*t)
1558         self.the_empty_object_type = QAPISchemaObjectType(
1559             'q_empty', None, None, None, [], None)
1560         self._def_entity(self.the_empty_object_type)
1561         qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
1562                                                 'qstring', 'qdict', 'qlist',
1563                                                 'qfloat', 'qbool'])
1564         self._def_entity(QAPISchemaEnumType('QType', None, None,
1565                                             qtype_values, 'QTYPE'))
1566
1567     def _make_enum_members(self, values):
1568         return [QAPISchemaMember(v) for v in values]
1569
1570     def _make_implicit_enum_type(self, name, info, values):
1571         # See also QAPISchemaObjectTypeMember._pretty_owner()
1572         name = name + 'Kind'   # Use namespace reserved by add_name()
1573         self._def_entity(QAPISchemaEnumType(
1574             name, info, None, self._make_enum_members(values), None))
1575         return name
1576
1577     def _make_array_type(self, element_type, info):
1578         name = element_type + 'List'   # Use namespace reserved by add_name()
1579         if not self.lookup_type(name):
1580             self._def_entity(QAPISchemaArrayType(name, info, element_type))
1581         return name
1582
1583     def _make_implicit_object_type(self, name, info, doc, role, members):
1584         if not members:
1585             return None
1586         # See also QAPISchemaObjectTypeMember._pretty_owner()
1587         name = 'q_obj_%s-%s' % (name, role)
1588         if not self.lookup_entity(name, QAPISchemaObjectType):
1589             self._def_entity(QAPISchemaObjectType(name, info, doc, None,
1590                                                   members, None))
1591         return name
1592
1593     def _def_enum_type(self, expr, info, doc):
1594         name = expr['enum']
1595         data = expr['data']
1596         prefix = expr.get('prefix')
1597         self._def_entity(QAPISchemaEnumType(
1598             name, info, doc, self._make_enum_members(data), prefix))
1599
1600     def _make_member(self, name, typ, info):
1601         optional = False
1602         if name.startswith('*'):
1603             name = name[1:]
1604             optional = True
1605         if isinstance(typ, list):
1606             assert len(typ) == 1
1607             typ = self._make_array_type(typ[0], info)
1608         return QAPISchemaObjectTypeMember(name, typ, optional)
1609
1610     def _make_members(self, data, info):
1611         return [self._make_member(key, value, info)
1612                 for (key, value) in data.iteritems()]
1613
1614     def _def_struct_type(self, expr, info, doc):
1615         name = expr['struct']
1616         base = expr.get('base')
1617         data = expr['data']
1618         self._def_entity(QAPISchemaObjectType(name, info, doc, base,
1619                                               self._make_members(data, info),
1620                                               None))
1621
1622     def _make_variant(self, case, typ):
1623         return QAPISchemaObjectTypeVariant(case, typ)
1624
1625     def _make_simple_variant(self, case, typ, info):
1626         if isinstance(typ, list):
1627             assert len(typ) == 1
1628             typ = self._make_array_type(typ[0], info)
1629         typ = self._make_implicit_object_type(
1630             typ, info, None, 'wrapper', [self._make_member('data', typ, info)])
1631         return QAPISchemaObjectTypeVariant(case, typ)
1632
1633     def _def_union_type(self, expr, info, doc):
1634         name = expr['union']
1635         data = expr['data']
1636         base = expr.get('base')
1637         tag_name = expr.get('discriminator')
1638         tag_member = None
1639         if isinstance(base, dict):
1640             base = (self._make_implicit_object_type(
1641                     name, info, doc, 'base', self._make_members(base, info)))
1642         if tag_name:
1643             variants = [self._make_variant(key, value)
1644                         for (key, value) in data.iteritems()]
1645             members = []
1646         else:
1647             variants = [self._make_simple_variant(key, value, info)
1648                         for (key, value) in data.iteritems()]
1649             typ = self._make_implicit_enum_type(name, info,
1650                                                 [v.name for v in variants])
1651             tag_member = QAPISchemaObjectTypeMember('type', typ, False)
1652             members = [tag_member]
1653         self._def_entity(
1654             QAPISchemaObjectType(name, info, doc, base, members,
1655                                  QAPISchemaObjectTypeVariants(tag_name,
1656                                                               tag_member,
1657                                                               variants)))
1658
1659     def _def_alternate_type(self, expr, info, doc):
1660         name = expr['alternate']
1661         data = expr['data']
1662         variants = [self._make_variant(key, value)
1663                     for (key, value) in data.iteritems()]
1664         tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
1665         self._def_entity(
1666             QAPISchemaAlternateType(name, info, doc,
1667                                     QAPISchemaObjectTypeVariants(None,
1668                                                                  tag_member,
1669                                                                  variants)))
1670
1671     def _def_command(self, expr, info, doc):
1672         name = expr['command']
1673         data = expr.get('data')
1674         rets = expr.get('returns')
1675         gen = expr.get('gen', True)
1676         success_response = expr.get('success-response', True)
1677         boxed = expr.get('boxed', False)
1678         if isinstance(data, OrderedDict):
1679             data = self._make_implicit_object_type(
1680                 name, info, doc, 'arg', self._make_members(data, info))
1681         if isinstance(rets, list):
1682             assert len(rets) == 1
1683             rets = self._make_array_type(rets[0], info)
1684         self._def_entity(QAPISchemaCommand(name, info, doc, data, rets,
1685                                            gen, success_response, boxed))
1686
1687     def _def_event(self, expr, info, doc):
1688         name = expr['event']
1689         data = expr.get('data')
1690         boxed = expr.get('boxed', False)
1691         if isinstance(data, OrderedDict):
1692             data = self._make_implicit_object_type(
1693                 name, info, doc, 'arg', self._make_members(data, info))
1694         self._def_entity(QAPISchemaEvent(name, info, doc, data, boxed))
1695
1696     def _def_exprs(self):
1697         for expr_elem in self.exprs:
1698             expr = expr_elem['expr']
1699             info = expr_elem['info']
1700             doc = expr_elem.get('doc')
1701             if 'enum' in expr:
1702                 self._def_enum_type(expr, info, doc)
1703             elif 'struct' in expr:
1704                 self._def_struct_type(expr, info, doc)
1705             elif 'union' in expr:
1706                 self._def_union_type(expr, info, doc)
1707             elif 'alternate' in expr:
1708                 self._def_alternate_type(expr, info, doc)
1709             elif 'command' in expr:
1710                 self._def_command(expr, info, doc)
1711             elif 'event' in expr:
1712                 self._def_event(expr, info, doc)
1713             else:
1714                 assert False
1715
1716     def check(self):
1717         for ent in self._entity_dict.values():
1718             ent.check(self)
1719
1720     def visit(self, visitor):
1721         visitor.visit_begin(self)
1722         for (name, entity) in sorted(self._entity_dict.items()):
1723             if visitor.visit_needed(entity):
1724                 entity.visit(visitor)
1725         visitor.visit_end()
1726
1727
1728 #
1729 # Code generation helpers
1730 #
1731
1732 def camel_case(name):
1733     new_name = ''
1734     first = True
1735     for ch in name:
1736         if ch in ['_', '-']:
1737             first = True
1738         elif first:
1739             new_name += ch.upper()
1740             first = False
1741         else:
1742             new_name += ch.lower()
1743     return new_name
1744
1745
1746 # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
1747 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
1748 # ENUM24_Name -> ENUM24_NAME
1749 def camel_to_upper(value):
1750     c_fun_str = c_name(value, False)
1751     if value.isupper():
1752         return c_fun_str
1753
1754     new_name = ''
1755     l = len(c_fun_str)
1756     for i in range(l):
1757         c = c_fun_str[i]
1758         # When c is upper and no '_' appears before, do more checks
1759         if c.isupper() and (i > 0) and c_fun_str[i - 1] != '_':
1760             if i < l - 1 and c_fun_str[i + 1].islower():
1761                 new_name += '_'
1762             elif c_fun_str[i - 1].isdigit():
1763                 new_name += '_'
1764         new_name += c
1765     return new_name.lstrip('_').upper()
1766
1767
1768 def c_enum_const(type_name, const_name, prefix=None):
1769     if prefix is not None:
1770         type_name = prefix
1771     return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
1772
1773 c_name_trans = string.maketrans('.-', '__')
1774
1775
1776 # Map @name to a valid C identifier.
1777 # If @protect, avoid returning certain ticklish identifiers (like
1778 # C keywords) by prepending 'q_'.
1779 #
1780 # Used for converting 'name' from a 'name':'type' qapi definition
1781 # into a generated struct member, as well as converting type names
1782 # into substrings of a generated C function name.
1783 # '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
1784 # protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
1785 def c_name(name, protect=True):
1786     # ANSI X3J11/88-090, 3.1.1
1787     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
1788                      'default', 'do', 'double', 'else', 'enum', 'extern',
1789                      'float', 'for', 'goto', 'if', 'int', 'long', 'register',
1790                      'return', 'short', 'signed', 'sizeof', 'static',
1791                      'struct', 'switch', 'typedef', 'union', 'unsigned',
1792                      'void', 'volatile', 'while'])
1793     # ISO/IEC 9899:1999, 6.4.1
1794     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
1795     # ISO/IEC 9899:2011, 6.4.1
1796     c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
1797                      '_Noreturn', '_Static_assert', '_Thread_local'])
1798     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
1799     # excluding _.*
1800     gcc_words = set(['asm', 'typeof'])
1801     # C++ ISO/IEC 14882:2003 2.11
1802     cpp_words = set(['bool', 'catch', 'class', 'const_cast', 'delete',
1803                      'dynamic_cast', 'explicit', 'false', 'friend', 'mutable',
1804                      'namespace', 'new', 'operator', 'private', 'protected',
1805                      'public', 'reinterpret_cast', 'static_cast', 'template',
1806                      'this', 'throw', 'true', 'try', 'typeid', 'typename',
1807                      'using', 'virtual', 'wchar_t',
1808                      # alternative representations
1809                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
1810                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
1811     # namespace pollution:
1812     polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
1813     name = name.translate(c_name_trans)
1814     if protect and (name in c89_words | c99_words | c11_words | gcc_words
1815                     | cpp_words | polluted_words):
1816         return 'q_' + name
1817     return name
1818
1819 eatspace = '\033EATSPACE.'
1820 pointer_suffix = ' *' + eatspace
1821
1822
1823 def genindent(count):
1824     ret = ''
1825     for _ in range(count):
1826         ret += ' '
1827     return ret
1828
1829 indent_level = 0
1830
1831
1832 def push_indent(indent_amount=4):
1833     global indent_level
1834     indent_level += indent_amount
1835
1836
1837 def pop_indent(indent_amount=4):
1838     global indent_level
1839     indent_level -= indent_amount
1840
1841
1842 # Generate @code with @kwds interpolated.
1843 # Obey indent_level, and strip eatspace.
1844 def cgen(code, **kwds):
1845     raw = code % kwds
1846     if indent_level:
1847         indent = genindent(indent_level)
1848         # re.subn() lacks flags support before Python 2.7, use re.compile()
1849         raw = re.subn(re.compile(r'^.', re.MULTILINE),
1850                       indent + r'\g<0>', raw)
1851         raw = raw[0]
1852     return re.sub(re.escape(eatspace) + r' *', '', raw)
1853
1854
1855 def mcgen(code, **kwds):
1856     if code[0] == '\n':
1857         code = code[1:]
1858     return cgen(code, **kwds)
1859
1860
1861 def guardname(filename):
1862     return c_name(filename, protect=False).upper()
1863
1864
1865 def guardstart(name):
1866     return mcgen('''
1867
1868 #ifndef %(name)s
1869 #define %(name)s
1870
1871 ''',
1872                  name=guardname(name))
1873
1874
1875 def guardend(name):
1876     return mcgen('''
1877
1878 #endif /* %(name)s */
1879
1880 ''',
1881                  name=guardname(name))
1882
1883
1884 def gen_enum_lookup(name, values, prefix=None):
1885     ret = mcgen('''
1886
1887 const char *const %(c_name)s_lookup[] = {
1888 ''',
1889                 c_name=c_name(name))
1890     for value in values:
1891         index = c_enum_const(name, value, prefix)
1892         ret += mcgen('''
1893     [%(index)s] = "%(value)s",
1894 ''',
1895                      index=index, value=value)
1896
1897     max_index = c_enum_const(name, '_MAX', prefix)
1898     ret += mcgen('''
1899     [%(max_index)s] = NULL,
1900 };
1901 ''',
1902                  max_index=max_index)
1903     return ret
1904
1905
1906 def gen_enum(name, values, prefix=None):
1907     # append automatically generated _MAX value
1908     enum_values = values + ['_MAX']
1909
1910     ret = mcgen('''
1911
1912 typedef enum %(c_name)s {
1913 ''',
1914                 c_name=c_name(name))
1915
1916     i = 0
1917     for value in enum_values:
1918         ret += mcgen('''
1919     %(c_enum)s = %(i)d,
1920 ''',
1921                      c_enum=c_enum_const(name, value, prefix),
1922                      i=i)
1923         i += 1
1924
1925     ret += mcgen('''
1926 } %(c_name)s;
1927 ''',
1928                  c_name=c_name(name))
1929
1930     ret += mcgen('''
1931
1932 extern const char *const %(c_name)s_lookup[];
1933 ''',
1934                  c_name=c_name(name))
1935     return ret
1936
1937
1938 def gen_params(arg_type, boxed, extra):
1939     if not arg_type:
1940         assert not boxed
1941         return extra
1942     ret = ''
1943     sep = ''
1944     if boxed:
1945         ret += '%s arg' % arg_type.c_param_type()
1946         sep = ', '
1947     else:
1948         assert not arg_type.variants
1949         for memb in arg_type.members:
1950             ret += sep
1951             sep = ', '
1952             if memb.optional:
1953                 ret += 'bool has_%s, ' % c_name(memb.name)
1954             ret += '%s %s' % (memb.type.c_param_type(),
1955                               c_name(memb.name))
1956     if extra:
1957         ret += sep + extra
1958     return ret
1959
1960
1961 #
1962 # Common command line parsing
1963 #
1964
1965
1966 def parse_command_line(extra_options='', extra_long_options=[]):
1967
1968     try:
1969         opts, args = getopt.gnu_getopt(sys.argv[1:],
1970                                        'chp:o:' + extra_options,
1971                                        ['source', 'header', 'prefix=',
1972                                         'output-dir='] + extra_long_options)
1973     except getopt.GetoptError as err:
1974         print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
1975         sys.exit(1)
1976
1977     output_dir = ''
1978     prefix = ''
1979     do_c = False
1980     do_h = False
1981     extra_opts = []
1982
1983     for oa in opts:
1984         o, a = oa
1985         if o in ('-p', '--prefix'):
1986             match = re.match(r'([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
1987             if match.end() != len(a):
1988                 print >>sys.stderr, \
1989                     "%s: 'funny character '%s' in argument of --prefix" \
1990                     % (sys.argv[0], a[match.end()])
1991                 sys.exit(1)
1992             prefix = a
1993         elif o in ('-o', '--output-dir'):
1994             output_dir = a + '/'
1995         elif o in ('-c', '--source'):
1996             do_c = True
1997         elif o in ('-h', '--header'):
1998             do_h = True
1999         else:
2000             extra_opts.append(oa)
2001
2002     if not do_c and not do_h:
2003         do_c = True
2004         do_h = True
2005
2006     if len(args) != 1:
2007         print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
2008         sys.exit(1)
2009     fname = args[0]
2010
2011     return (fname, output_dir, do_c, do_h, prefix, extra_opts)
2012
2013 #
2014 # Generate output files with boilerplate
2015 #
2016
2017
2018 def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
2019                 c_comment, h_comment):
2020     guard = guardname(prefix + h_file)
2021     c_file = output_dir + prefix + c_file
2022     h_file = output_dir + prefix + h_file
2023
2024     if output_dir:
2025         try:
2026             os.makedirs(output_dir)
2027         except os.error as e:
2028             if e.errno != errno.EEXIST:
2029                 raise
2030
2031     def maybe_open(really, name, opt):
2032         if really:
2033             return open(name, opt)
2034         else:
2035             import StringIO
2036             return StringIO.StringIO()
2037
2038     fdef = maybe_open(do_c, c_file, 'w')
2039     fdecl = maybe_open(do_h, h_file, 'w')
2040
2041     fdef.write(mcgen('''
2042 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2043 %(comment)s
2044 ''',
2045                      comment=c_comment))
2046
2047     fdecl.write(mcgen('''
2048 /* AUTOMATICALLY GENERATED, DO NOT MODIFY */
2049 %(comment)s
2050 #ifndef %(guard)s
2051 #define %(guard)s
2052
2053 ''',
2054                       comment=h_comment, guard=guard))
2055
2056     return (fdef, fdecl)
2057
2058
2059 def close_output(fdef, fdecl):
2060     fdecl.write('''
2061 #endif
2062 ''')
2063     fdecl.close()
2064     fdef.close()