ceph/src/seastar/scripts/seastar-json2code.py

   1 #!/usr/bin/env python3
   2
   3 # C++ Code generation utility from Swagger definitions.
   4 # This utility support Both the swagger 1.2 format
   5 #    https://github.com/OAI/OpenAPI-Specification/blob/master/versions/1.2.md
   6 # And the 2.0 format
   7 #    https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md
   8 #
   9 # Swagger 2.0 is not only different in its structure (apis have moved, and
  10 # models are now under definitions) It also moved from multiple file structure
  11 # to a single file.
  12 # To keep the multiple file support, each group of APIs will be placed in a single file
  13 # Each group can have a .def.json file with its definitions (What used to be models)
  14 # Because the APIs in definitions are snippets, they are not legal json objects
  15 # and need to be formated as such so that a json parser would work.
  16
  17 import json
  18 import sys
  19 import re
  20 import glob
  21 import argparse
  22 import os
  23 from string import Template
  24
  25 parser = argparse.ArgumentParser(description="""Generate C++ class for json
  26 handling from swagger definition""")
  27
  28 parser.add_argument('--outdir', help='the output directory', default='autogen')
  29 parser.add_argument('-o', help='Output file', default='')
  30 parser.add_argument('-f', help='input file', default='api-java.json')
  31 parser.add_argument('-ns', help="""namespace when set struct will be created
  32 under the namespace""", default='')
  33 parser.add_argument('-jsoninc', help='relative path to the jsaon include',
  34                     default='json/')
  35 parser.add_argument('-jsonns', help='set the json namespace', default='json')
  36 parser.add_argument('-indir', help="""when set all json file in the given
  37 directory will be parsed, do not use with -f""", default='')
  38 parser.add_argument('-debug', help='debug level 0 -quite,1-error,2-verbose',
  39                     default='1', type=int)
  40 parser.add_argument('-combined', help='set the name of the combined file',
  41                     default='autogen/pathautogen.ee')
  42 parser.add_argument('--create-cc', dest='create_cc', action='store_true', default=False,
  43                     help='Put global variables in a .cc file')
  44 config = parser.parse_args()
  45
  46
  47 valid_vars = {'string': 'sstring', 'int': 'int', 'double': 'double',
  48              'float': 'float', 'long': 'long', 'boolean': 'bool', 'char': 'char',
  49              'datetime': 'json::date_time'}
  50
  51 current_file = ''
  52
  53 spacing = "    "
  54 def getitem(d, key, name):
  55     if key in d:
  56         return d[key]
  57     else:
  58         raise Exception("'" + key + "' not found in " + name)
  59
  60 def fprint(f, *args):
  61     for arg in args:
  62         f.write(arg)
  63
  64 def fprintln(f, *args):
  65     for arg in args:
  66         f.write(arg)
  67     f.write('\n')
  68
  69
  70 def open_namespace(f, ns=config.ns):
  71     fprintln(f, "namespace ", ns , ' {\n')
  72
  73
  74 def close_namespace(f):
  75     fprintln(f, '}')
  76
  77
  78 def add_include(f, includes):
  79     for include in includes:
  80         fprintln(f, '#include ', include)
  81     fprintln(f, "")
  82
  83 def trace_verbose(*params):
  84     if config.debug > 1:
  85         print(''.join(params))
  86
  87
  88 def trace_err(*params):
  89     if config.debug > 0:
  90         print(current_file + ':' + ''.join(params))
  91
  92
  93 def valid_type(param):
  94     if param in valid_vars:
  95         return valid_vars[param]
  96     trace_err("Type [", param, "] not defined")
  97     return param
  98
  99
 100 def type_change(param, member):
 101     if param == "array":
 102         if "items" not in member:
 103             trace_err("array without item declaration in ", param)
 104             return ""
 105         item = member["items"]
 106         if "type" in item:
 107             t = item["type"]
 108         elif "$ref" in item:
 109             t = item["$ref"]
 110         else:
 111             trace_err("array items with no type or ref declaration ", param)
 112             return ""
 113         return "json_list< " + valid_type(t) + " >"
 114     return "json_element< " + valid_type(param) + " >"
 115
 116
 117
 118 def print_ind_comment(f, ind, *params):
 119     fprintln(f, ind, "/**")
 120     for s in params:
 121         fprintln(f, ind, " * ", s)
 122     fprintln(f, ind, " */")
 123
 124 def print_comment(f, *params):
 125     print_ind_comment(f, spacing, *params)
 126
 127 def print_copyrights(f):
 128     fprintln(f, "/*")
 129     fprintln(f, "* Copyright (C) 2014 Cloudius Systems, Ltd.")
 130     fprintln(f, "*")
 131     fprintln(f, "* This work is open source software, licensed under the",
 132            " terms of the")
 133     fprintln(f, "* BSD license as described in the LICENSE f in the top-",
 134            "level directory.")
 135     fprintln(f, "*")
 136     fprintln(f, "*  This is an Auto-Generated-code  ")
 137     fprintln(f, "*  Changes you do in this file will be erased on next",
 138            " code generation")
 139     fprintln(f, "*/\n")
 140
 141
 142 def print_h_file_headers(f, name):
 143     print_copyrights(f)
 144     fprintln(f, "#ifndef __JSON_AUTO_GENERATED_" + name)
 145     fprintln(f, "#define __JSON_AUTO_GENERATED_" + name + "\n")
 146
 147
 148 def clean_param(param):
 149     match = re.match(r"^\{\s*([^\}]+)\s*}", param)
 150     if match:
 151         return [match.group(1), False]
 152     return [param, True]
 153
 154
 155 def get_parameter_by_name(obj, name):
 156     for p in obj["parameters"]:
 157         if p["name"] == name:
 158             return p
 159     trace_err ("No Parameter declaration found for ", name)
 160
 161
 162 def clear_path_ending(path):
 163     if not path or path[-1] != '/':
 164         return path
 165     return path[0:-1]
 166
 167 # check if a parameter is query required.
 168 # It will return true if the required flag is set
 169 # and if it is a query parameter, both swagger 1.2 'paramType' and swagger 2.0 'in' attributes
 170 # are supported
 171 def is_required_query_param(param):
 172     return "required" in param and param["required"] and ("paramType" in param and param["paramType"] == "query" or "in" in param and param["in"] == "query")
 173
 174 def add_path(f, path, details):
 175     if "summary" in details:
 176         print_comment(f, details["summary"])
 177     param_starts = path.find("{")
 178     if param_starts >= 0:
 179         path_reminder = path[param_starts:]
 180         vals = path.split("/")
 181         vals.reverse()
 182         fprintln(f, spacing, 'path_description::add_path("', clear_path_ending(vals.pop()),
 183            '",', details["method"], ',"', details["nickname"], '")')
 184         while vals:
 185             param, is_url = clean_param(vals.pop())
 186             if is_url:
 187                 fprintln(f, spacing, '  ->pushurl("', param, '")')
 188             else:
 189                 param_type = get_parameter_by_name(details, param)
 190                 if ("allowMultiple" in param_type and
 191                     param_type["allowMultiple"] == True):
 192                     fprintln(f, spacing, '  ->pushparam("', param, '",true)')
 193                 else:
 194                     fprintln(f, spacing, '  ->pushparam("', param, '")')
 195     else:
 196         fprintln(f, spacing, 'path_description::add_path("', clear_path_ending(path), '",',
 197            details["method"], ',"', details["nickname"], '")')
 198     if "parameters" in details:
 199         for param in details["parameters"]:
 200             if is_required_query_param(param):
 201                 fprintln(f, spacing, '  ->pushmandatory_param("', param["name"], '")')
 202     fprintln(f, spacing, ";")
 203
 204
 205 def get_base_name(param):
 206     return os.path.basename(param)
 207
 208
 209 def is_model_valid(name, model):
 210     if name in valid_vars:
 211         return ""
 212     properties = getitem(model[name], "properties", name)
 213     for var in properties:
 214         type = getitem(properties[var], "type", name + ":" + var)
 215         if type == "array":
 216             items = getitem(properties[var], "items", name + ":" + var);
 217             try :
 218                 type = getitem(items, "type", name + ":" + var + ":items")
 219             except Exception as e:
 220                 try:
 221                     type = getitem(items, "$ref", name + ":" + var + ":items")
 222                 except:
 223                     raise e;
 224         if type not in valid_vars:
 225             if type not in model:
 226                 raise Exception("Unknown type '" + type + "' in Model '" + name + "'")
 227             return type
 228     valid_vars[name] = name
 229     return ""
 230
 231 def resolve_model_order(data):
 232     res = []
 233     models = set()
 234     for model_name in data:
 235         visited = set(model_name)
 236         missing = is_model_valid(model_name, data)
 237         resolved = missing == ''
 238         if not resolved:
 239             stack = [model_name]
 240             while not resolved:
 241                 if missing in visited:
 242                     raise Exception("Cyclic dependency found: " + missing)
 243                 missing_depends = is_model_valid(missing, data)
 244                 if missing_depends == '':
 245                     if missing not in models:
 246                         res.append(missing)
 247                         models.add(missing)
 248                     resolved = len(stack) == 0
 249                     if not resolved:
 250                         missing = stack.pop()
 251                 else:
 252                     stack.append(missing)
 253                     missing = missing_depends
 254         elif model_name not in models:
 255             res.append(model_name)
 256             models.add(model_name)
 257     return res
 258
 259 def create_enum_wrapper(model_name, name, values):
 260     enum_name = model_name + "_" + name
 261     res =  "  enum class " + enum_name + " {"
 262     for enum_entry in values:
 263         res = res +  "  " + enum_entry + ", "
 264     res = res +   "NUM_ITEMS};\n"
 265     wrapper = name + "_wrapper"
 266     res = res + Template("""  struct $wrapper : public json::jsonable  {
 267         $wrapper() = default;
 268         virtual std::string to_json() const {
 269             switch(v) {
 270         """).substitute({'wrapper' : wrapper})
 271     for enum_entry in values:
 272         res = res + "      case " + enum_name + "::" + enum_entry + ": return \"\\\"" + enum_entry + "\\\"\";\n"
 273     res = res + Template("""      default: return \"\\\"Unknown\\\"\";
 274         }
 275      }
 276     template<class T>
 277     $wrapper (const T& _v) {
 278     switch(_v) {
 279     """).substitute({'wrapper' : wrapper})
 280     for enum_entry in values:
 281         res = res +  "      case T::" + enum_entry + ": v = " + enum_name + "::" + enum_entry + "; break;\n"
 282     res = res + Template("""      default: v = $enum_name::NUM_ITEMS;
 283         }
 284     }
 285     template<class T>
 286     operator T() const {
 287         switch(v) {
 288     """).substitute({'enum_name': enum_name})
 289     for enum_entry in values:
 290         res = res + "      case " + enum_name + "::" + enum_entry + ": return T::" + enum_entry + ";\n"
 291     return res + Template("""      default: return T::$value;
 292           }
 293         }
 294         typedef typename std::underlying_type<$enum_name>::type pos_type;
 295         $wrapper& operator++() {
 296         v = static_cast<$enum_name>(static_cast<pos_type>(v) + 1);
 297             return *this;
 298         }
 299         $wrapper & operator++(int) {
 300             return ++(*this);
 301         }
 302         bool operator==(const  $wrapper& c) const {
 303             return v == c.v;
 304         }
 305         bool operator!=(const $wrapper& c) const {
 306             return v != c.v;
 307         }
 308         bool operator<=(const $wrapper& c) const {
 309             return static_cast<pos_type>(v) <= static_cast<pos_type>(c.v);
 310         }
 311         static $wrapper begin() {
 312             return $wrapper ($enum_name::$value);
 313         }
 314         static $wrapper end() {
 315             return $wrapper ($enum_name::NUM_ITEMS);
 316         }
 317         static boost::integer_range<$wrapper> all_items() {
 318             return boost::irange(begin(), end());
 319         }
 320         $enum_name v;
 321     };
 322     """).substitute({'enum_name': enum_name, 'wrapper' : wrapper, 'value':values[0]})
 323
 324 def to_operation(opr, data):
 325     data["method"] = opr.upper()
 326     data["nickname"] = data["operationId"]
 327     return data
 328
 329 def to_path(path, data):
 330     data["operations"] = [to_operation(k, data[k]) for k in data]
 331     data["path"] = path
 332
 333     return data
 334
 335 def create_h_file(data, hfile_name, api_name, init_method, base_api):
 336     if config.o != '':
 337         final_hfile_name = config.o
 338     else:
 339         final_hfile_name = config.outdir + "/" + hfile_name
 340     hfile = open(final_hfile_name, "w")
 341
 342     if config.create_cc:
 343         ccfile = open(final_hfile_name.rstrip('.hh') + ".cc", "w")
 344         add_include(ccfile, ['"{}"'.format(final_hfile_name)])
 345         open_namespace(ccfile, "seastar")
 346         open_namespace(ccfile, "httpd")
 347         open_namespace(ccfile, api_name)
 348     else:
 349         ccfile = hfile
 350     print_h_file_headers(hfile, api_name)
 351     add_include(hfile, ['<seastar/core/sstring.hh>',
 352                         '<seastar/json/json_elements.hh>',
 353                         '<seastar/http/json_path.hh>'])
 354
 355     add_include(hfile, ['<iostream>', '<boost/range/irange.hpp>'])
 356     open_namespace(hfile, "seastar")
 357     open_namespace(hfile, "httpd")
 358     open_namespace(hfile, api_name)
 359
 360     if "models" in data:
 361         models_order = resolve_model_order(data["models"])
 362         for model_name in models_order:
 363             model = data["models"][model_name]
 364             if 'description' in model:
 365                 print_ind_comment(hfile, "", model["description"])
 366             fprintln(hfile, "struct ", model_name, " : public json::json_base {")
 367             member_init = ''
 368             member_assignment = ''
 369             member_copy = ''
 370             for member_name in model["properties"]:
 371                 member = model["properties"][member_name]
 372                 if "description" in member:
 373                     print_comment(hfile, member["description"])
 374                 if "enum" in member:
 375                     enum_name = model_name + "_" + member_name
 376                     fprintln(hfile, create_enum_wrapper(model_name, member_name, member["enum"]))
 377                     fprintln(hfile, "  ", config.jsonns, "::json_element<",
 378                            member_name, "_wrapper> ",
 379                            member_name, ";\n")
 380                 else:
 381                     fprintln(hfile, "  ", config.jsonns, "::",
 382                            type_change(member["type"], member), " ",
 383                            member_name, ";\n")
 384                 member_init += "  add(&" + member_name + ',"'
 385                 member_init += member_name + '");\n'
 386                 member_assignment += "  " + member_name + " = " + "e." + member_name + ";\n"
 387                 member_copy += "  e." + member_name + " = " + member_name + ";\n"
 388             fprintln(hfile, "void register_params() {")
 389             fprintln(hfile, member_init)
 390             fprintln(hfile, '}')
 391
 392             fprintln(hfile, model_name, '() {')
 393             fprintln(hfile, '  register_params();')
 394             fprintln(hfile, '}')
 395             fprintln(hfile, model_name, '(const ' + model_name + ' & e) {')
 396             fprintln(hfile, '  register_params();')
 397             fprintln(hfile, member_assignment)
 398             fprintln(hfile, '}')
 399             fprintln(hfile, "template<class T>")
 400             fprintln(hfile, model_name, "& operator=(const ", "T& e) {")
 401             fprintln(hfile, member_assignment)
 402             fprintln(hfile, "  return *this;")
 403             fprintln(hfile, "}")
 404             fprintln(hfile, model_name, "& operator=(const ", model_name, "& e) {")
 405             fprintln(hfile, member_assignment)
 406             fprintln(hfile, "  return *this;")
 407             fprintln(hfile, "}")
 408             fprintln(hfile, "template<class T>")
 409             fprintln(hfile, model_name, "& update(T& e) {")
 410             fprintln(hfile, member_copy)
 411             fprintln(hfile, "  return *this;")
 412             fprintln(hfile, "}")
 413             fprintln(hfile, "};\n\n")
 414
 415  #   print_ind_comment(hfile, "", "Initialize the path")
 416 #    fprintln(hfile, init_method + "(const std::string& description);")
 417     fprintln(hfile, 'static const sstring name = "', base_api, '";')
 418     for item in data["apis"]:
 419         path = item["path"]
 420         if "operations" in item:
 421             for oper in item["operations"]:
 422                 if "summary" in oper:
 423                     print_comment(hfile, oper["summary"])
 424
 425                 param_starts = path.find("{")
 426                 base_url = path
 427                 vals = []
 428                 if param_starts >= 0:
 429                     vals = path[param_starts:].split("/")
 430                     vals.reverse()
 431                     base_url = path[:param_starts]
 432
 433                 varname = getitem(oper, "nickname", oper)
 434                 if config.create_cc:
 435                     fprintln(hfile, 'extern const path_description ', varname, ';')
 436                     maybe_static = ''
 437                 else:
 438                     maybe_static = 'static '
 439                 fprintln(ccfile, maybe_static, 'const path_description ', varname, '("', clear_path_ending(base_url),
 440                        '",', oper["method"], ',"', oper["nickname"], '",')
 441                 fprint(ccfile, '{')
 442                 first = True
 443                 while vals:
 444                     path_param, is_url = clean_param(vals.pop())
 445                     if path_param == "":
 446                         continue
 447                     if first == True:
 448                         first = False
 449                     else:
 450                         fprint(ccfile, "\n,")
 451                     if is_url:
 452                         fprint(ccfile, '{', '"/', path_param , '", path_description::url_component_type::FIXED_STRING', '}')
 453                     else:
 454                         path_param_type = get_parameter_by_name(oper, path_param)
 455                         if ("allowMultiple" in path_param_type and
 456                             path_param_type["allowMultiple"] == True):
 457                             fprint(ccfile, '{', '"', path_param , '", path_description::url_component_type::PARAM_UNTIL_END_OF_PATH', '}')
 458                         else:
 459                             fprint(ccfile, '{', '"', path_param , '", path_description::url_component_type::PARAM', '}')
 460                 fprint(ccfile, '}')
 461                 fprint(ccfile, ',{')
 462                 first = True
 463                 enum_definitions = ""
 464                 if "enum" in oper:
 465                     enum_definitions = ("namespace ns_" + oper["nickname"] + " {\n" +
 466                                        create_enum_wrapper(oper["nickname"], "return_type", oper["enum"]) +
 467                                        "}\n")
 468                 funcs = ""
 469                 if "parameters" in oper:
 470                     for param in oper["parameters"]:
 471                         if is_required_query_param(param):
 472                             if first == True:
 473                                 first = False
 474                             else:
 475                                 fprint(ccfile, "\n,")
 476                             fprint(ccfile, '"', param["name"], '"')
 477                         if "enum" in param:
 478                             enum_definitions = enum_definitions + 'namespace ns_' + oper["nickname"] + '{\n'
 479                             enm = param["name"]
 480                             enum_definitions = enum_definitions + 'enum class ' + enm + ' {'
 481                             for val in param["enum"]:
 482                                 enum_definitions = enum_definitions + val + ", "
 483                             enum_definitions = enum_definitions + 'NUM_ITEMS};\n'
 484                             enum_definitions = enum_definitions + enm + ' str2' + enm + '(const sstring& str);'
 485
 486                             funcs = funcs + enm + ' str2' + enm + '(const sstring& str) {\n'
 487                             funcs = funcs + '  static const sstring arr[] = {"' + '","'.join(param["enum"]) + '"};\n'
 488                             funcs = funcs + '  int i;\n'
 489                             funcs = funcs + '  for (i=0; i < ' + str(len(param["enum"])) + '; i++) {\n'
 490                             funcs = funcs + '    if (arr[i] == str) {return (' + enm + ')i;}\n}\n'
 491                             funcs = funcs + '  return (' + enm + ')i;\n'
 492                             funcs = funcs + '}\n'
 493
 494                             enum_definitions = enum_definitions + '}\n'
 495
 496                 fprintln(ccfile, '});')
 497                 fprintln(hfile, enum_definitions)
 498                 open_namespace(ccfile, 'ns_' + oper["nickname"])
 499                 fprintln(ccfile, funcs)
 500                 close_namespace(ccfile)
 501
 502     close_namespace(hfile)
 503     close_namespace(hfile)
 504     close_namespace(hfile)
 505     if config.create_cc:
 506         close_namespace(ccfile)
 507         close_namespace(ccfile)
 508         close_namespace(ccfile)
 509
 510     hfile.write("#endif //__JSON_AUTO_GENERATED_HEADERS\n")
 511     hfile.close()
 512
 513 def remove_leading_comma(data):
 514     return re.sub(r'^\s*,','', data)
 515
 516 def format_as_json_object(data):
 517     return "{" + remove_leading_comma(data) + "}"
 518
 519 def check_for_models(data, param):
 520     model_name = param.replace(".json", ".def.json")
 521     if not os.path.isfile(model_name):
 522         return
 523     try:
 524         with open(model_name) as myfile:
 525             json_data = myfile.read()
 526             def_data = json.loads(format_as_json_object(json_data))
 527             data["models"] = def_data
 528     except Exception as e:
 529         type, value, tb = sys.exc_info()
 530         print("Bad formatted JSON definition file '" + model_name + "' error ", value.message)
 531         sys.exit(-1)
 532
 533 def set_apis(data):
 534     return {"apis": [to_path(p, data[p]) for p in data]}
 535
 536 def parse_file(param, combined):
 537     global current_file
 538     trace_verbose("parsing ", param, " file")
 539     with open(param) as myfile:
 540         json_data = myfile.read()
 541     try:
 542         data = json.loads(json_data)
 543     except Exception as e:
 544         try:
 545             # the passed data is not a valid json, so maybe its a swagger 2.0
 546             # snippet, format it as json and try again
 547             # set_apis and check_for_models will create an object with a similiar format
 548             # to a swagger 1.2 so the code generation would work
 549             data = set_apis(json.loads(format_as_json_object(json_data)))
 550             check_for_models(data, param)
 551         except:
 552             # The problem is with the file,
 553             # just report the error and exit.
 554             type, value, tb = sys.exc_info()
 555             print("Bad formatted JSON file '" + param + "' error ", value.message)
 556             sys.exit(-1)
 557     try:
 558         base_file_name = get_base_name(param)
 559         current_file = base_file_name
 560         hfile_name = base_file_name + ".hh"
 561         api_name = base_file_name.replace('.', '_')
 562         base_api = base_file_name.replace('.json', '')
 563         init_method = "void " + api_name + "_init_path"
 564         trace_verbose("creating ", hfile_name)
 565         if (combined):
 566             fprintln(combined, '#include "', base_file_name, ".cc", '"')
 567         create_h_file(data, hfile_name, api_name, init_method, base_api)
 568     except:
 569         type, value, tb = sys.exc_info()
 570         print("Error while parsing JSON file '" + param + "' error ", value.message)
 571         sys.exit(-1)
 572
 573 if "indir" in config and config.indir != '':
 574     combined = open(config.combined, "w")
 575     for f in glob.glob(os.path.join(config.indir, "*.json")):
 576         parse_file(f, combined)
 577 else:
 578     parse_file(config.f, None)