ceph/src/pybind/mgr/rest/app/management/commands/api_docs.py

   1 #
   2 # If Django is installed locally, run the following command from the top of
   3 # the ceph source tree:
   4 #
   5 #   PYTHONPATH=src/pybind/mgr \
   6 #   DJANGO_SETTINGS_MODULE=rest.app.settings \
   7 #   CALAMARI_CONFIG=src/pybind/mgr/calamari.conf \
   8 #       django-admin api_docs
   9 #
  10 # This will create resources.rst (the API docs), and rest.log (which will
  11 # probably just be empty).
  12 #
  13 # TODO: Add the above to a makefile, so the docs land somewhere sane.
  14 #
  15
  16 from collections import defaultdict
  17 import json
  18 from optparse import make_option
  19 import os
  20 from django.core.management.base import NoArgsCommand
  21 import importlib
  22 from jinja2 import Environment
  23 import re
  24 import rest_framework.viewsets
  25 import traceback
  26 from django.core.urlresolvers import RegexURLPattern, RegexURLResolver
  27 import sys
  28 import codecs
  29
  30 class ceph_state:
  31     pass
  32
  33 sys.modules["ceph_state"] = ceph_state
  34
  35 # Needed to avoid weird import loops
  36 from rest.module import global_instance  # NOQA
  37
  38 from rest.app.serializers.v2 import ValidatingSerializer
  39
  40 GENERATED_PREFIX = "."
  41
  42
  43 EXAMPLES_FILE = os.path.join(GENERATED_PREFIX, "api_examples.json")
  44 RESOURCES_FILE = os.path.join("resources.rst")
  45 EXAMPLES_PREFIX = "api_example_"
  46
  47
  48 old_as_view = rest_framework.viewsets.ViewSetMixin.as_view
  49
  50
  51 @classmethod
  52 def as_view(cls, actions=None, **initkwargs):
  53     view = old_as_view.__func__(cls, actions, **initkwargs)
  54     view._actions = actions
  55     return view
  56
  57 rest_framework.viewsets.ViewSetMixin.as_view = as_view
  58
  59 # >>> RsT table code borrowed from http://stackoverflow.com/a/17203834/99876
  60
  61
  62 def make_table(grid):
  63     max_cols = [max(out) for out in map(list, zip(*[[len(item) for item in row] for row in grid]))]
  64     rst = table_div(max_cols, 1)
  65
  66     for i, row in enumerate(grid):
  67         header_flag = False
  68         if i == 0 or i == len(grid) - 1:
  69             header_flag = True
  70         rst += normalize_row(row, max_cols)
  71         rst += table_div(max_cols, header_flag)
  72     return rst
  73
  74
  75 def table_div(max_cols, header_flag=1):
  76     out = ""
  77     if header_flag == 1:
  78         style = "="
  79     else:
  80         style = "-"
  81
  82     for max_col in max_cols:
  83         out += max_col * style + " "
  84
  85     out += "\n"
  86     return out
  87
  88
  89 def normalize_row(row, max_cols):
  90     r = ""
  91     for i, max_col in enumerate(max_cols):
  92         r += row[i] + (max_col - len(row[i]) + 1) * " "
  93
  94     return r + "\n"
  95 # <<< RsT table code borrowed from http://stackoverflow.com/a/17203834/99876
  96
  97
  98 PAGE_TEMPLATE = """
  99
 100 :tocdepth: 3
 101
 102 API resources
 103 =============
 104
 105 URL summary
 106 -----------
 107
 108 {{url_summary_rst}}
 109
 110 API reference
 111 -------------
 112
 113 {{resources_rst}}
 114
 115 Examples
 116 --------
 117
 118 .. toctree::
 119    :maxdepth: 1
 120
 121 {% for example_doc in example_docs %}
 122    {{example_doc}}
 123 {% endfor %}
 124
 125
 126 """
 127
 128
 129 RESOURCE_TEMPLATE = """
 130
 131 .. _{{class_name}}:
 132
 133 {{name}}
 134 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 135
 136 {{help_text}}
 137
 138 URLs
 139 ____
 140
 141 {{url_table}}
 142
 143 Fields
 144 ______
 145
 146 {{field_table}}
 147
 148 """
 149
 150
 151 VERBS = ["GET", "PUT", "POST", "PATCH", "DELETE"]
 152
 153
 154 def _url_pattern_methods(url_pattern):
 155     view_class = url_pattern.callback.cls
 156
 157     if hasattr(url_pattern.callback, '_actions'):
 158         # An APIViewSet
 159         methods = [k.upper() for k in url_pattern.callback._actions.keys()]
 160     else:
 161         methods = view_class().allowed_methods
 162
 163     if not methods:
 164         # A view that isn't using django rest framework?
 165         raise RuntimeError("No methods for url %s" % url_pattern.regex.pattern)
 166
 167     return methods
 168
 169
 170 def _stripped_url(prefix, url_pattern):
 171     """
 172     Convert a URL regex into something for human eyes
 173
 174     ^server/(?P<pk>[^/]+)$ becomes server/<pk>
 175     """
 176     url = prefix + url_pattern.regex.pattern.strip("^$")
 177     url = re.sub("\(.+?<(.+?)>.+?\)", "<\\1>", url)
 178     return url
 179
 180
 181 def _pretty_url(prefix, url_pattern):
 182     return "%s" % _stripped_url(prefix, url_pattern).replace("<", "\\<").replace(">", "\\>")
 183
 184
 185 def _find_prefix(toplevel_mod, sub_mod):
 186     """
 187     Find the URL prefix of sub_mod in toplevel_mod
 188     """
 189     for toplevel_pattern in importlib.import_module(toplevel_mod).urlpatterns:
 190         if isinstance(toplevel_pattern, RegexURLResolver):
 191             if toplevel_pattern.urlconf_name.__name__ == sub_mod:
 192                 regex_str = toplevel_pattern.regex.pattern
 193                 return regex_str.strip("^")
 194
 195     raise RuntimeError("'%s' not included in '%s', cannot find prefix" % (sub_mod, toplevel_mod))
 196
 197
 198 class ApiIntrospector(object):
 199     def __init__(self, url_module):
 200         view_to_url_patterns = defaultdict(list)
 201
 202         def parse_urls(urls):
 203             for url_pattern in urls:
 204                 if isinstance(url_pattern, RegexURLResolver):
 205                     parse_urls(url_pattern.urlconf_module)
 206                 elif isinstance(url_pattern, RegexURLPattern):
 207                     if url_pattern.regex.pattern.endswith('\.(?P<format>[a-z0-9]+)$'):
 208                         # Suppress the .<format> urls that rest_framework generates
 209                         continue
 210
 211                     if hasattr(url_pattern.callback, 'cls'):
 212                         # This is a rest_framework as_view wrapper
 213                         view_cls = url_pattern.callback.cls
 214                         if view_cls.__name__.endswith("APIRoot"):
 215                             continue
 216                         view_to_url_patterns[view_cls].append(url_pattern)
 217
 218         self.prefix = _find_prefix("rest.app.urls", url_module)
 219         parse_urls(importlib.import_module(url_module).urlpatterns)
 220
 221         self.view_to_url_patterns = sorted(view_to_url_patterns.items(), cmp=lambda x, y: cmp(x[0].__name__, y[0].__name__))
 222
 223         self.all_url_patterns = []
 224         for view, url_patterns in self.view_to_url_patterns:
 225             self.all_url_patterns.extend(url_patterns)
 226         self.all_url_patterns = sorted(self.all_url_patterns,
 227                                        lambda a, b: cmp(_pretty_url(self.prefix, a), _pretty_url(self.prefix, b)))
 228
 229     def _view_rst(self, view, url_patterns):
 230         """
 231         Output RsT for one API view
 232         """
 233         name = view().get_view_name()
 234
 235         if view.__doc__:
 236             view_help_text = view.__doc__
 237         else:
 238             view_help_text = "*No description available*"
 239
 240         url_table = [["URL"] + VERBS]
 241         for url_pattern in url_patterns:
 242             methods = _url_pattern_methods(url_pattern)
 243
 244             row = [":doc:`%s <%s>`" % (_pretty_url(self.prefix, url_pattern),
 245                                        self._example_document_name(_stripped_url(self.prefix, url_pattern)))]
 246             for v in VERBS:
 247                 if v in methods:
 248                     row.append("Yes")
 249                 else:
 250                     row.append("")
 251             url_table.append(row)
 252
 253         url_table_rst = make_table(url_table)
 254
 255         if hasattr(view, 'serializer_class') and view.serializer_class:
 256             field_table = [["Name", "Type", "Readonly", "Create", "Modify", "Description"]]
 257
 258             serializer = view.serializer_class()
 259             if isinstance(serializer, ValidatingSerializer):
 260                 allowed_during_create = serializer.Meta.create_allowed
 261                 required_during_create = serializer.Meta.create_required
 262                 allowed_during_modify = serializer.Meta.modify_allowed
 263                 required_during_modify = serializer.Meta.modify_required
 264             else:
 265                 allowed_during_create = required_during_create = allowed_during_modify = required_during_modify = ()
 266
 267             fields = serializer.get_fields()
 268             for field_name, field in fields.items():
 269                 create = modify = ''
 270                 if field_name in allowed_during_create:
 271                     create = 'Allowed'
 272                 if field_name in required_during_create:
 273                     create = 'Required'
 274
 275                 if field_name in allowed_during_modify:
 276                     modify = 'Allowed'
 277                 if field_name in required_during_modify:
 278                     modify = 'Required'
 279
 280                 if hasattr(field, 'help_text'):
 281                     field_help_text = field.help_text
 282                 else:
 283                     field_help_text = ""
 284                 field_table.append(
 285                     [field_name,
 286                      field.__class__.__name__,
 287                      str(field.read_only),
 288                      create,
 289                      modify,
 290                      field_help_text if field_help_text else ""])
 291             field_table_rst = make_table(field_table)
 292         else:
 293             field_table_rst = "*No field data available*"
 294
 295         return Environment().from_string(RESOURCE_TEMPLATE).render(
 296             name=name,
 297             class_name=view.__name__,
 298             help_text=view_help_text,
 299             field_table=field_table_rst,
 300             url_table=url_table_rst
 301         )
 302
 303     def _url_table(self, url_patterns):
 304         url_table = [["URL", "View", "Examples"] + VERBS]
 305         for view, url_patterns in self.view_to_url_patterns:
 306             for url_pattern in url_patterns:
 307                 methods = _url_pattern_methods(url_pattern)
 308
 309                 row = [_pretty_url(self.prefix, url_pattern)]
 310
 311                 view_name = view().get_view_name()
 312                 row.append(
 313                     u":ref:`{0} <{1}>`".format(view_name.replace(" ", unichr(0x00a0)), view.__name__)
 314                 )
 315
 316                 example_doc_name = self._example_document_name(_stripped_url(self.prefix, url_pattern))
 317                 if os.path.exists("{0}/{1}.rst".format(GENERATED_PREFIX, example_doc_name)):
 318                     print "It exists: {0}".format(example_doc_name)
 319                     row.append(":doc:`%s <%s>`" % ("Example", example_doc_name))
 320                 else:
 321                     row.append("")
 322                 for v in VERBS:
 323                     if v in methods:
 324                         row.append("Yes")
 325                     else:
 326                         row.append("")
 327                 url_table.append(row)
 328
 329         return make_table(url_table)
 330
 331     def _flatten_path(self, path):
 332         """
 333         Escape a URL pattern to something suitable for use as a filename
 334         """
 335         return path.replace("/", "_").replace("<", "_").replace(">", "_")
 336
 337     def _example_document_name(self, pattern):
 338         return EXAMPLES_PREFIX + self._flatten_path(pattern)
 339
 340     def _write_example(self, example_pattern, example_results):
 341         """
 342         Write RsT file with API examples for a particular pattern
 343         """
 344         rst = ""
 345         title = "Examples for %s" % example_pattern
 346         rst += "%s\n%s\n\n" % (title, "=" * len(title))
 347         for url, content in example_results.items():
 348             rst += "%s\n" % url
 349             rst += "-" * len(url)
 350             rst += "\n\n.. code-block:: json\n\n"
 351             data_dump = json.dumps(json.loads(content), indent=2)
 352             data_dump = "\n".join(["   %s" % l for l in data_dump.split("\n")])
 353             rst += data_dump
 354             rst += "\n\n"
 355         codecs.open("{0}/{1}.rst".format(GENERATED_PREFIX, self._example_document_name(example_pattern)), 'w',
 356                     encoding="UTF-8").write(rst)
 357
 358     def write_docs(self, examples):
 359         resources_rst = ""
 360         for view, url_patterns in self.view_to_url_patterns:
 361             resources_rst += self._view_rst(view, url_patterns)
 362
 363         url_table_rst = self._url_table(self.all_url_patterns)
 364
 365         example_docs = [self._example_document_name(p) for p in examples.keys()]
 366
 367         resources_rst = Environment().from_string(PAGE_TEMPLATE).render(
 368             resources_rst=resources_rst, url_summary_rst=url_table_rst, example_docs=example_docs)
 369         codecs.open(RESOURCES_FILE, 'w', encoding="UTF-8").write(resources_rst)
 370
 371         for example_pattern, example_results in examples.items():
 372             self._write_example(example_pattern, example_results)
 373
 374     def get_url_list(self, method="GET"):
 375         return [_stripped_url(self.prefix, u)
 376                 for u in self.all_url_patterns
 377                 if method in _url_pattern_methods(u)]
 378
 379
 380 class Command(NoArgsCommand):
 381     help = "Print introspected REST API documentation"
 382     option_list = NoArgsCommand.option_list + (
 383         make_option('--list-urls',
 384                     action='store_true',
 385                     dest='list_urls',
 386                     default=False,
 387                     help='Print a list of URL patterns instead of RsT documentation'),
 388     )
 389
 390     def handle_noargs(self, list_urls, **options):
 391         introspector = ApiIntrospector("rest.app.urls.v2")
 392         if list_urls:
 393             # TODO: this just prints an empty array (not sure why)
 394             print json.dumps(introspector.get_url_list())
 395         else:
 396             try:
 397                 try:
 398                     examples = json.load(open(EXAMPLES_FILE, 'r'))
 399                 except IOError:
 400                     examples = {}
 401                     print >>sys.stderr, "Examples data '%s' not found, no examples will be generated" % EXAMPLES_FILE
 402
 403                 introspector.write_docs(examples)
 404             except:
 405                 print >>sys.stderr, traceback.format_exc()
 406                 raise