ceph/doc/_ext/ceph_commands.py

   1 import io
   2 import os
   3 import sys
   4 import contextlib
   5
   6 from docutils.parsers.rst import directives
   7 from docutils.parsers.rst import Directive
   8 from jinja2 import Template
   9 from pcpp.preprocessor import Preprocessor
  10 from sphinx.util import logging
  11 from sphinx.util.console import bold
  12
  13 logger = logging.getLogger(__name__)
  14
  15
  16 class Flags:
  17     NOFORWARD = (1 << 0)
  18     OBSOLETE = (1 << 1)
  19     DEPRECATED = (1 << 2)
  20     MGR = (1 << 3)
  21     POLL = (1 << 4)
  22     HIDDEN = (1 << 5)
  23
  24     VALS = {
  25         NOFORWARD: 'no_forward',
  26         OBSOLETE: 'obsolete',
  27         DEPRECATED: 'deprecated',
  28         MGR: 'mgr',
  29         POLL: 'poll',
  30         HIDDEN: 'hidden',
  31     }
  32
  33     def __init__(self, fs):
  34         self.fs = fs
  35
  36     def __contains__(self, other):
  37         return other in str(self)
  38
  39     def __str__(self):
  40         keys = Flags.VALS.keys()
  41         es = {Flags.VALS[k] for k in keys if self.fs & k == k}
  42         return ', '.join(sorted(es))
  43
  44     def __bool__(self):
  45         return bool(str(self))
  46
  47
  48 class CmdParam(object):
  49     t = {
  50         'CephInt': 'int',
  51         'CephString': 'str',
  52         'CephChoices': 'str',
  53         'CephPgid': 'str',
  54         'CephOsdName': 'str',
  55         'CephPoolname': 'str',
  56         'CephObjectname': 'str',
  57         'CephUUID': 'str',
  58         'CephEntityAddr': 'str',
  59         'CephIPAddr': 'str',
  60         'CephName': 'str',
  61         'CephBool': 'bool',
  62         'CephFloat': 'float',
  63         'CephFilepath': 'str',
  64     }
  65
  66     bash_example = {
  67         'CephInt': '1',
  68         'CephString': 'string',
  69         'CephChoices': 'choice',
  70         'CephPgid': '0',
  71         'CephOsdName': 'osd.0',
  72         'CephPoolname': 'poolname',
  73         'CephObjectname': 'objectname',
  74         'CephUUID': 'uuid',
  75         'CephEntityAddr': 'entityaddr',
  76         'CephIPAddr': '0.0.0.0',
  77         'CephName': 'name',
  78         'CephBool': 'true',
  79         'CephFloat': '0.0',
  80         'CephFilepath': '/path/to/file',
  81     }
  82
  83     def __init__(self, type, name,
  84                  who=None, n=None, req=True, range=None, strings=None,
  85                  goodchars=None):
  86         self.type = type
  87         self.name = name
  88         self.who = who
  89         self.n = n == 'N'
  90         self.req = req != 'false'
  91         self.range = range.split('|') if range else []
  92         self.strings = strings.split('|') if strings else []
  93         self.goodchars = goodchars
  94
  95         assert who == None
  96
  97     def help(self):
  98         advanced = []
  99         if self.type != 'CephString':
 100             advanced.append(self.type + ' ')
 101         if self.range:
 102             advanced.append('range= ``{}`` '.format('..'.join(self.range)))
 103         if self.strings:
 104             advanced.append('strings=({}) '.format(' '.join(self.strings)))
 105         if self.goodchars:
 106             advanced.append('goodchars= ``{}`` '.format(self.goodchars))
 107         if self.n:
 108             advanced.append('(can be repeated)')
 109
 110         advanced = advanced or ["(string)"]
 111         return ' '.join(advanced)
 112
 113     def mk_example_value(self):
 114         if self.type == 'CephChoices' and self.strings:
 115             return self.strings[0]
 116         if self.range:
 117             return self.range[0]
 118         return CmdParam.bash_example[self.type]
 119
 120     def mk_bash_example(self, simple):
 121         val = self.mk_example_value()
 122
 123         if self.type == 'CephBool':
 124             return '--' + self.name
 125         if simple:
 126             if self.type == "CephChoices" and self.strings:
 127                 return val
 128             elif self.type == "CephString" and self.name != 'who':
 129                 return 'my_' + self.name
 130             else:
 131                 return CmdParam.bash_example[self.type]
 132         else:
 133             return '--{}={}'.format(self.name, val)
 134
 135
 136 class CmdCommand(object):
 137     def __init__(self, prefix, args, desc,
 138                  module=None, perm=None, flags=0, poll=None):
 139         self.prefix = prefix
 140         self.params = sorted([CmdParam(**arg) for arg in args],
 141                              key=lambda p: p.req, reverse=True)
 142         self.help = desc
 143         self.module = module
 144         self.perm = perm
 145         self.flags = Flags(flags)
 146         self.needs_overload = False
 147
 148     def is_reasonably_simple(self):
 149         if len(self.params) > 3:
 150             return False
 151         if any(p.n for p in self.params):
 152             return False
 153         return True
 154
 155     def mk_bash_example(self):
 156         simple = self.is_reasonably_simple()
 157         line = ' '.join(['ceph', self.prefix] + [p.mk_bash_example(simple) for p in self.params])
 158         return line
 159
 160
 161 class Sig:
 162     @staticmethod
 163     def _parse_arg_desc(desc):
 164         try:
 165             return dict(kv.split('=') for kv in desc.split(',') if kv)
 166         except ValueError:
 167             return desc
 168
 169     @staticmethod
 170     def parse_cmd(cmd):
 171         parsed = [Sig._parse_arg_desc(s) or s for s in cmd.split()]
 172         prefix = [s for s in parsed if isinstance(s, str)]
 173         params = [s for s in parsed if not isinstance(s, str)]
 174         return ' '.join(prefix), params
 175
 176     @staticmethod
 177     def parse_args(args):
 178         return [Sig._parse_arg_desc(arg) for arg in args.split()]
 179
 180
 181 TEMPLATE = '''
 182 .. This file is automatically generated. do not modify
 183
 184 {% for command in commands %}
 185
 186 {{ command.prefix }}
 187 {{ command.prefix | length * '^' }}
 188
 189 {{ command.help | wordwrap(70)}}
 190
 191 Example command:
 192
 193 .. code-block:: bash
 194
 195     {{ command.mk_bash_example() }}
 196 {% if command.params %}
 197 Parameters:
 198
 199 {% for param in command.params %}* **{{param.name}}**: {{ param.help() | wordwrap(70) | indent(2) }}
 200 {% endfor %}{% endif %}
 201 Ceph Module:
 202
 203 * *{{ command.module }}*
 204
 205 Required Permissions:
 206
 207 * *{{ command.perm }}*
 208
 209 {% if command.flags %}Command Flags:
 210
 211 * *{{ command.flags }}*
 212 {% endif %}
 213 {% endfor %}
 214
 215 '''
 216
 217
 218 class CephMgrCommands(Directive):
 219     """
 220     extracts commands from specified mgr modules
 221     """
 222     has_content = True
 223     required_arguments = 1
 224     optional_arguments = 0
 225     final_argument_whitespace = False
 226     option_spec = {'python_path': directives.unchanged}
 227
 228     def _normalize_path(self, dirname):
 229         my_dir = os.path.dirname(os.path.realpath(__file__))
 230         src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
 231         return os.path.join(src_dir, dirname)
 232
 233     def _is_mgr_module(self, dirname, name):
 234         if not os.path.isdir(os.path.join(dirname, name)):
 235             return False
 236         if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
 237             return False
 238         return name not in ['tests']
 239
 240     @contextlib.contextmanager
 241     def mocked_modules(self):
 242         # src/pybind/mgr/tests
 243         from tests import mock
 244         mock_imports = ['rados',
 245                         'rbd',
 246                         'cephfs',
 247                         'dateutil',
 248                         'dateutil.parser']
 249         # make dashboard happy
 250         mock_imports += ['OpenSSL',
 251                          'jwt',
 252                          'bcrypt',
 253                          'scipy',
 254                          'jsonpatch',
 255                          'rook.rook_client',
 256                          'rook.rook_client.ceph',
 257                          'cherrypy=3.2.3']
 258
 259         # make restful happy
 260         mock_imports += ['pecan',
 261                          'pecan.rest',
 262                          'pecan.hooks',
 263                          'werkzeug',
 264                          'werkzeug.serving']
 265
 266         for m in mock_imports:
 267             args = {}
 268             parts = m.split('=', 1)
 269             mocked = parts[0]
 270             if len(parts) > 1:
 271                 args['__version__'] = parts[1]
 272             sys.modules[mocked] = mock.Mock(**args)
 273
 274         try:
 275             yield
 276         finally:
 277             for m in mock_imports:
 278                 mocked = m.split('=', 1)[0]
 279                 sys.modules.pop(mocked)
 280
 281     def _collect_module_commands(self, name):
 282         with self.mocked_modules():
 283             logger.info(bold(f"loading mgr module '{name}'..."))
 284             mgr_mod = __import__(name, globals(), locals(), [], 0)
 285             from tests import M
 286
 287             def subclass(x):
 288                 try:
 289                     return issubclass(x, M)
 290                 except TypeError:
 291                     return False
 292             ms = [c for c in mgr_mod.__dict__.values()
 293                   if subclass(c) and 'Standby' not in c.__name__]
 294             [m] = ms
 295             assert isinstance(m.COMMANDS, list)
 296             return m.COMMANDS
 297
 298     def _normalize_command(self, command):
 299         if 'handler' in command:
 300             del command['handler']
 301         if 'cmd' in command:
 302             command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
 303             del command['cmd']
 304         else:
 305             command['args'] = Sig.parse_args(command['args'])
 306         command['flags'] = (1 << 3)
 307         command['module'] = 'mgr'
 308         return command
 309
 310     def _render_cmds(self, commands):
 311         rendered = Template(TEMPLATE).render(commands=list(commands))
 312         lines = rendered.split("\n")
 313         assert lines
 314         lineno = self.lineno - self.state_machine.input_offset - 1
 315         source = self.state_machine.input_lines.source(lineno)
 316         self.state_machine.insert_input(lines, source)
 317
 318     def run(self):
 319         module_path = self._normalize_path(self.arguments[0])
 320         sys.path.insert(0, module_path)
 321         for path in self.options.get('python_path', '').split(':'):
 322             sys.path.insert(0, self._normalize_path(path))
 323         os.environ['UNITTEST'] = 'true'
 324         modules = [name for name in os.listdir(module_path)
 325                    if self._is_mgr_module(module_path, name)]
 326         commands = sum([self._collect_module_commands(name) for name in modules], [])
 327         cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
 328         cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
 329         cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
 330         self._render_cmds(cmds)
 331         return []
 332
 333
 334 class MyProcessor(Preprocessor):
 335     def __init__(self):
 336         super().__init__()
 337         self.cmds = []
 338         self.undef('__DATE__')
 339         self.undef('__TIME__')
 340         self.expand_linemacro = False
 341         self.expand_filemacro = False
 342         self.expand_countermacro = False
 343         self.line_directive = '#line'
 344         self.define("__PCPP_VERSION__ " + '')
 345         self.define("__PCPP_ALWAYS_FALSE__ 0")
 346         self.define("__PCPP_ALWAYS_TRUE__ 1")
 347
 348     def eval(self, src):
 349         _cmds = []
 350
 351         NONE = 0
 352         NOFORWARD = (1 << 0)
 353         OBSOLETE = (1 << 1)
 354         DEPRECATED = (1 << 2)
 355         MGR = (1 << 3)
 356         POLL = (1 << 4)
 357         HIDDEN = (1 << 5)
 358         TELL = (1 << 6)
 359
 360         def FLAG(a):
 361             return a
 362
 363         def COMMAND(cmd, desc, module, perm):
 364             _cmds.append({
 365                 'cmd': cmd,
 366                 'desc': desc,
 367                 'module': module,
 368                 'perm': perm
 369             })
 370
 371         def COMMAND_WITH_FLAG(cmd, desc, module, perm, flag):
 372             _cmds.append({
 373                 'cmd': cmd,
 374                 'desc': desc,
 375                 'module': module,
 376                 'perm': perm,
 377                 'flags': flag
 378             })
 379
 380         self.parse(src)
 381         out = io.StringIO()
 382         self.write(out)
 383         out.seek(0)
 384         s = out.read()
 385         exec(s, globals(), locals())
 386         return _cmds
 387
 388
 389 class CephMonCommands(Directive):
 390     """
 391     extracts commands from specified header file
 392     """
 393     has_content = True
 394     required_arguments = 1
 395     optional_arguments = 0
 396     final_argument_whitespace = True
 397
 398     def _src_dir(self):
 399         my_dir = os.path.dirname(os.path.realpath(__file__))
 400         return os.path.abspath(os.path.join(my_dir, '../..'))
 401
 402     def _parse_headers(self, headers):
 403         src_dir = self._src_dir()
 404         src = '\n'.join(f'#include "{src_dir}/{header}"' for header in headers)
 405         return MyProcessor().eval(src)
 406
 407     def _normalize_command(self, command):
 408         if 'handler' in command:
 409             del command['handler']
 410         command['prefix'], command['args'] = Sig.parse_cmd(command['cmd'])
 411         del command['cmd']
 412         return command
 413
 414     def _render_cmds(self, commands):
 415         rendered = Template(TEMPLATE).render(commands=list(commands))
 416         lines = rendered.split("\n")
 417         assert lines
 418         lineno = self.lineno - self.state_machine.input_offset - 1
 419         source = self.state_machine.input_lines.source(lineno)
 420         self.state_machine.insert_input(lines, source)
 421
 422     def run(self):
 423         headers = self.arguments[0].split()
 424         commands = self._parse_headers(headers)
 425         cmds = [CmdCommand(**self._normalize_command(c)) for c in commands]
 426         cmds = [cmd for cmd in cmds if 'hidden' not in cmd.flags]
 427         cmds = sorted(cmds, key=lambda cmd: cmd.prefix)
 428         self._render_cmds(cmds)
 429         return []
 430
 431
 432 def setup(app):
 433     app.add_directive("ceph-mgr-commands", CephMgrCommands)
 434     app.add_directive("ceph-mon-commands", CephMonCommands)
 435
 436     return {
 437         'version': '0.1',
 438         'parallel_read_safe': True,
 439         'parallel_write_safe': True,
 440     }