ceph/doc/_ext/ceph_confval.py

   1 import io
   2 import contextlib
   3 import os
   4 import sys
   5 from typing import Any, Dict, List, Union
   6
   7 from docutils.nodes import Node
   8 from docutils.parsers.rst import directives
   9 from docutils.statemachine import StringList
  10
  11 from sphinx import addnodes
  12 from sphinx.directives import ObjectDescription
  13 from sphinx.domains.python import PyField
  14 from sphinx.environment import BuildEnvironment
  15 from sphinx.locale import _
  16 from sphinx.util import logging, status_iterator, ws_re
  17 from sphinx.util.docutils import switch_source_input, SphinxDirective
  18 from sphinx.util.docfields import Field
  19 from sphinx.util.nodes import make_id
  20 import jinja2
  21 import jinja2.filters
  22 import yaml
  23
  24 logger = logging.getLogger(__name__)
  25
  26
  27 TEMPLATE = '''
  28 {% if desc %}
  29    {{ desc | wordwrap(70) | indent(3) }}
  30 {% endif %}
  31    :type: ``{{opt.type}}``
  32 {%- if default is not none %}
  33   {%- if opt.type == 'size' %}
  34    :default: ``{{ default | eval_size | iec_size }}``
  35   {%- elif opt.type == 'secs' %}
  36    :default: ``{{ default | readable_duration(opt.type) }}``
  37   {%- elif opt.type in ('uint', 'int', 'float') %}
  38    :default: ``{{ default | readable_num(opt.type) }}``
  39   {%- elif opt.type == 'millisecs' %}
  40    :default: ``{{ default }}`` milliseconds
  41   {%- elif opt.type == 'bool' %}
  42    :default: ``{{ default | string | lower }}``
  43   {%- else %}
  44    :default: {{ default | literal }}
  45   {%- endif -%}
  46 {%- endif %}
  47 {%- if opt.enum_values %}
  48    :valid choices:{% for enum_value in opt.enum_values -%}
  49 {{" -" | indent(18, not loop.first) }} {{ enum_value | literal }}
  50 {% endfor %}
  51 {%- endif %}
  52 {%- if opt.min is defined and opt.max is defined %}
  53    :allowed range: ``[{{ opt.min }}, {{ opt.max }}]``
  54 {%- elif opt.min is defined %}
  55    :min: ``{{ opt.min }}``
  56 {%- elif opt.max is defined %}
  57    :max: ``{{ opt.max }}``
  58 {%- endif %}
  59 {%- if opt.constraint %}
  60    :constraint: {{ opt.constraint }}
  61 {% endif %}
  62 {%- if opt.policies %}
  63    :policies: {{ opt.policies }}
  64 {% endif %}
  65 {%- if opt.example %}
  66    :example: {{ opt.example }}
  67 {%- endif %}
  68 {%- if opt.see_also %}
  69    :see also: {{ opt.see_also | map('ref_confval') | join(', ') }}
  70 {%- endif %}
  71 {% if opt.note %}
  72    .. note::
  73       {{ opt.note }}
  74 {%- endif -%}
  75 {%- if opt.warning %}
  76    .. warning::
  77       {{ opt.warning }}
  78 {%- endif %}
  79 '''
  80
  81
  82 def eval_size(value) -> int:
  83     try:
  84         return int(value)
  85     except ValueError:
  86         times = dict(_K=1 << 10,
  87                      _M=1 << 20,
  88                      _G=1 << 30,
  89                      _T=1 << 40)
  90         for unit, m in times.items():
  91             if value.endswith(unit):
  92                 return int(value[:-len(unit)]) * m
  93         raise ValueError(f'unknown value: {value}')
  94
  95
  96 def readable_duration(value: str, typ: str) -> str:
  97     try:
  98         if typ == 'sec':
  99             v = int(value)
 100             postfix = 'second' if v == 1 else 'seconds'
 101             return f'{v} {postfix}'
 102         elif typ == 'float':
 103             return str(float(value))
 104         else:
 105             return str(int(value))
 106     except ValueError:
 107         times = dict(_min=['minute', 'minutes'],
 108                      _hr=['hour', 'hours'],
 109                      _day=['day', 'days'])
 110         for unit, readables in times.items():
 111             if value.endswith(unit):
 112                 v = int(value[:-len(unit)])
 113                 postfix = readables[0 if v == 1 else 1]
 114                 return f'{v} {postfix}'
 115         raise ValueError(f'unknown value: {value}')
 116
 117
 118 def do_plain_num(value: str, typ: str) -> str:
 119     if typ == 'float':
 120         return str(float(value))
 121     else:
 122         return str(int(value))
 123
 124
 125 def iec_size(value: int) -> str:
 126     if value == 0:
 127         return '0B'
 128     units = dict(Ei=60,
 129                  Pi=50,
 130                  Ti=40,
 131                  Gi=30,
 132                  Mi=20,
 133                  Ki=10,
 134                  B=0)
 135     for unit, bits in units.items():
 136         m = 1 << bits
 137         if value % m == 0:
 138             value //= m
 139             return f'{value}{unit}'
 140     raise Exception(f'iec_size() failed to convert {value}')
 141
 142
 143 def do_fileize_num(value: str, typ: str) -> str:
 144     v = eval_size(value)
 145     return iec_size(v)
 146
 147
 148 def readable_num(value: str, typ: str) -> str:
 149     e = ValueError()
 150     for eval_func in [do_plain_num,
 151                       readable_duration,
 152                       do_fileize_num]:
 153         try:
 154             return eval_func(value, typ)
 155         except ValueError as ex:
 156             e = ex
 157     raise e
 158
 159
 160 def literal(name) -> str:
 161     if name:
 162         return f'``{name}``'
 163     else:
 164         return f'<empty string>'
 165
 166
 167 def ref_confval(name) -> str:
 168     return f':confval:`{name}`'
 169
 170
 171 def jinja_template() -> jinja2.Template:
 172     env = jinja2.Environment()
 173     env.filters['eval_size'] = eval_size
 174     env.filters['iec_size'] = iec_size
 175     env.filters['readable_duration'] = readable_duration
 176     env.filters['readable_num'] = readable_num
 177     env.filters['literal'] = literal
 178     env.filters['ref_confval'] = ref_confval
 179     return env.from_string(TEMPLATE)
 180
 181
 182 FieldValueT = Union[bool, float, int, str]
 183
 184
 185 class CephModule(SphinxDirective):
 186     """
 187     Directive to name the mgr module for which options are documented.
 188     """
 189     has_content = False
 190     required_arguments = 1
 191     optional_arguments = 0
 192     final_argument_whitespace = False
 193
 194     def run(self) -> List[Node]:
 195         module = self.arguments[0].strip()
 196         if module == 'None':
 197             self.env.ref_context.pop('ceph:module', None)
 198         else:
 199             self.env.ref_context['ceph:module'] = module
 200         return []
 201
 202
 203 class CephOption(ObjectDescription):
 204     """
 205     emit option loaded from given command/options/<name>.yaml.in file
 206     """
 207     has_content = True
 208     required_arguments = 1
 209     optional_arguments = 0
 210     final_argument_whitespace = False
 211     option_spec = {
 212         'module': directives.unchanged,
 213         'default': directives.unchanged
 214     }
 215
 216
 217     doc_field_types = [
 218         Field('default',
 219               label=_('Default'),
 220               has_arg=False,
 221               names=('default',)),
 222         Field('type',
 223               label=_('Type'),
 224               has_arg=False,
 225               names=('type',),
 226               bodyrolename='class'),
 227     ]
 228
 229     template = jinja_template()
 230     opts: Dict[str, Dict[str, FieldValueT]] = {}
 231     mgr_opts: Dict[str,            # module name
 232                    Dict[str,       # option name
 233                         Dict[str,  # field_name
 234                              FieldValueT]]] = {}
 235
 236     def _load_yaml(self) -> Dict[str, Dict[str, FieldValueT]]:
 237         if CephOption.opts:
 238             return CephOption.opts
 239         opts = []
 240         for fn in status_iterator(self.config.ceph_confval_imports,
 241                                   'loading options...', 'red',
 242                                   len(self.config.ceph_confval_imports),
 243                                   self.env.app.verbosity):
 244             self.env.note_dependency(fn)
 245             try:
 246                 with open(fn, 'r') as f:
 247                     yaml_in = io.StringIO()
 248                     for line in f:
 249                         if '@' not in line:
 250                             yaml_in.write(line)
 251                     yaml_in.seek(0)
 252                     opts += yaml.safe_load(yaml_in)['options']
 253             except OSError as e:
 254                 message = f'Unable to open option file "{fn}": {e}'
 255                 raise self.error(message)
 256         CephOption.opts = dict((opt['name'], opt) for opt in opts)
 257         return CephOption.opts
 258
 259     def _normalize_path(self, dirname):
 260         my_dir = os.path.dirname(os.path.realpath(__file__))
 261         src_dir = os.path.abspath(os.path.join(my_dir, '../..'))
 262         return os.path.join(src_dir, dirname)
 263
 264     def _is_mgr_module(self, dirname, name):
 265         if not os.path.isdir(os.path.join(dirname, name)):
 266             return False
 267         if not os.path.isfile(os.path.join(dirname, name, '__init__.py')):
 268             return False
 269         return name not in ['tests']
 270
 271     @contextlib.contextmanager
 272     def mocked_modules(self):
 273         # src/pybind/mgr/tests
 274         from tests import mock
 275         mock_imports = ['rados',
 276                         'rbd',
 277                         'cephfs',
 278                         'dateutil',
 279                         'dateutil.parser']
 280         # make dashboard happy
 281         mock_imports += ['OpenSSL',
 282                          'jwt',
 283                          'bcrypt',
 284                          'jsonpatch',
 285                          'rook.rook_client',
 286                          'rook.rook_client.ceph',
 287                          'rook.rook_client._helper',
 288                          'cherrypy=3.2.3']
 289         # make diskprediction_local happy
 290         mock_imports += ['numpy',
 291                          'scipy']
 292         # make restful happy
 293         mock_imports += ['pecan',
 294                          'pecan.rest',
 295                          'pecan.hooks',
 296                          'werkzeug',
 297                          'werkzeug.serving']
 298
 299         for m in mock_imports:
 300             args = {}
 301             parts = m.split('=', 1)
 302             mocked = parts[0]
 303             if len(parts) > 1:
 304                 args['__version__'] = parts[1]
 305             sys.modules[mocked] = mock.Mock(**args)
 306
 307         try:
 308             yield
 309         finally:
 310             for m in mock_imports:
 311                 mocked = m.split('=', 1)[0]
 312                 sys.modules.pop(mocked)
 313
 314     def _collect_options_from_module(self, name):
 315         with self.mocked_modules():
 316             mgr_mod = __import__(name, globals(), locals(), [], 0)
 317             # import 'M' from src/pybind/mgr/tests
 318             from tests import M
 319
 320             def subclass(x):
 321                 try:
 322                     return issubclass(x, M)
 323                 except TypeError:
 324                     return False
 325             ms = [c for c in mgr_mod.__dict__.values()
 326                   if subclass(c) and 'Standby' not in c.__name__]
 327             [m] = ms
 328             assert isinstance(m.MODULE_OPTIONS, list)
 329             return m.MODULE_OPTIONS
 330
 331     def _load_module(self, module) -> Dict[str, Dict[str, FieldValueT]]:
 332         mgr_opts = CephOption.mgr_opts.get(module)
 333         if mgr_opts is not None:
 334             return mgr_opts
 335         python_path = self.config.ceph_confval_mgr_python_path
 336         for path in python_path.split(':'):
 337             sys.path.insert(0, self._normalize_path(path))
 338         module_path = self.env.config.ceph_confval_mgr_module_path
 339         module_path = self._normalize_path(module_path)
 340         sys.path.insert(0, module_path)
 341         if not self._is_mgr_module(module_path, module):
 342             raise self.error(f'module "{module}" not found under {module_path}')
 343         fn = os.path.join(module_path, module, 'module.py')
 344         if os.path.exists(fn):
 345             self.env.note_dependency(fn)
 346         os.environ['UNITTEST'] = 'true'
 347         opts = self._collect_options_from_module(module)
 348         CephOption.mgr_opts[module] = dict((opt['name'], opt) for opt in opts)
 349         return CephOption.mgr_opts[module]
 350
 351     def _current_module(self) -> str:
 352         return self.options.get('module',
 353                                 self.env.ref_context.get('ceph:module'))
 354
 355     def _render_option(self, name) -> str:
 356         cur_module = self._current_module()
 357         if cur_module:
 358             try:
 359                 opt = self._load_module(cur_module).get(name)
 360             except Exception as e:
 361                 message = f'Unable to load module "{cur_module}": {e}'
 362                 raise self.error(message)
 363         else:
 364             opt = self._load_yaml().get(name)
 365         if opt is None:
 366             raise self.error(f'Option "{name}" not found!')
 367         if cur_module and 'type' not in opt:
 368             # the type of module option defaults to 'str'
 369             opt['type'] = 'str'
 370         desc = opt.get('fmt_desc') or opt.get('long_desc') or opt.get('desc')
 371         opt_default = opt.get('default')
 372         default = self.options.get('default', opt_default)
 373         try:
 374             return self.template.render(opt=opt,
 375                                         desc=desc,
 376                                         default=default)
 377         except Exception as e:
 378             message = (f'Unable to render option "{name}": {e}. ',
 379                        f'opt={opt}, desc={desc}, default={default}')
 380             raise self.error(message)
 381
 382     def handle_signature(self,
 383                          sig: str,
 384                          signode: addnodes.desc_signature) -> str:
 385         signode.clear()
 386         signode += addnodes.desc_name(sig, sig)
 387         # normalize whitespace like XRefRole does
 388         name = ws_re.sub(' ', sig)
 389         cur_module = self._current_module()
 390         if cur_module:
 391             return '/'.join(['mgr', cur_module, name])
 392         else:
 393             return name
 394
 395     def transform_content(self, contentnode: addnodes.desc_content) -> None:
 396         name = self.arguments[0]
 397         source, lineno = self.get_source_info()
 398         source = f'{source}:{lineno}:<confval>'
 399         fields = StringList(self._render_option(name).splitlines() + [''],
 400                             source=source, parent_offset=lineno)
 401         with switch_source_input(self.state, fields):
 402             self.state.nested_parse(fields, 0, contentnode)
 403
 404     def add_target_and_index(self,
 405                              name: str,
 406                              sig: str,
 407                              signode: addnodes.desc_signature) -> None:
 408         node_id = make_id(self.env, self.state.document, self.objtype, name)
 409         signode['ids'].append(node_id)
 410         self.state.document.note_explicit_target(signode)
 411         entry = f'{name}; configuration option'
 412         self.indexnode['entries'].append(('pair', entry, node_id, '', None))
 413         std = self.env.get_domain('std')
 414         std.note_object(self.objtype, name, node_id, location=signode)
 415
 416
 417 def _reset_ref_context(app, env, docname):
 418     env.ref_context.pop('ceph:module', None)
 419
 420
 421 def setup(app) -> Dict[str, Any]:
 422     app.add_config_value('ceph_confval_imports',
 423                          default=[],
 424                          rebuild='html',
 425                          types=[str])
 426     app.add_config_value('ceph_confval_mgr_module_path',
 427                          default=[],
 428                          rebuild='html',
 429                          types=[str])
 430     app.add_config_value('ceph_confval_mgr_python_path',
 431                          default=[],
 432                          rebuild='',
 433                          types=[str])
 434     app.add_object_type(
 435         'confsec',
 436         'confsec',
 437         objname='configuration section',
 438         indextemplate='pair: %s; configuration section',
 439         doc_field_types=[
 440             Field(
 441                 'example',
 442                 label=_('Example'),
 443                 has_arg=False,
 444             )]
 445     )
 446     app.add_object_type(
 447         'confval',
 448         'confval',
 449         objname='configuration option',
 450     )
 451     app.add_directive_to_domain('std', 'mgr_module', CephModule)
 452     app.add_directive_to_domain('std', 'confval', CephOption, override=True)
 453     app.connect('env-purge-doc', _reset_ref_context)
 454
 455     return {
 456         'version': 'builtin',
 457         'parallel_read_safe': True,
 458         'parallel_write_safe': True,
 459     }