]>
git.proxmox.com Git - ceph.git/blob - ceph/src/pybind/mgr/object_format.py
4a843e5c26cf69bc2ffde1dc9a6977ee90d2df3a
1 # object_format.py provides types and functions for working with
2 # requested output formats such as JSON, YAML, etc.
3 """tools for writing formatting-friendly mgr module functions
5 Currently, the ceph mgr code in python is most commonly written by adding mgr
6 modules and corresponding classes and then adding methods to those classes that
7 are decorated using `@CLICommand` from `mgr_module.py`. These methods (that
8 will be called endpoints subsequently) then implement the logic that is
9 executed when the mgr receives a command from a client. These endpoints are
10 currently responsible for forming a response tuple of (int, str, str) where the
11 int represents a return value (error code) and the first string the "body" of
12 the response. The mgr supports a generic `format` parameter (`--format` on the
13 ceph cli) that each endpoint must then explicitly handle. At the time of this
14 writing, many endpoints do not handle alternate formats and are each
15 implementing formatting/serialization of values in various different ways.
17 The `object_format` module aims to make the process of writing endpoint
18 functions easier, more consistent, and (hopefully) better documented. At the
19 highest level, the module provides a new decorator `Responder` that must be
20 placed below the `CLICommand` decorator (so that it decorates the endpoint
21 before `CLICommand`). This decorator helps automatically convert Python objects
22 to response tuples expected by the manager, while handling the `format`
23 parameter automatically.
25 In addition to the decorator the module provides a few other types and methods
26 that intended to interoperate with the decorator and make small customizations
27 and error handling easier.
31 The simple and intended way to use the decorator is as follows:
32 @CLICommand("command name", perm="r")
34 def create_something(self, name: str) -> Dict[str, str]:
36 return {"name": name, "id": new_id}
38 In this case the `create_something` method return a python dict,
39 and does not return a response tuple directly. Instead, the
40 dict is converted to either JSON or YAML depending on what the
41 client requested. Assuming no exception is raised by the
42 implementation then the response code is always zero (success).
44 The object_format module provides an exception type `ErrorResponse`
45 that assists in returning "clean" error conditions to the client.
46 Extending the previous example to use this exception:
47 @CLICommand("command name", perm="r")
49 def create_something(self, name: str) -> Dict[str, str]:
52 return {"name": name, "id": new_id}
53 except KeyError as kerr:
54 # explicitly set the return value to ENOENT for KeyError
55 raise ErrorResponse.wrap(kerr, return_value=-errno.ENOENT)
56 except (BusinessLogcError, OSError) as err:
57 # return value is based on err when possible
58 raise ErrorResponse.wrap(err)
60 Most uses of ErrorResponse are expected to use the `wrap` classmethod,
61 as it will aid in the handling of an existing exception but `ErrorResponse`
62 can be used directly too.
64 == Customizing Response Formatting ==
66 The `Responder` is built using two additional mid-layer types. The
67 `ObjectFormatAdapter` and the `ReturnValueAdapter` by default. These types
68 implement the `CommonFormatter` protocol and `ReturnValueProvider` protocols
69 respectively. Most cases will not need to customize the `ReturnValueAdapter` as
70 returning zero on success is expected. However, if there's a need to return a
71 non-zero error code outside of an exception, you can add the `mgr_return_value`
72 function to the returned type of the endpoint function - causing it to meet the
73 `ReturnValueProvider` protocol. Whatever integer that function returns will
74 then be used in the response tuple.
76 The `ObjectFormatAdapter` can operate in two modes. By default, any type
77 returned from the endpoint function will be checked for a `to_simplified`
78 method (the type matches the SimpleDataProvider` protocol) and if it exists
79 the method will be called and the result serialized. Example:
81 def __init__(self, temperature: int, quantity: int) -> None:
82 self.temperature = temperature
83 self.quantity = quantity
84 def to_simplified(self) -> Dict[str, int]:
85 return {"temp": self.temperature, "qty": self.quantity}
87 @CLICommand("command name", perm="r")
89 def create_something_cool(self) -> CoolStuff:
90 cool_stuff: CoolStuff = self._make_cool_stuff() # implementation
93 In order to serialize the result, the object returned from the wrapped
94 function must provide the `to_simplified` method (or the compatibility methods,
95 see below) or already be a "simplified type". Valid types include lists and
96 dicts that contain other lists and dicts and ints, strs, bools -- basic objects
97 that can be directly converted to json (via json.dumps) without any additional
98 conversions. The `to_simplified` method must always return such types.
100 To be compatible with many existing types in the ceph mgr codebase one can pass
101 `compatible=True` to the `ObjectFormatAdapter`. If the type provides a
102 `to_json` and/or `to_yaml` method that returns basic python types (dict, list,
103 str, etc...) but *not* already serialized JSON or YAML this flag can be
104 enabled. Note that Responder takes as an argument any callable that returns a
105 `CommonFormatter`. In this example below we enable the flag using
107 class MyExistingClass:
108 def to_json(self) -> Dict[str, Any]:
109 return {"name": self.name, "height": self.height}
111 @CLICommand("command name", perm="r")
112 Responder(functools.partial(ObjectFormatAdapter, compatible=True))
113 def create_an_item(self) -> MyExistingClass:
114 item: MyExistingClass = self._new_item() # implementation
118 For cases that need to return xml or plain text formatted responses one can
119 create a new class that matches the `CommonFormatter` protocol (provides a
120 valid_formats method) and one or more `format_x` method where x is the name of
121 a format ("json", "yaml", "xml", "plain", etc...).
122 class MyCustomFormatAdapter:
123 def __init__(self, obj_to_format: Any) -> None:
125 def valid_formats(self) -> Iterable[str]:
127 def format_json(self) -> str:
129 def format_xml(self) -> str:
133 Of course, the Responder itself can be used as a base class and aspects of the
134 Responder altered for specific use cases. Inheriting from `Responder` and
135 customizing it is an exercise left for those brave enough to read the code in
136 `object_format.py` :-).
144 from functools
import wraps
161 # this uses a version check as opposed to a try/except because this
162 # form makes mypy happy and try/except doesn't.
163 if sys
.version_info
>= (3, 8):
164 from typing
import Protocol
166 # typing_extensions will not be available for the real mgr server
167 from typing_extensions
import Protocol
169 # fallback type that is acceptable to older python on prod. builds
170 class Protocol
: # type: ignore
173 from mgr_module
import HandlerFuncType
176 DEFAULT_JSON_INDENT
: int = 2
179 class Format(str, enum
.Enum
):
182 json_pretty
= "json-pretty"
184 xml_pretty
= "xml-pretty"
188 # SimpleData is a type alias for Any unless we can determine the
189 # exact set of subtypes we want to support. But it is explicit!
193 ObjectResponseFuncType
= Union
[
194 Callable
[..., Dict
[Any
, Any
]],
195 Callable
[..., List
[Any
]],
199 class SimpleDataProvider(Protocol
):
200 def to_simplified(self
) -> SimpleData
:
201 """Return a simplified representation of the current object.
202 The simplified representation should be trivially serializable.
204 ... # pragma: no cover
207 class JSONDataProvider(Protocol
):
208 def to_json(self
) -> Any
:
209 """Return a python object that can be serialized into JSON.
210 This function does _not_ return a JSON string.
212 ... # pragma: no cover
215 class YAMLDataProvider(Protocol
):
216 def to_yaml(self
) -> Any
:
217 """Return a python object that can be serialized into YAML.
218 This function does _not_ return a string of YAML.
220 ... # pragma: no cover
223 class JSONFormatter(Protocol
):
224 def format_json(self
) -> str:
225 """Return a JSON formatted representation of an object."""
226 ... # pragma: no cover
229 class YAMLFormatter(Protocol
):
230 def format_yaml(self
) -> str:
231 """Return a JSON formatted representation of an object."""
232 ... # pragma: no cover
235 class ReturnValueProvider(Protocol
):
236 def mgr_return_value(self
) -> int:
237 """Return an integer value to provide the Ceph MGR with a error code
238 for the MGR's response tuple. Zero means success. Return an negative
241 ... # pragma: no cover
244 class CommonFormatter(Protocol
):
245 """A protocol that indicates the type is a formatter for multiple
249 def valid_formats(self
) -> Iterable
[str]:
250 """Return the names of known valid formats."""
251 ... # pragma: no cover
254 # The _is_name_of_protocol_type functions below are here because the production
255 # builds of the ceph manager are lower than python 3.8 and do not have
256 # typing_extensions available in the resulting images. This means that
257 # runtime_checkable is not available and isinstance can not be used with a
258 # protocol type. These could be replaced by isinstance in a later version of
259 # python. Note that these functions *can not* be methods of the protocol types
260 # for neatness - including methods on the protocl types makes mypy consider
261 # those methods as part of the protcol & a required method. Using decorators
262 # did not change that - I checked.
265 def _is_simple_data_provider(obj
: SimpleDataProvider
) -> bool:
266 """Return true if obj is usable as a SimpleDataProvider."""
267 return callable(getattr(obj
, 'to_simplified', None))
270 def _is_json_data_provider(obj
: JSONDataProvider
) -> bool:
271 """Return true if obj is usable as a JSONDataProvider."""
272 return callable(getattr(obj
, 'to_json', None))
275 def _is_yaml_data_provider(obj
: YAMLDataProvider
) -> bool:
276 """Return true if obj is usable as a YAMLDataProvider."""
277 return callable(getattr(obj
, 'to_yaml', None))
280 def _is_return_value_provider(obj
: ReturnValueProvider
) -> bool:
281 """Return true if obj is usable as a YAMLDataProvider."""
282 return callable(getattr(obj
, 'mgr_return_value', None))
285 class ObjectFormatAdapter
:
286 """A format adapater for a single object.
287 Given an input object, this type will adapt the object, or a simplified
288 representation of the object, to either JSON or YAML when the format_json or
289 format_yaml methods are used.
291 If the compatible flag is true and the object provided to the adapter has
292 methods such as `to_json` and/or `to_yaml` these methods will be called in
293 order to get a JSON/YAML compatible simplified representation of the
296 If the above case is not satisfied and the object provided to the adapter
297 has a method `to_simplified`, this method will be called to acquire a
298 simplified representation of the object.
300 If none of the above cases is true, the object itself will be used for
301 serialization. If the object can not be safely serialized an exception will
304 NOTE: Some code may use methods named like `to_json` to return a JSON
305 string. If that is the case, you should not use that method with the
306 ObjectFormatAdapter. Do not set compatible=True for objects of this type.
312 json_indent
: Optional
[int] = DEFAULT_JSON_INDENT
,
313 compatible
: bool = False,
316 self
._compatible
= compatible
317 self
.json_indent
= json_indent
319 def _fetch_json_data(self
) -> Any
:
320 # if the data object provides a specific simplified representation for
321 # JSON (and compatible mode is enabled) get the data via that method
322 if self
._compatible
and _is_json_data_provider(self
.obj
):
323 return self
.obj
.to_json()
324 # otherwise we use our specific method `to_simplified` if it exists
325 if _is_simple_data_provider(self
.obj
):
326 return self
.obj
.to_simplified()
327 # and fall back to the "raw" object
330 def format_json(self
) -> str:
331 """Return a JSON formatted string representing the input object."""
333 self
._fetch
_json
_data
(), indent
=self
.json_indent
, sort_keys
=True
336 def _fetch_yaml_data(self
) -> Any
:
337 if self
._compatible
and _is_yaml_data_provider(self
.obj
):
338 return self
.obj
.to_yaml()
339 # nothing specific to YAML was found. use the simplified representation
340 # for JSON, as all valid JSON is valid YAML.
341 return self
._fetch
_json
_data
()
343 def format_yaml(self
) -> str:
344 """Return a YAML formatted string representing the input object."""
345 return yaml
.safe_dump(self
._fetch
_yaml
_data
())
347 format_json_pretty
= format_json
349 def valid_formats(self
) -> Iterable
[str]:
350 """Return valid format names."""
351 return set(str(v
) for v
in Format
.__members
__)
354 class ReturnValueAdapter
:
355 """A return-value adapter for an object.
356 Given an input object, this type will attempt to get a mgr return value
357 from the object if provides a `mgr_return_value` function.
358 If not it returns a default return value, typically 0.
367 self
.default_return_value
= default
369 def mgr_return_value(self
) -> int:
370 if _is_return_value_provider(self
.obj
):
371 return int(self
.obj
.mgr_return_value())
372 return self
.default_return_value
375 class ErrorResponseBase(Exception):
376 """An exception that can directly be converted to a mgr reponse."""
378 def format_response(self
) -> Tuple
[int, str, str]:
379 raise NotImplementedError()
382 class UnknownFormat(ErrorResponseBase
):
383 """Raised if the format name is unexpected.
384 This can help distinguish typos from formats that are known but
388 def __init__(self
, format_name
: str) -> None:
389 self
.format_name
= format_name
391 def format_response(self
) -> Tuple
[int, str, str]:
392 return -errno
.EINVAL
, "", f
"Unknown format name: {self.format_name}"
395 class UnsupportedFormat(ErrorResponseBase
):
396 """Raised if the format name does not correspond to any valid
397 conversion functions.
400 def __init__(self
, format_name
: str) -> None:
401 self
.format_name
= format_name
403 def format_response(self
) -> Tuple
[int, str, str]:
404 return -errno
.EINVAL
, "", f
"Unsupported format: {self.format_name}"
407 class ErrorResponse(ErrorResponseBase
):
408 """General exception convertible to a mgr response."""
410 E
= TypeVar("E", bound
="ErrorResponse")
412 def __init__(self
, status
: str, return_value
: Optional
[int] = None) -> None:
413 self
.return_value
= (
414 return_value
if return_value
is not None else -errno
.EINVAL
418 def format_response(self
) -> Tuple
[int, str, str]:
419 return (self
.return_value
, "", self
.status
)
421 def mgr_return_value(self
) -> int:
422 return self
.return_value
425 def errno(self
) -> int:
426 rv
= self
.return_value
427 return -rv
if rv
< 0 else rv
429 def __repr__(self
) -> str:
430 return f
"ErrorResponse({self.status!r}, {self.return_value!r})"
434 cls
: Type
[E
], exc
: Exception, return_value
: Optional
[int] = None
436 if return_value
is None:
438 return_value
= -int(getattr(exc
, "errno"))
439 except (AttributeError, ValueError):
441 err
= cls(str(exc
), return_value
=return_value
)
442 setattr(err
, "__cause__", exc
)
446 def _get_requested_format(f
: ObjectResponseFuncType
, kw
: Dict
[str, Any
]) -> str:
447 # todo: leave 'format' in kw dict iff its part of f's signature
448 return kw
.pop("format", None)
452 """A decorator type intended to assist in converting Python return types
453 into valid responses for the Ceph MGR.
455 A function that returns a Python object will have the object converted into
456 a return value and formatted response body, based on the `format` argument
457 passed to the mgr. When used from the ceph cli tool the `--format=[name]`
458 argument is mapped to a `format` keyword argument. The decorated function
459 may provide a `format` argument (type str). If the decorated function does
460 not provide a `format` argument itself, the Responder decorator will
461 implicitly add one to the MGR's "CLI arguments" handling stack.
463 The Responder object is callable and is expected to be used as a decorator.
467 self
, formatter
: Optional
[Callable
[..., CommonFormatter
]] = None
469 self
.formatter
= formatter
470 self
.default_format
= "json"
472 def _formatter(self
, obj
: Any
) -> CommonFormatter
:
473 """Return the formatter/format-adapter for the object."""
474 if self
.formatter
is not None:
475 return self
.formatter(obj
)
476 return ObjectFormatAdapter(obj
)
478 def _retval_provider(self
, obj
: Any
) -> ReturnValueProvider
:
479 """Return a ReturnValueProvider for the given object."""
480 return ReturnValueAdapter(obj
)
482 def _get_format_func(
483 self
, obj
: Any
, format_req
: Optional
[str] = None
485 formatter
= self
._formatter
(obj
)
486 if format_req
is None:
487 format_req
= self
.default_format
488 if format_req
not in formatter
.valid_formats():
489 raise UnknownFormat(format_req
)
490 req
= str(format_req
).replace("-", "_")
491 ffunc
= getattr(formatter
, f
"format_{req}", None)
493 raise UnsupportedFormat(format_req
)
496 def _dry_run(self
, format_req
: Optional
[str] = None) -> None:
497 """Raise an exception if the format_req is not supported."""
498 # call with an empty dict to see if format_req is valid and supported
499 self
._get
_format
_func
({}, format_req
)
501 def _formatted(self
, obj
: Any
, format_req
: Optional
[str] = None) -> str:
502 """Return the object formatted/serialized."""
503 ffunc
= self
._get
_format
_func
(obj
, format_req
)
506 def _return_value(self
, obj
: Any
) -> int:
507 """Return a mgr return-value for the given object (usually zero)."""
508 return self
._retval
_provider
(obj
).mgr_return_value()
510 def __call__(self
, f
: ObjectResponseFuncType
) -> HandlerFuncType
:
511 """Wrap a python function so that the original function's return value
512 becomes the source for an automatically formatted mgr response.
516 def _format_response(*args
: Any
, **kwargs
: Any
) -> Tuple
[int, str, str]:
517 format_req
= _get_requested_format(f
, kwargs
)
519 self
._dry
_run
(format_req
)
520 robj
= f(*args
, **kwargs
)
521 body
= self
._formatted
(robj
, format_req
)
522 retval
= self
._return
_value
(robj
)
523 except ErrorResponseBase
as e
:
524 return e
.format_response()
525 return retval
, body
, ""
527 # set the extra args on our wrapper function. this will be consumed by
528 # the CLICommand decorator and added to the set of optional arguments
529 # on the ceph cli/api
530 setattr(_format_response
, "extra_args", {"format": str})
531 return _format_response