]>
git.proxmox.com Git - ceph.git/blob - ceph/src/pybind/ceph_argparse.py
f58fca3afd80016d3f4fe4ad31b5894e7ac5b26b
2 Types and routines used by the ceph CLI as well as the RESTful
3 interface. These have to do with querying the daemons for
4 command-description information, validating user command input against
5 those descriptions, and submitting the command to the appropriate
8 Copyright (C) 2013 Inktank Storage, Inc.
10 LGPL-2.1 or LGPL-3.0. See file COPYING.
25 from collections
import abc
26 from typing
import Any
, Callable
, Dict
, Generic
, List
, Optional
, Sequence
, Tuple
, Union
28 if sys
.version_info
>= (3, 8):
29 from typing
import get_args
, get_origin
35 return getattr(tp
, '__args__', ())
38 return getattr(tp
, '__origin__', None)
41 # Flags are from MonCommand.h
51 KWARG_EQUALS
= "--([^=]+)=(.+)"
52 KWARG_SPACE
= "--([^=]+)"
60 class ArgumentError(Exception):
62 Something wrong with arguments
67 class ArgumentNumber(ArgumentError
):
69 Wrong number of a repeated argument
74 class ArgumentFormat(ArgumentError
):
76 Argument value has wrong format
81 class ArgumentMissing(ArgumentError
):
83 Argument value missing in a command
88 class ArgumentValid(ArgumentError
):
90 Argument value is otherwise invalid (doesn't match choices, for instance)
95 class ArgumentTooFew(ArgumentError
):
97 Fewer arguments than descriptors in signature; may mean to continue
98 the search, so gets a special exception type
102 class ArgumentPrefix(ArgumentError
):
104 Special for mismatched prefix; less severe, don't report by default
109 class JsonFormat(Exception):
111 some syntactic or semantic issue with the JSON
116 class CephArgtype(object):
118 Base class for all Ceph argument types
120 Instantiating an object sets any validation parameters
121 (allowable strings, numeric ranges, etc.). The 'valid'
122 method validates a string against that initialized instance,
123 throwing ArgumentError if there's a problem.
125 def __init__(self
, **kwargs
):
127 set any per-instance validation parameters here
128 from kwargs (fixed string sets, integer ranges, etc)
132 def valid(self
, s
, partial
=False):
134 Run validation against given string s (generally one word);
135 partial means to accept partial string matches (begins-with).
136 If cool, set self.val to the value that should be returned
137 (a copy of the input string, or a numeric or boolean interpretation
138 thereof, for example)
139 if not, throw ArgumentError(msg-as-to-why)
145 return string representation of description of type. Note,
146 this is not a representation of the actual value. Subclasses
147 probably also override __str__() to give a more user-friendly
148 'name/type' description for use in command format help messages.
151 if hasattr(self
, 'typeargs'):
153 return '{0}(\'{1}\')'.format(self
.__class
__.__name
__, a
)
157 where __repr__ (ideally) returns a string that could be used to
158 reproduce the object, __str__ returns one you'd like to see in
159 print messages. Use __str__ to format the argtype descriptor
160 as it would be useful in a command usage message.
162 return '<{0}>'.format(self
.__class
__.__name
__)
164 def __call__(self
, v
):
167 def complete(self
, s
):
171 def _compound_type_to_argdesc(tp
, attrs
):
172 # generate argdesc from Sequence[T], Tuple[T,..] and Optional[T]
173 orig_type
= get_origin(tp
)
174 type_args
= get_args(tp
)
175 if orig_type
in (abc
.Sequence
, Sequence
, List
, list):
176 assert len(type_args
) == 1
178 return CephArgtype
.to_argdesc(type_args
[0], attrs
)
179 elif orig_type
is Tuple
:
180 assert len(type_args
) >= 1
181 inner_tp
= type_args
[0]
182 assert type_args
.count(inner_tp
) == len(type_args
), \
183 f
'all elements in {tp} should be identical'
184 attrs
['n'] = str(len(type_args
))
185 return CephArgtype
.to_argdesc(inner_tp
, attrs
)
186 elif get_origin(tp
) is Union
:
187 # should be Union[t, NoneType]
188 assert len(type_args
) == 2 and isinstance(None, type_args
[1])
189 return CephArgtype
.to_argdesc(type_args
[0], attrs
, True)
191 raise ValueError(f
"unknown type '{tp}': '{attrs}'")
194 def to_argdesc(tp
, attrs
, has_default
=False):
196 attrs
['req'] = 'false'
204 return CEPH_ARG_TYPES
[tp
]().argdesc(attrs
)
206 if isinstance(tp
, CephArgtype
):
207 return tp
.argdesc(attrs
)
208 elif isinstance(tp
, type) and issubclass(tp
, enum
.Enum
):
209 return CephChoices(tp
=tp
).argdesc(attrs
)
211 return CephArgtype
._compound
_type
_to
_argdesc
(tp
, attrs
)
213 def argdesc(self
, attrs
):
214 attrs
['type'] = type(self
).__name
__
215 return ','.join(f
'{k}={v}' for k
, v
in attrs
.items())
218 def _cast_to_compound_type(tp
, v
):
219 orig_type
= get_origin(tp
)
220 type_args
= get_args(tp
)
221 if orig_type
in (abc
.Sequence
, Sequence
, List
, list):
222 return [CephArgtype
.cast_to(type_args
[0], e
) for e
in v
]
223 elif orig_type
is Tuple
:
224 return tuple(CephArgtype
.cast_to(type_args
[0], e
) for e
in v
)
225 elif get_origin(tp
) is Union
:
226 # should be Union[t, NoneType]
227 assert len(type_args
) == 2 and isinstance(None, type_args
[1])
228 return CephArgtype
.cast_to(type_args
[0], v
)
230 raise ValueError(f
"unknown type '{tp}': '{v}'")
240 if tp
in PYTHON_TYPES
:
242 elif isinstance(tp
, type) and issubclass(tp
, enum
.Enum
):
245 return CephArgtype
._cast
_to
_compound
_type
(tp
, v
)
248 class CephInt(CephArgtype
):
250 range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+
251 range: list of 1 or 2 ints, [min] or [min,max]
253 def __init__(self
, range=''):
257 self
.range = list(range.split('|'))
258 self
.range = [int(x
) for x
in self
.range]
260 def valid(self
, s
, partial
=False):
264 raise ArgumentValid("{0} doesn't represent an int".format(s
))
265 if len(self
.range) == 2:
266 if val
< self
.range[0] or val
> self
.range[1]:
267 raise ArgumentValid(f
"{val} not in range {self.range}")
268 elif len(self
.range) == 1:
269 if val
< self
.range[0]:
270 raise ArgumentValid(f
"{val} not in range {self.range}")
275 if len(self
.range) == 1:
276 r
= '[{0}-]'.format(self
.range[0])
277 if len(self
.range) == 2:
278 r
= '[{0}-{1}]'.format(self
.range[0], self
.range[1])
280 return '<int{0}>'.format(r
)
282 def argdesc(self
, attrs
):
284 attrs
['range'] = '|'.join(self
.range)
285 return super().argdesc(attrs
)
288 class CephFloat(CephArgtype
):
290 range-limited float type
291 range: list of 1 or 2 floats, [min] or [min, max]
293 def __init__(self
, range=''):
297 self
.range = list(range.split('|'))
298 self
.range = [float(x
) for x
in self
.range]
300 def valid(self
, s
, partial
=False):
304 raise ArgumentValid("{0} doesn't represent a float".format(s
))
305 if len(self
.range) == 2:
306 if val
< self
.range[0] or val
> self
.range[1]:
307 raise ArgumentValid(f
"{val} not in range {self.range}")
308 elif len(self
.range) == 1:
309 if val
< self
.range[0]:
310 raise ArgumentValid(f
"{val} not in range {self.range}")
315 if len(self
.range) == 1:
316 r
= '[{0}-]'.format(self
.range[0])
317 if len(self
.range) == 2:
318 r
= '[{0}-{1}]'.format(self
.range[0], self
.range[1])
319 return '<float{0}>'.format(r
)
321 def argdesc(self
, attrs
):
323 attrs
['range'] = '|'.join(self
.range)
324 return super().argdesc(attrs
)
327 class CephString(CephArgtype
):
329 String; pretty generic. goodchars is a RE char class of valid chars
331 def __init__(self
, goodchars
=''):
332 from string
import printable
334 re
.compile(goodchars
)
336 raise ValueError('CephString(): "{0}" is not a valid RE'.
338 self
.goodchars
= goodchars
339 self
.goodset
= frozenset(
340 [c
for c
in printable
if re
.match(goodchars
, c
)]
343 def valid(self
, s
, partial
=False):
345 if self
.goodset
and not sset
<= self
.goodset
:
346 raise ArgumentFormat("invalid chars {0} in {1}".
347 format(''.join(sset
- self
.goodset
), s
))
353 b
+= '(goodchars {0})'.format(self
.goodchars
)
354 return '<string{0}>'.format(b
)
356 def complete(self
, s
):
362 def argdesc(self
, attrs
):
364 attrs
['goodchars'] = self
.goodchars
365 return super().argdesc(attrs
)
368 class CephSocketpath(CephArgtype
):
370 Admin socket path; check that it's readable and S_ISSOCK
372 def valid(self
, s
, partial
=False):
373 mode
= os
.stat(s
).st_mode
374 if not stat
.S_ISSOCK(mode
):
375 raise ArgumentValid('socket path {0} is not a socket'.format(s
))
379 return '<admin-socket-path>'
382 class CephIPAddr(CephArgtype
):
384 IP address (v4 or v6) with optional port
386 def valid(self
, s
, partial
=False):
387 # parse off port, use socket to validate addr
389 if s
.startswith('['):
391 elif s
.find('.') != -1:
399 raise ArgumentValid('{0}: invalid IPv4 port'.format(p
))
404 socket
.inet_pton(socket
.AF_INET
, a
)
406 raise ArgumentValid('{0}: invalid IPv4 address'.format(a
))
409 if s
.startswith('['):
412 raise ArgumentFormat('{0} missing terminating ]'.format(s
))
413 if s
[end
+ 1] == ':':
417 raise ArgumentValid('{0}: bad port number'.format(s
))
423 socket
.inet_pton(socket
.AF_INET6
, a
)
425 raise ArgumentValid('{0} not valid IPv6 address'.format(s
))
426 if p
is not None and int(p
) > 65535:
427 raise ArgumentValid("{0} not a valid port number".format(p
))
433 return '<IPaddr[:port]>'
436 class CephEntityAddr(CephIPAddr
):
438 EntityAddress, that is, IP address[/nonce]
440 def valid(self
, s
, partial
=False):
443 ip
, nonce
= s
.split('/')
446 super(self
.__class
__, self
).valid(ip
)
450 nonce_int
= int(nonce
)
453 if nonce_int
is None or nonce_int
< 0:
455 '{0}: invalid entity, nonce {1} not integer > 0'.
461 return '<EntityAddr>'
464 class CephPoolname(CephArgtype
):
466 Pool name; very little utility
472 class CephObjectname(CephArgtype
):
474 Object name. Maybe should be combined with Pool name as they're always
475 present in pairs, and then could be checked for presence
478 return '<objectname>'
481 class CephPgid(CephArgtype
):
483 pgid, in form N.xxx (N = pool number, xxx = hex pgnum)
485 def valid(self
, s
, partial
=False):
486 if s
.find('.') == -1:
487 raise ArgumentFormat('pgid has no .')
488 poolid
, pgnum
= s
.split('.', 1)
492 raise ArgumentFormat('pool {0} not integer'.format(poolid
))
494 raise ArgumentFormat('pool {0} < 0'.format(poolid
))
496 pgnum
= int(pgnum
, 16)
498 raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum
))
505 class CephName(CephArgtype
):
507 Name (type.id) where:
508 type is osd|mon|client|mds
509 id is a base10 int, if type == osd, or a string otherwise
517 def valid(self
, s
, partial
=False):
522 self
.nametype
= "mgr"
526 self
.nametype
= "mon"
529 if s
.find('.') == -1:
530 raise ArgumentFormat('CephName: no . in {0}'.format(s
))
532 t
, i
= s
.split('.', 1)
533 if t
not in ('osd', 'mon', 'client', 'mds', 'mgr'):
534 raise ArgumentValid('unknown type ' + t
)
540 raise ArgumentFormat('osd id ' + i
+ ' not integer')
546 return '<name (type.id)>'
549 class CephOsdName(CephArgtype
):
551 Like CephName, but specific to osds: allow <id> alone
553 osd.<id>, or <id>, or *, where id is a base10 int
559 def valid(self
, s
, partial
=False):
563 if s
.find('.') != -1:
564 t
, i
= s
.split('.', 1)
566 raise ArgumentValid('unknown type ' + t
)
573 raise ArgumentFormat('osd id ' + i
+ ' not integer')
575 raise ArgumentFormat('osd id {0} is less than 0'.format(i
))
581 return '<osdname (id|osd.id)>'
584 class CephChoices(CephArgtype
):
586 Set of string literals; init with valid choices
588 def __init__(self
, strings
='', tp
=None, **kwargs
):
589 self
.strings
= strings
.split('|')
591 if self
.enum
is not None:
592 self
.strings
= list(e
.value
for e
in self
.enum
)
594 def valid(self
, s
, partial
=False):
596 if s
not in self
.strings
:
597 # show as __str__ does: {s1|s2..}
598 raise ArgumentValid("{0} not in {1}".format(s
, self
))
603 for t
in self
.strings
:
607 raise ArgumentValid("{0} not in {1}". format(s
, self
))
610 if len(self
.strings
) == 1:
611 return '{0}'.format(self
.strings
[0])
613 return '{0}'.format('|'.join(self
.strings
))
615 def __call__(self
, v
):
616 if self
.enum
is None:
621 def complete(self
, s
):
622 all_elems
= [token
for token
in self
.strings
if token
.startswith(s
)]
625 def argdesc(self
, attrs
):
626 attrs
['strings'] = '|'.join(self
.strings
)
627 return super().argdesc(attrs
)
630 class CephBool(CephArgtype
):
632 A boolean argument, values may be case insensitive 'true', 'false', '0',
633 '1'. In keyword form, value may be left off (implies true).
635 def __init__(self
, strings
='', **kwargs
):
636 self
.strings
= strings
.split('|')
638 def valid(self
, s
, partial
=False):
639 lower_case
= s
.lower()
640 if lower_case
in ['true', '1']:
642 elif lower_case
in ['false', '0']:
645 raise ArgumentValid("{0} not one of 'true', 'false'".format(s
))
651 class CephFilepath(CephArgtype
):
655 def valid(self
, s
, partial
=False):
656 # set self.val if the specified path is readable or writable
657 s
= os
.path
.abspath(s
)
658 if not os
.access(s
, os
.R_OK
):
659 self
._validate
_writable
_file
(s
)
662 def _validate_writable_file(self
, fname
):
663 if os
.path
.exists(fname
):
664 if os
.path
.isfile(fname
):
665 if not os
.access(fname
, os
.W_OK
):
666 raise ArgumentValid('{0} is not writable'.format(fname
))
668 raise ArgumentValid('{0} is not file'.format(fname
))
670 dirname
= os
.path
.dirname(fname
)
671 if not os
.access(dirname
, os
.W_OK
):
672 raise ArgumentValid('cannot create file in {0}'.format(dirname
))
675 return '<outfilename>'
678 class CephFragment(CephArgtype
):
682 def valid(self
, s
, partial
=False):
683 if s
.find('/') == -1:
684 raise ArgumentFormat('{0}: no /'.format(s
))
685 val
, bits
= s
.split('/')
687 if not val
.startswith('0x'):
688 raise ArgumentFormat("{0} not a hex integer".format(val
))
692 raise ArgumentFormat('can\'t convert {0} to integer'.format(val
))
696 raise ArgumentFormat('can\'t convert {0} to integer'.format(bits
))
700 return "<CephFS fragment ID (0xvvv/bbb)>"
703 class CephUUID(CephArgtype
):
705 CephUUID: pretty self-explanatory
707 def valid(self
, s
, partial
=False):
710 except Exception as e
:
711 raise ArgumentFormat('invalid UUID {0}: {1}'.format(s
, e
))
718 class CephPrefix(CephArgtype
):
720 CephPrefix: magic type for "all the first n fixed strings"
722 def __init__(self
, prefix
=''):
725 def valid(self
, s
, partial
=False):
728 if isinstance(s
, bytes
):
729 # `prefix` can always be converted into unicode when being compared,
730 # but `s` could be anything passed by user.
731 s
= s
.decode('ascii')
732 except UnicodeEncodeError:
733 raise ArgumentPrefix(u
"no match for {0}".format(s
))
734 except UnicodeDecodeError:
735 raise ArgumentPrefix("no match for {0}".format(s
))
738 if self
.prefix
.startswith(s
):
746 raise ArgumentPrefix("no match for {0}".format(s
))
751 def complete(self
, s
):
752 if self
.prefix
.startswith(s
):
753 return [self
.prefix
.rstrip(' ')]
758 class argdesc(object):
760 argdesc(typename, name='name', n=numallowed|N,
761 req=False, helptext=helptext, **kwargs (type-specific))
764 typename: type(**kwargs) will be constructed
765 later, type.valid(w) will be called with a word in that position
767 name is used for parse errors and for constructing JSON output
768 n is a numeric literal or 'n|N', meaning "at least one, but maybe more"
769 req=False means the argument need not be present in the list
770 helptext is the associated help for the command
771 anything else are arguments to pass to the type constructor.
773 self.instance is an instance of type t constructed with typeargs.
775 valid() will later be called with input to validate against it,
776 and will store the validated value in self.instance.val for extraction.
778 def __init__(self
, t
, name
=None, n
=1, req
=True, **kwargs
):
779 if isinstance(t
, basestring
):
781 self
.typeargs
= {'prefix': t
}
785 self
.typeargs
= kwargs
786 self
.req
= req
in (True, 'True', 'true')
789 self
.N
= (n
in ['n', 'N'])
797 self
.instance
= self
.t(**self
.typeargs
)
800 r
= 'argdesc(' + str(self
.t
) + ', '
801 internals
= ['N', 'typeargs', 'instance', 't']
802 for (k
, v
) in self
.__dict
__.items():
803 if k
.startswith('__') or k
in internals
:
806 # undo modification from __init__
807 if k
== 'n' and self
.N
:
809 r
+= '{0}={1}, '.format(k
, v
)
810 for (k
, v
) in self
.typeargs
.items():
811 r
+= '{0}={1}, '.format(k
, v
)
815 if ((self
.t
== CephChoices
and len(self
.instance
.strings
) == 1)
816 or (self
.t
== CephPrefix
)):
817 s
= str(self
.instance
)
819 s
= '{0}({1})'.format(self
.name
, str(self
.instance
))
828 like str(), but omit parameter names (except for CephString,
829 which really needs them)
831 if self
.t
== CephBool
:
832 chunk
= "--{0}".format(self
.name
.replace("_", "-"))
833 elif self
.t
== CephPrefix
:
834 chunk
= str(self
.instance
)
835 elif self
.t
== CephChoices
:
836 if self
.name
== 'format':
837 chunk
= f
'--{self.name} {{{str(self.instance)}}}'
839 chunk
= str(self
.instance
)
840 elif self
.t
== CephOsdName
:
841 # it just so happens all CephOsdName commands are named 'id' anyway,
842 # so <id|osd.id> is perfect.
843 chunk
= '<id|osd.id>'
844 elif self
.t
== CephName
:
845 # CephName commands similarly only have one arg of the
846 # type, so <type.id> is good.
848 elif self
.t
== CephInt
:
849 chunk
= '<{0}:int>'.format(self
.name
)
850 elif self
.t
== CephFloat
:
851 chunk
= '<{0}:float>'.format(self
.name
)
853 chunk
= '<{0}>'.format(self
.name
)
861 def complete(self
, s
):
862 return self
.instance
.complete(s
)
865 def concise_sig(sig
):
867 Return string representation of sig useful for syntax reference in help
869 return ' '.join([d
.helpstr() for d
in sig
])
872 def descsort_key(sh
):
874 sort descriptors by prefixes, defined as the concatenation of all simple
875 strings in the descriptor; this works out to just the leading strings.
877 return concise_sig(sh
['sig'])
880 def descsort(sh1
, sh2
):
882 Deprecated; use (key=descsort_key) instead of (cmp=descsort)
884 return cmp(descsort_key(sh1
), descsort_key(sh2
))
887 def parse_funcsig(sig
: Sequence
[Union
[str, Dict
[str, str]]]) -> List
[argdesc
]:
889 parse a single descriptor (array of strings or dicts) into a
890 dict of function descriptor/validators (objects of CephXXX type)
892 :returns: list of ``argdesc``
898 if isinstance(desc
, basestring
):
900 desc
= {'type': t
, 'name': 'prefix', 'prefix': desc
}
902 # not a simple string, must be dict
903 if 'type' not in desc
:
904 s
= 'JSON descriptor {0} has no type'.format(sig
)
906 # look up type string in our globals() dict; if it's an
907 # object of type `type`, it must be a
908 # locally-defined class. otherwise, we haven't a clue.
909 if desc
['type'] in globals():
910 t
= globals()[desc
['type']]
911 if not isinstance(t
, type):
912 s
= 'unknown type {0}'.format(desc
['type'])
915 s
= 'unknown type {0}'.format(desc
['type'])
919 for key
, val
in desc
.items():
920 if key
not in ['type', 'name', 'n', 'req']:
922 newsig
.append(argdesc(t
,
923 name
=desc
.get('name', None),
925 req
=desc
.get('req', True),
930 def parse_json_funcsigs(s
: str,
931 consumer
: str) -> Dict
[str, Dict
[str, List
[argdesc
]]]:
933 A function signature is mostly an array of argdesc; it's represented
936 "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false <other param>], "help":helptext, "module":modulename, "perm":perms, "avail":availability}
942 A set of sigs is in an dict mapped by a unique number:
945 "sig": ["type.. ], "help":helptext...
948 "sig": [.. ], "help":helptext...
952 Parse the string s and return a dict of dicts, keyed by opcode;
953 each dict contains 'sig' with the array of descriptors, and 'help'
954 with the helptext, 'module' with the module name, 'perm' with a
955 string representing required permissions in that module to execute
956 this command (and also whether it is a read or write command from
957 the cluster state perspective), and 'avail' as a hint for
958 whether the command should be advertised by CLI, REST, or both.
959 If avail does not contain 'consumer', don't include the command
960 in the returned dict.
963 overall
= json
.loads(s
)
964 except Exception as e
:
965 print("Couldn't parse JSON {0}: {1}".format(s
, e
), file=sys
.stderr
)
968 for cmdtag
, cmd
in overall
.items():
970 s
= "JSON descriptor {0} has no 'sig'".format(cmdtag
)
972 # check 'avail' and possibly ignore this command
974 if consumer
not in cmd
['avail']:
976 # rewrite the 'sig' item with the argdesc-ized version, and...
977 cmd
['sig'] = parse_funcsig(cmd
['sig'])
978 # just take everything else as given
979 sigdict
[cmdtag
] = cmd
983 def validate_one(word
, desc
, partial
=False):
985 validate_one(word, desc, partial=False)
987 validate word against the constructed instance of the type
988 in desc. May raise exception. If it returns false (and doesn't
989 raise an exception), desc.instance.val will
990 contain the validated value (in the appropriate type).
992 desc
.instance
.valid(word
, partial
)
995 desc
.n
= desc
.numseen
+ 1
998 def matchnum(args
, signature
, partial
=False):
1000 matchnum(s, signature, partial=False)
1002 Returns number of arguments matched in s against signature.
1003 Can be used to determine most-likely command for full or partial
1004 matches (partial applies to string matches).
1007 mysig
= copy
.deepcopy(signature
)
1011 while desc
.numseen
< desc
.n
:
1012 # if there are no more arguments, return
1018 # only allow partial matching if we're on the last supplied
1019 # word; avoid matching foo bar and foot bar just because
1021 validate_one(word
, desc
, partial
and (len(words
) == 0))
1023 except ArgumentError
:
1024 # matchnum doesn't care about type of error
1029 # this wasn't required, so word may match the next desc
1030 words
.insert(0, word
)
1033 # it was required, and didn't match, return
1040 ValidatedArgs
= Dict
[str, Union
[bool, int, float, str,
1045 def store_arg(desc
: argdesc
, d
: ValidatedArgs
):
1047 Store argument described by, and held in, thanks to valid(),
1048 desc into the dictionary d, keyed by desc.name. Three cases:
1050 1) desc.N is set: value in d is a list
1051 2) prefix: multiple args are joined with ' ' into one d{} item
1052 3) single prefix or other arg: store as simple value
1054 Used in validate() below.
1057 # value should be a list
1059 d
[desc
.name
] += [desc
.instance
.val
]
1061 d
[desc
.name
] = [desc
.instance
.val
]
1062 elif (desc
.t
== CephPrefix
) and (desc
.name
in d
):
1063 # prefixes' values should be a space-joined concatenation
1064 d
[desc
.name
] += ' ' + desc
.instance
.val
1066 # if first CephPrefix or any other type, just set it
1067 d
[desc
.name
] = desc
.instance
.val
1070 def validate(args
: List
[str],
1071 signature
: Sequence
[argdesc
],
1072 flags
: Optional
[int] = 0,
1073 partial
: Optional
[bool] = False) -> ValidatedArgs
:
1075 validate(args, signature, flags=0, partial=False)
1077 args is a list of strings representing a possible
1078 command input following format of signature. Runs a validation; no
1079 exception means it's OK. Return a dict containing all arguments keyed
1080 by their descriptor name, with duplicate args per name accumulated
1081 into a list (or space-separated value for CephPrefix).
1083 Mismatches of prefix are non-fatal, as this probably just means the
1084 search hasn't hit the correct command. Mismatches of non-prefix
1085 arguments are treated as fatal, and an exception raised.
1087 This matching is modified if partial is set: allow partial matching
1088 (with partial dict returned); in this case, there are no exceptions
1092 myargs
= copy
.deepcopy(args
)
1093 mysig
= copy
.deepcopy(signature
)
1094 reqsiglen
= len([desc
for desc
in mysig
if desc
.req
])
1097 save_exception
= None
1099 arg_descs_by_name
= dict([desc
.name
, desc
] for desc
in mysig
1100 if desc
.t
!= CephPrefix
)
1102 # Special case: detect "injectargs" (legacy way of modifying daemon
1103 # configs) and permit "--" string arguments if so.
1104 injectargs
= myargs
and myargs
[0] == "injectargs"
1106 # Make a pass through all arguments
1110 while desc
.numseen
< desc
.n
:
1112 myarg
= myargs
.pop(0)
1116 # no arg, but not required? Continue consuming mysig
1117 # in case there are later required args
1118 if myarg
in (None, []) and not desc
.req
:
1121 # A keyword argument?
1123 # argdesc for the keyword argument, if we find one
1126 # Track whether we need to push value back onto
1127 # myargs in the case that this isn't a valid k=v
1128 consumed_next
= False
1130 # Try both styles of keyword argument
1131 kwarg_match
= re
.match(KWARG_EQUALS
, myarg
)
1133 # We have a "--foo=bar" style argument
1134 kwarg_k
, kwarg_v
= kwarg_match
.groups()
1136 # Either "--foo-bar" or "--foo_bar" style is accepted
1137 kwarg_k
= kwarg_k
.replace('-', '_')
1139 kwarg_desc
= arg_descs_by_name
.get(kwarg_k
, None)
1141 # Maybe this is a "--foo bar" or "--bool" style argument
1142 key_match
= re
.match(KWARG_SPACE
, myarg
)
1144 kwarg_k
= key_match
.group(1)
1146 # Permit --foo-bar=123 form or --foo_bar=123 form,
1147 # assuming all command definitions use foo_bar argument
1149 kwarg_k
= kwarg_k
.replace('-', '_')
1151 kwarg_desc
= arg_descs_by_name
.get(kwarg_k
, None)
1153 if kwarg_desc
.t
== CephBool
:
1155 elif len(myargs
): # Some trailing arguments exist
1156 kwarg_v
= myargs
.pop(0)
1158 # Forget it, this is not a valid kwarg
1162 validate_one(kwarg_v
, kwarg_desc
)
1164 store_arg(kwarg_desc
, d
)
1167 # Don't handle something as a positional argument if it
1168 # has a leading "--" unless it's a CephChoices (used for
1169 # "--yes-i-really-mean-it")
1170 if myarg
and myarg
.startswith("--"):
1171 # Special cases for instances of confirmation flags
1172 # that were defined as CephString/CephChoices instead of CephBool
1173 # in pre-nautilus versions of Ceph daemons.
1174 is_value
= desc
.t
== CephChoices \
1175 or myarg
== "--yes-i-really-mean-it" \
1176 or myarg
== "--yes-i-really-really-mean-it" \
1177 or myarg
== "--yes-i-really-really-mean-it-not-faking" \
1178 or myarg
== "--force" \
1182 # Didn't get caught by kwarg handling, but has a "--", so
1183 # we must assume it's something invalid, to avoid naively
1184 # passing through mis-typed options as the values of
1185 # positional arguments.
1186 raise ArgumentValid("Unexpected argument '{0}'".format(
1189 # out of arguments for a required param?
1190 # Either return (if partial validation) or raise
1191 if myarg
in (None, []) and desc
.req
:
1192 if desc
.N
and desc
.numseen
< 1:
1193 # wanted N, didn't even get 1
1196 raise ArgumentNumber(
1197 'saw {0} of {1}, expected at least 1'.
1198 format(desc
.numseen
, desc
)
1200 elif not desc
.N
and desc
.numseen
< desc
.n
:
1201 # wanted n, got too few
1204 # special-case the "0 expected 1" case
1205 if desc
.numseen
== 0 and desc
.n
== 1:
1206 raise ArgumentMissing(
1207 'missing required parameter {0}'.format(desc
)
1209 raise ArgumentNumber(
1210 'saw {0} of {1}, expected {2}'.
1211 format(desc
.numseen
, desc
, desc
.n
)
1215 # Have an arg; validate it
1217 validate_one(myarg
, desc
)
1218 except ArgumentError
as e
:
1221 # if not required, just push back; it might match
1223 save_exception
= [myarg
, e
]
1224 myargs
.insert(0, myarg
)
1227 # hm, it was required, so time to return/raise
1232 # Whew, valid arg acquired. Store in dict
1235 # Clear prior exception
1236 save_exception
= None
1238 # Done with entire list of argdescs
1239 if matchcnt
< reqsiglen
:
1240 raise ArgumentTooFew("not enough arguments given")
1242 if myargs
and not partial
:
1244 print(save_exception
[0], 'not valid: ', save_exception
[1], file=sys
.stderr
)
1245 raise ArgumentError("unused arguments: " + str(myargs
))
1247 if flags
& Flag
.MGR
:
1248 d
['target'] = ('mon-mgr', '')
1250 if flags
& Flag
.POLL
:
1257 def validate_command(sigdict
: Dict
[str, Dict
[str, Any
]],
1258 args
: Sequence
[str],
1259 verbose
: Optional
[bool] = False) -> ValidatedArgs
:
1261 Parse positional arguments into a parameter dict, according to
1262 the command descriptions.
1264 Writes advice about nearly-matching commands ``sys.stderr`` if
1265 the arguments do not match any command.
1267 :param sigdict: A command description dictionary, as returned
1268 from Ceph daemons by the get_command_descriptions
1270 :param args: List of strings, should match one of the command
1271 signatures in ``sigdict``
1273 :returns: A dict of parsed parameters (including ``prefix``),
1274 or an empty dict if the args did not match any signature
1277 print("validate_command: " + " ".join(args
), file=sys
.stderr
)
1281 # look for best match, accumulate possibles in bestcmds
1282 # (so we can maybe give a more-useful error message)
1285 for cmd
in sigdict
.values():
1286 flags
= cmd
.get('flags', 0)
1287 if flags
& Flag
.OBSOLETE
:
1290 matched
= matchnum(args
, sig
, partial
=True)
1291 if (matched
>= math
.floor(best_match_cnt
) and
1292 matched
== matchnum(args
, sig
, partial
=False)):
1293 # prefer those fully matched over partial patch
1295 if matched
< best_match_cnt
:
1298 print("better match: {0} > {1}: {2} ".format(
1299 matched
, best_match_cnt
, concise_sig(sig
)
1301 if matched
> best_match_cnt
:
1302 best_match_cnt
= matched
1305 bestcmds
.append(cmd
)
1307 # Sort bestcmds by number of req args so we can try shortest first
1308 # (relies on a cmdsig being key,val where val is a list of len 1)
1311 # prefer optional arguments over required ones
1313 return sum(map(lambda sig
: sig
.req
, sigs
))
1315 bestcmds_sorted
= sorted(bestcmds
, key
=grade
)
1317 print("bestcmds_sorted: ", file=sys
.stderr
)
1318 pprint
.PrettyPrinter(stream
=sys
.stderr
).pprint(bestcmds_sorted
)
1321 # for everything in bestcmds, look for a true match
1322 for cmd
in bestcmds_sorted
:
1325 valid_dict
= validate(args
, sig
, flags
=cmd
.get('flags', 0))
1328 except ArgumentPrefix
:
1329 # ignore prefix mismatches; we just haven't found
1330 # the right command yet
1332 except ArgumentMissing
as e
:
1334 if len(bestcmds
) == 1:
1337 except ArgumentTooFew
:
1338 # It looked like this matched the beginning, but it
1339 # didn't have enough args supplied. If we're out of
1340 # cmdsigs we'll fall out unfound; if we're not, maybe
1341 # the next one matches completely. Whine, but pass.
1343 print('Not enough args supplied for ',
1344 concise_sig(sig
), file=sys
.stderr
)
1345 except ArgumentError
as e
:
1347 # Solid mismatch on an arg (type, range, etc.)
1348 # Stop now, because we have the right command but
1349 # some other input is invalid
1355 print("Invalid command:", ex
, file=sys
.stderr
)
1356 print(concise_sig(sig
), ': ', cmd
['help'], file=sys
.stderr
)
1358 bestcmds
= [c
for c
in bestcmds
1359 if not c
.get('flags', 0) & (Flag
.DEPRECATED | Flag
.HIDDEN
)]
1360 bestcmds
= bestcmds
[:10] # top 10
1361 print('no valid command found; {0} closest matches:'.format(len(bestcmds
)),
1363 for cmd
in bestcmds
:
1364 print(concise_sig(cmd
['sig']), file=sys
.stderr
)
1368 def find_cmd_target(childargs
: List
[str]) -> Tuple
[str, str]:
1370 Using a minimal validation, figure out whether the command
1371 should be sent to a monitor or an osd. We do this before even
1372 asking for the 'real' set of command signatures, so we can ask the
1374 Returns ('osd', osdid), ('pg', pgid), ('mgr', '') or ('mon', '')
1376 sig
= parse_funcsig(['tell', {'name': 'target', 'type': 'CephName'}])
1378 valid_dict
= validate(childargs
, sig
, partial
=True)
1379 except ArgumentError
:
1382 if len(valid_dict
) == 2:
1383 # revalidate to isolate type and id
1385 # if this fails, something is horribly wrong, as it just
1386 # validated successfully above
1387 name
.valid(valid_dict
['target'])
1388 return name
.nametype
, name
.nameid
1390 sig
= parse_funcsig(['tell', {'name': 'pgid', 'type': 'CephPgid'}])
1392 valid_dict
= validate(childargs
, sig
, partial
=True)
1393 except ArgumentError
:
1396 if len(valid_dict
) == 2:
1397 # pg doesn't need revalidation; the string is fine
1398 return 'pg', valid_dict
['pgid']
1400 # If we reached this far it must mean that so far we've been unable to
1401 # obtain a proper target from childargs. This may mean that we are not
1402 # dealing with a 'tell' command, or that the specified target is invalid.
1403 # If the latter, we likely were unable to catch it because we were not
1404 # really looking for it: first we tried to parse a 'CephName' (osd, mon,
1405 # mds, followed by and id); given our failure to parse, we tried to parse
1406 # a 'CephPgid' instead (e.g., 0.4a). Considering we got this far though
1407 # we were unable to do so.
1409 # We will now check if this is a tell and, if so, forcefully validate the
1410 # target as a 'CephName'. This must be so because otherwise we will end
1411 # up sending garbage to a monitor, which is the default target when a
1412 # target is not explicitly specified.
1414 # 'ceph status' -> target is any one monitor
1415 # 'ceph tell mon.* status -> target is all monitors
1416 # 'ceph tell foo status -> target is invalid!
1417 if len(childargs
) > 1 and childargs
[0] == 'tell':
1419 # CephName.valid() raises on validation error; find_cmd_target()'s
1420 # caller should handle them
1421 name
.valid(childargs
[1])
1422 return name
.nametype
, name
.nameid
1424 sig
= parse_funcsig(['pg', {'name': 'pgid', 'type': 'CephPgid'}])
1426 valid_dict
= validate(childargs
, sig
, partial
=True)
1427 except ArgumentError
:
1430 if len(valid_dict
) == 2:
1431 return 'pg', valid_dict
['pgid']
1436 class RadosThread(threading
.Thread
):
1437 def __init__(self
, func
, *args
, **kwargs
):
1439 self
.kwargs
= kwargs
1441 self
.exception
= None
1442 threading
.Thread
.__init
__(self
)
1446 self
.retval
= self
.func(*self
.args
, **self
.kwargs
)
1447 except Exception as e
:
1451 def run_in_thread(func
: Callable
[[Any
, Any
], int],
1452 *args
: Any
, **kwargs
: Any
) -> int:
1453 timeout
= kwargs
.pop('timeout', 0)
1454 if timeout
== 0 or timeout
is None:
1455 # python threading module will just get blocked if timeout is `None`,
1456 # otherwise it will keep polling until timeout or thread stops.
1457 # timeout in integer when converting it to nanoseconds, but since
1458 # python3 uses `int64_t` for the deadline before timeout expires,
1459 # we have to use a safe value which does not overflow after being
1460 # added to current time in microseconds.
1461 timeout
= 24 * 60 * 60
1462 t
= RadosThread(func
, *args
, **kwargs
)
1464 # allow the main thread to exit (presumably, avoid a join() on this
1465 # subthread) before this thread terminates. This allows SIGINT
1466 # exit of a blocked call. See below.
1470 t
.join(timeout
=timeout
)
1471 # ..but allow SIGINT to terminate the waiting. Note: this
1472 # relies on the Linux kernel behavior of delivering the signal
1473 # to the main thread in preference to any subthread (all that's
1474 # strictly guaranteed is that *some* thread that has the signal
1475 # unblocked will receive it). But there doesn't seem to be
1476 # any interface to create a thread with SIGINT blocked.
1478 raise Exception("timed out")
1485 def send_command_retry(*args
: Any
, **kwargs
: Any
) -> Tuple
[int, bytes
, str]:
1488 return send_command(*args
, **kwargs
)
1489 except Exception as e
:
1490 # If our librados instance has not reached state 'connected'
1491 # yet, we'll see an exception like this and retry
1492 if ('get_command_descriptions' in str(e
) and
1493 'object in state configuring' in str(e
)):
1499 def send_command(cluster
,
1500 target
: Optional
[Tuple
[str, str]] = ('mon', ''),
1501 cmd
: Optional
[List
[str]] = None,
1502 inbuf
: Optional
[bytes
] = b
'',
1503 timeout
: Optional
[int] = 0,
1504 verbose
: Optional
[bool] = False) -> Tuple
[int, bytes
, str]:
1506 Send a command to a daemon using librados's
1507 mon_command, osd_command, mgr_command, or pg_command. Any bulk input data
1510 Returns (ret, outbuf, outs); ret is the return code, outbuf is
1511 the outbl "bulk useful output" buffer, and outs is any status
1512 or error message (intended for stderr).
1514 If target is osd.N, send command to that osd (except for pgid cmds)
1518 if target
[0] == 'osd':
1522 print('submit {0} to osd.{1}'.format(cmd
, osdid
),
1524 ret
, outbuf
, outs
= run_in_thread(
1525 cluster
.osd_command
, osdid
, cmd
, inbuf
, timeout
=timeout
)
1527 elif target
[0] == 'mgr':
1528 name
= '' # non-None empty string means "current active mgr"
1529 if len(target
) > 1 and target
[1] is not None:
1532 print('submit {0} to {1} name {2}'.format(cmd
, target
[0], name
),
1534 ret
, outbuf
, outs
= run_in_thread(
1535 cluster
.mgr_command
, cmd
, inbuf
, timeout
=timeout
, target
=name
)
1537 elif target
[0] == 'mon-mgr':
1539 print('submit {0} to {1}'.format(cmd
, target
[0]),
1541 ret
, outbuf
, outs
= run_in_thread(
1542 cluster
.mgr_command
, cmd
, inbuf
, timeout
=timeout
)
1544 elif target
[0] == 'pg':
1546 # pgid will already be in the command for the pg <pgid>
1547 # form, but for tell <pgid>, we need to put it in
1549 cmddict
= json
.loads(cmd
)
1550 cmddict
['pgid'] = pgid
1552 cmddict
= dict(pgid
=pgid
)
1553 cmd
= json
.dumps(cmddict
)
1555 print('submit {0} for pgid {1}'.format(cmd
, pgid
),
1557 ret
, outbuf
, outs
= run_in_thread(
1558 cluster
.pg_command
, pgid
, cmd
, inbuf
, timeout
=timeout
)
1560 elif target
[0] == 'mon':
1562 print('{0} to {1}'.format(cmd
, target
[0]),
1564 if len(target
) < 2 or target
[1] == '':
1565 ret
, outbuf
, outs
= run_in_thread(
1566 cluster
.mon_command
, cmd
, inbuf
, timeout
=timeout
)
1568 ret
, outbuf
, outs
= run_in_thread(
1569 cluster
.mon_command
, cmd
, inbuf
, timeout
=timeout
, target
=target
[1])
1570 elif target
[0] == 'mds':
1571 mds_spec
= target
[1]
1574 print('submit {0} to mds.{1}'.format(cmd
, mds_spec
),
1578 from cephfs
import LibCephFS
1580 raise RuntimeError("CephFS unavailable, have you installed libcephfs?")
1582 filesystem
= LibCephFS(rados_inst
=cluster
)
1584 ret
, outbuf
, outs
= \
1585 filesystem
.mds_command(mds_spec
, cmd
, inbuf
)
1586 filesystem
.shutdown()
1588 raise ArgumentValid("Bad target type '{0}'".format(target
[0]))
1590 except Exception as e
:
1591 if not isinstance(e
, ArgumentError
):
1592 raise RuntimeError('"{0}": exception {1}'.format(cmd
, e
))
1596 return ret
, outbuf
, outs
1599 def json_command(cluster
,
1600 target
: Optional
[Tuple
[str, str]] = ('mon', ''),
1601 prefix
: Optional
[str] = None,
1602 argdict
: Optional
[Dict
[str, str]] = None,
1603 inbuf
: Optional
[bytes
] = b
'',
1604 timeout
: Optional
[int] = 0,
1605 verbose
: Optional
[bool] = False) -> Tuple
[int, bytes
, str]:
1607 Serialize a command and up a JSON command and send it with send_command() above.
1608 Prefix may be supplied separately or in argdict. Any bulk input
1609 data comes in inbuf.
1611 If target is osd.N, send command to that osd (except for pgid cmds)
1613 :param cluster: ``rados.Rados`` instance
1614 :param prefix: String to inject into command arguments as 'prefix'
1615 :param argdict: Command arguments
1619 cmddict
.update({'prefix': prefix
})
1622 cmddict
.update(argdict
)
1623 if 'target' in argdict
:
1624 target
= argdict
.get('target')
1627 if target
[0] == 'osd':
1628 osdtarg
= CephName()
1629 osdtarget
= '{0}.{1}'.format(*target
)
1630 # prefer target from cmddict if present and valid
1631 if 'target' in cmddict
:
1632 osdtarget
= cmddict
.pop('target')
1634 osdtarg
.valid(osdtarget
)
1635 target
= ('osd', osdtarg
.nameid
)
1637 # use the target we were originally given
1639 ret
, outbuf
, outs
= send_command_retry(cluster
,
1640 target
, json
.dumps(cmddict
),
1641 inbuf
, timeout
, verbose
)
1643 except Exception as e
:
1644 if not isinstance(e
, ArgumentError
):
1645 raise RuntimeError('"{0}": exception {1}'.format(argdict
, e
))
1649 return ret
, outbuf
, outs