ceph/src/pybind/ceph_argparse.py

   1 """
   2 Types and routines used by the ceph CLI as well as the RESTful
   3 interface.  These have to do with querying the daemons for
   4 command-description information, validating user command input against
   5 those descriptions, and submitting the command to the appropriate
   6 daemon.
   7
   8 Copyright (C) 2013 Inktank Storage, Inc.
   9
  10 LGPL-2.1 or LGPL-3.0.  See file COPYING.
  11 """
  12 import copy
  13 import enum
  14 import math
  15 import json
  16 import os
  17 import pprint
  18 import re
  19 import socket
  20 import stat
  21 import sys
  22 import threading
  23 import uuid
  24
  25 from collections import abc
  26 from typing import cast, Any, Callable, Dict, Generic, List, Optional, Sequence, Tuple, Union
  27
  28 if sys.version_info >= (3, 8):
  29     from typing import get_args, get_origin
  30 else:
  31     def get_args(tp):
  32         if tp is Generic:
  33             return tp
  34         else:
  35             return getattr(tp, '__args__', ())
  36
  37     def get_origin(tp):
  38         return getattr(tp, '__origin__', None)
  39
  40
  41 # Flags are from MonCommand.h
  42 class Flag:
  43     NOFORWARD = (1 << 0)
  44     OBSOLETE = (1 << 1)
  45     DEPRECATED = (1 << 2)
  46     MGR = (1 << 3)
  47     POLL = (1 << 4)
  48     HIDDEN = (1 << 5)
  49
  50
  51 KWARG_EQUALS = "--([^=]+)=(.+)"
  52 KWARG_SPACE = "--([^=]+)"
  53
  54 try:
  55     basestring
  56 except NameError:
  57     basestring = str
  58
  59
  60 class ArgumentError(Exception):
  61     """
  62     Something wrong with arguments
  63     """
  64     pass
  65
  66
  67 class ArgumentNumber(ArgumentError):
  68     """
  69     Wrong number of a repeated argument
  70     """
  71     pass
  72
  73
  74 class ArgumentFormat(ArgumentError):
  75     """
  76     Argument value has wrong format
  77     """
  78     pass
  79
  80
  81 class ArgumentMissing(ArgumentError):
  82     """
  83     Argument value missing in a command
  84     """
  85     pass
  86
  87
  88 class ArgumentValid(ArgumentError):
  89     """
  90     Argument value is otherwise invalid (doesn't match choices, for instance)
  91     """
  92     pass
  93
  94
  95 class ArgumentTooFew(ArgumentError):
  96     """
  97     Fewer arguments than descriptors in signature; may mean to continue
  98     the search, so gets a special exception type
  99     """
 100
 101
 102 class ArgumentPrefix(ArgumentError):
 103     """
 104     Special for mismatched prefix; less severe, don't report by default
 105     """
 106     pass
 107
 108
 109 class JsonFormat(Exception):
 110     """
 111     some syntactic or semantic issue with the JSON
 112     """
 113     pass
 114
 115
 116 class CephArgtype(object):
 117     """
 118     Base class for all Ceph argument types
 119
 120     Instantiating an object sets any validation parameters
 121     (allowable strings, numeric ranges, etc.).  The 'valid'
 122     method validates a string against that initialized instance,
 123     throwing ArgumentError if there's a problem.
 124     """
 125     def __init__(self, **kwargs):
 126         """
 127         set any per-instance validation parameters here
 128         from kwargs (fixed string sets, integer ranges, etc)
 129         """
 130         pass
 131
 132     def valid(self, s, partial=False):
 133         """
 134         Run validation against given string s (generally one word);
 135         partial means to accept partial string matches (begins-with).
 136         If cool, set self.val to the value that should be returned
 137         (a copy of the input string, or a numeric or boolean interpretation
 138         thereof, for example)
 139         if not, throw ArgumentError(msg-as-to-why)
 140         """
 141         self.val = s
 142
 143     def __repr__(self):
 144         """
 145         return string representation of description of type.  Note,
 146         this is not a representation of the actual value.  Subclasses
 147         probably also override __str__() to give a more user-friendly
 148         'name/type' description for use in command format help messages.
 149         """
 150         return self.__class__.__name__
 151
 152     def __str__(self):
 153         """
 154         where __repr__ (ideally) returns a string that could be used to
 155         reproduce the object, __str__ returns one you'd like to see in
 156         print messages.  Use __str__ to format the argtype descriptor
 157         as it would be useful in a command usage message.
 158         """
 159         return '<{0}>'.format(self.__class__.__name__)
 160
 161     def __call__(self, v):
 162         return v
 163
 164     def complete(self, s):
 165         return []
 166
 167     @staticmethod
 168     def _compound_type_to_argdesc(tp, attrs, positional):
 169         # generate argdesc from Sequence[T], Tuple[T,..] and Optional[T]
 170         orig_type = get_origin(tp)
 171         type_args = get_args(tp)
 172         if orig_type in (abc.Sequence, Sequence, List, list):
 173             assert len(type_args) == 1
 174             attrs['n'] = 'N'
 175             return CephArgtype.to_argdesc(type_args[0], attrs, positional=positional)
 176         elif orig_type is Tuple:
 177             assert len(type_args) >= 1
 178             inner_tp = type_args[0]
 179             assert type_args.count(inner_tp) == len(type_args), \
 180                 f'all elements in {tp} should be identical'
 181             attrs['n'] = str(len(type_args))
 182             return CephArgtype.to_argdesc(inner_tp, attrs, positional=positional)
 183         elif get_origin(tp) is Union:
 184             # should be Union[t, NoneType]
 185             assert len(type_args) == 2 and isinstance(None, type_args[1])
 186             return CephArgtype.to_argdesc(type_args[0], attrs, True, positional)
 187         else:
 188             raise ValueError(f"unknown type '{tp}': '{attrs}'")
 189
 190     @staticmethod
 191     def to_argdesc(tp, attrs, has_default=False, positional=True):
 192         if has_default:
 193             attrs['req'] = 'false'
 194         if not positional:
 195             attrs['positional'] = 'false'
 196         CEPH_ARG_TYPES = {
 197             str: CephString,
 198             int: CephInt,
 199             float: CephFloat,
 200             bool: CephBool
 201         }
 202         try:
 203             return CEPH_ARG_TYPES[tp]().argdesc(attrs)
 204         except KeyError:
 205             if isinstance(tp, CephArgtype):
 206                 return tp.argdesc(attrs)
 207             elif isinstance(tp, type) and issubclass(tp, enum.Enum):
 208                 return CephChoices(tp=tp).argdesc(attrs)
 209             else:
 210                 return CephArgtype._compound_type_to_argdesc(tp, attrs, positional)
 211
 212     def argdesc(self, attrs):
 213         attrs['type'] = type(self).__name__
 214         return ','.join(f'{k}={v}' for k, v in attrs.items())
 215
 216     @staticmethod
 217     def _cast_to_compound_type(tp, v):
 218         orig_type = get_origin(tp)
 219         type_args = get_args(tp)
 220         if orig_type in (abc.Sequence, Sequence, List, list):
 221             if v is None:
 222                 return None
 223             return [CephArgtype.cast_to(type_args[0], e) for e in v]
 224         elif orig_type is Tuple:
 225             return tuple(CephArgtype.cast_to(type_args[0], e) for e in v)
 226         elif get_origin(tp) is Union:
 227             # should be Union[t, NoneType]
 228             assert len(type_args) == 2 and isinstance(None, type_args[1])
 229             return CephArgtype.cast_to(type_args[0], v)
 230         else:
 231             raise ValueError(f"unknown type '{tp}': '{v}'")
 232
 233     @staticmethod
 234     def cast_to(tp, v):
 235         PYTHON_TYPES = (
 236             str,
 237             int,
 238             float,
 239             bool
 240         )
 241         if tp in PYTHON_TYPES:
 242             return tp(v)
 243         elif isinstance(tp, type) and issubclass(tp, enum.Enum):
 244             return tp(v)
 245         else:
 246             return CephArgtype._cast_to_compound_type(tp, v)
 247
 248
 249 class CephInt(CephArgtype):
 250     """
 251     range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+
 252     range: list of 1 or 2 ints, [min] or [min,max]
 253     """
 254     def __init__(self, range=''):
 255         if range == '':
 256             self.range = list()
 257         else:
 258             self.range = [int(x) for x in range.split('|')]
 259
 260     def valid(self, s, partial=False):
 261         try:
 262             val = int(s, 0)
 263         except ValueError:
 264             raise ArgumentValid("{0} doesn't represent an int".format(s))
 265         if len(self.range) == 2:
 266             if val < self.range[0] or val > self.range[1]:
 267                 raise ArgumentValid(f"{val} not in range {self.range}")
 268         elif len(self.range) == 1:
 269             if val < self.range[0]:
 270                 raise ArgumentValid(f"{val} not in range {self.range}")
 271         self.val = val
 272
 273     def __str__(self):
 274         r = ''
 275         if len(self.range) == 1:
 276             r = '[{0}-]'.format(self.range[0])
 277         if len(self.range) == 2:
 278             r = '[{0}-{1}]'.format(self.range[0], self.range[1])
 279
 280         return '<int{0}>'.format(r)
 281
 282     def argdesc(self, attrs):
 283         if self.range:
 284             attrs['range'] = '|'.join(str(v) for v in self.range)
 285         return super().argdesc(attrs)
 286
 287
 288 class CephFloat(CephArgtype):
 289     """
 290     range-limited float type
 291     range: list of 1 or 2 floats, [min] or [min, max]
 292     """
 293     def __init__(self, range=''):
 294         if range == '':
 295             self.range = list()
 296         else:
 297             self.range = [float(x) for x in range.split('|')]
 298
 299     def valid(self, s, partial=False):
 300         try:
 301             val = float(s)
 302         except ValueError:
 303             raise ArgumentValid("{0} doesn't represent a float".format(s))
 304         if len(self.range) == 2:
 305             if val < self.range[0] or val > self.range[1]:
 306                 raise ArgumentValid(f"{val} not in range {self.range}")
 307         elif len(self.range) == 1:
 308             if val < self.range[0]:
 309                 raise ArgumentValid(f"{val} not in range {self.range}")
 310         self.val = val
 311
 312     def __str__(self):
 313         r = ''
 314         if len(self.range) == 1:
 315             r = '[{0}-]'.format(self.range[0])
 316         if len(self.range) == 2:
 317             r = '[{0}-{1}]'.format(self.range[0], self.range[1])
 318         return '<float{0}>'.format(r)
 319
 320     def argdesc(self, attrs):
 321         if self.range:
 322             attrs['range'] = '|'.join(str(v) for v in self.range)
 323         return super().argdesc(attrs)
 324
 325
 326 class CephString(CephArgtype):
 327     """
 328     String; pretty generic.  goodchars is a RE char class of valid chars
 329     """
 330     def __init__(self, goodchars=''):
 331         from string import printable
 332         try:
 333             re.compile(goodchars)
 334         except re.error:
 335             raise ValueError('CephString(): "{0}" is not a valid RE'.
 336                              format(goodchars))
 337         self.goodchars = goodchars
 338         self.goodset = frozenset(
 339             [c for c in printable if re.match(goodchars, c)]
 340         )
 341
 342     def valid(self, s: str, partial: bool = False) -> None:
 343         sset = set(s)
 344         if self.goodset and not sset <= self.goodset:
 345             raise ArgumentFormat("invalid chars {0} in {1}".
 346                                  format(''.join(sset - self.goodset), s))
 347         self.val = s
 348
 349     def __str__(self) -> str:
 350         b = ''
 351         if self.goodchars:
 352             b += '(goodchars {0})'.format(self.goodchars)
 353         return '<string{0}>'.format(b)
 354
 355     def complete(self, s) -> List[str]:
 356         if s == '':
 357             return []
 358         else:
 359             return [s]
 360
 361     def argdesc(self, attrs):
 362         if self.goodchars:
 363             attrs['goodchars'] = self.goodchars
 364         return super().argdesc(attrs)
 365
 366
 367 class CephSocketpath(CephArgtype):
 368     """
 369     Admin socket path; check that it's readable and S_ISSOCK
 370     """
 371     def valid(self, s: str, partial: bool = False) -> None:
 372         mode = os.stat(s).st_mode
 373         if not stat.S_ISSOCK(mode):
 374             raise ArgumentValid('socket path {0} is not a socket'.format(s))
 375         self.val = s
 376
 377     def __str__(self) -> str:
 378         return '<admin-socket-path>'
 379
 380
 381 class CephIPAddr(CephArgtype):
 382     """
 383     IP address (v4 or v6) with optional port
 384     """
 385     def valid(self, s, partial=False):
 386         # parse off port, use socket to validate addr
 387         type = 6
 388         p: Optional[str] = None
 389         if s.startswith('['):
 390             type = 6
 391         elif s.find('.') != -1:
 392             type = 4
 393         if type == 4:
 394             port = s.find(':')
 395             if port != -1:
 396                 a = s[:port]
 397                 p = s[port + 1:]
 398                 if int(p) > 65535:
 399                     raise ArgumentValid('{0}: invalid IPv4 port'.format(p))
 400             else:
 401                 a = s
 402                 p = None
 403             try:
 404                 socket.inet_pton(socket.AF_INET, a)
 405             except OSError:
 406                 raise ArgumentValid('{0}: invalid IPv4 address'.format(a))
 407         else:
 408             # v6
 409             if s.startswith('['):
 410                 end = s.find(']')
 411                 if end == -1:
 412                     raise ArgumentFormat('{0} missing terminating ]'.format(s))
 413                 if s[end + 1] == ':':
 414                     try:
 415                         p = s[end + 2]
 416                     except ValueError:
 417                         raise ArgumentValid('{0}: bad port number'.format(s))
 418                 a = s[1:end]
 419             else:
 420                 a = s
 421                 p = None
 422             try:
 423                 socket.inet_pton(socket.AF_INET6, a)
 424             except OSError:
 425                 raise ArgumentValid('{0} not valid IPv6 address'.format(s))
 426         if p is not None and int(p) > 65535:
 427             raise ArgumentValid("{0} not a valid port number".format(p))
 428         self.val = s
 429         self.addr = a
 430         self.port = p
 431
 432     def __str__(self):
 433         return '<IPaddr[:port]>'
 434
 435
 436 class CephEntityAddr(CephIPAddr):
 437     """
 438     EntityAddress, that is, IP address[/nonce]
 439     """
 440     def valid(self, s: str, partial: bool = False) -> None:
 441         nonce = None
 442         if '/' in s:
 443             ip, nonce = s.split('/')
 444         else:
 445             ip = s
 446         super(self.__class__, self).valid(ip)
 447         if nonce:
 448             nonce_int = None
 449             try:
 450                 nonce_int = int(nonce)
 451             except ValueError:
 452                 pass
 453             if nonce_int is None or nonce_int < 0:
 454                 raise ArgumentValid(
 455                     '{0}: invalid entity, nonce {1} not integer > 0'.
 456                     format(s, nonce)
 457                 )
 458         self.val = s
 459
 460     def __str__(self) -> str:
 461         return '<EntityAddr>'
 462
 463
 464 class CephPoolname(CephArgtype):
 465     """
 466     Pool name; very little utility
 467     """
 468     def __str__(self) -> str:
 469         return '<poolname>'
 470
 471
 472 class CephObjectname(CephArgtype):
 473     """
 474     Object name.  Maybe should be combined with Pool name as they're always
 475     present in pairs, and then could be checked for presence
 476     """
 477     def __str__(self) -> str:
 478         return '<objectname>'
 479
 480
 481 class CephPgid(CephArgtype):
 482     """
 483     pgid, in form N.xxx (N = pool number, xxx = hex pgnum)
 484     """
 485     def valid(self, s, partial=False):
 486         if s.find('.') == -1:
 487             raise ArgumentFormat('pgid has no .')
 488         poolid_s, pgnum_s = s.split('.', 1)
 489         try:
 490             poolid = int(poolid_s)
 491         except ValueError:
 492             raise ArgumentFormat('pool {0} not integer'.format(poolid))
 493         if poolid < 0:
 494             raise ArgumentFormat('pool {0} < 0'.format(poolid))
 495         try:
 496             pgnum = int(pgnum_s, 16)
 497         except ValueError:
 498             raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum))
 499         self.val = s
 500
 501     def __str__(self):
 502         return '<pgid>'
 503
 504
 505 class CephName(CephArgtype):
 506     """
 507     Name (type.id) where:
 508     type is osd|mon|client|mds
 509     id is a base10 int, if type == osd, or a string otherwise
 510
 511     Also accept '*'
 512     """
 513     def __init__(self) -> None:
 514         self.nametype: Optional[str] = None
 515         self.nameid: Optional[str] = None
 516
 517     def valid(self, s, partial=False):
 518         if s == '*':
 519             self.val = s
 520             return
 521         elif s == "mgr":
 522             self.nametype = "mgr"
 523             self.val = s
 524             return
 525         elif s == "mon":
 526             self.nametype = "mon"
 527             self.val = s
 528             return
 529         if s.find('.') == -1:
 530             raise ArgumentFormat('CephName: no . in {0}'.format(s))
 531         else:
 532             t, i = s.split('.', 1)
 533             if t not in ('osd', 'mon', 'client', 'mds', 'mgr'):
 534                 raise ArgumentValid('unknown type ' + t)
 535             if t == 'osd':
 536                 if i != '*':
 537                     try:
 538                         int(i)
 539                     except ValueError:
 540                         raise ArgumentFormat(f'osd id {i} not integer')
 541             self.nametype = t
 542         self.val = s
 543         self.nameid = i
 544
 545     def __str__(self):
 546         return '<name (type.id)>'
 547
 548
 549 class CephOsdName(CephArgtype):
 550     """
 551     Like CephName, but specific to osds: allow <id> alone
 552
 553     osd.<id>, or <id>, or *, where id is a base10 int
 554     """
 555     def __init__(self):
 556         self.nametype: Optional[str] = None
 557         self.nameid: Optional[int] = None
 558
 559     def valid(self, s, partial=False):
 560         if s == '*':
 561             self.val = s
 562             return
 563         if s.find('.') != -1:
 564             t, i = s.split('.', 1)
 565             if t != 'osd':
 566                 raise ArgumentValid('unknown type ' + t)
 567         else:
 568             t = 'osd'
 569             i = s
 570         try:
 571             v = int(i)
 572         except ValueError:
 573             raise ArgumentFormat(f'osd id {i} not integer')
 574         if v < 0:
 575             raise ArgumentFormat(f'osd id {v} is less than 0')
 576         self.nametype = t
 577         self.nameid = v
 578         self.val = v
 579
 580     def __str__(self):
 581         return '<osdname (id|osd.id)>'
 582
 583
 584 class CephChoices(CephArgtype):
 585     """
 586     Set of string literals; init with valid choices
 587     """
 588     def __init__(self, strings='', tp=None, **kwargs):
 589         self.strings = strings.split('|')
 590         self.enum = tp
 591         if self.enum is not None:
 592             self.strings = list(e.value for e in self.enum)
 593
 594     def valid(self, s, partial=False):
 595         if not partial:
 596             if s not in self.strings:
 597                 # show as __str__ does: {s1|s2..}
 598                 raise ArgumentValid("{0} not in {1}".format(s, self))
 599             self.val = s
 600             return
 601
 602         # partial
 603         for t in self.strings:
 604             if t.startswith(s):
 605                 self.val = s
 606                 return
 607         raise ArgumentValid("{0} not in {1}".  format(s, self))
 608
 609     def __str__(self):
 610         if len(self.strings) == 1:
 611             return '{0}'.format(self.strings[0])
 612         else:
 613             return '{0}'.format('|'.join(self.strings))
 614
 615     def __call__(self, v):
 616         if self.enum is None:
 617             return v
 618         else:
 619             return self.enum[v]
 620
 621     def complete(self, s):
 622         all_elems = [token for token in self.strings if token.startswith(s)]
 623         return all_elems
 624
 625     def argdesc(self, attrs):
 626         attrs['strings'] = '|'.join(self.strings)
 627         return super().argdesc(attrs)
 628
 629
 630 class CephBool(CephArgtype):
 631     """
 632     A boolean argument, values may be case insensitive 'true', 'false', '0',
 633     '1'.  In keyword form, value may be left off (implies true).
 634     """
 635     def __init__(self, strings='', **kwargs):
 636         self.strings = strings.split('|')
 637
 638     def valid(self, s, partial=False):
 639         lower_case = s.lower()
 640         if lower_case in ['true', '1']:
 641             self.val = True
 642         elif lower_case in ['false', '0']:
 643             self.val = False
 644         else:
 645             raise ArgumentValid("{0} not one of 'true', 'false'".format(s))
 646
 647     def __str__(self):
 648         return '<bool>'
 649
 650
 651 class CephFilepath(CephArgtype):
 652     """
 653     Openable file
 654     """
 655     def valid(self, s, partial=False):
 656         # set self.val if the specified path is readable or writable
 657         s = os.path.abspath(s)
 658         if not os.access(s, os.R_OK):
 659             self._validate_writable_file(s)
 660         self.val = s
 661
 662     def _validate_writable_file(self, fname):
 663         if os.path.exists(fname):
 664             if os.path.isfile(fname):
 665                 if not os.access(fname, os.W_OK):
 666                     raise ArgumentValid('{0} is not writable'.format(fname))
 667             else:
 668                 raise ArgumentValid('{0} is not file'.format(fname))
 669         else:
 670             dirname = os.path.dirname(fname)
 671             if not os.access(dirname, os.W_OK):
 672                 raise ArgumentValid('cannot create file in {0}'.format(dirname))
 673
 674     def __str__(self):
 675         return '<outfilename>'
 676
 677
 678 class CephFragment(CephArgtype):
 679     """
 680     'Fragment' ??? XXX
 681     """
 682     def valid(self, s, partial=False):
 683         if s.find('/') == -1:
 684             raise ArgumentFormat('{0}: no /'.format(s))
 685         val, bits = s.split('/')
 686         # XXX is this right?
 687         if not val.startswith('0x'):
 688             raise ArgumentFormat("{0} not a hex integer".format(val))
 689         try:
 690             int(val)
 691         except ValueError:
 692             raise ArgumentFormat('can\'t convert {0} to integer'.format(val))
 693         try:
 694             int(bits)
 695         except ValueError:
 696             raise ArgumentFormat('can\'t convert {0} to integer'.format(bits))
 697         self.val = s
 698
 699     def __str__(self):
 700         return "<CephFS fragment ID (0xvvv/bbb)>"
 701
 702
 703 class CephUUID(CephArgtype):
 704     """
 705     CephUUID: pretty self-explanatory
 706     """
 707     def valid(self, s: str, partial: bool = False) -> None:
 708         try:
 709             uuid.UUID(s)
 710         except Exception as e:
 711             raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e))
 712         self.val = s
 713
 714     def __str__(self) -> str:
 715         return '<uuid>'
 716
 717
 718 class CephPrefix(CephArgtype):
 719     """
 720     CephPrefix: magic type for "all the first n fixed strings"
 721     """
 722     def __init__(self, prefix: str = '') -> None:
 723         self.prefix = prefix
 724
 725     def valid(self, s: str, partial: bool = False) -> None:
 726         try:
 727             s = str(s)
 728             if isinstance(s, bytes):
 729                 # `prefix` can always be converted into unicode when being compared,
 730                 # but `s` could be anything passed by user.
 731                 s = s.decode('ascii')
 732         except UnicodeEncodeError:
 733             raise ArgumentPrefix(u"no match for {0}".format(s))
 734         except UnicodeDecodeError:
 735             raise ArgumentPrefix("no match for {0}".format(s))
 736
 737         if partial:
 738             if self.prefix.startswith(s):
 739                 self.val = s
 740                 return
 741         else:
 742             if s == self.prefix:
 743                 self.val = s
 744                 return
 745
 746         raise ArgumentPrefix("no match for {0}".format(s))
 747
 748     def __str__(self) -> str:
 749         return self.prefix
 750
 751     def complete(self, s) -> List[str]:
 752         if self.prefix.startswith(s):
 753             return [self.prefix.rstrip(' ')]
 754         else:
 755             return []
 756
 757
 758 class argdesc(object):
 759     """
 760     argdesc(typename, name='name', n=numallowed|N,
 761             req=False, positional=True,
 762             helptext=helptext, **kwargs (type-specific))
 763
 764     validation rules:
 765     typename: type(**kwargs) will be constructed
 766     later, type.valid(w) will be called with a word in that position
 767
 768     name is used for parse errors and for constructing JSON output
 769     n is a numeric literal or 'n|N', meaning "at least one, but maybe more"
 770     req=False means the argument need not be present in the list
 771     positional=False means the argument name must be specified, e.g. "--myoption value"
 772     helptext is the associated help for the command
 773     anything else are arguments to pass to the type constructor.
 774
 775     self.instance is an instance of type t constructed with typeargs.
 776
 777     valid() will later be called with input to validate against it,
 778     and will store the validated value in self.instance.val for extraction.
 779     """
 780     def __init__(self, t, name=None, n=1, req=True, positional=True, **kwargs) -> None:
 781         if isinstance(t, basestring):
 782             self.t = CephPrefix
 783             self.typeargs = {'prefix': t}
 784             self.req = True
 785             self.positional = True
 786         else:
 787             self.t = t
 788             self.typeargs = kwargs
 789             self.req = req in (True, 'True', 'true')
 790             self.positional = positional in (True, 'True', 'true')
 791             if not positional:
 792                 assert not req
 793
 794         self.name = name
 795         self.N = (n in ['n', 'N'])
 796         if self.N:
 797             self.n = 1
 798         else:
 799             self.n = int(n)
 800
 801         self.numseen = 0
 802
 803         self.instance = self.t(**self.typeargs)
 804
 805     def __repr__(self):
 806         r = 'argdesc(' + str(self.t) + ', '
 807         internals = ['N', 'typeargs', 'instance', 't']
 808         for (k, v) in self.__dict__.items():
 809             if k.startswith('__') or k in internals:
 810                 pass
 811             else:
 812                 # undo modification from __init__
 813                 if k == 'n' and self.N:
 814                     v = 'N'
 815                 r += '{0}={1}, '.format(k, v)
 816         for (k, v) in self.typeargs.items():
 817             r += '{0}={1}, '.format(k, v)
 818         return r[:-2] + ')'
 819
 820     def __str__(self):
 821         if ((self.t == CephChoices and len(self.instance.strings) == 1)
 822            or (self.t == CephPrefix)):
 823             s = str(self.instance)
 824         else:
 825             s = '{0}({1})'.format(self.name, str(self.instance))
 826             if self.N:
 827                 s += '...'
 828         if not self.req:
 829             s = '[' + s + ']'
 830         return s
 831
 832     def helpstr(self):
 833         """
 834         like str(), but omit parameter names (except for CephString,
 835         which really needs them)
 836         """
 837         if self.positional:
 838             if self.t == CephBool:
 839                 chunk = "--{0}".format(self.name.replace("_", "-"))
 840             elif self.t == CephPrefix:
 841                 chunk = str(self.instance)
 842             elif self.t == CephChoices:
 843                 if self.name == 'format':
 844                     # this is for talking to legacy clusters only; new clusters
 845                     # should properly mark format args as non-positional.
 846                     chunk = f'--{self.name} {{{str(self.instance)}}}'
 847                 else:
 848                     chunk = f'<{self.name}:{self.instance}>'
 849             elif self.t == CephOsdName:
 850                 # it just so happens all CephOsdName commands are named 'id' anyway,
 851                 # so <id|osd.id> is perfect.
 852                 chunk = '<id|osd.id>'
 853             elif self.t == CephName:
 854                 # CephName commands similarly only have one arg of the
 855                 # type, so <type.id> is good.
 856                 chunk = '<type.id>'
 857             elif self.t == CephInt:
 858                 chunk = '<{0}:int>'.format(self.name)
 859             elif self.t == CephFloat:
 860                 chunk = '<{0}:float>'.format(self.name)
 861             else:
 862                 chunk = '<{0}>'.format(self.name)
 863             s = chunk
 864             if self.N:
 865                 s += '...'
 866             if not self.req:
 867                 s = '[' + s + ']'
 868         else:
 869             # non-positional
 870             if self.t == CephBool:
 871                 chunk = "--{0}".format(self.name.replace("_", "-"))
 872             elif self.t == CephPrefix:
 873                 chunk = str(self.instance)
 874             elif self.t == CephChoices:
 875                 chunk = f'--{self.name} {{{str(self.instance)}}}'
 876             elif self.t == CephOsdName:
 877                 chunk = f'--{self.name} <id|osd.id>'
 878             elif self.t == CephName:
 879                 chunk = f'--{self.name} <type.id>'
 880             elif self.t == CephInt:
 881                 chunk = f'--{self.name} <int>'
 882             elif self.t == CephFloat:
 883                 chunk = f'--{self.name} <float>'
 884             else:
 885                 chunk = f'--{self.name} <value>'
 886             s = chunk
 887             if self.N:
 888                 s += '...'
 889             if not self.req:  # req should *always* be false
 890                 s = '[' + s + ']'
 891
 892         return s
 893
 894     def complete(self, s):
 895         return self.instance.complete(s)
 896
 897
 898 def concise_sig(sig):
 899     """
 900     Return string representation of sig useful for syntax reference in help
 901     """
 902     return ' '.join([d.helpstr() for d in sig])
 903
 904
 905 def descsort_key(sh):
 906     """
 907     sort descriptors by prefixes, defined as the concatenation of all simple
 908     strings in the descriptor; this works out to just the leading strings.
 909     """
 910     return concise_sig(sh['sig'])
 911
 912
 913 def parse_funcsig(sig: Sequence[Union[str, Dict[str, Any]]]) -> List[argdesc]:
 914     """
 915     parse a single descriptor (array of strings or dicts) into a
 916     dict of function descriptor/validators (objects of CephXXX type)
 917
 918     :returns: list of ``argdesc``
 919     """
 920     newsig = []
 921     argnum = 0
 922     for desc in sig:
 923         argnum += 1
 924         if isinstance(desc, basestring):
 925             t = CephPrefix
 926             desc = {'type': t, 'name': 'prefix', 'prefix': desc}
 927         else:
 928             # not a simple string, must be dict
 929             if 'type' not in desc:
 930                 s = 'JSON descriptor {0} has no type'.format(sig)
 931                 raise JsonFormat(s)
 932             # look up type string in our globals() dict; if it's an
 933             # object of type `type`, it must be a
 934             # locally-defined class. otherwise, we haven't a clue.
 935             if desc['type'] in globals():
 936                 t = globals()[desc['type']]
 937                 if not isinstance(t, type):
 938                     s = 'unknown type {0}'.format(desc['type'])
 939                     raise JsonFormat(s)
 940             else:
 941                 s = 'unknown type {0}'.format(desc['type'])
 942                 raise JsonFormat(s)
 943
 944         kwargs = dict()
 945         for key, val in desc.items():
 946             if key not in ['type', 'name', 'n', 'req', 'positional']:
 947                 kwargs[key] = val
 948         newsig.append(argdesc(t,
 949                               name=desc.get('name', None),
 950                               n=desc.get('n', 1),
 951                               req=desc.get('req', True),
 952                               positional=desc.get('positional', True),
 953                               **kwargs))
 954     return newsig
 955
 956
 957 def parse_json_funcsigs(s: str,
 958                         consumer: str) -> Dict[str, Dict[str, List[argdesc]]]:
 959     """
 960     A function signature is mostly an array of argdesc; it's represented
 961     in JSON as
 962     {
 963       "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false <other param>], "help":helptext, "module":modulename, "perm":perms, "avail":availability}
 964        .
 965        .
 966        .
 967       ]
 968
 969     A set of sigs is in an dict mapped by a unique number:
 970     {
 971       "cmd1": {
 972          "sig": ["type.. ], "help":helptext...
 973       }
 974       "cmd2"{
 975          "sig": [.. ], "help":helptext...
 976       }
 977     }
 978
 979     Parse the string s and return a dict of dicts, keyed by opcode;
 980     each dict contains 'sig' with the array of descriptors, and 'help'
 981     with the helptext, 'module' with the module name, 'perm' with a
 982     string representing required permissions in that module to execute
 983     this command (and also whether it is a read or write command from
 984     the cluster state perspective), and 'avail' as a hint for
 985     whether the command should be advertised by CLI, REST, or both.
 986     If avail does not contain 'consumer', don't include the command
 987     in the returned dict.
 988     """
 989     try:
 990         overall = json.loads(s)
 991     except Exception as e:
 992         print("Couldn't parse JSON {0}: {1}".format(s, e), file=sys.stderr)
 993         raise e
 994     sigdict = {}
 995     for cmdtag, cmd in overall.items():
 996         if 'sig' not in cmd:
 997             s = "JSON descriptor {0} has no 'sig'".format(cmdtag)
 998             raise JsonFormat(s)
 999         # check 'avail' and possibly ignore this command
1000         if 'avail' in cmd:
1001             if consumer not in cmd['avail']:
1002                 continue
1003         # rewrite the 'sig' item with the argdesc-ized version, and...
1004         cmd['sig'] = parse_funcsig(cmd['sig'])
1005         # just take everything else as given
1006         sigdict[cmdtag] = cmd
1007     return sigdict
1008
1009
1010 ArgValT = Union[bool, int, float, str, Tuple[str, str]]
1011
1012 def validate_one(word: str,
1013                  desc,
1014                  is_kwarg: bool,
1015                  partial: bool = False) -> List[ArgValT]:
1016     """
1017     validate_one(word, desc, is_kwarg, partial=False)
1018
1019     validate word against the constructed instance of the type
1020     in desc.  May raise exception.  If it returns false (and doesn't
1021     raise an exception), desc.instance.val will
1022     contain the validated value (in the appropriate type).
1023     """
1024     vals = []
1025     # CephString option might contain "," in it
1026     allow_csv = is_kwarg or desc.t is not CephString
1027     if desc.N and allow_csv:
1028         for part in word.split(','):
1029             desc.instance.valid(part, partial)
1030             vals.append(desc.instance.val)
1031     else:
1032         desc.instance.valid(word, partial)
1033         vals.append(desc.instance.val)
1034     desc.numseen += 1
1035     if desc.N:
1036         desc.n = desc.numseen + 1
1037     return vals
1038
1039
1040 def matchnum(args: List[str],
1041              signature: List[argdesc],
1042              partial: bool = False) -> int:
1043     """
1044     matchnum(s, signature, partial=False)
1045
1046     Returns number of arguments matched in s against signature.
1047     Can be used to determine most-likely command for full or partial
1048     matches (partial applies to string matches).
1049     """
1050     words = args[:]
1051     mysig = copy.deepcopy(signature)
1052     matchcnt = 0
1053     for desc in mysig:
1054         desc.numseen = 0
1055         while desc.numseen < desc.n:
1056             # if there are no more arguments, return
1057             if not words:
1058                 return matchcnt
1059             word = words.pop(0)
1060
1061             try:
1062                 # only allow partial matching if we're on the last supplied
1063                 # word; avoid matching foo bar and foot bar just because
1064                 # partial is set
1065                 validate_one(word, desc, False, partial and (len(words) == 0))
1066                 valid = True
1067             except ArgumentError:
1068                 # matchnum doesn't care about type of error
1069                 valid = False
1070
1071             if not valid:
1072                 if not desc.req:
1073                     # this wasn't required, so word may match the next desc
1074                     words.insert(0, word)
1075                     break
1076                 else:
1077                     # it was required, and didn't match, return
1078                     return matchcnt
1079         if desc.req:
1080             matchcnt += 1
1081     return matchcnt
1082
1083
1084 ValidatedArg = Union[bool, int, float, str,
1085                      Tuple[str, str],
1086                      Sequence[str]]
1087 ValidatedArgs = Dict[str, ValidatedArg]
1088
1089
1090 def store_arg(desc: argdesc, args: Sequence[ValidatedArg], d: ValidatedArgs):
1091     '''
1092     Store argument described by, and held in, thanks to valid(),
1093     desc into the dictionary d, keyed by desc.name.  Three cases:
1094
1095     1) desc.N is set: use args for arg value in "d", desc.instance.val
1096        only contains the last parsed arg in the "args" list
1097     2) prefix: multiple args are joined with ' ' into one d{} item
1098     3) single prefix or other arg: store as simple value
1099
1100     Used in validate() below.
1101     '''
1102     if desc.N:
1103         # value should be a list
1104         if desc.name in d:
1105             d[desc.name] += args
1106         else:
1107             d[desc.name] = args
1108     elif (desc.t == CephPrefix) and (desc.name in d):
1109         # prefixes' values should be a space-joined concatenation
1110         d[desc.name] += ' ' + desc.instance.val
1111     else:
1112         # if first CephPrefix or any other type, just set it
1113         d[desc.name] = desc.instance.val
1114
1115
1116 def validate(args: List[str],
1117              signature: Sequence[argdesc],
1118              flags: int = 0,
1119              partial: Optional[bool] = False) -> ValidatedArgs:
1120     """
1121     validate(args, signature, flags=0, partial=False)
1122
1123     args is a list of strings representing a possible
1124     command input following format of signature.  Runs a validation; no
1125     exception means it's OK.  Return a dict containing all arguments keyed
1126     by their descriptor name, with duplicate args per name accumulated
1127     into a list (or space-separated value for CephPrefix).
1128
1129     Mismatches of prefix are non-fatal, as this probably just means the
1130     search hasn't hit the correct command.  Mismatches of non-prefix
1131     arguments are treated as fatal, and an exception raised.
1132
1133     This matching is modified if partial is set: allow partial matching
1134     (with partial dict returned); in this case, there are no exceptions
1135     raised.
1136     """
1137
1138     myargs = copy.deepcopy(args)
1139     mysig = copy.deepcopy(signature)
1140     reqsiglen = len([desc for desc in mysig if desc.req])
1141     matchcnt = 0
1142     d: ValidatedArgs = dict()
1143     save_exception = None
1144
1145     arg_descs_by_name: Dict[str, argdesc] = \
1146         dict((desc.name, desc) for desc in mysig if desc.t != CephPrefix)
1147
1148     # Special case: detect "injectargs" (legacy way of modifying daemon
1149     # configs) and permit "--" string arguments if so.
1150     injectargs = myargs and myargs[0] == "injectargs"
1151
1152     # Make a pass through all arguments
1153     for desc in mysig:
1154         desc.numseen = 0
1155
1156         while desc.numseen < desc.n:
1157             if myargs:
1158                 myarg: Optional[str] = myargs.pop(0)
1159             else:
1160                 myarg = None
1161
1162             # no arg, but not required?  Continue consuming mysig
1163             # in case there are later required args
1164             if myarg in (None, []):
1165                 if not desc.req:
1166                     break
1167                 # did we already get this argument (as a named arg, earlier?)
1168                 if desc.name in d:
1169                     break
1170
1171             # A keyword argument?
1172             if myarg:
1173                 # argdesc for the keyword argument, if we find one
1174                 kwarg_desc = None
1175
1176                 # Try both styles of keyword argument
1177                 kwarg_match = re.match(KWARG_EQUALS, myarg)
1178                 if kwarg_match:
1179                     # We have a "--foo=bar" style argument
1180                     kwarg_k, kwarg_v = kwarg_match.groups()
1181
1182                     # Either "--foo-bar" or "--foo_bar" style is accepted
1183                     kwarg_k = kwarg_k.replace('-', '_')
1184
1185                     kwarg_desc = arg_descs_by_name.get(kwarg_k, None)
1186                 else:
1187                     # Maybe this is a "--foo bar" or "--bool" style argument
1188                     key_match = re.match(KWARG_SPACE, myarg)
1189                     if key_match:
1190                         kwarg_k = key_match.group(1)
1191
1192                         # Permit --foo-bar=123 form or --foo_bar=123 form,
1193                         # assuming all command definitions use foo_bar argument
1194                         # naming style
1195                         kwarg_k = kwarg_k.replace('-', '_')
1196
1197                         kwarg_desc = arg_descs_by_name.get(kwarg_k, None)
1198                         if kwarg_desc:
1199                             if kwarg_desc.t == CephBool:
1200                                 kwarg_v = 'true'
1201                             elif len(myargs):  # Some trailing arguments exist
1202                                 kwarg_v = myargs.pop(0)
1203                             else:
1204                                 # Forget it, this is not a valid kwarg
1205                                 kwarg_desc = None
1206
1207                 if kwarg_desc:
1208                     args = validate_one(kwarg_v, kwarg_desc, True)
1209                     matchcnt += 1
1210                     store_arg(kwarg_desc, args, d)
1211                     continue
1212
1213             if not desc.positional:
1214                 # No more positional args!
1215                 raise ArgumentValid(f"Unexpected argument '{myarg}'")
1216
1217             # Don't handle something as a positional argument if it
1218             # has a leading "--" unless it's a CephChoices (used for
1219             # "--yes-i-really-mean-it")
1220             if myarg and myarg.startswith("--"):
1221                 # Special cases for instances of confirmation flags
1222                 # that were defined as CephString/CephChoices instead of CephBool
1223                 # in pre-nautilus versions of Ceph daemons.
1224                 is_value = desc.t == CephChoices \
1225                         or myarg == "--yes-i-really-mean-it" \
1226                         or myarg == "--yes-i-really-really-mean-it" \
1227                         or myarg == "--yes-i-really-really-mean-it-not-faking" \
1228                         or myarg == "--force" \
1229                         or injectargs
1230
1231                 if not is_value:
1232                     # Didn't get caught by kwarg handling, but has a "--", so
1233                     # we must assume it's something invalid, to avoid naively
1234                     # passing through mis-typed options as the values of
1235                     # positional arguments.
1236                     raise ArgumentValid("Unexpected argument '{0}'".format(
1237                         myarg))
1238
1239             # out of arguments for a required param?
1240             # Either return (if partial validation) or raise
1241             if myarg in (None, []) and desc.req:
1242                 if desc.N and desc.numseen < 1:
1243                     # wanted N, didn't even get 1
1244                     if partial:
1245                         return d
1246                     raise ArgumentNumber(
1247                         'saw {0} of {1}, expected at least 1'.
1248                         format(desc.numseen, desc)
1249                     )
1250                 elif not desc.N and desc.numseen < desc.n:
1251                     # wanted n, got too few
1252                     if partial:
1253                         return d
1254                     # special-case the "0 expected 1" case
1255                     if desc.numseen == 0 and desc.n == 1:
1256                         raise ArgumentMissing(
1257                             'missing required parameter {0}'.format(desc)
1258                         )
1259                     raise ArgumentNumber(
1260                         'saw {0} of {1}, expected {2}'.
1261                         format(desc.numseen, desc, desc.n)
1262                     )
1263                 break
1264
1265             # Have an arg; validate it
1266             assert myarg is not None
1267             try:
1268                 args = validate_one(myarg, desc, False)
1269             except ArgumentError as e:
1270                 # argument mismatch
1271                 if not desc.req:
1272                     # if not required, just push back; it might match
1273                     # the next arg
1274                     save_exception = [myarg, e]
1275                     myargs.insert(0, myarg)
1276                     break
1277                 else:
1278                     # hm, it was required, so time to return/raise
1279                     if partial:
1280                         return d
1281                     raise
1282
1283             # Whew, valid arg acquired.  Store in dict
1284             matchcnt += 1
1285             store_arg(desc, args, d)
1286             # Clear prior exception
1287             save_exception = None
1288
1289     # Done with entire list of argdescs
1290     if matchcnt < reqsiglen:
1291         raise ArgumentTooFew("not enough arguments given")
1292
1293     if myargs and not partial:
1294         if save_exception:
1295             print(save_exception[0], 'not valid: ', save_exception[1], file=sys.stderr)
1296         raise ArgumentError("unused arguments: " + str(myargs))
1297
1298     if flags & Flag.MGR:
1299         d['target'] = ('mon-mgr', '')
1300
1301     if flags & Flag.POLL:
1302         d['poll'] = True
1303
1304     # Finally, success
1305     return d
1306
1307
1308 def validate_command(sigdict: Dict[str, Dict[str, Any]],
1309                      args: List[str],
1310                      verbose: Optional[bool] = False) -> ValidatedArgs:
1311     """
1312     Parse positional arguments into a parameter dict, according to
1313     the command descriptions.
1314
1315     Writes advice about nearly-matching commands ``sys.stderr`` if
1316     the arguments do not match any command.
1317
1318     :param sigdict: A command description dictionary, as returned
1319                     from Ceph daemons by the get_command_descriptions
1320                     command.
1321     :param args: List of strings, should match one of the command
1322                  signatures in ``sigdict``
1323
1324     :returns: A dict of parsed parameters (including ``prefix``),
1325               or an empty dict if the args did not match any signature
1326     """
1327     if verbose:
1328         print("validate_command: " + " ".join(args), file=sys.stderr)
1329     found: Optional[Dict[str, Any]] = None
1330     valid_dict = {}
1331
1332     # look for best match, accumulate possibles in bestcmds
1333     # (so we can maybe give a more-useful error message)
1334     best_match_cnt = 0.0
1335     bestcmds: List[Dict[str, Any]] = []
1336     for cmd in sigdict.values():
1337         flags = cmd.get('flags', 0)
1338         if flags & Flag.OBSOLETE:
1339             continue
1340         sig = cmd['sig']
1341         matched: float = matchnum(args, sig, partial=True)
1342         if (matched >= math.floor(best_match_cnt) and
1343             matched == matchnum(args, sig, partial=False)):
1344             # prefer those fully matched over partial patch
1345             matched += 0.5
1346         if matched < best_match_cnt:
1347             continue
1348         if verbose:
1349             print("better match: {0} > {1}: {2} ".format(
1350                 matched, best_match_cnt, concise_sig(sig)
1351             ), file=sys.stderr)
1352         if matched > best_match_cnt:
1353             best_match_cnt = matched
1354             bestcmds = [cmd]
1355         else:
1356             bestcmds.append(cmd)
1357
1358     # Sort bestcmds by number of req args so we can try shortest first
1359     # (relies on a cmdsig being key,val where val is a list of len 1)
1360
1361     def grade(cmd):
1362         # prefer optional arguments over required ones
1363         sigs = cmd['sig']
1364         return sum(map(lambda sig: sig.req, sigs))
1365
1366     bestcmds_sorted = sorted(bestcmds, key=grade)
1367     if verbose:
1368         print("bestcmds_sorted: ", file=sys.stderr)
1369         pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted)
1370
1371     ex: Optional[ArgumentError] = None
1372     # for everything in bestcmds, look for a true match
1373     for cmd in bestcmds_sorted:
1374         sig = cmd['sig']
1375         try:
1376             valid_dict = validate(args, sig, flags=cmd.get('flags', 0))
1377             found = cmd
1378             break
1379         except ArgumentPrefix:
1380             # ignore prefix mismatches; we just haven't found
1381             # the right command yet
1382             pass
1383         except ArgumentMissing as e:
1384             ex = e
1385             if len(bestcmds) == 1:
1386                 found = cmd
1387             break
1388         except ArgumentTooFew:
1389             # It looked like this matched the beginning, but it
1390             # didn't have enough args supplied.  If we're out of
1391             # cmdsigs we'll fall out unfound; if we're not, maybe
1392             # the next one matches completely.  Whine, but pass.
1393             if verbose:
1394                 print('Not enough args supplied for ',
1395                       concise_sig(sig), file=sys.stderr)
1396         except ArgumentError as e:
1397             ex = e
1398             # Solid mismatch on an arg (type, range, etc.)
1399             # Stop now, because we have the right command but
1400             # some other input is invalid
1401             found = cmd
1402             break
1403
1404     if found:
1405         if not valid_dict:
1406             print("Invalid command:", ex, file=sys.stderr)
1407             print(concise_sig(sig), ': ', cmd['help'], file=sys.stderr)
1408     else:
1409         bestcmds = [c for c in bestcmds
1410                     if not c.get('flags', 0) & (Flag.DEPRECATED | Flag.HIDDEN)]
1411         bestcmds = bestcmds[:10]  # top 10
1412         print('no valid command found; {0} closest matches:'.format(len(bestcmds)),
1413               file=sys.stderr)
1414         for cmd in bestcmds:
1415             print(concise_sig(cmd['sig']), file=sys.stderr)
1416     return valid_dict
1417
1418
1419 def find_cmd_target(childargs: List[str]) -> Tuple[str, Optional[str]]:
1420     """
1421     Using a minimal validation, figure out whether the command
1422     should be sent to a monitor or an osd.  We do this before even
1423     asking for the 'real' set of command signatures, so we can ask the
1424     right daemon.
1425     Returns ('osd', osdid), ('pg', pgid), ('mgr', '') or ('mon', '')
1426     """
1427     sig = parse_funcsig(['tell', {'name': 'target', 'type': 'CephName'}])
1428     try:
1429         valid_dict = validate(childargs, sig, partial=True)
1430     except ArgumentError:
1431         pass
1432     else:
1433         if len(valid_dict) == 2:
1434             # revalidate to isolate type and id
1435             name = CephName()
1436             # if this fails, something is horribly wrong, as it just
1437             # validated successfully above
1438             name.valid(valid_dict['target'])
1439             assert name.nametype is not None
1440             return name.nametype, name.nameid
1441
1442     sig = parse_funcsig(['tell', {'name': 'pgid', 'type': 'CephPgid'}])
1443     try:
1444         valid_dict = validate(childargs, sig, partial=True)
1445     except ArgumentError:
1446         pass
1447     else:
1448         if len(valid_dict) == 2:
1449             # pg doesn't need revalidation; the string is fine
1450             pgid = valid_dict['pgid']
1451             assert isinstance(pgid, str)
1452             return 'pg', pgid
1453
1454     # If we reached this far it must mean that so far we've been unable to
1455     # obtain a proper target from childargs.  This may mean that we are not
1456     # dealing with a 'tell' command, or that the specified target is invalid.
1457     # If the latter, we likely were unable to catch it because we were not
1458     # really looking for it: first we tried to parse a 'CephName' (osd, mon,
1459     # mds, followed by and id); given our failure to parse, we tried to parse
1460     # a 'CephPgid' instead (e.g., 0.4a).  Considering we got this far though
1461     # we were unable to do so.
1462     #
1463     # We will now check if this is a tell and, if so, forcefully validate the
1464     # target as a 'CephName'.  This must be so because otherwise we will end
1465     # up sending garbage to a monitor, which is the default target when a
1466     # target is not explicitly specified.
1467     # e.g.,
1468     #   'ceph status' -> target is any one monitor
1469     #   'ceph tell mon.* status -> target is all monitors
1470     #   'ceph tell foo status -> target is invalid!
1471     if len(childargs) > 1 and childargs[0] == 'tell':
1472         name = CephName()
1473         # CephName.valid() raises on validation error; find_cmd_target()'s
1474         # caller should handle them
1475         name.valid(childargs[1])
1476         assert name.nametype is not None
1477         assert name.nameid is not None
1478         return name.nametype, name.nameid
1479
1480     sig = parse_funcsig(['pg', {'name': 'pgid', 'type': 'CephPgid'}])
1481     try:
1482         valid_dict = validate(childargs, sig, partial=True)
1483     except ArgumentError:
1484         pass
1485     else:
1486         if len(valid_dict) == 2:
1487             pgid = valid_dict['pgid']
1488             assert isinstance(pgid, str)
1489             return 'pg', pgid
1490
1491     return 'mon', ''
1492
1493
1494 class RadosThread(threading.Thread):
1495     def __init__(self, func, *args, **kwargs):
1496         self.args = args
1497         self.kwargs = kwargs
1498         self.func = func
1499         self.exception = None
1500         threading.Thread.__init__(self)
1501
1502     def run(self):
1503         try:
1504             self.retval = self.func(*self.args, **self.kwargs)
1505         except Exception as e:
1506             self.exception = e
1507
1508
1509 def run_in_thread(func: Callable[[Any, Any], Tuple[int, bytes, str]],
1510                   *args: Any, **kwargs: Any) -> Tuple[int, bytes, str]:
1511     timeout = kwargs.pop('timeout', 0)
1512     if timeout == 0 or timeout is None:
1513         # python threading module will just get blocked if timeout is `None`,
1514         # otherwise it will keep polling until timeout or thread stops.
1515         # timeout in integer when converting it to nanoseconds, but since
1516         # python3 uses `int64_t` for the deadline before timeout expires,
1517         # we have to use a safe value which does not overflow after being
1518         # added to current time in microseconds.
1519         timeout = 24 * 60 * 60
1520     t = RadosThread(func, *args, **kwargs)
1521
1522     # allow the main thread to exit (presumably, avoid a join() on this
1523     # subthread) before this thread terminates.  This allows SIGINT
1524     # exit of a blocked call.  See below.
1525     t.daemon = True
1526
1527     t.start()
1528     t.join(timeout=timeout)
1529     # ..but allow SIGINT to terminate the waiting.  Note: this
1530     # relies on the Linux kernel behavior of delivering the signal
1531     # to the main thread in preference to any subthread (all that's
1532     # strictly guaranteed is that *some* thread that has the signal
1533     # unblocked will receive it).  But there doesn't seem to be
1534     # any interface to create a thread with SIGINT blocked.
1535     if t.is_alive():
1536         raise Exception("timed out")
1537     elif t.exception:
1538         raise t.exception
1539     else:
1540         return t.retval
1541
1542
1543 def send_command_retry(*args: Any, **kwargs: Any) -> Tuple[int, bytes, str]:
1544     while True:
1545         try:
1546             return send_command(*args, **kwargs)
1547         except Exception as e:
1548             # If our librados instance has not reached state 'connected'
1549             # yet, we'll see an exception like this and retry
1550             if ('get_command_descriptions' in str(e) and
1551                 'object in state configuring' in str(e)):
1552                 continue
1553             else:
1554                 raise
1555
1556
1557 def send_command(cluster,
1558                  target: Tuple[str, Optional[str]] = ('mon', ''),
1559                  cmd: Optional[str] = None,
1560                  inbuf: Optional[bytes] = b'',
1561                  timeout: Optional[int] = 0,
1562                  verbose: Optional[bool] = False) -> Tuple[int, bytes, str]:
1563     """
1564     Send a command to a daemon using librados's
1565     mon_command, osd_command, mgr_command, or pg_command.  Any bulk input data
1566     comes in inbuf.
1567
1568     Returns (ret, outbuf, outs); ret is the return code, outbuf is
1569     the outbl "bulk useful output" buffer, and outs is any status
1570     or error message (intended for stderr).
1571
1572     If target is osd.N, send command to that osd (except for pgid cmds)
1573     """
1574     try:
1575         if target[0] == 'osd':
1576             osdid = target[1]
1577             assert osdid is not None
1578
1579             if verbose:
1580                 print('submit {0} to osd.{1}'.format(cmd, osdid),
1581                       file=sys.stderr)
1582             ret, outbuf, outs = run_in_thread(
1583                 cluster.osd_command, int(osdid), cmd, inbuf, timeout=timeout)
1584
1585         elif target[0] == 'mgr':
1586             name = ''     # non-None empty string means "current active mgr"
1587             if len(target) > 1 and target[1] is not None:
1588                 name = target[1]
1589             if verbose:
1590                 print('submit {0} to {1} name {2}'.format(cmd, target[0], name),
1591                       file=sys.stderr)
1592             ret, outbuf, outs = run_in_thread(
1593                 cluster.mgr_command, cmd, inbuf, timeout=timeout, target=name)
1594
1595         elif target[0] == 'mon-mgr':
1596             if verbose:
1597                 print('submit {0} to {1}'.format(cmd, target[0]),
1598                       file=sys.stderr)
1599             ret, outbuf, outs = run_in_thread(
1600                 cluster.mgr_command, cmd, inbuf, timeout=timeout)
1601
1602         elif target[0] == 'pg':
1603             pgid = target[1]
1604             # pgid will already be in the command for the pg <pgid>
1605             # form, but for tell <pgid>, we need to put it in
1606             if cmd:
1607                 cmddict = json.loads(cmd)
1608                 cmddict['pgid'] = pgid
1609             else:
1610                 cmddict = dict(pgid=pgid)
1611             cmd = json.dumps(cmddict)
1612             if verbose:
1613                 print('submit {0} for pgid {1}'.format(cmd, pgid),
1614                       file=sys.stderr)
1615             ret, outbuf, outs = run_in_thread(
1616                 cluster.pg_command, pgid, cmd, inbuf, timeout=timeout)
1617
1618         elif target[0] == 'mon':
1619             if verbose:
1620                 print('{0} to {1}'.format(cmd, target[0]),
1621                       file=sys.stderr)
1622             if len(target) < 2 or target[1] == '':
1623                 ret, outbuf, outs = run_in_thread(
1624                     cluster.mon_command, cmd, inbuf, timeout=timeout)
1625             else:
1626                 ret, outbuf, outs = run_in_thread(
1627                     cluster.mon_command, cmd, inbuf, timeout=timeout, target=target[1])
1628         elif target[0] == 'mds':
1629             mds_spec = target[1]
1630
1631             if verbose:
1632                 print('submit {0} to mds.{1}'.format(cmd, mds_spec),
1633                       file=sys.stderr)
1634
1635             try:
1636                 from cephfs import LibCephFS
1637             except ImportError:
1638                 raise RuntimeError("CephFS unavailable, have you installed libcephfs?")
1639
1640             filesystem = LibCephFS(rados_inst=cluster)
1641             filesystem.init()
1642             ret, outbuf, outs = \
1643                 filesystem.mds_command(mds_spec, cmd, inbuf)
1644             filesystem.shutdown()
1645         else:
1646             raise ArgumentValid("Bad target type '{0}'".format(target[0]))
1647
1648     except Exception as e:
1649         if not isinstance(e, ArgumentError):
1650             raise RuntimeError('"{0}": exception {1}'.format(cmd, e))
1651         else:
1652             raise
1653
1654     return ret, outbuf, outs
1655
1656
1657 def json_command(cluster,
1658                  target: Tuple[str, Optional[str]] = ('mon', ''),
1659                  prefix: Optional[str] = None,
1660                  argdict: Optional[ValidatedArgs] = None,
1661                  inbuf: Optional[bytes] = b'',
1662                  timeout: Optional[int] = 0,
1663                  verbose: Optional[bool] = False) -> Tuple[int, bytes, str]:
1664     """
1665     Serialize a command and up a JSON command and send it with send_command() above.
1666     Prefix may be supplied separately or in argdict.  Any bulk input
1667     data comes in inbuf.
1668
1669     If target is osd.N, send command to that osd (except for pgid cmds)
1670
1671     :param cluster: ``rados.Rados`` instance
1672     :param prefix: String to inject into command arguments as 'prefix'
1673     :param argdict: Command arguments
1674     """
1675     cmddict: ValidatedArgs = {}
1676     if prefix:
1677         cmddict.update({'prefix': prefix})
1678
1679     if argdict:
1680         cmddict.update(argdict)
1681         if 'target' in argdict:
1682             target = cast(Tuple[str, str], argdict['target'])
1683
1684     try:
1685         if target[0] == 'osd':
1686             osdtarg = CephName()
1687             osdtarget = '{0}.{1}'.format(*target)
1688             # prefer target from cmddict if present and valid
1689             if 'target' in cmddict:
1690                 osdtarget = cast(str, cmddict.pop('target'))
1691             try:
1692                 osdtarg.valid(osdtarget)
1693                 target = ('osd', osdtarg.nameid)
1694             except:
1695                 # use the target we were originally given
1696                 pass
1697         ret, outbuf, outs = send_command_retry(cluster,
1698                                                target, json.dumps(cmddict),
1699                                                inbuf, timeout, verbose)
1700
1701     except Exception as e:
1702         if not isinstance(e, ArgumentError):
1703             raise RuntimeError('"{0}": exception {1}'.format(argdict, e))
1704         else:
1705             raise
1706
1707     return ret, outbuf, outs