ceph/src/pybind/ceph_argparse.py

   1 """
   2 Types and routines used by the ceph CLI as well as the RESTful
   3 interface.  These have to do with querying the daemons for
   4 command-description information, validating user command input against
   5 those descriptions, and submitting the command to the appropriate
   6 daemon.
   7
   8 Copyright (C) 2013 Inktank Storage, Inc.
   9
  10 LGPL-2.1 or LGPL-3.0.  See file COPYING.
  11 """
  12 import copy
  13 import enum
  14 import math
  15 import json
  16 import os
  17 import pprint
  18 import re
  19 import socket
  20 import stat
  21 import sys
  22 import threading
  23 import uuid
  24
  25 from collections import abc
  26 from typing import Any, Callable, Dict, Generic, List, Optional, Sequence, Tuple, Union
  27
  28 if sys.version_info >= (3, 8):
  29     from typing import get_args, get_origin
  30 else:
  31     def get_args(tp):
  32         if tp is Generic:
  33             return tp
  34         else:
  35             return getattr(tp, '__args__', ())
  36
  37     def get_origin(tp):
  38         return getattr(tp, '__origin__', None)
  39
  40
  41 # Flags are from MonCommand.h
  42 class Flag:
  43     NOFORWARD = (1 << 0)
  44     OBSOLETE = (1 << 1)
  45     DEPRECATED = (1 << 2)
  46     MGR = (1 << 3)
  47     POLL = (1 << 4)
  48     HIDDEN = (1 << 5)
  49
  50
  51 KWARG_EQUALS = "--([^=]+)=(.+)"
  52 KWARG_SPACE = "--([^=]+)"
  53
  54 try:
  55     basestring
  56 except NameError:
  57     basestring = str
  58
  59
  60 class ArgumentError(Exception):
  61     """
  62     Something wrong with arguments
  63     """
  64     pass
  65
  66
  67 class ArgumentNumber(ArgumentError):
  68     """
  69     Wrong number of a repeated argument
  70     """
  71     pass
  72
  73
  74 class ArgumentFormat(ArgumentError):
  75     """
  76     Argument value has wrong format
  77     """
  78     pass
  79
  80
  81 class ArgumentMissing(ArgumentError):
  82     """
  83     Argument value missing in a command
  84     """
  85     pass
  86
  87
  88 class ArgumentValid(ArgumentError):
  89     """
  90     Argument value is otherwise invalid (doesn't match choices, for instance)
  91     """
  92     pass
  93
  94
  95 class ArgumentTooFew(ArgumentError):
  96     """
  97     Fewer arguments than descriptors in signature; may mean to continue
  98     the search, so gets a special exception type
  99     """
 100
 101
 102 class ArgumentPrefix(ArgumentError):
 103     """
 104     Special for mismatched prefix; less severe, don't report by default
 105     """
 106     pass
 107
 108
 109 class JsonFormat(Exception):
 110     """
 111     some syntactic or semantic issue with the JSON
 112     """
 113     pass
 114
 115
 116 class CephArgtype(object):
 117     """
 118     Base class for all Ceph argument types
 119
 120     Instantiating an object sets any validation parameters
 121     (allowable strings, numeric ranges, etc.).  The 'valid'
 122     method validates a string against that initialized instance,
 123     throwing ArgumentError if there's a problem.
 124     """
 125     def __init__(self, **kwargs):
 126         """
 127         set any per-instance validation parameters here
 128         from kwargs (fixed string sets, integer ranges, etc)
 129         """
 130         pass
 131
 132     def valid(self, s, partial=False):
 133         """
 134         Run validation against given string s (generally one word);
 135         partial means to accept partial string matches (begins-with).
 136         If cool, set self.val to the value that should be returned
 137         (a copy of the input string, or a numeric or boolean interpretation
 138         thereof, for example)
 139         if not, throw ArgumentError(msg-as-to-why)
 140         """
 141         self.val = s
 142
 143     def __repr__(self):
 144         """
 145         return string representation of description of type.  Note,
 146         this is not a representation of the actual value.  Subclasses
 147         probably also override __str__() to give a more user-friendly
 148         'name/type' description for use in command format help messages.
 149         """
 150         a = ''
 151         if hasattr(self, 'typeargs'):
 152             a = self.typeargs
 153         return '{0}(\'{1}\')'.format(self.__class__.__name__, a)
 154
 155     def __str__(self):
 156         """
 157         where __repr__ (ideally) returns a string that could be used to
 158         reproduce the object, __str__ returns one you'd like to see in
 159         print messages.  Use __str__ to format the argtype descriptor
 160         as it would be useful in a command usage message.
 161         """
 162         return '<{0}>'.format(self.__class__.__name__)
 163
 164     def __call__(self, v):
 165         return v
 166
 167     def complete(self, s):
 168         return []
 169
 170     @staticmethod
 171     def _compound_type_to_argdesc(tp, attrs):
 172         # generate argdesc from Sequence[T], Tuple[T,..] and Optional[T]
 173         orig_type = get_origin(tp)
 174         type_args = get_args(tp)
 175         if orig_type in (abc.Sequence, Sequence, List, list):
 176             assert len(type_args) == 1
 177             attrs['n'] = 'N'
 178             return CephArgtype.to_argdesc(type_args[0], attrs)
 179         elif orig_type is Tuple:
 180             assert len(type_args) >= 1
 181             inner_tp = type_args[0]
 182             assert type_args.count(inner_tp) == len(type_args), \
 183                 f'all elements in {tp} should be identical'
 184             attrs['n'] = str(len(type_args))
 185             return CephArgtype.to_argdesc(inner_tp, attrs)
 186         elif get_origin(tp) is Union:
 187             # should be Union[t, NoneType]
 188             assert len(type_args) == 2 and isinstance(None, type_args[1])
 189             return CephArgtype.to_argdesc(type_args[0], attrs, True)
 190         else:
 191             raise ValueError(f"unknown type '{tp}': '{attrs}'")
 192
 193     @staticmethod
 194     def to_argdesc(tp, attrs, has_default=False):
 195         if has_default:
 196             attrs['req'] = 'false'
 197         CEPH_ARG_TYPES = {
 198             str: CephString,
 199             int: CephInt,
 200             float: CephFloat,
 201             bool: CephBool
 202         }
 203         try:
 204             return CEPH_ARG_TYPES[tp]().argdesc(attrs)
 205         except KeyError:
 206             if isinstance(tp, CephArgtype):
 207                 return tp.argdesc(attrs)
 208             elif isinstance(tp, type) and issubclass(tp, enum.Enum):
 209                 return CephChoices(tp=tp).argdesc(attrs)
 210             else:
 211                 return CephArgtype._compound_type_to_argdesc(tp, attrs)
 212
 213     def argdesc(self, attrs):
 214         attrs['type'] = type(self).__name__
 215         return ','.join(f'{k}={v}' for k, v in attrs.items())
 216
 217     @staticmethod
 218     def _cast_to_compound_type(tp, v):
 219         orig_type = get_origin(tp)
 220         type_args = get_args(tp)
 221         if orig_type in (abc.Sequence, Sequence, List, list):
 222             return [CephArgtype.cast_to(type_args[0], e) for e in v]
 223         elif orig_type is Tuple:
 224             return tuple(CephArgtype.cast_to(type_args[0], e) for e in v)
 225         elif get_origin(tp) is Union:
 226             # should be Union[t, NoneType]
 227             assert len(type_args) == 2 and isinstance(None, type_args[1])
 228             return CephArgtype.cast_to(type_args[0], v)
 229         else:
 230             raise ValueError(f"unknown type '{tp}': '{v}'")
 231
 232     @staticmethod
 233     def cast_to(tp, v):
 234         PYTHON_TYPES = (
 235             str,
 236             int,
 237             float,
 238             bool
 239         )
 240         if tp in PYTHON_TYPES:
 241             return tp(v)
 242         elif isinstance(tp, type) and issubclass(tp, enum.Enum):
 243             return tp(v)
 244         else:
 245             return CephArgtype._cast_to_compound_type(tp, v)
 246
 247
 248 class CephInt(CephArgtype):
 249     """
 250     range-limited integers, [+|-][0-9]+ or 0x[0-9a-f]+
 251     range: list of 1 or 2 ints, [min] or [min,max]
 252     """
 253     def __init__(self, range=''):
 254         if range == '':
 255             self.range = list()
 256         else:
 257             self.range = list(range.split('|'))
 258             self.range = [int(x) for x in self.range]
 259
 260     def valid(self, s, partial=False):
 261         try:
 262             val = int(s, 0)
 263         except ValueError:
 264             raise ArgumentValid("{0} doesn't represent an int".format(s))
 265         if len(self.range) == 2:
 266             if val < self.range[0] or val > self.range[1]:
 267                 raise ArgumentValid(f"{val} not in range {self.range}")
 268         elif len(self.range) == 1:
 269             if val < self.range[0]:
 270                 raise ArgumentValid(f"{val} not in range {self.range}")
 271         self.val = val
 272
 273     def __str__(self):
 274         r = ''
 275         if len(self.range) == 1:
 276             r = '[{0}-]'.format(self.range[0])
 277         if len(self.range) == 2:
 278             r = '[{0}-{1}]'.format(self.range[0], self.range[1])
 279
 280         return '<int{0}>'.format(r)
 281
 282     def argdesc(self, attrs):
 283         if self.range:
 284             attrs['range'] = '|'.join(self.range)
 285         return super().argdesc(attrs)
 286
 287
 288 class CephFloat(CephArgtype):
 289     """
 290     range-limited float type
 291     range: list of 1 or 2 floats, [min] or [min, max]
 292     """
 293     def __init__(self, range=''):
 294         if range == '':
 295             self.range = list()
 296         else:
 297             self.range = list(range.split('|'))
 298             self.range = [float(x) for x in self.range]
 299
 300     def valid(self, s, partial=False):
 301         try:
 302             val = float(s)
 303         except ValueError:
 304             raise ArgumentValid("{0} doesn't represent a float".format(s))
 305         if len(self.range) == 2:
 306             if val < self.range[0] or val > self.range[1]:
 307                 raise ArgumentValid(f"{val} not in range {self.range}")
 308         elif len(self.range) == 1:
 309             if val < self.range[0]:
 310                 raise ArgumentValid(f"{val} not in range {self.range}")
 311         self.val = val
 312
 313     def __str__(self):
 314         r = ''
 315         if len(self.range) == 1:
 316             r = '[{0}-]'.format(self.range[0])
 317         if len(self.range) == 2:
 318             r = '[{0}-{1}]'.format(self.range[0], self.range[1])
 319         return '<float{0}>'.format(r)
 320
 321     def argdesc(self, attrs):
 322         if self.range:
 323             attrs['range'] = '|'.join(self.range)
 324         return super().argdesc(attrs)
 325
 326
 327 class CephString(CephArgtype):
 328     """
 329     String; pretty generic.  goodchars is a RE char class of valid chars
 330     """
 331     def __init__(self, goodchars=''):
 332         from string import printable
 333         try:
 334             re.compile(goodchars)
 335         except re.error:
 336             raise ValueError('CephString(): "{0}" is not a valid RE'.
 337                              format(goodchars))
 338         self.goodchars = goodchars
 339         self.goodset = frozenset(
 340             [c for c in printable if re.match(goodchars, c)]
 341         )
 342
 343     def valid(self, s, partial=False):
 344         sset = set(s)
 345         if self.goodset and not sset <= self.goodset:
 346             raise ArgumentFormat("invalid chars {0} in {1}".
 347                                  format(''.join(sset - self.goodset), s))
 348         self.val = s
 349
 350     def __str__(self):
 351         b = ''
 352         if self.goodchars:
 353             b += '(goodchars {0})'.format(self.goodchars)
 354         return '<string{0}>'.format(b)
 355
 356     def complete(self, s):
 357         if s == '':
 358             return []
 359         else:
 360             return [s]
 361
 362     def argdesc(self, attrs):
 363         if self.goodchars:
 364             attrs['goodchars'] = self.goodchars
 365         return super().argdesc(attrs)
 366
 367
 368 class CephSocketpath(CephArgtype):
 369     """
 370     Admin socket path; check that it's readable and S_ISSOCK
 371     """
 372     def valid(self, s, partial=False):
 373         mode = os.stat(s).st_mode
 374         if not stat.S_ISSOCK(mode):
 375             raise ArgumentValid('socket path {0} is not a socket'.format(s))
 376         self.val = s
 377
 378     def __str__(self):
 379         return '<admin-socket-path>'
 380
 381
 382 class CephIPAddr(CephArgtype):
 383     """
 384     IP address (v4 or v6) with optional port
 385     """
 386     def valid(self, s, partial=False):
 387         # parse off port, use socket to validate addr
 388         type = 6
 389         if s.startswith('['):
 390             type = 6
 391         elif s.find('.') != -1:
 392             type = 4
 393         if type == 4:
 394             port = s.find(':')
 395             if port != -1:
 396                 a = s[:port]
 397                 p = s[port + 1:]
 398                 if int(p) > 65535:
 399                     raise ArgumentValid('{0}: invalid IPv4 port'.format(p))
 400             else:
 401                 a = s
 402                 p = None
 403             try:
 404                 socket.inet_pton(socket.AF_INET, a)
 405             except OSError:
 406                 raise ArgumentValid('{0}: invalid IPv4 address'.format(a))
 407         else:
 408             # v6
 409             if s.startswith('['):
 410                 end = s.find(']')
 411                 if end == -1:
 412                     raise ArgumentFormat('{0} missing terminating ]'.format(s))
 413                 if s[end + 1] == ':':
 414                     try:
 415                         p = int(s[end + 2])
 416                     except ValueError:
 417                         raise ArgumentValid('{0}: bad port number'.format(s))
 418                 a = s[1:end]
 419             else:
 420                 a = s
 421                 p = None
 422             try:
 423                 socket.inet_pton(socket.AF_INET6, a)
 424             except OSError:
 425                 raise ArgumentValid('{0} not valid IPv6 address'.format(s))
 426         if p is not None and int(p) > 65535:
 427             raise ArgumentValid("{0} not a valid port number".format(p))
 428         self.val = s
 429         self.addr = a
 430         self.port = p
 431
 432     def __str__(self):
 433         return '<IPaddr[:port]>'
 434
 435
 436 class CephEntityAddr(CephIPAddr):
 437     """
 438     EntityAddress, that is, IP address[/nonce]
 439     """
 440     def valid(self, s, partial=False):
 441         nonce = None
 442         if '/' in s:
 443             ip, nonce = s.split('/')
 444         else:
 445             ip = s
 446         super(self.__class__, self).valid(ip)
 447         if nonce:
 448             nonce_int = None
 449             try:
 450                 nonce_int = int(nonce)
 451             except ValueError:
 452                 pass
 453             if nonce_int is None or nonce_int < 0:
 454                 raise ArgumentValid(
 455                     '{0}: invalid entity, nonce {1} not integer > 0'.
 456                     format(s, nonce)
 457                 )
 458         self.val = s
 459
 460     def __str__(self):
 461         return '<EntityAddr>'
 462
 463
 464 class CephPoolname(CephArgtype):
 465     """
 466     Pool name; very little utility
 467     """
 468     def __str__(self):
 469         return '<poolname>'
 470
 471
 472 class CephObjectname(CephArgtype):
 473     """
 474     Object name.  Maybe should be combined with Pool name as they're always
 475     present in pairs, and then could be checked for presence
 476     """
 477     def __str__(self):
 478         return '<objectname>'
 479
 480
 481 class CephPgid(CephArgtype):
 482     """
 483     pgid, in form N.xxx (N = pool number, xxx = hex pgnum)
 484     """
 485     def valid(self, s, partial=False):
 486         if s.find('.') == -1:
 487             raise ArgumentFormat('pgid has no .')
 488         poolid, pgnum = s.split('.', 1)
 489         try:
 490             poolid = int(poolid)
 491         except ValueError:
 492             raise ArgumentFormat('pool {0} not integer'.format(poolid))
 493         if poolid < 0:
 494             raise ArgumentFormat('pool {0} < 0'.format(poolid))
 495         try:
 496             pgnum = int(pgnum, 16)
 497         except ValueError:
 498             raise ArgumentFormat('pgnum {0} not hex integer'.format(pgnum))
 499         self.val = s
 500
 501     def __str__(self):
 502         return '<pgid>'
 503
 504
 505 class CephName(CephArgtype):
 506     """
 507     Name (type.id) where:
 508     type is osd|mon|client|mds
 509     id is a base10 int, if type == osd, or a string otherwise
 510
 511     Also accept '*'
 512     """
 513     def __init__(self):
 514         self.nametype = None
 515         self.nameid = None
 516
 517     def valid(self, s, partial=False):
 518         if s == '*':
 519             self.val = s
 520             return
 521         elif s == "mgr":
 522             self.nametype = "mgr"
 523             self.val = s
 524             return
 525         elif s == "mon":
 526             self.nametype = "mon"
 527             self.val = s
 528             return
 529         if s.find('.') == -1:
 530             raise ArgumentFormat('CephName: no . in {0}'.format(s))
 531         else:
 532             t, i = s.split('.', 1)
 533             if t not in ('osd', 'mon', 'client', 'mds', 'mgr'):
 534                 raise ArgumentValid('unknown type ' + t)
 535             if t == 'osd':
 536                 if i != '*':
 537                     try:
 538                         i = int(i)
 539                     except ValueError:
 540                         raise ArgumentFormat('osd id ' + i + ' not integer')
 541             self.nametype = t
 542         self.val = s
 543         self.nameid = i
 544
 545     def __str__(self):
 546         return '<name (type.id)>'
 547
 548
 549 class CephOsdName(CephArgtype):
 550     """
 551     Like CephName, but specific to osds: allow <id> alone
 552
 553     osd.<id>, or <id>, or *, where id is a base10 int
 554     """
 555     def __init__(self):
 556         self.nametype = None
 557         self.nameid = None
 558
 559     def valid(self, s, partial=False):
 560         if s == '*':
 561             self.val = s
 562             return
 563         if s.find('.') != -1:
 564             t, i = s.split('.', 1)
 565             if t != 'osd':
 566                 raise ArgumentValid('unknown type ' + t)
 567         else:
 568             t = 'osd'
 569             i = s
 570         try:
 571             i = int(i)
 572         except ValueError:
 573             raise ArgumentFormat('osd id ' + i + ' not integer')
 574         if i < 0:
 575             raise ArgumentFormat('osd id {0} is less than 0'.format(i))
 576         self.nametype = t
 577         self.nameid = i
 578         self.val = i
 579
 580     def __str__(self):
 581         return '<osdname (id|osd.id)>'
 582
 583
 584 class CephChoices(CephArgtype):
 585     """
 586     Set of string literals; init with valid choices
 587     """
 588     def __init__(self, strings='', tp=None, **kwargs):
 589         self.strings = strings.split('|')
 590         self.enum = tp
 591         if self.enum is not None:
 592             self.strings = list(e.value for e in self.enum)
 593
 594     def valid(self, s, partial=False):
 595         if not partial:
 596             if s not in self.strings:
 597                 # show as __str__ does: {s1|s2..}
 598                 raise ArgumentValid("{0} not in {1}".format(s, self))
 599             self.val = s
 600             return
 601
 602         # partial
 603         for t in self.strings:
 604             if t.startswith(s):
 605                 self.val = s
 606                 return
 607         raise ArgumentValid("{0} not in {1}".  format(s, self))
 608
 609     def __str__(self):
 610         if len(self.strings) == 1:
 611             return '{0}'.format(self.strings[0])
 612         else:
 613             return '{0}'.format('|'.join(self.strings))
 614
 615     def __call__(self, v):
 616         if self.enum is None:
 617             return v
 618         else:
 619             return self.enum[v]
 620
 621     def complete(self, s):
 622         all_elems = [token for token in self.strings if token.startswith(s)]
 623         return all_elems
 624
 625     def argdesc(self, attrs):
 626         attrs['strings'] = '|'.join(self.strings)
 627         return super().argdesc(attrs)
 628
 629
 630 class CephBool(CephArgtype):
 631     """
 632     A boolean argument, values may be case insensitive 'true', 'false', '0',
 633     '1'.  In keyword form, value may be left off (implies true).
 634     """
 635     def __init__(self, strings='', **kwargs):
 636         self.strings = strings.split('|')
 637
 638     def valid(self, s, partial=False):
 639         lower_case = s.lower()
 640         if lower_case in ['true', '1']:
 641             self.val = True
 642         elif lower_case in ['false', '0']:
 643             self.val = False
 644         else:
 645             raise ArgumentValid("{0} not one of 'true', 'false'".format(s))
 646
 647     def __str__(self):
 648         return '<bool>'
 649
 650
 651 class CephFilepath(CephArgtype):
 652     """
 653     Openable file
 654     """
 655     def valid(self, s, partial=False):
 656         # set self.val if the specified path is readable or writable
 657         s = os.path.abspath(s)
 658         if not os.access(s, os.R_OK):
 659             self._validate_writable_file(s)
 660         self.val = s
 661
 662     def _validate_writable_file(self, fname):
 663         if os.path.exists(fname):
 664             if os.path.isfile(fname):
 665                 if not os.access(fname, os.W_OK):
 666                     raise ArgumentValid('{0} is not writable'.format(fname))
 667             else:
 668                 raise ArgumentValid('{0} is not file'.format(fname))
 669         else:
 670             dirname = os.path.dirname(fname)
 671             if not os.access(dirname, os.W_OK):
 672                 raise ArgumentValid('cannot create file in {0}'.format(dirname))
 673
 674     def __str__(self):
 675         return '<outfilename>'
 676
 677
 678 class CephFragment(CephArgtype):
 679     """
 680     'Fragment' ??? XXX
 681     """
 682     def valid(self, s, partial=False):
 683         if s.find('/') == -1:
 684             raise ArgumentFormat('{0}: no /'.format(s))
 685         val, bits = s.split('/')
 686         # XXX is this right?
 687         if not val.startswith('0x'):
 688             raise ArgumentFormat("{0} not a hex integer".format(val))
 689         try:
 690             int(val)
 691         except ValueError:
 692             raise ArgumentFormat('can\'t convert {0} to integer'.format(val))
 693         try:
 694             int(bits)
 695         except ValueError:
 696             raise ArgumentFormat('can\'t convert {0} to integer'.format(bits))
 697         self.val = s
 698
 699     def __str__(self):
 700         return "<CephFS fragment ID (0xvvv/bbb)>"
 701
 702
 703 class CephUUID(CephArgtype):
 704     """
 705     CephUUID: pretty self-explanatory
 706     """
 707     def valid(self, s, partial=False):
 708         try:
 709             uuid.UUID(s)
 710         except Exception as e:
 711             raise ArgumentFormat('invalid UUID {0}: {1}'.format(s, e))
 712         self.val = s
 713
 714     def __str__(self):
 715         return '<uuid>'
 716
 717
 718 class CephPrefix(CephArgtype):
 719     """
 720     CephPrefix: magic type for "all the first n fixed strings"
 721     """
 722     def __init__(self, prefix=''):
 723         self.prefix = prefix
 724
 725     def valid(self, s, partial=False):
 726         try:
 727             s = str(s)
 728             if isinstance(s, bytes):
 729                 # `prefix` can always be converted into unicode when being compared,
 730                 # but `s` could be anything passed by user.
 731                 s = s.decode('ascii')
 732         except UnicodeEncodeError:
 733             raise ArgumentPrefix(u"no match for {0}".format(s))
 734         except UnicodeDecodeError:
 735             raise ArgumentPrefix("no match for {0}".format(s))
 736
 737         if partial:
 738             if self.prefix.startswith(s):
 739                 self.val = s
 740                 return
 741         else:
 742             if s == self.prefix:
 743                 self.val = s
 744                 return
 745
 746         raise ArgumentPrefix("no match for {0}".format(s))
 747
 748     def __str__(self):
 749         return self.prefix
 750
 751     def complete(self, s):
 752         if self.prefix.startswith(s):
 753             return [self.prefix.rstrip(' ')]
 754         else:
 755             return []
 756
 757
 758 class argdesc(object):
 759     """
 760     argdesc(typename, name='name', n=numallowed|N,
 761             req=False, helptext=helptext, **kwargs (type-specific))
 762
 763     validation rules:
 764     typename: type(**kwargs) will be constructed
 765     later, type.valid(w) will be called with a word in that position
 766
 767     name is used for parse errors and for constructing JSON output
 768     n is a numeric literal or 'n|N', meaning "at least one, but maybe more"
 769     req=False means the argument need not be present in the list
 770     helptext is the associated help for the command
 771     anything else are arguments to pass to the type constructor.
 772
 773     self.instance is an instance of type t constructed with typeargs.
 774
 775     valid() will later be called with input to validate against it,
 776     and will store the validated value in self.instance.val for extraction.
 777     """
 778     def __init__(self, t, name=None, n=1, req=True, **kwargs):
 779         if isinstance(t, basestring):
 780             self.t = CephPrefix
 781             self.typeargs = {'prefix': t}
 782             self.req = True
 783         else:
 784             self.t = t
 785             self.typeargs = kwargs
 786             self.req = req in (True, 'True', 'true')
 787
 788         self.name = name
 789         self.N = (n in ['n', 'N'])
 790         if self.N:
 791             self.n = 1
 792         else:
 793             self.n = int(n)
 794
 795         self.numseen = 0
 796
 797         self.instance = self.t(**self.typeargs)
 798
 799     def __repr__(self):
 800         r = 'argdesc(' + str(self.t) + ', '
 801         internals = ['N', 'typeargs', 'instance', 't']
 802         for (k, v) in self.__dict__.items():
 803             if k.startswith('__') or k in internals:
 804                 pass
 805             else:
 806                 # undo modification from __init__
 807                 if k == 'n' and self.N:
 808                     v = 'N'
 809                 r += '{0}={1}, '.format(k, v)
 810         for (k, v) in self.typeargs.items():
 811             r += '{0}={1}, '.format(k, v)
 812         return r[:-2] + ')'
 813
 814     def __str__(self):
 815         if ((self.t == CephChoices and len(self.instance.strings) == 1)
 816            or (self.t == CephPrefix)):
 817             s = str(self.instance)
 818         else:
 819             s = '{0}({1})'.format(self.name, str(self.instance))
 820             if self.N:
 821                 s += '...'
 822         if not self.req:
 823             s = '[' + s + ']'
 824         return s
 825
 826     def helpstr(self):
 827         """
 828         like str(), but omit parameter names (except for CephString,
 829         which really needs them)
 830         """
 831         if self.t == CephBool:
 832             chunk = "--{0}".format(self.name.replace("_", "-"))
 833         elif self.t == CephPrefix:
 834             chunk = str(self.instance)
 835         elif self.t == CephChoices:
 836             if self.name == 'format':
 837                 chunk = f'--{self.name} {{{str(self.instance)}}}'
 838             else:
 839                 chunk = str(self.instance)
 840         elif self.t == CephOsdName:
 841             # it just so happens all CephOsdName commands are named 'id' anyway,
 842             # so <id|osd.id> is perfect.
 843             chunk = '<id|osd.id>'
 844         elif self.t == CephName:
 845             # CephName commands similarly only have one arg of the
 846             # type, so <type.id> is good.
 847             chunk = '<type.id>'
 848         elif self.t == CephInt:
 849             chunk = '<{0}:int>'.format(self.name)
 850         elif self.t == CephFloat:
 851             chunk = '<{0}:float>'.format(self.name)
 852         else:
 853             chunk = '<{0}>'.format(self.name)
 854         s = chunk
 855         if self.N:
 856             s += '...'
 857         if not self.req:
 858             s = '[' + s + ']'
 859         return s
 860
 861     def complete(self, s):
 862         return self.instance.complete(s)
 863
 864
 865 def concise_sig(sig):
 866     """
 867     Return string representation of sig useful for syntax reference in help
 868     """
 869     return ' '.join([d.helpstr() for d in sig])
 870
 871
 872 def descsort_key(sh):
 873     """
 874     sort descriptors by prefixes, defined as the concatenation of all simple
 875     strings in the descriptor; this works out to just the leading strings.
 876     """
 877     return concise_sig(sh['sig'])
 878
 879
 880 def descsort(sh1, sh2):
 881     """
 882     Deprecated; use (key=descsort_key) instead of (cmp=descsort)
 883     """
 884     return cmp(descsort_key(sh1), descsort_key(sh2))
 885
 886
 887 def parse_funcsig(sig: Sequence[Union[str, Dict[str, str]]]) -> List[argdesc]:
 888     """
 889     parse a single descriptor (array of strings or dicts) into a
 890     dict of function descriptor/validators (objects of CephXXX type)
 891
 892     :returns: list of ``argdesc``
 893     """
 894     newsig = []
 895     argnum = 0
 896     for desc in sig:
 897         argnum += 1
 898         if isinstance(desc, basestring):
 899             t = CephPrefix
 900             desc = {'type': t, 'name': 'prefix', 'prefix': desc}
 901         else:
 902             # not a simple string, must be dict
 903             if 'type' not in desc:
 904                 s = 'JSON descriptor {0} has no type'.format(sig)
 905                 raise JsonFormat(s)
 906             # look up type string in our globals() dict; if it's an
 907             # object of type `type`, it must be a
 908             # locally-defined class. otherwise, we haven't a clue.
 909             if desc['type'] in globals():
 910                 t = globals()[desc['type']]
 911                 if not isinstance(t, type):
 912                     s = 'unknown type {0}'.format(desc['type'])
 913                     raise JsonFormat(s)
 914             else:
 915                 s = 'unknown type {0}'.format(desc['type'])
 916                 raise JsonFormat(s)
 917
 918         kwargs = dict()
 919         for key, val in desc.items():
 920             if key not in ['type', 'name', 'n', 'req']:
 921                 kwargs[key] = val
 922         newsig.append(argdesc(t,
 923                               name=desc.get('name', None),
 924                               n=desc.get('n', 1),
 925                               req=desc.get('req', True),
 926                               **kwargs))
 927     return newsig
 928
 929
 930 def parse_json_funcsigs(s: str,
 931                         consumer: str) -> Dict[str, Dict[str, List[argdesc]]]:
 932     """
 933     A function signature is mostly an array of argdesc; it's represented
 934     in JSON as
 935     {
 936       "cmd001": {"sig":[ "type": type, "name": name, "n": num, "req":true|false <other param>], "help":helptext, "module":modulename, "perm":perms, "avail":availability}
 937        .
 938        .
 939        .
 940       ]
 941
 942     A set of sigs is in an dict mapped by a unique number:
 943     {
 944       "cmd1": {
 945          "sig": ["type.. ], "help":helptext...
 946       }
 947       "cmd2"{
 948          "sig": [.. ], "help":helptext...
 949       }
 950     }
 951
 952     Parse the string s and return a dict of dicts, keyed by opcode;
 953     each dict contains 'sig' with the array of descriptors, and 'help'
 954     with the helptext, 'module' with the module name, 'perm' with a
 955     string representing required permissions in that module to execute
 956     this command (and also whether it is a read or write command from
 957     the cluster state perspective), and 'avail' as a hint for
 958     whether the command should be advertised by CLI, REST, or both.
 959     If avail does not contain 'consumer', don't include the command
 960     in the returned dict.
 961     """
 962     try:
 963         overall = json.loads(s)
 964     except Exception as e:
 965         print("Couldn't parse JSON {0}: {1}".format(s, e), file=sys.stderr)
 966         raise e
 967     sigdict = {}
 968     for cmdtag, cmd in overall.items():
 969         if 'sig' not in cmd:
 970             s = "JSON descriptor {0} has no 'sig'".format(cmdtag)
 971             raise JsonFormat(s)
 972         # check 'avail' and possibly ignore this command
 973         if 'avail' in cmd:
 974             if consumer not in cmd['avail']:
 975                 continue
 976         # rewrite the 'sig' item with the argdesc-ized version, and...
 977         cmd['sig'] = parse_funcsig(cmd['sig'])
 978         # just take everything else as given
 979         sigdict[cmdtag] = cmd
 980     return sigdict
 981
 982
 983 def validate_one(word, desc, is_kwarg, partial=False):
 984     """
 985     validate_one(word, desc, is_kwarg, partial=False)
 986
 987     validate word against the constructed instance of the type
 988     in desc.  May raise exception.  If it returns false (and doesn't
 989     raise an exception), desc.instance.val will
 990     contain the validated value (in the appropriate type).
 991     """
 992     vals = []
 993     # CephString option might contain "," in it
 994     allow_csv = is_kwarg or desc.t is not CephString
 995     if desc.N and allow_csv:
 996         for part in word.split(','):
 997             desc.instance.valid(part, partial)
 998             vals.append(desc.instance.val)
 999     else:
1000         desc.instance.valid(word, partial)
1001         vals.append(desc.instance.val)
1002     desc.numseen += 1
1003     if desc.N:
1004         desc.n = desc.numseen + 1
1005     return vals
1006
1007
1008 def matchnum(args, signature, partial=False):
1009     """
1010     matchnum(s, signature, partial=False)
1011
1012     Returns number of arguments matched in s against signature.
1013     Can be used to determine most-likely command for full or partial
1014     matches (partial applies to string matches).
1015     """
1016     words = args[:]
1017     mysig = copy.deepcopy(signature)
1018     matchcnt = 0
1019     for desc in mysig:
1020         desc.numseen = 0
1021         while desc.numseen < desc.n:
1022             # if there are no more arguments, return
1023             if not words:
1024                 return matchcnt
1025             word = words.pop(0)
1026
1027             try:
1028                 # only allow partial matching if we're on the last supplied
1029                 # word; avoid matching foo bar and foot bar just because
1030                 # partial is set
1031                 validate_one(word, desc, False, partial and (len(words) == 0))
1032                 valid = True
1033             except ArgumentError:
1034                 # matchnum doesn't care about type of error
1035                 valid = False
1036
1037             if not valid:
1038                 if not desc.req:
1039                     # this wasn't required, so word may match the next desc
1040                     words.insert(0, word)
1041                     break
1042                 else:
1043                     # it was required, and didn't match, return
1044                     return matchcnt
1045         if desc.req:
1046             matchcnt += 1
1047     return matchcnt
1048
1049
1050 ValidatedArg = Union[bool, int, float, str,
1051                      Tuple[str, str],
1052                      Sequence[str]]
1053 ValidatedArgs = Dict[str, ValidatedArg]
1054
1055
1056 def store_arg(desc: argdesc, args: List[ValidatedArg], d: ValidatedArgs):
1057     '''
1058     Store argument described by, and held in, thanks to valid(),
1059     desc into the dictionary d, keyed by desc.name.  Three cases:
1060
1061     1) desc.N is set: use args for arg value in "d", desc.instance.val
1062        only contains the last parsed arg in the "args" list
1063     2) prefix: multiple args are joined with ' ' into one d{} item
1064     3) single prefix or other arg: store as simple value
1065
1066     Used in validate() below.
1067     '''
1068     if desc.N:
1069         # value should be a list
1070         if desc.name in d:
1071             d[desc.name] += args
1072         else:
1073             d[desc.name] = args
1074     elif (desc.t == CephPrefix) and (desc.name in d):
1075         # prefixes' values should be a space-joined concatenation
1076         d[desc.name] += ' ' + desc.instance.val
1077     else:
1078         # if first CephPrefix or any other type, just set it
1079         d[desc.name] = desc.instance.val
1080
1081
1082 def validate(args: List[str],
1083              signature: Sequence[argdesc],
1084              flags: Optional[int] = 0,
1085              partial: Optional[bool] = False) -> ValidatedArgs:
1086     """
1087     validate(args, signature, flags=0, partial=False)
1088
1089     args is a list of strings representing a possible
1090     command input following format of signature.  Runs a validation; no
1091     exception means it's OK.  Return a dict containing all arguments keyed
1092     by their descriptor name, with duplicate args per name accumulated
1093     into a list (or space-separated value for CephPrefix).
1094
1095     Mismatches of prefix are non-fatal, as this probably just means the
1096     search hasn't hit the correct command.  Mismatches of non-prefix
1097     arguments are treated as fatal, and an exception raised.
1098
1099     This matching is modified if partial is set: allow partial matching
1100     (with partial dict returned); in this case, there are no exceptions
1101     raised.
1102     """
1103
1104     myargs = copy.deepcopy(args)
1105     mysig = copy.deepcopy(signature)
1106     reqsiglen = len([desc for desc in mysig if desc.req])
1107     matchcnt = 0
1108     d = dict()
1109     save_exception = None
1110
1111     arg_descs_by_name = dict([desc.name, desc] for desc in mysig
1112                              if desc.t != CephPrefix)
1113
1114     # Special case: detect "injectargs" (legacy way of modifying daemon
1115     # configs) and permit "--" string arguments if so.
1116     injectargs = myargs and myargs[0] == "injectargs"
1117
1118     # Make a pass through all arguments
1119     for desc in mysig:
1120         desc.numseen = 0
1121
1122         while desc.numseen < desc.n:
1123             if myargs:
1124                 myarg = myargs.pop(0)
1125             else:
1126                 myarg = None
1127
1128             # no arg, but not required?  Continue consuming mysig
1129             # in case there are later required args
1130             if myarg in (None, []) and not desc.req:
1131                 break
1132
1133             # A keyword argument?
1134             if myarg:
1135                 # argdesc for the keyword argument, if we find one
1136                 kwarg_desc = None
1137
1138                 # Track whether we need to push value back onto
1139                 # myargs in the case that this isn't a valid k=v
1140                 consumed_next = False
1141
1142                 # Try both styles of keyword argument
1143                 kwarg_match = re.match(KWARG_EQUALS, myarg)
1144                 if kwarg_match:
1145                     # We have a "--foo=bar" style argument
1146                     kwarg_k, kwarg_v = kwarg_match.groups()
1147
1148                     # Either "--foo-bar" or "--foo_bar" style is accepted
1149                     kwarg_k = kwarg_k.replace('-', '_')
1150
1151                     kwarg_desc = arg_descs_by_name.get(kwarg_k, None)
1152                 else:
1153                     # Maybe this is a "--foo bar" or "--bool" style argument
1154                     key_match = re.match(KWARG_SPACE, myarg)
1155                     if key_match:
1156                         kwarg_k = key_match.group(1)
1157
1158                         # Permit --foo-bar=123 form or --foo_bar=123 form,
1159                         # assuming all command definitions use foo_bar argument
1160                         # naming style
1161                         kwarg_k = kwarg_k.replace('-', '_')
1162
1163                         kwarg_desc = arg_descs_by_name.get(kwarg_k, None)
1164                         if kwarg_desc:
1165                             if kwarg_desc.t == CephBool:
1166                                 kwarg_v = 'true'
1167                             elif len(myargs):  # Some trailing arguments exist
1168                                 kwarg_v = myargs.pop(0)
1169                             else:
1170                                 # Forget it, this is not a valid kwarg
1171                                 kwarg_desc = None
1172
1173                 if kwarg_desc:
1174                     args = validate_one(kwarg_v, kwarg_desc, True)
1175                     matchcnt += 1
1176                     store_arg(kwarg_desc, args, d)
1177                     continue
1178
1179             # Don't handle something as a positional argument if it
1180             # has a leading "--" unless it's a CephChoices (used for
1181             # "--yes-i-really-mean-it")
1182             if myarg and myarg.startswith("--"):
1183                 # Special cases for instances of confirmation flags
1184                 # that were defined as CephString/CephChoices instead of CephBool
1185                 # in pre-nautilus versions of Ceph daemons.
1186                 is_value = desc.t == CephChoices \
1187                         or myarg == "--yes-i-really-mean-it" \
1188                         or myarg == "--yes-i-really-really-mean-it" \
1189                         or myarg == "--yes-i-really-really-mean-it-not-faking" \
1190                         or myarg == "--force" \
1191                         or injectargs
1192
1193                 if not is_value:
1194                     # Didn't get caught by kwarg handling, but has a "--", so
1195                     # we must assume it's something invalid, to avoid naively
1196                     # passing through mis-typed options as the values of
1197                     # positional arguments.
1198                     raise ArgumentValid("Unexpected argument '{0}'".format(
1199                         myarg))
1200
1201             # out of arguments for a required param?
1202             # Either return (if partial validation) or raise
1203             if myarg in (None, []) and desc.req:
1204                 if desc.N and desc.numseen < 1:
1205                     # wanted N, didn't even get 1
1206                     if partial:
1207                         return d
1208                     raise ArgumentNumber(
1209                         'saw {0} of {1}, expected at least 1'.
1210                         format(desc.numseen, desc)
1211                     )
1212                 elif not desc.N and desc.numseen < desc.n:
1213                     # wanted n, got too few
1214                     if partial:
1215                         return d
1216                     # special-case the "0 expected 1" case
1217                     if desc.numseen == 0 and desc.n == 1:
1218                         raise ArgumentMissing(
1219                             'missing required parameter {0}'.format(desc)
1220                         )
1221                     raise ArgumentNumber(
1222                         'saw {0} of {1}, expected {2}'.
1223                         format(desc.numseen, desc, desc.n)
1224                     )
1225                 break
1226
1227             # Have an arg; validate it
1228             try:
1229                 args = validate_one(myarg, desc, False)
1230             except ArgumentError as e:
1231                 # argument mismatch
1232                 if not desc.req:
1233                     # if not required, just push back; it might match
1234                     # the next arg
1235                     save_exception = [myarg, e]
1236                     myargs.insert(0, myarg)
1237                     break
1238                 else:
1239                     # hm, it was required, so time to return/raise
1240                     if partial:
1241                         return d
1242                     raise
1243
1244             # Whew, valid arg acquired.  Store in dict
1245             matchcnt += 1
1246             store_arg(desc, args, d)
1247             # Clear prior exception
1248             save_exception = None
1249
1250     # Done with entire list of argdescs
1251     if matchcnt < reqsiglen:
1252         raise ArgumentTooFew("not enough arguments given")
1253
1254     if myargs and not partial:
1255         if save_exception:
1256             print(save_exception[0], 'not valid: ', save_exception[1], file=sys.stderr)
1257         raise ArgumentError("unused arguments: " + str(myargs))
1258
1259     if flags & Flag.MGR:
1260         d['target'] = ('mon-mgr', '')
1261
1262     if flags & Flag.POLL:
1263         d['poll'] = True
1264
1265     # Finally, success
1266     return d
1267
1268
1269 def validate_command(sigdict: Dict[str, Dict[str, Any]],
1270                      args: Sequence[str],
1271                      verbose: Optional[bool] = False) -> ValidatedArgs:
1272     """
1273     Parse positional arguments into a parameter dict, according to
1274     the command descriptions.
1275
1276     Writes advice about nearly-matching commands ``sys.stderr`` if
1277     the arguments do not match any command.
1278
1279     :param sigdict: A command description dictionary, as returned
1280                     from Ceph daemons by the get_command_descriptions
1281                     command.
1282     :param args: List of strings, should match one of the command
1283                  signatures in ``sigdict``
1284
1285     :returns: A dict of parsed parameters (including ``prefix``),
1286               or an empty dict if the args did not match any signature
1287     """
1288     if verbose:
1289         print("validate_command: " + " ".join(args), file=sys.stderr)
1290     found = []
1291     valid_dict = {}
1292
1293     # look for best match, accumulate possibles in bestcmds
1294     # (so we can maybe give a more-useful error message)
1295     best_match_cnt = 0
1296     bestcmds = []
1297     for cmd in sigdict.values():
1298         flags = cmd.get('flags', 0)
1299         if flags & Flag.OBSOLETE:
1300             continue
1301         sig = cmd['sig']
1302         matched = matchnum(args, sig, partial=True)
1303         if (matched >= math.floor(best_match_cnt) and
1304             matched == matchnum(args, sig, partial=False)):
1305             # prefer those fully matched over partial patch
1306             matched += 0.5
1307         if matched < best_match_cnt:
1308             continue
1309         if verbose:
1310             print("better match: {0} > {1}: {2} ".format(
1311                 matched, best_match_cnt, concise_sig(sig)
1312             ), file=sys.stderr)
1313         if matched > best_match_cnt:
1314             best_match_cnt = matched
1315             bestcmds = [cmd]
1316         else:
1317             bestcmds.append(cmd)
1318
1319     # Sort bestcmds by number of req args so we can try shortest first
1320     # (relies on a cmdsig being key,val where val is a list of len 1)
1321
1322     def grade(cmd):
1323         # prefer optional arguments over required ones
1324         sigs = cmd['sig']
1325         return sum(map(lambda sig: sig.req, sigs))
1326
1327     bestcmds_sorted = sorted(bestcmds, key=grade)
1328     if verbose:
1329         print("bestcmds_sorted: ", file=sys.stderr)
1330         pprint.PrettyPrinter(stream=sys.stderr).pprint(bestcmds_sorted)
1331
1332     ex = None
1333     # for everything in bestcmds, look for a true match
1334     for cmd in bestcmds_sorted:
1335         sig = cmd['sig']
1336         try:
1337             valid_dict = validate(args, sig, flags=cmd.get('flags', 0))
1338             found = cmd
1339             break
1340         except ArgumentPrefix:
1341             # ignore prefix mismatches; we just haven't found
1342             # the right command yet
1343             pass
1344         except ArgumentMissing as e:
1345             ex = e
1346             if len(bestcmds) == 1:
1347                 found = cmd
1348             break
1349         except ArgumentTooFew:
1350             # It looked like this matched the beginning, but it
1351             # didn't have enough args supplied.  If we're out of
1352             # cmdsigs we'll fall out unfound; if we're not, maybe
1353             # the next one matches completely.  Whine, but pass.
1354             if verbose:
1355                 print('Not enough args supplied for ',
1356                       concise_sig(sig), file=sys.stderr)
1357         except ArgumentError as e:
1358             ex = e
1359             # Solid mismatch on an arg (type, range, etc.)
1360             # Stop now, because we have the right command but
1361             # some other input is invalid
1362             found = cmd
1363             break
1364
1365     if found:
1366         if not valid_dict:
1367             print("Invalid command:", ex, file=sys.stderr)
1368             print(concise_sig(sig), ': ', cmd['help'], file=sys.stderr)
1369     else:
1370         bestcmds = [c for c in bestcmds
1371                     if not c.get('flags', 0) & (Flag.DEPRECATED | Flag.HIDDEN)]
1372         bestcmds = bestcmds[:10]  # top 10
1373         print('no valid command found; {0} closest matches:'.format(len(bestcmds)),
1374               file=sys.stderr)
1375         for cmd in bestcmds:
1376             print(concise_sig(cmd['sig']), file=sys.stderr)
1377     return valid_dict
1378
1379
1380 def find_cmd_target(childargs: List[str]) -> Tuple[str, str]:
1381     """
1382     Using a minimal validation, figure out whether the command
1383     should be sent to a monitor or an osd.  We do this before even
1384     asking for the 'real' set of command signatures, so we can ask the
1385     right daemon.
1386     Returns ('osd', osdid), ('pg', pgid), ('mgr', '') or ('mon', '')
1387     """
1388     sig = parse_funcsig(['tell', {'name': 'target', 'type': 'CephName'}])
1389     try:
1390         valid_dict = validate(childargs, sig, partial=True)
1391     except ArgumentError:
1392         pass
1393     else:
1394         if len(valid_dict) == 2:
1395             # revalidate to isolate type and id
1396             name = CephName()
1397             # if this fails, something is horribly wrong, as it just
1398             # validated successfully above
1399             name.valid(valid_dict['target'])
1400             return name.nametype, name.nameid
1401
1402     sig = parse_funcsig(['tell', {'name': 'pgid', 'type': 'CephPgid'}])
1403     try:
1404         valid_dict = validate(childargs, sig, partial=True)
1405     except ArgumentError:
1406         pass
1407     else:
1408         if len(valid_dict) == 2:
1409             # pg doesn't need revalidation; the string is fine
1410             return 'pg', valid_dict['pgid']
1411
1412     # If we reached this far it must mean that so far we've been unable to
1413     # obtain a proper target from childargs.  This may mean that we are not
1414     # dealing with a 'tell' command, or that the specified target is invalid.
1415     # If the latter, we likely were unable to catch it because we were not
1416     # really looking for it: first we tried to parse a 'CephName' (osd, mon,
1417     # mds, followed by and id); given our failure to parse, we tried to parse
1418     # a 'CephPgid' instead (e.g., 0.4a).  Considering we got this far though
1419     # we were unable to do so.
1420     #
1421     # We will now check if this is a tell and, if so, forcefully validate the
1422     # target as a 'CephName'.  This must be so because otherwise we will end
1423     # up sending garbage to a monitor, which is the default target when a
1424     # target is not explicitly specified.
1425     # e.g.,
1426     #   'ceph status' -> target is any one monitor
1427     #   'ceph tell mon.* status -> target is all monitors
1428     #   'ceph tell foo status -> target is invalid!
1429     if len(childargs) > 1 and childargs[0] == 'tell':
1430         name = CephName()
1431         # CephName.valid() raises on validation error; find_cmd_target()'s
1432         # caller should handle them
1433         name.valid(childargs[1])
1434         return name.nametype, name.nameid
1435
1436     sig = parse_funcsig(['pg', {'name': 'pgid', 'type': 'CephPgid'}])
1437     try:
1438         valid_dict = validate(childargs, sig, partial=True)
1439     except ArgumentError:
1440         pass
1441     else:
1442         if len(valid_dict) == 2:
1443             return 'pg', valid_dict['pgid']
1444
1445     return 'mon', ''
1446
1447
1448 class RadosThread(threading.Thread):
1449     def __init__(self, func, *args, **kwargs):
1450         self.args = args
1451         self.kwargs = kwargs
1452         self.func = func
1453         self.exception = None
1454         threading.Thread.__init__(self)
1455
1456     def run(self):
1457         try:
1458             self.retval = self.func(*self.args, **self.kwargs)
1459         except Exception as e:
1460             self.exception = e
1461
1462
1463 def run_in_thread(func: Callable[[Any, Any], int],
1464                   *args: Any, **kwargs: Any) -> int:
1465     timeout = kwargs.pop('timeout', 0)
1466     if timeout == 0 or timeout is None:
1467         # python threading module will just get blocked if timeout is `None`,
1468         # otherwise it will keep polling until timeout or thread stops.
1469         # timeout in integer when converting it to nanoseconds, but since
1470         # python3 uses `int64_t` for the deadline before timeout expires,
1471         # we have to use a safe value which does not overflow after being
1472         # added to current time in microseconds.
1473         timeout = 24 * 60 * 60
1474     t = RadosThread(func, *args, **kwargs)
1475
1476     # allow the main thread to exit (presumably, avoid a join() on this
1477     # subthread) before this thread terminates.  This allows SIGINT
1478     # exit of a blocked call.  See below.
1479     t.daemon = True
1480
1481     t.start()
1482     t.join(timeout=timeout)
1483     # ..but allow SIGINT to terminate the waiting.  Note: this
1484     # relies on the Linux kernel behavior of delivering the signal
1485     # to the main thread in preference to any subthread (all that's
1486     # strictly guaranteed is that *some* thread that has the signal
1487     # unblocked will receive it).  But there doesn't seem to be
1488     # any interface to create a thread with SIGINT blocked.
1489     if t.is_alive():
1490         raise Exception("timed out")
1491     elif t.exception:
1492         raise t.exception
1493     else:
1494         return t.retval
1495
1496
1497 def send_command_retry(*args: Any, **kwargs: Any) -> Tuple[int, bytes, str]:
1498     while True:
1499         try:
1500             return send_command(*args, **kwargs)
1501         except Exception as e:
1502             # If our librados instance has not reached state 'connected'
1503             # yet, we'll see an exception like this and retry
1504             if ('get_command_descriptions' in str(e) and
1505                 'object in state configuring' in str(e)):
1506                 continue
1507             else:
1508                 raise
1509
1510
1511 def send_command(cluster,
1512                  target: Optional[Tuple[str, str]] = ('mon', ''),
1513                  cmd: Optional[List[str]] = None,
1514                  inbuf: Optional[bytes] = b'',
1515                  timeout: Optional[int] = 0,
1516                  verbose: Optional[bool] = False) -> Tuple[int, bytes, str]:
1517     """
1518     Send a command to a daemon using librados's
1519     mon_command, osd_command, mgr_command, or pg_command.  Any bulk input data
1520     comes in inbuf.
1521
1522     Returns (ret, outbuf, outs); ret is the return code, outbuf is
1523     the outbl "bulk useful output" buffer, and outs is any status
1524     or error message (intended for stderr).
1525
1526     If target is osd.N, send command to that osd (except for pgid cmds)
1527     """
1528     cmd = cmd or []
1529     try:
1530         if target[0] == 'osd':
1531             osdid = target[1]
1532
1533             if verbose:
1534                 print('submit {0} to osd.{1}'.format(cmd, osdid),
1535                       file=sys.stderr)
1536             ret, outbuf, outs = run_in_thread(
1537                 cluster.osd_command, osdid, cmd, inbuf, timeout=timeout)
1538
1539         elif target[0] == 'mgr':
1540             name = ''     # non-None empty string means "current active mgr"
1541             if len(target) > 1 and target[1] is not None:
1542                 name = target[1]
1543             if verbose:
1544                 print('submit {0} to {1} name {2}'.format(cmd, target[0], name),
1545                       file=sys.stderr)
1546             ret, outbuf, outs = run_in_thread(
1547                 cluster.mgr_command, cmd, inbuf, timeout=timeout, target=name)
1548
1549         elif target[0] == 'mon-mgr':
1550             if verbose:
1551                 print('submit {0} to {1}'.format(cmd, target[0]),
1552                       file=sys.stderr)
1553             ret, outbuf, outs = run_in_thread(
1554                 cluster.mgr_command, cmd, inbuf, timeout=timeout)
1555
1556         elif target[0] == 'pg':
1557             pgid = target[1]
1558             # pgid will already be in the command for the pg <pgid>
1559             # form, but for tell <pgid>, we need to put it in
1560             if cmd:
1561                 cmddict = json.loads(cmd)
1562                 cmddict['pgid'] = pgid
1563             else:
1564                 cmddict = dict(pgid=pgid)
1565             cmd = json.dumps(cmddict)
1566             if verbose:
1567                 print('submit {0} for pgid {1}'.format(cmd, pgid),
1568                       file=sys.stderr)
1569             ret, outbuf, outs = run_in_thread(
1570                 cluster.pg_command, pgid, cmd, inbuf, timeout=timeout)
1571
1572         elif target[0] == 'mon':
1573             if verbose:
1574                 print('{0} to {1}'.format(cmd, target[0]),
1575                       file=sys.stderr)
1576             if len(target) < 2 or target[1] == '':
1577                 ret, outbuf, outs = run_in_thread(
1578                     cluster.mon_command, cmd, inbuf, timeout=timeout)
1579             else:
1580                 ret, outbuf, outs = run_in_thread(
1581                     cluster.mon_command, cmd, inbuf, timeout=timeout, target=target[1])
1582         elif target[0] == 'mds':
1583             mds_spec = target[1]
1584
1585             if verbose:
1586                 print('submit {0} to mds.{1}'.format(cmd, mds_spec),
1587                       file=sys.stderr)
1588
1589             try:
1590                 from cephfs import LibCephFS
1591             except ImportError:
1592                 raise RuntimeError("CephFS unavailable, have you installed libcephfs?")
1593
1594             filesystem = LibCephFS(rados_inst=cluster)
1595             filesystem.init()
1596             ret, outbuf, outs = \
1597                 filesystem.mds_command(mds_spec, cmd, inbuf)
1598             filesystem.shutdown()
1599         else:
1600             raise ArgumentValid("Bad target type '{0}'".format(target[0]))
1601
1602     except Exception as e:
1603         if not isinstance(e, ArgumentError):
1604             raise RuntimeError('"{0}": exception {1}'.format(cmd, e))
1605         else:
1606             raise
1607
1608     return ret, outbuf, outs
1609
1610
1611 def json_command(cluster,
1612                  target: Optional[Tuple[str, str]] = ('mon', ''),
1613                  prefix: Optional[str] = None,
1614                  argdict: Optional[Dict[str, str]] = None,
1615                  inbuf: Optional[bytes] = b'',
1616                  timeout: Optional[int] = 0,
1617                  verbose: Optional[bool] = False) -> Tuple[int, bytes, str]:
1618     """
1619     Serialize a command and up a JSON command and send it with send_command() above.
1620     Prefix may be supplied separately or in argdict.  Any bulk input
1621     data comes in inbuf.
1622
1623     If target is osd.N, send command to that osd (except for pgid cmds)
1624
1625     :param cluster: ``rados.Rados`` instance
1626     :param prefix: String to inject into command arguments as 'prefix'
1627     :param argdict: Command arguments
1628     """
1629     cmddict = {}
1630     if prefix:
1631         cmddict.update({'prefix': prefix})
1632
1633     if argdict:
1634         cmddict.update(argdict)
1635         if 'target' in argdict:
1636             target = argdict.get('target')
1637
1638     try:
1639         if target[0] == 'osd':
1640             osdtarg = CephName()
1641             osdtarget = '{0}.{1}'.format(*target)
1642             # prefer target from cmddict if present and valid
1643             if 'target' in cmddict:
1644                 osdtarget = cmddict.pop('target')
1645             try:
1646                 osdtarg.valid(osdtarget)
1647                 target = ('osd', osdtarg.nameid)
1648             except:
1649                 # use the target we were originally given
1650                 pass
1651         ret, outbuf, outs = send_command_retry(cluster,
1652                                                target, json.dumps(cmddict),
1653                                                inbuf, timeout, verbose)
1654
1655     except Exception as e:
1656         if not isinstance(e, ArgumentError):
1657             raise RuntimeError('"{0}": exception {1}'.format(argdict, e))
1658         else:
1659             raise
1660
1661     return ret, outbuf, outs