ceph/src/pybind/mgr/mgr_util.py

   1 import contextlib
   2 import datetime
   3 import os
   4 import socket
   5 import logging
   6 import time
   7 from functools import wraps
   8
   9 try:
  10     from typing import Tuple, Any, Callable
  11 except ImportError:
  12     TYPE_CHECKING = False  # just for type checking
  13
  14 (
  15     BLACK,
  16     RED,
  17     GREEN,
  18     YELLOW,
  19     BLUE,
  20     MAGENTA,
  21     CYAN,
  22     GRAY
  23 ) = range(8)
  24
  25 RESET_SEQ = "\033[0m"
  26 COLOR_SEQ = "\033[1;%dm"
  27 COLOR_DARK_SEQ = "\033[0;%dm"
  28 BOLD_SEQ = "\033[1m"
  29 UNDERLINE_SEQ = "\033[4m"
  30
  31 logger = logging.getLogger(__name__)
  32
  33
  34 def colorize(msg, color, dark=False):
  35     """
  36     Decorate `msg` with escape sequences to give the requested color
  37     """
  38     return (COLOR_DARK_SEQ if dark else COLOR_SEQ) % (30 + color) \
  39         + msg + RESET_SEQ
  40
  41
  42 def bold(msg):
  43     """
  44     Decorate `msg` with escape sequences to make it appear bold
  45     """
  46     return BOLD_SEQ + msg + RESET_SEQ
  47
  48
  49 def format_units(n, width, colored, decimal):
  50     """
  51     Format a number without units, so as to fit into `width` characters, substituting
  52     an appropriate unit suffix.
  53
  54     Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
  55     """
  56
  57     factor = 1000 if decimal else 1024
  58     units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
  59     unit = 0
  60     while len("%s" % (int(n) // (factor**unit))) > width - 1:
  61         unit += 1
  62
  63     if unit > 0:
  64         truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
  65         if truncated_float[-1] == '.':
  66             truncated_float = " " + truncated_float[0:-1]
  67     else:
  68         truncated_float = "%{wid}d".format(wid=width - 1) % n
  69     formatted = "%s%s" % (truncated_float, units[unit])
  70
  71     if colored:
  72         if n == 0:
  73             color = BLACK, False
  74         else:
  75             color = YELLOW, False
  76         return bold(colorize(formatted[0:-1], color[0], color[1])) \
  77             + bold(colorize(formatted[-1], BLACK, False))
  78     else:
  79         return formatted
  80
  81
  82 def format_dimless(n, width, colored=False):
  83     return format_units(n, width, colored, decimal=True)
  84
  85
  86 def format_bytes(n, width, colored=False):
  87     return format_units(n, width, colored, decimal=False)
  88
  89
  90 def merge_dicts(*args):
  91     # type: (dict) -> dict
  92     """
  93     >>> merge_dicts({1:2}, {3:4})
  94     {1: 2, 3: 4}
  95
  96     You can also overwrite keys:
  97     >>> merge_dicts({1:2}, {1:4})
  98     {1: 4}
  99
 100     :rtype: dict[str, Any]
 101     """
 102     ret = {}
 103     for arg in args:
 104         ret.update(arg)
 105     return ret
 106
 107
 108 def get_default_addr():
 109     # type: () -> str
 110     def is_ipv6_enabled():
 111         try:
 112             sock = socket.socket(socket.AF_INET6)
 113             with contextlib.closing(sock):
 114                 sock.bind(("::1", 0))
 115                 return True
 116         except (AttributeError, socket.error) as e:
 117            return False
 118
 119     try:
 120         return get_default_addr.result  # type: ignore
 121     except AttributeError:
 122         result = '::' if is_ipv6_enabled() else '0.0.0.0'
 123         get_default_addr.result = result  # type: ignore
 124         return result
 125
 126
 127 class ServerConfigException(Exception):
 128     pass
 129
 130
 131 def create_self_signed_cert(organisation='Ceph', common_name='mgr') -> Tuple[str, str]:
 132     """Returns self-signed PEM certificates valid for 10 years.
 133     :return cert, pkey
 134     """
 135
 136     from OpenSSL import crypto
 137     from uuid import uuid4
 138
 139     # create a key pair
 140     pkey = crypto.PKey()
 141     pkey.generate_key(crypto.TYPE_RSA, 2048)
 142
 143     # create a self-signed cert
 144     cert = crypto.X509()
 145     cert.get_subject().O = organisation
 146     cert.get_subject().CN = common_name
 147     cert.set_serial_number(int(uuid4()))
 148     cert.gmtime_adj_notBefore(0)
 149     cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)  # 10 years
 150     cert.set_issuer(cert.get_subject())
 151     cert.set_pubkey(pkey)
 152     cert.sign(pkey, 'sha512')
 153
 154     cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
 155     pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
 156
 157     return cert.decode('utf-8'), pkey.decode('utf-8')
 158
 159
 160 def verify_cacrt_content(crt):
 161     # type: (str) -> None
 162     from OpenSSL import crypto
 163     try:
 164         x509 = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
 165         if x509.has_expired():
 166             logger.warning('Certificate has expired: {}'.format(crt))
 167     except (ValueError, crypto.Error) as e:
 168         raise ServerConfigException(
 169             'Invalid certificate: {}'.format(str(e)))
 170
 171
 172 def verify_cacrt(cert_fname):
 173     # type: (str) -> None
 174     """Basic validation of a ca cert"""
 175
 176     if not cert_fname:
 177         raise ServerConfigException("CA cert not configured")
 178     if not os.path.isfile(cert_fname):
 179         raise ServerConfigException("Certificate {} does not exist".format(cert_fname))
 180
 181     try:
 182         with open(cert_fname) as f:
 183             verify_cacrt_content(f.read())
 184     except ValueError as e:
 185         raise ServerConfigException(
 186             'Invalid certificate {}: {}'.format(cert_fname, str(e)))
 187
 188
 189 def verify_tls(crt, key):
 190     # type: (str, str) -> None
 191     verify_cacrt_content(crt)
 192
 193     from OpenSSL import crypto, SSL
 194     try:
 195         _key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
 196         _key.check()
 197     except (ValueError, crypto.Error) as e:
 198         raise ServerConfigException(
 199             'Invalid private key: {}'.format(str(e)))
 200     try:
 201         _crt = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
 202     except ValueError as e:
 203         raise ServerConfigException(
 204             'Invalid certificate key: {}'.format(str(e))
 205         )
 206
 207     try:
 208         context = SSL.Context(SSL.TLSv1_METHOD)
 209         context.use_certificate(_crt)
 210         context.use_privatekey(_key)
 211         context.check_privatekey()
 212     except crypto.Error as e:
 213         logger.warning(
 214             'Private key and certificate do not match up: {}'.format(str(e)))
 215
 216
 217 def verify_tls_files(cert_fname, pkey_fname):
 218     # type: (str, str) -> None
 219     """Basic checks for TLS certificate and key files
 220
 221     Do some validations to the private key and certificate:
 222     - Check the type and format
 223     - Check the certificate expiration date
 224     - Check the consistency of the private key
 225     - Check that the private key and certificate match up
 226
 227     :param cert_fname: Name of the certificate file
 228     :param pkey_fname: name of the certificate public key file
 229
 230     :raises ServerConfigException: An error with a message
 231
 232     """
 233
 234     if not cert_fname or not pkey_fname:
 235         raise ServerConfigException('no certificate configured')
 236
 237     verify_cacrt(cert_fname)
 238
 239     if not os.path.isfile(pkey_fname):
 240         raise ServerConfigException('private key %s does not exist' % pkey_fname)
 241
 242     from OpenSSL import crypto, SSL
 243
 244     try:
 245         with open(pkey_fname) as f:
 246             pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
 247             pkey.check()
 248     except (ValueError, crypto.Error) as e:
 249         raise ServerConfigException(
 250             'Invalid private key {}: {}'.format(pkey_fname, str(e)))
 251     try:
 252         context = SSL.Context(SSL.TLSv1_METHOD)
 253         context.use_certificate_file(cert_fname, crypto.FILETYPE_PEM)
 254         context.use_privatekey_file(pkey_fname, crypto.FILETYPE_PEM)
 255         context.check_privatekey()
 256     except crypto.Error as e:
 257         logger.warning(
 258             'Private key {} and certificate {} do not match up: {}'.format(
 259                 pkey_fname, cert_fname, str(e)))
 260
 261 def get_most_recent_rate(rates):
 262     """ Get most recent rate from rates
 263
 264     :param rates: The derivative between all time series data points [time in seconds, value]
 265     :type rates: list[tuple[int, float]]
 266
 267     :return: The last derivative or 0.0 if none exists
 268     :rtype: float
 269
 270     >>> get_most_recent_rate(None)
 271     0.0
 272     >>> get_most_recent_rate([])
 273     0.0
 274     >>> get_most_recent_rate([(1, -2.0)])
 275     -2.0
 276     >>> get_most_recent_rate([(1, 2.0), (2, 1.5), (3, 5.0)])
 277     5.0
 278     """
 279     if not rates:
 280         return 0.0
 281     return rates[-1][1]
 282
 283 def get_time_series_rates(data):
 284     """ Rates from time series data
 285
 286     :param data: Time series data [time in seconds, value]
 287     :type data: list[tuple[int, float]]
 288
 289     :return: The derivative between all time series data points [time in seconds, value]
 290     :rtype: list[tuple[int, float]]
 291
 292     >>> logger.debug = lambda s,x,y: print(s % (x,y))
 293     >>> get_time_series_rates([])
 294     []
 295     >>> get_time_series_rates([[0, 1], [1, 3]])
 296     [(1, 2.0)]
 297     >>> get_time_series_rates([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
 298     Duplicate timestamp in time series data: [0, 2], [0, 3]
 299     Duplicate timestamp in time series data: [0, 3], [0, 1]
 300     Duplicate timestamp in time series data: [1, 2], [1, 3]
 301     [(1, 2.0)]
 302     >>> get_time_series_rates([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
 303     [(2, 2.0), (4, 4.0), (5, 5.0), (6, 6.0)]
 304     """
 305     data = _filter_time_series(data)
 306     if not data:
 307         return []
 308     return [(data2[0], _derivative(data1, data2)) for data1, data2 in
 309             _pairwise(data)]
 310
 311 def _filter_time_series(data):
 312     """ Filters time series data
 313
 314     Filters out samples with the same timestamp in given time series data.
 315     It also enforces the list to contain at least two samples.
 316
 317     All filtered values will be shown in the debug log. If values were filtered it's a bug in the
 318     time series data collector, please report it.
 319
 320     :param data: Time series data [time in seconds, value]
 321     :type data: list[tuple[int, float]]
 322
 323     :return: Filtered time series data [time in seconds, value]
 324     :rtype: list[tuple[int, float]]
 325
 326     >>> logger.debug = lambda s,x,y: print(s % (x,y))
 327     >>> _filter_time_series([])
 328     []
 329     >>> _filter_time_series([[1, 42]])
 330     []
 331     >>> _filter_time_series([[10, 2], [10, 3]])
 332     Duplicate timestamp in time series data: [10, 2], [10, 3]
 333     []
 334     >>> _filter_time_series([[0, 1], [1, 2]])
 335     [[0, 1], [1, 2]]
 336     >>> _filter_time_series([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
 337     Duplicate timestamp in time series data: [0, 2], [0, 3]
 338     Duplicate timestamp in time series data: [0, 3], [0, 1]
 339     Duplicate timestamp in time series data: [1, 2], [1, 3]
 340     [[0, 1], [1, 3]]
 341     >>> _filter_time_series([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
 342     [[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]]
 343     """
 344     filtered = []
 345     for i in range(len(data) - 1):
 346         if data[i][0] == data[i + 1][0]:  # Same timestamp
 347             logger.debug("Duplicate timestamp in time series data: %s, %s", data[i], data[i + 1])
 348             continue
 349         filtered.append(data[i])
 350     if not filtered:
 351         return []
 352     filtered.append(data[-1])
 353     return filtered
 354
 355 def _derivative(p1, p2):
 356     """ Derivative between two time series data points
 357
 358     :param p1: Time series data [time in seconds, value]
 359     :type p1: tuple[int, float]
 360     :param p2: Time series data [time in seconds, value]
 361     :type p2: tuple[int, float]
 362
 363     :return: Derivative between both points
 364     :rtype: float
 365
 366     >>> _derivative([0, 0], [2, 1])
 367     0.5
 368     >>> _derivative([0, 1], [2, 0])
 369     -0.5
 370     >>> _derivative([0, 0], [3, 1])
 371     0.3333333333333333
 372     """
 373     return (p2[1] - p1[1]) / float(p2[0] - p1[0])
 374
 375 def _pairwise(iterable):
 376     it = iter(iterable)
 377     a = next(it, None)
 378
 379     for b in it:
 380         yield (a, b)
 381         a = b
 382
 383 def to_pretty_timedelta(n):
 384     if n < datetime.timedelta(seconds=120):
 385         return str(n.seconds) + 's'
 386     if n < datetime.timedelta(minutes=120):
 387         return str(n.seconds // 60) + 'm'
 388     if n < datetime.timedelta(hours=48):
 389         return str(n.seconds // 3600) + 'h'
 390     if n < datetime.timedelta(days=14):
 391         return str(n.days) + 'd'
 392     if n < datetime.timedelta(days=7*12):
 393         return str(n.days // 7) + 'w'
 394     if n < datetime.timedelta(days=365*2):
 395         return str(n.days // 30) + 'M'
 396     return str(n.days // 365) + 'y'
 397
 398
 399 def profile_method(skip_attribute=False):
 400     """
 401     Decorator for methods of the Module class. Logs the name of the given
 402     function f with the time it takes to execute it.
 403     """
 404     def outer(f):
 405         @wraps(f)
 406         def wrapper(*args, **kwargs):
 407             self = args[0]
 408             t = time.time()
 409             self.log.debug('Starting method {}.'.format(f.__name__))
 410             result = f(*args, **kwargs)
 411             duration = time.time() - t
 412             if not skip_attribute:
 413                 wrapper._execution_duration = duration  # type: ignore
 414             self.log.debug('Method {} ran {:.3f} seconds.'.format(f.__name__, duration))
 415             return result
 416         return wrapper
 417     return outer