ceph/src/pybind/mgr/mgr_util.py

   1 import contextlib
   2 import datetime
   3 import os
   4 import socket
   5 import logging
   6
   7 try:
   8     from typing import Tuple
   9 except ImportError:
  10     TYPE_CHECKING = False  # just for type checking
  11
  12 (
  13     BLACK,
  14     RED,
  15     GREEN,
  16     YELLOW,
  17     BLUE,
  18     MAGENTA,
  19     CYAN,
  20     GRAY
  21 ) = range(8)
  22
  23 RESET_SEQ = "\033[0m"
  24 COLOR_SEQ = "\033[1;%dm"
  25 COLOR_DARK_SEQ = "\033[0;%dm"
  26 BOLD_SEQ = "\033[1m"
  27 UNDERLINE_SEQ = "\033[4m"
  28
  29 logger = logging.getLogger(__name__)
  30
  31
  32 def colorize(msg, color, dark=False):
  33     """
  34     Decorate `msg` with escape sequences to give the requested color
  35     """
  36     return (COLOR_DARK_SEQ if dark else COLOR_SEQ) % (30 + color) \
  37         + msg + RESET_SEQ
  38
  39
  40 def bold(msg):
  41     """
  42     Decorate `msg` with escape sequences to make it appear bold
  43     """
  44     return BOLD_SEQ + msg + RESET_SEQ
  45
  46
  47 def format_units(n, width, colored, decimal):
  48     """
  49     Format a number without units, so as to fit into `width` characters, substituting
  50     an appropriate unit suffix.
  51
  52     Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
  53     """
  54
  55     factor = 1000 if decimal else 1024
  56     units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
  57     unit = 0
  58     while len("%s" % (int(n) // (factor**unit))) > width - 1:
  59         unit += 1
  60
  61     if unit > 0:
  62         truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
  63         if truncated_float[-1] == '.':
  64             truncated_float = " " + truncated_float[0:-1]
  65     else:
  66         truncated_float = "%{wid}d".format(wid=width - 1) % n
  67     formatted = "%s%s" % (truncated_float, units[unit])
  68
  69     if colored:
  70         if n == 0:
  71             color = BLACK, False
  72         else:
  73             color = YELLOW, False
  74         return bold(colorize(formatted[0:-1], color[0], color[1])) \
  75             + bold(colorize(formatted[-1], BLACK, False))
  76     else:
  77         return formatted
  78
  79
  80 def format_dimless(n, width, colored=False):
  81     return format_units(n, width, colored, decimal=True)
  82
  83
  84 def format_bytes(n, width, colored=False):
  85     return format_units(n, width, colored, decimal=False)
  86
  87
  88 def merge_dicts(*args):
  89     # type: (dict) -> dict
  90     """
  91     >>> merge_dicts({1:2}, {3:4})
  92     {1: 2, 3: 4}
  93
  94     You can also overwrite keys:
  95     >>> merge_dicts({1:2}, {1:4})
  96     {1: 4}
  97
  98     :rtype: dict[str, Any]
  99     """
 100     ret = {}
 101     for arg in args:
 102         ret.update(arg)
 103     return ret
 104
 105
 106 def get_default_addr():
 107     # type: () -> str
 108     def is_ipv6_enabled():
 109         try:
 110             sock = socket.socket(socket.AF_INET6)
 111             with contextlib.closing(sock):
 112                 sock.bind(("::1", 0))
 113                 return True
 114         except (AttributeError, socket.error) as e:
 115            return False
 116
 117     try:
 118         return get_default_addr.result  # type: ignore
 119     except AttributeError:
 120         result = '::' if is_ipv6_enabled() else '0.0.0.0'
 121         get_default_addr.result = result  # type: ignore
 122         return result
 123
 124
 125 class ServerConfigException(Exception):
 126     pass
 127
 128
 129 def create_self_signed_cert(organisation='Ceph', common_name='mgr') -> Tuple[str, str]:
 130     """Returns self-signed PEM certificates valid for 10 years.
 131     :return cert, pkey
 132     """
 133
 134     from OpenSSL import crypto
 135     from uuid import uuid4
 136
 137     # create a key pair
 138     pkey = crypto.PKey()
 139     pkey.generate_key(crypto.TYPE_RSA, 2048)
 140
 141     # create a self-signed cert
 142     cert = crypto.X509()
 143     cert.get_subject().O = organisation
 144     cert.get_subject().CN = common_name
 145     cert.set_serial_number(int(uuid4()))
 146     cert.gmtime_adj_notBefore(0)
 147     cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)  # 10 years
 148     cert.set_issuer(cert.get_subject())
 149     cert.set_pubkey(pkey)
 150     cert.sign(pkey, 'sha512')
 151
 152     cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
 153     pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
 154
 155     return cert.decode('utf-8'), pkey.decode('utf-8')
 156
 157
 158 def verify_cacrt_content(crt):
 159     # type: (str) -> None
 160     from OpenSSL import crypto
 161     try:
 162         x509 = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
 163         if x509.has_expired():
 164             logger.warning('Certificate has expired: {}'.format(crt))
 165     except (ValueError, crypto.Error) as e:
 166         raise ServerConfigException(
 167             'Invalid certificate: {}'.format(str(e)))
 168
 169
 170 def verify_cacrt(cert_fname):
 171     # type: (str) -> None
 172     """Basic validation of a ca cert"""
 173
 174     if not cert_fname:
 175         raise ServerConfigException("CA cert not configured")
 176     if not os.path.isfile(cert_fname):
 177         raise ServerConfigException("Certificate {} does not exist".format(cert_fname))
 178
 179     try:
 180         with open(cert_fname) as f:
 181             verify_cacrt_content(f.read())
 182     except ValueError as e:
 183         raise ServerConfigException(
 184             'Invalid certificate {}: {}'.format(cert_fname, str(e)))
 185
 186
 187 def verify_tls(crt, key):
 188     # type: (str, str) -> None
 189     verify_cacrt_content(crt)
 190
 191     from OpenSSL import crypto, SSL
 192     try:
 193         _key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
 194         _key.check()
 195     except (ValueError, crypto.Error) as e:
 196         raise ServerConfigException(
 197             'Invalid private key: {}'.format(str(e)))
 198     try:
 199         _crt = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
 200     except ValueError as e:
 201         raise ServerConfigException(
 202             'Invalid certificate key: {}'.format(str(e))
 203         )
 204
 205     try:
 206         context = SSL.Context(SSL.TLSv1_METHOD)
 207         context.use_certificate(_crt)
 208         context.use_privatekey(_key)
 209         context.check_privatekey()
 210     except crypto.Error as e:
 211         logger.warning(
 212             'Private key and certificate do not match up: {}'.format(str(e)))
 213
 214
 215 def verify_tls_files(cert_fname, pkey_fname):
 216     # type: (str, str) -> None
 217     """Basic checks for TLS certificate and key files
 218
 219     Do some validations to the private key and certificate:
 220     - Check the type and format
 221     - Check the certificate expiration date
 222     - Check the consistency of the private key
 223     - Check that the private key and certificate match up
 224
 225     :param cert_fname: Name of the certificate file
 226     :param pkey_fname: name of the certificate public key file
 227
 228     :raises ServerConfigException: An error with a message
 229
 230     """
 231
 232     if not cert_fname or not pkey_fname:
 233         raise ServerConfigException('no certificate configured')
 234
 235     verify_cacrt(cert_fname)
 236
 237     if not os.path.isfile(pkey_fname):
 238         raise ServerConfigException('private key %s does not exist' % pkey_fname)
 239
 240     from OpenSSL import crypto, SSL
 241
 242     try:
 243         with open(pkey_fname) as f:
 244             pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
 245             pkey.check()
 246     except (ValueError, crypto.Error) as e:
 247         raise ServerConfigException(
 248             'Invalid private key {}: {}'.format(pkey_fname, str(e)))
 249     try:
 250         context = SSL.Context(SSL.TLSv1_METHOD)
 251         context.use_certificate_file(cert_fname, crypto.FILETYPE_PEM)
 252         context.use_privatekey_file(pkey_fname, crypto.FILETYPE_PEM)
 253         context.check_privatekey()
 254     except crypto.Error as e:
 255         logger.warning(
 256             'Private key {} and certificate {} do not match up: {}'.format(
 257                 pkey_fname, cert_fname, str(e)))
 258
 259 def get_most_recent_rate(rates):
 260     """ Get most recent rate from rates
 261
 262     :param rates: The derivative between all time series data points [time in seconds, value]
 263     :type rates: list[tuple[int, float]]
 264
 265     :return: The last derivative or 0.0 if none exists
 266     :rtype: float
 267
 268     >>> get_most_recent_rate(None)
 269     0.0
 270     >>> get_most_recent_rate([])
 271     0.0
 272     >>> get_most_recent_rate([(1, -2.0)])
 273     -2.0
 274     >>> get_most_recent_rate([(1, 2.0), (2, 1.5), (3, 5.0)])
 275     5.0
 276     """
 277     if not rates:
 278         return 0.0
 279     return rates[-1][1]
 280
 281 def get_time_series_rates(data):
 282     """ Rates from time series data
 283
 284     :param data: Time series data [time in seconds, value]
 285     :type data: list[tuple[int, float]]
 286
 287     :return: The derivative between all time series data points [time in seconds, value]
 288     :rtype: list[tuple[int, float]]
 289
 290     >>> logger.debug = lambda s,x,y: print(s % (x,y))
 291     >>> get_time_series_rates([])
 292     []
 293     >>> get_time_series_rates([[0, 1], [1, 3]])
 294     [(1, 2.0)]
 295     >>> get_time_series_rates([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
 296     Duplicate timestamp in time series data: [0, 2], [0, 3]
 297     Duplicate timestamp in time series data: [0, 3], [0, 1]
 298     Duplicate timestamp in time series data: [1, 2], [1, 3]
 299     [(1, 2.0)]
 300     >>> get_time_series_rates([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
 301     [(2, 2.0), (4, 4.0), (5, 5.0), (6, 6.0)]
 302     """
 303     data = _filter_time_series(data)
 304     if not data:
 305         return []
 306     return [(data2[0], _derivative(data1, data2)) for data1, data2 in
 307             _pairwise(data)]
 308
 309 def _filter_time_series(data):
 310     """ Filters time series data
 311
 312     Filters out samples with the same timestamp in given time series data.
 313     It also enforces the list to contain at least two samples.
 314
 315     All filtered values will be shown in the debug log. If values were filtered it's a bug in the
 316     time series data collector, please report it.
 317
 318     :param data: Time series data [time in seconds, value]
 319     :type data: list[tuple[int, float]]
 320
 321     :return: Filtered time series data [time in seconds, value]
 322     :rtype: list[tuple[int, float]]
 323
 324     >>> logger.debug = lambda s,x,y: print(s % (x,y))
 325     >>> _filter_time_series([])
 326     []
 327     >>> _filter_time_series([[1, 42]])
 328     []
 329     >>> _filter_time_series([[10, 2], [10, 3]])
 330     Duplicate timestamp in time series data: [10, 2], [10, 3]
 331     []
 332     >>> _filter_time_series([[0, 1], [1, 2]])
 333     [[0, 1], [1, 2]]
 334     >>> _filter_time_series([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
 335     Duplicate timestamp in time series data: [0, 2], [0, 3]
 336     Duplicate timestamp in time series data: [0, 3], [0, 1]
 337     Duplicate timestamp in time series data: [1, 2], [1, 3]
 338     [[0, 1], [1, 3]]
 339     >>> _filter_time_series([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
 340     [[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]]
 341     """
 342     filtered = []
 343     for i in range(len(data) - 1):
 344         if data[i][0] == data[i + 1][0]:  # Same timestamp
 345             logger.debug("Duplicate timestamp in time series data: %s, %s", data[i], data[i + 1])
 346             continue
 347         filtered.append(data[i])
 348     if not filtered:
 349         return []
 350     filtered.append(data[-1])
 351     return filtered
 352
 353 def _derivative(p1, p2):
 354     """ Derivative between two time series data points
 355
 356     :param p1: Time series data [time in seconds, value]
 357     :type p1: tuple[int, float]
 358     :param p2: Time series data [time in seconds, value]
 359     :type p2: tuple[int, float]
 360
 361     :return: Derivative between both points
 362     :rtype: float
 363
 364     >>> _derivative([0, 0], [2, 1])
 365     0.5
 366     >>> _derivative([0, 1], [2, 0])
 367     -0.5
 368     >>> _derivative([0, 0], [3, 1])
 369     0.3333333333333333
 370     """
 371     return (p2[1] - p1[1]) / float(p2[0] - p1[0])
 372
 373 def _pairwise(iterable):
 374     it = iter(iterable)
 375     a = next(it, None)
 376
 377     for b in it:
 378         yield (a, b)
 379         a = b
 380
 381 def to_pretty_timedelta(n):
 382     if n < datetime.timedelta(seconds=120):
 383         return str(n.seconds) + 's'
 384     if n < datetime.timedelta(minutes=120):
 385         return str(n.seconds // 60) + 'm'
 386     if n < datetime.timedelta(hours=48):
 387         return str(n.seconds // 3600) + 'h'
 388     if n < datetime.timedelta(days=14):
 389         return str(n.days) + 'd'
 390     if n < datetime.timedelta(days=7*12):
 391         return str(n.days // 7) + 'w'
 392     if n < datetime.timedelta(days=365*2):
 393         return str(n.days // 30) + 'M'
 394     return str(n.days // 365) + 'y'