[ceph.git] / ceph / src / pybind / mgr / mgr_util.py

import contextlib
import datetime
import os
import socket
import logging
import time
from functools import wraps

try:
    from typing import Tuple, Any, Callable
except ImportError:
    TYPE_CHECKING = False  # just for type checking

(
    BLACK,
    RED,
    GREEN,
    YELLOW,
    BLUE,
    MAGENTA,
    CYAN,
    GRAY
) = range(8)

RESET_SEQ = "\033[0m"
COLOR_SEQ = "\033[1;%dm"
COLOR_DARK_SEQ = "\033[0;%dm"
BOLD_SEQ = "\033[1m"
UNDERLINE_SEQ = "\033[4m"

logger = logging.getLogger(__name__)


def colorize(msg, color, dark=False):
    """
    Decorate `msg` with escape sequences to give the requested color
    """
    return (COLOR_DARK_SEQ if dark else COLOR_SEQ) % (30 + color) \
        + msg + RESET_SEQ


def bold(msg):
    """
    Decorate `msg` with escape sequences to make it appear bold
    """
    return BOLD_SEQ + msg + RESET_SEQ


def format_units(n, width, colored, decimal):
    """
    Format a number without units, so as to fit into `width` characters, substituting
    an appropriate unit suffix.

    Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
    """

    factor = 1000 if decimal else 1024
    units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
    unit = 0
    while len("%s" % (int(n) // (factor**unit))) > width - 1:
        unit += 1

    if unit > 0:
        truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
        if truncated_float[-1] == '.':
            truncated_float = " " + truncated_float[0:-1]
    else:
        truncated_float = "%{wid}d".format(wid=width - 1) % n
    formatted = "%s%s" % (truncated_float, units[unit])

    if colored:
        if n == 0:
            color = BLACK, False
        else:
            color = YELLOW, False
        return bold(colorize(formatted[0:-1], color[0], color[1])) \
            + bold(colorize(formatted[-1], BLACK, False))
    else:
        return formatted


def format_dimless(n, width, colored=False):
    return format_units(n, width, colored, decimal=True)


def format_bytes(n, width, colored=False):
    return format_units(n, width, colored, decimal=False)


def merge_dicts(*args):
    # type: (dict) -> dict
    """
    >>> merge_dicts({1:2}, {3:4})
    {1: 2, 3: 4}

    You can also overwrite keys:
    >>> merge_dicts({1:2}, {1:4})
    {1: 4}

    :rtype: dict[str, Any]
    """
    ret = {}
    for arg in args:
        ret.update(arg)
    return ret


def get_default_addr():
    # type: () -> str
    def is_ipv6_enabled():
        try:
            sock = socket.socket(socket.AF_INET6)
            with contextlib.closing(sock):
                sock.bind(("::1", 0))
                return True
        except (AttributeError, socket.error) as e:
           return False

    try:
        return get_default_addr.result  # type: ignore
    except AttributeError:
        result = '::' if is_ipv6_enabled() else '0.0.0.0'
        get_default_addr.result = result  # type: ignore
        return result


class ServerConfigException(Exception):
    pass


def create_self_signed_cert(organisation='Ceph', common_name='mgr') -> Tuple[str, str]:
    """Returns self-signed PEM certificates valid for 10 years.
    :return cert, pkey
    """

    from OpenSSL import crypto
    from uuid import uuid4

    # create a key pair
    pkey = crypto.PKey()
    pkey.generate_key(crypto.TYPE_RSA, 2048)

    # create a self-signed cert
    cert = crypto.X509()
    cert.get_subject().O = organisation
    cert.get_subject().CN = common_name
    cert.set_serial_number(int(uuid4()))
    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)  # 10 years
    cert.set_issuer(cert.get_subject())
    cert.set_pubkey(pkey)
    cert.sign(pkey, 'sha512')

    cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
    pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)

    return cert.decode('utf-8'), pkey.decode('utf-8')


def verify_cacrt_content(crt):
    # type: (str) -> None
    from OpenSSL import crypto
    try:
        x509 = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
        if x509.has_expired():
            logger.warning('Certificate has expired: {}'.format(crt))
    except (ValueError, crypto.Error) as e:
        raise ServerConfigException(
            'Invalid certificate: {}'.format(str(e)))


def verify_cacrt(cert_fname):
    # type: (str) -> None
    """Basic validation of a ca cert"""

    if not cert_fname:
        raise ServerConfigException("CA cert not configured")
    if not os.path.isfile(cert_fname):
        raise ServerConfigException("Certificate {} does not exist".format(cert_fname))

    try:
        with open(cert_fname) as f:
            verify_cacrt_content(f.read())
    except ValueError as e:
        raise ServerConfigException(
            'Invalid certificate {}: {}'.format(cert_fname, str(e)))


def verify_tls(crt, key):
    # type: (str, str) -> None
    verify_cacrt_content(crt)

    from OpenSSL import crypto, SSL
    try:
        _key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
        _key.check()
    except (ValueError, crypto.Error) as e:
        raise ServerConfigException(
            'Invalid private key: {}'.format(str(e)))
    try:
        _crt = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
    except ValueError as e:
        raise ServerConfigException(
            'Invalid certificate key: {}'.format(str(e))
        )

    try:
        context = SSL.Context(SSL.TLSv1_METHOD)
        context.use_certificate(_crt)
        context.use_privatekey(_key)
        context.check_privatekey()
    except crypto.Error as e:
        logger.warning(
            'Private key and certificate do not match up: {}'.format(str(e)))


def verify_tls_files(cert_fname, pkey_fname):
    # type: (str, str) -> None
    """Basic checks for TLS certificate and key files

    Do some validations to the private key and certificate:
    - Check the type and format
    - Check the certificate expiration date
    - Check the consistency of the private key
    - Check that the private key and certificate match up

    :param cert_fname: Name of the certificate file
    :param pkey_fname: name of the certificate public key file

    :raises ServerConfigException: An error with a message

    """

    if not cert_fname or not pkey_fname:
        raise ServerConfigException('no certificate configured')

    verify_cacrt(cert_fname)

    if not os.path.isfile(pkey_fname):
        raise ServerConfigException('private key %s does not exist' % pkey_fname)

    from OpenSSL import crypto, SSL

    try:
        with open(pkey_fname) as f:
            pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
            pkey.check()
    except (ValueError, crypto.Error) as e:
        raise ServerConfigException(
            'Invalid private key {}: {}'.format(pkey_fname, str(e)))
    try:
        context = SSL.Context(SSL.TLSv1_METHOD)
        context.use_certificate_file(cert_fname, crypto.FILETYPE_PEM)
        context.use_privatekey_file(pkey_fname, crypto.FILETYPE_PEM)
        context.check_privatekey()
    except crypto.Error as e:
        logger.warning(
            'Private key {} and certificate {} do not match up: {}'.format(
                pkey_fname, cert_fname, str(e)))

def get_most_recent_rate(rates):
    """ Get most recent rate from rates

    :param rates: The derivative between all time series data points [time in seconds, value]
    :type rates: list[tuple[int, float]]

    :return: The last derivative or 0.0 if none exists
    :rtype: float

    >>> get_most_recent_rate(None)
    0.0
    >>> get_most_recent_rate([])
    0.0
    >>> get_most_recent_rate([(1, -2.0)])
    -2.0
    >>> get_most_recent_rate([(1, 2.0), (2, 1.5), (3, 5.0)])
    5.0
    """
    if not rates:
        return 0.0
    return rates[-1][1]

def get_time_series_rates(data):
    """ Rates from time series data

    :param data: Time series data [time in seconds, value]
    :type data: list[tuple[int, float]]

    :return: The derivative between all time series data points [time in seconds, value]
    :rtype: list[tuple[int, float]]

    >>> logger.debug = lambda s,x,y: print(s % (x,y))
    >>> get_time_series_rates([])
    []
    >>> get_time_series_rates([[0, 1], [1, 3]])
    [(1, 2.0)]
    >>> get_time_series_rates([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
    Duplicate timestamp in time series data: [0, 2], [0, 3]
    Duplicate timestamp in time series data: [0, 3], [0, 1]
    Duplicate timestamp in time series data: [1, 2], [1, 3]
    [(1, 2.0)]
    >>> get_time_series_rates([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
    [(2, 2.0), (4, 4.0), (5, 5.0), (6, 6.0)]
    """
    data = _filter_time_series(data)
    if not data:
        return []
    return [(data2[0], _derivative(data1, data2)) for data1, data2 in
            _pairwise(data)]

def _filter_time_series(data):
    """ Filters time series data

    Filters out samples with the same timestamp in given time series data.
    It also enforces the list to contain at least two samples.

    All filtered values will be shown in the debug log. If values were filtered it's a bug in the
    time series data collector, please report it.

    :param data: Time series data [time in seconds, value]
    :type data: list[tuple[int, float]]

    :return: Filtered time series data [time in seconds, value]
    :rtype: list[tuple[int, float]]

    >>> logger.debug = lambda s,x,y: print(s % (x,y))
    >>> _filter_time_series([])
    []
    >>> _filter_time_series([[1, 42]])
    []
    >>> _filter_time_series([[10, 2], [10, 3]])
    Duplicate timestamp in time series data: [10, 2], [10, 3]
    []
    >>> _filter_time_series([[0, 1], [1, 2]])
    [[0, 1], [1, 2]]
    >>> _filter_time_series([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
    Duplicate timestamp in time series data: [0, 2], [0, 3]
    Duplicate timestamp in time series data: [0, 3], [0, 1]
    Duplicate timestamp in time series data: [1, 2], [1, 3]
    [[0, 1], [1, 3]]
    >>> _filter_time_series([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
    [[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]]
    """
    filtered = []
    for i in range(len(data) - 1):
        if data[i][0] == data[i + 1][0]:  # Same timestamp
            logger.debug("Duplicate timestamp in time series data: %s, %s", data[i], data[i + 1])
            continue
        filtered.append(data[i])
    if not filtered:
        return []
    filtered.append(data[-1])
    return filtered

def _derivative(p1, p2):
    """ Derivative between two time series data points

    :param p1: Time series data [time in seconds, value]
    :type p1: tuple[int, float]
    :param p2: Time series data [time in seconds, value]
    :type p2: tuple[int, float]

    :return: Derivative between both points
    :rtype: float

    >>> _derivative([0, 0], [2, 1])
    0.5
    >>> _derivative([0, 1], [2, 0])
    -0.5
    >>> _derivative([0, 0], [3, 1])
    0.3333333333333333
    """
    return (p2[1] - p1[1]) / float(p2[0] - p1[0])

def _pairwise(iterable):
    it = iter(iterable)
    a = next(it, None)

    for b in it:
        yield (a, b)
        a = b

def to_pretty_timedelta(n):
    if n < datetime.timedelta(seconds=120):
        return str(n.seconds) + 's'
    if n < datetime.timedelta(minutes=120):
        return str(n.seconds // 60) + 'm'
    if n < datetime.timedelta(hours=48):
        return str(n.seconds // 3600) + 'h'
    if n < datetime.timedelta(days=14):
        return str(n.days) + 'd'
    if n < datetime.timedelta(days=7*12):
        return str(n.days // 7) + 'w'
    if n < datetime.timedelta(days=365*2):
        return str(n.days // 30) + 'M'
    return str(n.days // 365) + 'y'


def profile_method(skip_attribute=False):
    """
    Decorator for methods of the Module class. Logs the name of the given
    function f with the time it takes to execute it.
    """
    def outer(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            self = args[0]
            t = time.time()
            self.log.debug('Starting method {}.'.format(f.__name__))
            result = f(*args, **kwargs)
            duration = time.time() - t
            if not skip_attribute:
                wrapper._execution_duration = duration  # type: ignore
            self.log.debug('Method {} ran {:.3f} seconds.'.format(f.__name__, duration))
            return result
        return wrapper
    return outer
Commit	Line	Data
494da23a	1	import contextlib
9f95a23c	2	import datetime
eafe8130	3	import os
494da23a	4	import socket
eafe8130	5	import logging
f6b5b4d7 TL	6	import time
f6b5b4d7 TL	7	from functools import wraps
11fdf7f2	8
9f95a23c	9	try:
f6b5b4d7	10	from typing import Tuple, Any, Callable
9f95a23c TL	11	except ImportError:
	12	TYPE_CHECKING = False # just for type checking
	13
11fdf7f2 TL	14	(
	15	BLACK,
	16	RED,
	17	GREEN,
	18	YELLOW,
	19	BLUE,
	20	MAGENTA,
	21	CYAN,
	22	GRAY
	23	) = range(8)
	24
	25	RESET_SEQ = "\033[0m"
	26	COLOR_SEQ = "\033[1;%dm"
	27	COLOR_DARK_SEQ = "\033[0;%dm"
	28	BOLD_SEQ = "\033[1m"
	29	UNDERLINE_SEQ = "\033[4m"
	30
eafe8130 TL	31	logger = logging.getLogger(__name__)
eafe8130 TL	32
11fdf7f2 TL	33
	34	def colorize(msg, color, dark=False):
	35	"""
	36	Decorate `msg` with escape sequences to give the requested color
	37	"""
	38	return (COLOR_DARK_SEQ if dark else COLOR_SEQ) % (30 + color) \
	39	+ msg + RESET_SEQ
	40
	41
	42	def bold(msg):
	43	"""
	44	Decorate `msg` with escape sequences to make it appear bold
	45	"""
	46	return BOLD_SEQ + msg + RESET_SEQ
	47
	48
	49	def format_units(n, width, colored, decimal):
	50	"""
	51	Format a number without units, so as to fit into `width` characters, substituting
	52	an appropriate unit suffix.
	53
	54	Use decimal for dimensionless things, use base 2 (decimal=False) for byte sizes/rates.
	55	"""
	56
	57	factor = 1000 if decimal else 1024
	58	units = [' ', 'k', 'M', 'G', 'T', 'P', 'E']
	59	unit = 0
	60	while len("%s" % (int(n) // (factor**unit))) > width - 1:
	61	unit += 1
	62
	63	if unit > 0:
	64	truncated_float = ("%f" % (n / (float(factor) ** unit)))[0:width - 1]
	65	if truncated_float[-1] == '.':
	66	truncated_float = " " + truncated_float[0:-1]
	67	else:
	68	truncated_float = "%{wid}d".format(wid=width - 1) % n
	69	formatted = "%s%s" % (truncated_float, units[unit])
	70
	71	if colored:
	72	if n == 0:
	73	color = BLACK, False
	74	else:
	75	color = YELLOW, False
	76	return bold(colorize(formatted[0:-1], color[0], color[1])) \
	77	+ bold(colorize(formatted[-1], BLACK, False))
	78	else:
	79	return formatted
	80
	81
92f5a8d4	82	def format_dimless(n, width, colored=False):
11fdf7f2 TL	83	return format_units(n, width, colored, decimal=True)
	84
	85
92f5a8d4	86	def format_bytes(n, width, colored=False):
11fdf7f2	87	return format_units(n, width, colored, decimal=False)
81eedcae TL	88
	89
	90	def merge_dicts(*args):
	91	# type: (dict) -> dict
	92	"""
9f95a23c TL	93	>>> merge_dicts({1:2}, {3:4})
	94	{1: 2, 3: 4}
	95
	96	You can also overwrite keys:
	97	>>> merge_dicts({1:2}, {1:4})
	98	{1: 4}
	99
81eedcae TL	100	:rtype: dict[str, Any]
	101	"""
	102	ret = {}
	103	for arg in args:
	104	ret.update(arg)
	105	return ret
494da23a TL	106
	107
	108	def get_default_addr():
9f95a23c	109	# type: () -> str
494da23a TL	110	def is_ipv6_enabled():
	111	try:
	112	sock = socket.socket(socket.AF_INET6)
	113	with contextlib.closing(sock):
	114	sock.bind(("::1", 0))
	115	return True
	116	except (AttributeError, socket.error) as e:
	117	return False
	118
	119	try:
9f95a23c	120	return get_default_addr.result # type: ignore
494da23a TL	121	except AttributeError:
494da23a TL	122	result = '::' if is_ipv6_enabled() else '0.0.0.0'
9f95a23c	123	get_default_addr.result = result # type: ignore
494da23a TL	124	return result
494da23a TL	125
eafe8130 TL	126
	127	class ServerConfigException(Exception):
	128	pass
	129
9f95a23c TL	130
	131	def create_self_signed_cert(organisation='Ceph', common_name='mgr') -> Tuple[str, str]:
	132	"""Returns self-signed PEM certificates valid for 10 years.
	133	:return cert, pkey
	134	"""
	135
	136	from OpenSSL import crypto
	137	from uuid import uuid4
	138
	139	# create a key pair
	140	pkey = crypto.PKey()
	141	pkey.generate_key(crypto.TYPE_RSA, 2048)
	142
	143	# create a self-signed cert
	144	cert = crypto.X509()
	145	cert.get_subject().O = organisation
	146	cert.get_subject().CN = common_name
	147	cert.set_serial_number(int(uuid4()))
	148	cert.gmtime_adj_notBefore(0)
	149	cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years
	150	cert.set_issuer(cert.get_subject())
	151	cert.set_pubkey(pkey)
	152	cert.sign(pkey, 'sha512')
	153
	154	cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
	155	pkey = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
	156
	157	return cert.decode('utf-8'), pkey.decode('utf-8')
	158
	159
	160	def verify_cacrt_content(crt):
	161	# type: (str) -> None
	162	from OpenSSL import crypto
	163	try:
	164	x509 = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
	165	if x509.has_expired():
	166	logger.warning('Certificate has expired: {}'.format(crt))
	167	except (ValueError, crypto.Error) as e:
	168	raise ServerConfigException(
	169	'Invalid certificate: {}'.format(str(e)))
	170
	171
eafe8130	172	def verify_cacrt(cert_fname):
9f95a23c	173	# type: (str) -> None
eafe8130 TL	174	"""Basic validation of a ca cert"""
	175
	176	if not cert_fname:
	177	raise ServerConfigException("CA cert not configured")
	178	if not os.path.isfile(cert_fname):
	179	raise ServerConfigException("Certificate {} does not exist".format(cert_fname))
	180
eafe8130 TL	181	try:
eafe8130 TL	182	with open(cert_fname) as f:
9f95a23c TL	183	verify_cacrt_content(f.read())
9f95a23c TL	184	except ValueError as e:
eafe8130 TL	185	raise ServerConfigException(
	186	'Invalid certificate {}: {}'.format(cert_fname, str(e)))
	187
	188
9f95a23c TL	189	def verify_tls(crt, key):
	190	# type: (str, str) -> None
	191	verify_cacrt_content(crt)
	192
	193	from OpenSSL import crypto, SSL
	194	try:
	195	_key = crypto.load_privatekey(crypto.FILETYPE_PEM, key)
	196	_key.check()
	197	except (ValueError, crypto.Error) as e:
	198	raise ServerConfigException(
	199	'Invalid private key: {}'.format(str(e)))
	200	try:
	201	_crt = crypto.load_certificate(crypto.FILETYPE_PEM, crt)
	202	except ValueError as e:
	203	raise ServerConfigException(
	204	'Invalid certificate key: {}'.format(str(e))
	205	)
	206
	207	try:
	208	context = SSL.Context(SSL.TLSv1_METHOD)
	209	context.use_certificate(_crt)
	210	context.use_privatekey(_key)
	211	context.check_privatekey()
	212	except crypto.Error as e:
	213	logger.warning(
	214	'Private key and certificate do not match up: {}'.format(str(e)))
	215
	216
eafe8130	217	def verify_tls_files(cert_fname, pkey_fname):
9f95a23c	218	# type: (str, str) -> None
eafe8130 TL	219	"""Basic checks for TLS certificate and key files
	220
	221	Do some validations to the private key and certificate:
	222	- Check the type and format
	223	- Check the certificate expiration date
	224	- Check the consistency of the private key
	225	- Check that the private key and certificate match up
	226
	227	:param cert_fname: Name of the certificate file
	228	:param pkey_fname: name of the certificate public key file
	229
	230	:raises ServerConfigException: An error with a message
	231
	232	"""
	233
	234	if not cert_fname or not pkey_fname:
	235	raise ServerConfigException('no certificate configured')
	236
	237	verify_cacrt(cert_fname)
	238
	239	if not os.path.isfile(pkey_fname):
	240	raise ServerConfigException('private key %s does not exist' % pkey_fname)
	241
	242	from OpenSSL import crypto, SSL
	243
	244	try:
	245	with open(pkey_fname) as f:
	246	pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read())
	247	pkey.check()
	248	except (ValueError, crypto.Error) as e:
	249	raise ServerConfigException(
	250	'Invalid private key {}: {}'.format(pkey_fname, str(e)))
	251	try:
	252	context = SSL.Context(SSL.TLSv1_METHOD)
	253	context.use_certificate_file(cert_fname, crypto.FILETYPE_PEM)
	254	context.use_privatekey_file(pkey_fname, crypto.FILETYPE_PEM)
	255	context.check_privatekey()
	256	except crypto.Error as e:
	257	logger.warning(
	258	'Private key {} and certificate {} do not match up: {}'.format(
	259	pkey_fname, cert_fname, str(e)))
9f95a23c TL	260
	261	def get_most_recent_rate(rates):
	262	""" Get most recent rate from rates
	263
	264	:param rates: The derivative between all time series data points [time in seconds, value]
	265	:type rates: list[tuple[int, float]]
	266
	267	:return: The last derivative or 0.0 if none exists
	268	:rtype: float
	269
	270	>>> get_most_recent_rate(None)
	271	0.0
	272	>>> get_most_recent_rate([])
	273	0.0
	274	>>> get_most_recent_rate([(1, -2.0)])
	275	-2.0
	276	>>> get_most_recent_rate([(1, 2.0), (2, 1.5), (3, 5.0)])
	277	5.0
	278	"""
	279	if not rates:
	280	return 0.0
	281	return rates[-1][1]
	282
	283	def get_time_series_rates(data):
	284	""" Rates from time series data
	285
	286	:param data: Time series data [time in seconds, value]
	287	:type data: list[tuple[int, float]]
	288
	289	:return: The derivative between all time series data points [time in seconds, value]
	290	:rtype: list[tuple[int, float]]
	291
	292	>>> logger.debug = lambda s,x,y: print(s % (x,y))
	293	>>> get_time_series_rates([])
	294	[]
	295	>>> get_time_series_rates([[0, 1], [1, 3]])
	296	[(1, 2.0)]
	297	>>> get_time_series_rates([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
	298	Duplicate timestamp in time series data: [0, 2], [0, 3]
	299	Duplicate timestamp in time series data: [0, 3], [0, 1]
	300	Duplicate timestamp in time series data: [1, 2], [1, 3]
	301	[(1, 2.0)]
	302	>>> get_time_series_rates([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
	303	[(2, 2.0), (4, 4.0), (5, 5.0), (6, 6.0)]
	304	"""
	305	data = _filter_time_series(data)
	306	if not data:
	307	return []
	308	return [(data2[0], _derivative(data1, data2)) for data1, data2 in
	309	_pairwise(data)]
	310
	311	def _filter_time_series(data):
	312	""" Filters time series data
	313
	314	Filters out samples with the same timestamp in given time series data.
	315	It also enforces the list to contain at least two samples.
	316
	317	All filtered values will be shown in the debug log. If values were filtered it's a bug in the
	318	time series data collector, please report it.
	319
	320	:param data: Time series data [time in seconds, value]
	321	:type data: list[tuple[int, float]]
	322
	323	:return: Filtered time series data [time in seconds, value]
324	:rtype: list[tuple[int, float]]
325
326	>>> logger.debug = lambda s,x,y: print(s % (x,y))
327	>>> _filter_time_series([])
328	[]
329	>>> _filter_time_series([[1, 42]])
330	[]
331	>>> _filter_time_series([[10, 2], [10, 3]])
332	Duplicate timestamp in time series data: [10, 2], [10, 3]
333	[]
334	>>> _filter_time_series([[0, 1], [1, 2]])
335	[[0, 1], [1, 2]]
336	>>> _filter_time_series([[0, 2], [0, 3], [0, 1], [1, 2], [1, 3]])
337	Duplicate timestamp in time series data: [0, 2], [0, 3]
338	Duplicate timestamp in time series data: [0, 3], [0, 1]
339	Duplicate timestamp in time series data: [1, 2], [1, 3]
340	[[0, 1], [1, 3]]
341	>>> _filter_time_series([[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]])
342	[[1, 1], [2, 3], [4, 11], [5, 16], [6, 22]]
343	"""
344	filtered = []
345	for i in range(len(data) - 1):
346	if data[i][0] == data[i + 1][0]: # Same timestamp
347	logger.debug("Duplicate timestamp in time series data: %s, %s", data[i], data[i + 1])
348	continue
349	filtered.append(data[i])
350	if not filtered:
351	return []
352	filtered.append(data[-1])
353	return filtered
354
355	def _derivative(p1, p2):
356	""" Derivative between two time series data points
357
358	:param p1: Time series data [time in seconds, value]
359	:type p1: tuple[int, float]
360	:param p2: Time series data [time in seconds, value]
361	:type p2: tuple[int, float]
362
363	:return: Derivative between both points
364	:rtype: float
365
366	>>> _derivative([0, 0], [2, 1])
367	0.5
368	>>> _derivative([0, 1], [2, 0])
369	-0.5
370	>>> _derivative([0, 0], [3, 1])
371	0.3333333333333333
372	"""
373	return (p2[1] - p1[1]) / float(p2[0] - p1[0])
374
375	def _pairwise(iterable):
376	it = iter(iterable)
377	a = next(it, None)
378
379	for b in it:
380	yield (a, b)
381	a = b
382
383	def to_pretty_timedelta(n):
384	if n < datetime.timedelta(seconds=120):
385	return str(n.seconds) + 's'
386	if n < datetime.timedelta(minutes=120):
387	return str(n.seconds // 60) + 'm'
388	if n < datetime.timedelta(hours=48):
389	return str(n.seconds // 3600) + 'h'
390	if n < datetime.timedelta(days=14):
391	return str(n.days) + 'd'
392	if n < datetime.timedelta(days=7*12):
393	return str(n.days // 7) + 'w'
394	if n < datetime.timedelta(days=365*2):
395	return str(n.days // 30) + 'M'
396	return str(n.days // 365) + 'y'
f6b5b4d7 TL	397
	398
	399	def profile_method(skip_attribute=False):
	400	"""
	401	Decorator for methods of the Module class. Logs the name of the given
	402	function f with the time it takes to execute it.
	403	"""
	404	def outer(f):
	405	@wraps(f)
	406	def wrapper(args, *kwargs):
	407	self = args[0]
	408	t = time.time()
	409	self.log.debug('Starting method {}.'.format(f.__name__))
	410	result = f(args, *kwargs)
	411	duration = time.time() - t
	412	if not skip_attribute:
	413	wrapper._execution_duration = duration # type: ignore
	414	self.log.debug('Method {} ran {:.3f} seconds.'.format(f.__name__, duration))
	415	return result
	416	return wrapper
	417	return outer