ceph/src/pybind/mgr/dashboard/tools.py

   1 # -*- coding: utf-8 -*-
   2 from __future__ import absolute_import
   3
   4 import sys
   5 import inspect
   6 import json
   7 import functools
   8 import ipaddress
   9 import logging
  10
  11 import collections
  12 from datetime import datetime, timedelta
  13 from distutils.util import strtobool
  14 import fnmatch
  15 import time
  16 import threading
  17 import six
  18 from six.moves import urllib
  19 import cherrypy
  20
  21 try:
  22     from urlparse import urljoin
  23 except ImportError:
  24     from urllib.parse import urljoin
  25
  26 from . import mgr
  27 from .exceptions import ViewCacheNoDataException
  28 from .settings import Settings
  29 from .services.auth import JwtManager
  30
  31 try:
  32     from typing import Any, AnyStr, Callable, DefaultDict, Deque,\
  33         Dict, List, Set, Tuple, Union  # noqa pylint: disable=unused-import
  34 except ImportError:
  35     pass  # For typing only
  36
  37
  38 class RequestLoggingTool(cherrypy.Tool):
  39     def __init__(self):
  40         cherrypy.Tool.__init__(self, 'before_handler', self.request_begin,
  41                                priority=10)
  42         self.logger = logging.getLogger('request')
  43
  44     def _setup(self):
  45         cherrypy.Tool._setup(self)
  46         cherrypy.request.hooks.attach('on_end_request', self.request_end,
  47                                       priority=5)
  48         cherrypy.request.hooks.attach('after_error_response', self.request_error,
  49                                       priority=5)
  50
  51     def request_begin(self):
  52         req = cherrypy.request
  53         user = JwtManager.get_username()
  54         # Log the request.
  55         self.logger.debug('[%s:%s] [%s] [%s] %s', req.remote.ip, req.remote.port,
  56                           req.method, user, req.path_info)
  57         # Audit the request.
  58         if Settings.AUDIT_API_ENABLED and req.method not in ['GET']:
  59             url = build_url(req.remote.ip, scheme=req.scheme,
  60                             port=req.remote.port)
  61             msg = '[DASHBOARD] from=\'{}\' path=\'{}\' method=\'{}\' ' \
  62                 'user=\'{}\''.format(url, req.path_info, req.method, user)
  63             if Settings.AUDIT_API_LOG_PAYLOAD:
  64                 params = dict(req.params or {}, **get_request_body_params(req))
  65                 # Hide sensitive data like passwords, secret keys, ...
  66                 # Extend the list of patterns to search for if necessary.
  67                 # Currently parameters like this are processed:
  68                 # - secret_key
  69                 # - user_password
  70                 # - new_passwd_to_login
  71                 keys = []
  72                 for key in ['password', 'passwd', 'secret']:
  73                     keys.extend([x for x in params.keys() if key in x])
  74                 for key in keys:
  75                     params[key] = '***'
  76                 msg = '{} params=\'{}\''.format(msg, json.dumps(params))
  77             mgr.cluster_log('audit', mgr.CLUSTER_LOG_PRIO_INFO, msg)
  78
  79     def request_error(self):
  80         self._request_log(self.logger.error)
  81         self.logger.error(cherrypy.response.body)
  82
  83     def request_end(self):
  84         status = cherrypy.response.status[:3]
  85         if status in ["401", "403"]:
  86             # log unauthorized accesses
  87             self._request_log(self.logger.warning)
  88         else:
  89             self._request_log(self.logger.info)
  90
  91     def _format_bytes(self, num):
  92         units = ['B', 'K', 'M', 'G']
  93
  94         if isinstance(num, str):
  95             try:
  96                 num = int(num)
  97             except ValueError:
  98                 return "n/a"
  99
 100         format_str = "{:.0f}{}"
 101         for i, unit in enumerate(units):
 102             div = 2**(10*i)
 103             if num < 2**(10*(i+1)):
 104                 if num % div == 0:
 105                     format_str = "{}{}"
 106                 else:
 107                     div = float(div)
 108                     format_str = "{:.1f}{}"
 109                 return format_str.format(num/div, unit[0])
 110
 111         # content-length bigger than 1T!! return value in bytes
 112         return "{}B".format(num)
 113
 114     def _request_log(self, logger_fn):
 115         req = cherrypy.request
 116         res = cherrypy.response
 117         lat = time.time() - res.time
 118         user = JwtManager.get_username()
 119         status = res.status[:3] if isinstance(res.status, str) else res.status
 120         if 'Content-Length' in res.headers:
 121             length = self._format_bytes(res.headers['Content-Length'])
 122         else:
 123             length = self._format_bytes(0)
 124         if user:
 125             logger_fn("[%s:%s] [%s] [%s] [%s] [%s] [%s] %s", req.remote.ip,
 126                       req.remote.port, req.method, status,
 127                       "{0:.3f}s".format(lat), user, length, req.path_info)
 128         else:
 129             logger_fn("[%s:%s] [%s] [%s] [%s] [%s] [%s] %s", req.remote.ip,
 130                       req.remote.port, req.method, status,
 131                       "{0:.3f}s".format(lat), length, getattr(req, 'unique_id', '-'), req.path_info)
 132
 133
 134 # pylint: disable=too-many-instance-attributes
 135 class ViewCache(object):
 136     VALUE_OK = 0
 137     VALUE_STALE = 1
 138     VALUE_NONE = 2
 139
 140     class GetterThread(threading.Thread):
 141         def __init__(self, view, fn, args, kwargs):
 142             super(ViewCache.GetterThread, self).__init__()
 143             self._view = view
 144             self.event = threading.Event()
 145             self.fn = fn
 146             self.args = args
 147             self.kwargs = kwargs
 148
 149         # pylint: disable=broad-except
 150         def run(self):
 151             t0 = 0.0
 152             t1 = 0.0
 153             try:
 154                 t0 = time.time()
 155                 self._view.logger.debug("starting execution of %s", self.fn)
 156                 val = self.fn(*self.args, **self.kwargs)
 157                 t1 = time.time()
 158             except Exception as ex:
 159                 with self._view.lock:
 160                     self._view.logger.exception("Error while calling fn=%s ex=%s", self.fn,
 161                                                 str(ex))
 162                     self._view.value = None
 163                     self._view.value_when = None
 164                     self._view.getter_thread = None
 165                     self._view.exception = ex
 166             else:
 167                 with self._view.lock:
 168                     self._view.latency = t1 - t0
 169                     self._view.value = val
 170                     self._view.value_when = datetime.now()
 171                     self._view.getter_thread = None
 172                     self._view.exception = None
 173
 174             self._view.logger.debug("execution of %s finished in: %s", self.fn,
 175                                     t1 - t0)
 176             self.event.set()
 177
 178     class RemoteViewCache(object):
 179         # Return stale data if
 180         STALE_PERIOD = 1.0
 181
 182         def __init__(self, timeout):
 183             self.getter_thread = None
 184             # Consider data within 1s old to be sufficiently fresh
 185             self.timeout = timeout
 186             self.event = threading.Event()
 187             self.value_when = None
 188             self.value = None
 189             self.latency = 0
 190             self.exception = None
 191             self.lock = threading.Lock()
 192             self.logger = logging.getLogger('viewcache')
 193
 194         def reset(self):
 195             with self.lock:
 196                 self.value_when = None
 197                 self.value = None
 198
 199         def run(self, fn, args, kwargs):
 200             """
 201             If data less than `stale_period` old is available, return it
 202             immediately.
 203             If an attempt to fetch data does not complete within `timeout`, then
 204             return the most recent data available, with a status to indicate that
 205             it is stale.
 206
 207             Initialization does not count towards the timeout, so the first call
 208             on one of these objects during the process lifetime may be slower
 209             than subsequent calls.
 210
 211             :return: 2-tuple of value status code, value
 212             """
 213             with self.lock:
 214                 now = datetime.now()
 215                 if self.value_when and now - self.value_when < timedelta(
 216                         seconds=self.STALE_PERIOD):
 217                     return ViewCache.VALUE_OK, self.value
 218
 219                 if self.getter_thread is None:
 220                     self.getter_thread = ViewCache.GetterThread(self, fn, args,
 221                                                                 kwargs)
 222                     self.getter_thread.start()
 223                 else:
 224                     self.logger.debug("getter_thread still alive for: %s", fn)
 225
 226                 ev = self.getter_thread.event
 227
 228             success = ev.wait(timeout=self.timeout)
 229
 230             with self.lock:
 231                 if success:
 232                     # We fetched the data within the timeout
 233                     if self.exception:
 234                         # execution raised an exception
 235                         # pylint: disable=raising-bad-type
 236                         raise self.exception
 237                     return ViewCache.VALUE_OK, self.value
 238                 if self.value_when is not None:
 239                     # We have some data, but it doesn't meet freshness requirements
 240                     return ViewCache.VALUE_STALE, self.value
 241                 # We have no data, not even stale data
 242                 raise ViewCacheNoDataException()
 243
 244     def __init__(self, timeout=5):
 245         self.timeout = timeout
 246         self.cache_by_args = {}
 247
 248     def __call__(self, fn):
 249         def wrapper(*args, **kwargs):
 250             rvc = self.cache_by_args.get(args, None)
 251             if not rvc:
 252                 rvc = ViewCache.RemoteViewCache(self.timeout)
 253                 self.cache_by_args[args] = rvc
 254             return rvc.run(fn, args, kwargs)
 255         wrapper.reset = self.reset  # type: ignore
 256         return wrapper
 257
 258     def reset(self):
 259         for _, rvc in self.cache_by_args.items():
 260             rvc.reset()
 261
 262
 263 class NotificationQueue(threading.Thread):
 264     _ALL_TYPES_ = '__ALL__'
 265     _listeners = collections.defaultdict(set)  # type: DefaultDict[str, Set[Tuple[int, Callable]]]
 266     _lock = threading.Lock()
 267     _cond = threading.Condition()
 268     _queue = collections.deque()  # type: Deque[Tuple[str, Any]]
 269     _running = False
 270     _instance = None
 271
 272     def __init__(self):
 273         super(NotificationQueue, self).__init__()
 274
 275     @classmethod
 276     def start_queue(cls):
 277         with cls._lock:
 278             if cls._instance:
 279                 # the queue thread is already running
 280                 return
 281             cls._running = True
 282             cls._instance = NotificationQueue()
 283         cls.logger = logging.getLogger('notification_queue')  # type: ignore
 284         cls.logger.debug("starting notification queue")  # type: ignore
 285         cls._instance.start()
 286
 287     @classmethod
 288     def stop(cls):
 289         with cls._lock:
 290             if not cls._instance:
 291                 # the queue thread was not started
 292                 return
 293             instance = cls._instance
 294             cls._instance = None
 295             cls._running = False
 296         with cls._cond:
 297             cls._cond.notify()
 298         cls.logger.debug("waiting for notification queue to finish")  # type: ignore
 299         instance.join()
 300         cls.logger.debug("notification queue stopped")  # type: ignore
 301
 302     @classmethod
 303     def _registered_handler(cls, func, n_types):
 304         for _, reg_func in cls._listeners[n_types]:
 305             if reg_func == func:
 306                 return True
 307         return False
 308
 309     @classmethod
 310     def register(cls, func, n_types=None, priority=1):
 311         """Registers function to listen for notifications
 312
 313         If the second parameter `n_types` is omitted, the function in `func`
 314         parameter will be called for any type of notifications.
 315
 316         Args:
 317             func (function): python function ex: def foo(val)
 318             n_types (str|list): the single type to listen, or a list of types
 319             priority (int): the priority level (1=max, +inf=min)
 320         """
 321         with cls._lock:
 322             if not n_types:
 323                 n_types = [cls._ALL_TYPES_]
 324             elif isinstance(n_types, str):
 325                 n_types = [n_types]
 326             elif not isinstance(n_types, list):
 327                 raise Exception("n_types param is neither a string nor a list")
 328             for ev_type in n_types:
 329                 if not cls._registered_handler(func, ev_type):
 330                     cls._listeners[ev_type].add((priority, func))
 331                     cls.logger.debug(  # type: ignore
 332                         "function %s was registered for events of type %s",
 333                         func, ev_type
 334                     )
 335
 336     @classmethod
 337     def deregister(cls, func, n_types=None):
 338         # type: (Callable, Union[str, list, None]) -> None
 339         """Removes the listener function from this notification queue
 340
 341         If the second parameter `n_types` is omitted, the function is removed
 342         from all event types, otherwise the function is removed only for the
 343         specified event types.
 344
 345         Args:
 346             func (function): python function
 347             n_types (str|list): the single event type, or a list of event types
 348         """
 349         with cls._lock:
 350             if not n_types:
 351                 n_types = list(cls._listeners.keys())
 352             elif isinstance(n_types, str):
 353                 n_types = [n_types]
 354             elif not isinstance(n_types, list):
 355                 raise Exception("n_types param is neither a string nor a list")
 356             for ev_type in n_types:
 357                 listeners = cls._listeners[ev_type]
 358                 to_remove = None
 359                 for pr, fn in listeners:
 360                     if fn == func:
 361                         to_remove = (pr, fn)
 362                         break
 363                 if to_remove:
 364                     listeners.discard(to_remove)
 365                     cls.logger.debug(  # type: ignore
 366                         "function %s was deregistered for events of type %s",
 367                         func, ev_type
 368                     )
 369
 370     @classmethod
 371     def new_notification(cls, notify_type, notify_value):
 372         # type: (str, Any) -> None
 373         with cls._cond:
 374             cls._queue.append((notify_type, notify_value))
 375             cls._cond.notify()
 376
 377     @classmethod
 378     def _notify_listeners(cls, events):
 379         for ev in events:
 380             notify_type, notify_value = ev
 381             with cls._lock:
 382                 listeners = list(cls._listeners[notify_type])
 383                 listeners.extend(cls._listeners[cls._ALL_TYPES_])
 384             listeners.sort(key=lambda lis: lis[0])
 385             for listener in listeners:
 386                 listener[1](notify_value)
 387
 388     def run(self):
 389         self.logger.debug("notification queue started")  # type: ignore
 390         while self._running:
 391             private_buffer = []
 392             self.logger.debug("processing queue: %s", len(self._queue))  # type: ignore
 393             try:
 394                 while True:
 395                     private_buffer.append(self._queue.popleft())
 396             except IndexError:
 397                 pass
 398             self._notify_listeners(private_buffer)
 399             with self._cond:
 400                 while self._running and not self._queue:
 401                     self._cond.wait()
 402         # flush remaining events
 403         self.logger.debug("flush remaining events: %s", len(self._queue))  # type: ignore
 404         self._notify_listeners(self._queue)
 405         self._queue.clear()
 406         self.logger.debug("notification queue finished")  # type: ignore
 407
 408
 409 # pylint: disable=too-many-arguments, protected-access
 410 class TaskManager(object):
 411     FINISHED_TASK_SIZE = 10
 412     FINISHED_TASK_TTL = 60.0
 413
 414     VALUE_DONE = "done"
 415     VALUE_EXECUTING = "executing"
 416
 417     _executing_tasks = set()  # type: Set[Task]
 418     _finished_tasks = []  # type: List[Task]
 419     _lock = threading.Lock()
 420
 421     _task_local_data = threading.local()
 422
 423     @classmethod
 424     def init(cls):
 425         cls.logger = logging.getLogger('taskmgr')  # type: ignore
 426         NotificationQueue.register(cls._handle_finished_task, 'cd_task_finished')
 427
 428     @classmethod
 429     def _handle_finished_task(cls, task):
 430         cls.logger.info("finished %s", task)  # type: ignore
 431         with cls._lock:
 432             cls._executing_tasks.remove(task)
 433             cls._finished_tasks.append(task)
 434
 435     @classmethod
 436     def run(cls, name, metadata, fn, args=None, kwargs=None, executor=None,
 437             exception_handler=None):
 438         if not args:
 439             args = []
 440         if not kwargs:
 441             kwargs = {}
 442         if not executor:
 443             executor = ThreadedExecutor()
 444         task = Task(name, metadata, fn, args, kwargs, executor,
 445                     exception_handler)
 446         with cls._lock:
 447             if task in cls._executing_tasks:
 448                 cls.logger.debug("task already executing: %s", task)  # type: ignore
 449                 for t in cls._executing_tasks:
 450                     if t == task:
 451                         return t
 452             cls.logger.debug("created %s", task)  # type: ignore
 453             cls._executing_tasks.add(task)
 454         cls.logger.info("running %s", task)  # type: ignore
 455         task._run()
 456         return task
 457
 458     @classmethod
 459     def current_task(cls):
 460         """
 461         Returns the current task object.
 462         This method should only be called from a threaded task operation code.
 463         """
 464         return cls._task_local_data.task
 465
 466     @classmethod
 467     def _cleanup_old_tasks(cls, task_list):
 468         """
 469         The cleanup rule is: maintain the FINISHED_TASK_SIZE more recent
 470         finished tasks, and the rest is maintained up to the FINISHED_TASK_TTL
 471         value.
 472         """
 473         now = datetime.now()
 474         for idx, t in enumerate(task_list):
 475             if idx < cls.FINISHED_TASK_SIZE:
 476                 continue
 477             if now - datetime.fromtimestamp(t[1].end_time) > \
 478                     timedelta(seconds=cls.FINISHED_TASK_TTL):
 479                 del cls._finished_tasks[t[0]]
 480
 481     @classmethod
 482     def list(cls, name_glob=None):
 483         executing_tasks = []
 484         finished_tasks = []
 485         with cls._lock:
 486             for task in cls._executing_tasks:
 487                 if not name_glob or fnmatch.fnmatch(task.name, name_glob):
 488                     executing_tasks.append(task)
 489             for idx, task in enumerate(cls._finished_tasks):
 490                 if not name_glob or fnmatch.fnmatch(task.name, name_glob):
 491                     finished_tasks.append((idx, task))
 492             finished_tasks.sort(key=lambda t: t[1].end_time, reverse=True)
 493             cls._cleanup_old_tasks(finished_tasks)
 494         executing_tasks.sort(key=lambda t: t.begin_time, reverse=True)
 495         return executing_tasks, [t[1] for t in finished_tasks]
 496
 497     @classmethod
 498     def list_serializable(cls, ns_glob=None):
 499         ex_t, fn_t = cls.list(ns_glob)
 500         return [{
 501             'name': t.name,
 502             'metadata': t.metadata,
 503             'begin_time': "{}Z".format(datetime.fromtimestamp(t.begin_time).isoformat()),
 504             'progress': t.progress
 505         } for t in ex_t if t.begin_time], [{
 506             'name': t.name,
 507             'metadata': t.metadata,
 508             'begin_time': "{}Z".format(datetime.fromtimestamp(t.begin_time).isoformat()),
 509             'end_time': "{}Z".format(datetime.fromtimestamp(t.end_time).isoformat()),
 510             'duration': t.duration,
 511             'progress': t.progress,
 512             'success': not t.exception,
 513             'ret_value': t.ret_value if not t.exception else None,
 514             'exception': t.ret_value if t.exception and t.ret_value else (
 515                 {'detail': str(t.exception)} if t.exception else None)
 516         } for t in fn_t]
 517
 518
 519 # pylint: disable=protected-access
 520 class TaskExecutor(object):
 521     def __init__(self):
 522         self.logger = logging.getLogger('taskexec')
 523         self.task = None
 524
 525     def init(self, task):
 526         self.task = task
 527
 528     # pylint: disable=broad-except
 529     def start(self):
 530         self.logger.debug("executing task %s", self.task)
 531         try:
 532             self.task.fn(*self.task.fn_args, **self.task.fn_kwargs)  # type: ignore
 533         except Exception as ex:
 534             self.logger.exception("Error while calling %s", self.task)
 535             self.finish(None, ex)
 536
 537     def finish(self, ret_value, exception):
 538         if not exception:
 539             self.logger.debug("successfully finished task: %s", self.task)
 540         else:
 541             self.logger.debug("task finished with exception: %s", self.task)
 542         self.task._complete(ret_value, exception)  # type: ignore
 543
 544
 545 # pylint: disable=protected-access
 546 class ThreadedExecutor(TaskExecutor):
 547     def __init__(self):
 548         super(ThreadedExecutor, self).__init__()
 549         self._thread = threading.Thread(target=self._run)
 550
 551     def start(self):
 552         self._thread.start()
 553
 554     # pylint: disable=broad-except
 555     def _run(self):
 556         TaskManager._task_local_data.task = self.task
 557         try:
 558             self.logger.debug("executing task %s", self.task)
 559             val = self.task.fn(*self.task.fn_args, **self.task.fn_kwargs)  # type: ignore
 560         except Exception as ex:
 561             self.logger.exception("Error while calling %s", self.task)
 562             self.finish(None, ex)
 563         else:
 564             self.finish(val, None)
 565
 566
 567 class Task(object):
 568     def __init__(self, name, metadata, fn, args, kwargs, executor,
 569                  exception_handler=None):
 570         self.name = name
 571         self.metadata = metadata
 572         self.fn = fn
 573         self.fn_args = args
 574         self.fn_kwargs = kwargs
 575         self.executor = executor
 576         self.ex_handler = exception_handler
 577         self.running = False
 578         self.event = threading.Event()
 579         self.progress = None
 580         self.ret_value = None
 581         self.begin_time = None
 582         self.end_time = None
 583         self.duration = 0
 584         self.exception = None
 585         self.logger = logging.getLogger('task')
 586         self.lock = threading.Lock()
 587
 588     def __hash__(self):
 589         return hash((self.name, tuple(sorted(self.metadata.items()))))
 590
 591     def __eq__(self, other):
 592         return self.name == other.name and self.metadata == other.metadata
 593
 594     def __str__(self):
 595         return "Task(ns={}, md={})" \
 596                .format(self.name, self.metadata)
 597
 598     def __repr__(self):
 599         return str(self)
 600
 601     def _run(self):
 602         NotificationQueue.register(self._handle_task_finished, 'cd_task_finished', 100)
 603         with self.lock:
 604             assert not self.running
 605             self.executor.init(self)
 606             self.set_progress(0, in_lock=True)
 607             self.begin_time = time.time()
 608             self.running = True
 609         self.executor.start()
 610
 611     def _complete(self, ret_value, exception=None):
 612         now = time.time()
 613         if exception and self.ex_handler:
 614             # pylint: disable=broad-except
 615             try:
 616                 ret_value = self.ex_handler(exception, task=self)
 617             except Exception as ex:
 618                 exception = ex
 619         with self.lock:
 620             assert self.running, "_complete cannot be called before _run"
 621             self.end_time = now
 622             self.ret_value = ret_value
 623             self.exception = exception
 624             self.duration = now - self.begin_time  # type: ignore
 625             if not self.exception:
 626                 self.set_progress(100, True)
 627         NotificationQueue.new_notification('cd_task_finished', self)
 628         self.logger.debug("execution of %s finished in: %s s", self,
 629                           self.duration)
 630
 631     def _handle_task_finished(self, task):
 632         if self == task:
 633             NotificationQueue.deregister(self._handle_task_finished)
 634             self.event.set()
 635
 636     def wait(self, timeout=None):
 637         with self.lock:
 638             assert self.running, "wait cannot be called before _run"
 639             ev = self.event
 640
 641         success = ev.wait(timeout=timeout)
 642         with self.lock:
 643             if success:
 644                 # the action executed within the timeout
 645                 if self.exception:
 646                     # pylint: disable=raising-bad-type
 647                     # execution raised an exception
 648                     raise self.exception
 649                 return TaskManager.VALUE_DONE, self.ret_value
 650             # the action is still executing
 651             return TaskManager.VALUE_EXECUTING, None
 652
 653     def inc_progress(self, delta, in_lock=False):
 654         if not isinstance(delta, int) or delta < 0:
 655             raise Exception("Progress delta value must be a positive integer")
 656         if not in_lock:
 657             self.lock.acquire()
 658         prog = self.progress + delta  # type: ignore
 659         self.progress = prog if prog <= 100 else 100
 660         if not in_lock:
 661             self.lock.release()
 662
 663     def set_progress(self, percentage, in_lock=False):
 664         if not isinstance(percentage, int) or percentage < 0 or percentage > 100:
 665             raise Exception("Progress value must be in percentage "
 666                             "(0 <= percentage <= 100)")
 667         if not in_lock:
 668             self.lock.acquire()
 669         self.progress = percentage
 670         if not in_lock:
 671             self.lock.release()
 672
 673
 674 def build_url(host, scheme=None, port=None):
 675     """
 676     Build a valid URL. IPv6 addresses specified in host will be enclosed in brackets
 677     automatically.
 678
 679     >>> build_url('example.com', 'https', 443)
 680     'https://example.com:443'
 681
 682     >>> build_url(host='example.com', port=443)
 683     '//example.com:443'
 684
 685     >>> build_url('fce:9af7:a667:7286:4917:b8d3:34df:8373', port=80, scheme='http')
 686     'http://[fce:9af7:a667:7286:4917:b8d3:34df:8373]:80'
 687
 688     :param scheme: The scheme, e.g. http, https or ftp.
 689     :type scheme: str
 690     :param host: Consisting of either a registered name (including but not limited to
 691                  a hostname) or an IP address.
 692     :type host: str
 693     :type port: int
 694     :rtype: str
 695     """
 696     try:
 697         try:
 698             u_host = six.u(host)
 699         except TypeError:
 700             u_host = host
 701
 702         ipaddress.IPv6Address(u_host)
 703         netloc = '[{}]'.format(host)
 704     except ValueError:
 705         netloc = host
 706     if port:
 707         netloc += ':{}'.format(port)
 708     pr = urllib.parse.ParseResult(
 709         scheme=scheme if scheme else '',
 710         netloc=netloc,
 711         path='',
 712         params='',
 713         query='',
 714         fragment='')
 715     return pr.geturl()
 716
 717
 718 def prepare_url_prefix(url_prefix):
 719     """
 720     return '' if no prefix, or '/prefix' without slash in the end.
 721     """
 722     url_prefix = urljoin('/', url_prefix)
 723     return url_prefix.rstrip('/')
 724
 725
 726 def dict_contains_path(dct, keys):
 727     """
 728     Tests whether the keys exist recursively in `dictionary`.
 729
 730     :type dct: dict
 731     :type keys: list
 732     :rtype: bool
 733     """
 734     if keys:
 735         if not isinstance(dct, dict):
 736             return False
 737         key = keys.pop(0)
 738         if key in dct:
 739             dct = dct[key]
 740             return dict_contains_path(dct, keys)
 741         return False
 742     return True
 743
 744
 745 def dict_get(obj, path, default=None):
 746     """
 747     Get the value at any depth of a nested object based on the path
 748     described by `path`. If path doesn't exist, `default` is returned.
 749     """
 750     current = obj
 751     for part in path.split('.'):
 752         if not isinstance(current, dict):
 753             return default
 754         if part not in current.keys():
 755             return default
 756         current = current.get(part, {})
 757     return current
 758
 759
 760 if sys.version_info > (3, 0):
 761     wraps = functools.wraps
 762     _getargspec = inspect.getfullargspec
 763 else:
 764     def wraps(func):
 765         def decorator(wrapper):
 766             new_wrapper = functools.wraps(func)(wrapper)
 767             new_wrapper.__wrapped__ = func  # set __wrapped__ even for Python 2
 768             return new_wrapper
 769         return decorator
 770
 771     _getargspec = inspect.getargspec
 772
 773
 774 def getargspec(func):
 775     try:
 776         while True:
 777             func = func.__wrapped__
 778     except AttributeError:
 779         pass
 780     # pylint: disable=deprecated-method
 781     return _getargspec(func)
 782
 783
 784 def str_to_bool(val):
 785     """
 786     Convert a string representation of truth to True or False.
 787
 788     >>> str_to_bool('true') and str_to_bool('yes') and str_to_bool('1') and str_to_bool(True)
 789     True
 790
 791     >>> str_to_bool('false') and str_to_bool('no') and str_to_bool('0') and str_to_bool(False)
 792     False
 793
 794     >>> str_to_bool('xyz')
 795     Traceback (most recent call last):
 796         ...
 797     ValueError: invalid truth value 'xyz'
 798
 799     :param val: The value to convert.
 800     :type val: str|bool
 801     :rtype: bool
 802     """
 803     if isinstance(val, bool):
 804         return val
 805     return bool(strtobool(val))
 806
 807
 808 def json_str_to_object(value):  # type: (AnyStr) -> Any
 809     """
 810     It converts a JSON valid string representation to object.
 811
 812     >>> result = json_str_to_object('{"a": 1}')
 813     >>> result == {'a': 1}
 814     True
 815     """
 816     if value == '':
 817         return value
 818
 819     try:
 820         # json.loads accepts binary input from version >=3.6
 821         value = value.decode('utf-8')  # type: ignore
 822     except AttributeError:
 823         pass
 824
 825     return json.loads(value)
 826
 827
 828 def partial_dict(orig, keys):  # type: (Dict, List[str]) -> Dict
 829     """
 830     It returns Dict containing only the selected keys of original Dict.
 831
 832     >>> partial_dict({'a': 1, 'b': 2}, ['b'])
 833     {'b': 2}
 834     """
 835     return {k: orig[k] for k in keys}
 836
 837
 838 def get_request_body_params(request):
 839     """
 840     Helper function to get parameters from the request body.
 841     :param request The CherryPy request object.
 842     :type request: cherrypy.Request
 843     :return: A dictionary containing the parameters.
 844     :rtype: dict
 845     """
 846     params = {}  # type: dict
 847     if request.method not in request.methods_with_bodies:
 848         return params
 849
 850     content_type = request.headers.get('Content-Type', '')
 851     if content_type in ['application/json', 'text/javascript']:
 852         if not hasattr(request, 'json'):
 853             raise cherrypy.HTTPError(400, 'Expected JSON body')
 854         if isinstance(request.json, str):
 855             params.update(json.loads(request.json))
 856         else:
 857             params.update(request.json)
 858
 859     return params
 860
 861
 862 def find_object_in_list(key, value, iterable):
 863     """
 864     Get the first occurrence of an object within a list with
 865     the specified key/value.
 866
 867     >>> find_object_in_list('name', 'bar', [{'name': 'foo'}, {'name': 'bar'}])
 868     {'name': 'bar'}
 869
 870     >>> find_object_in_list('name', 'xyz', [{'name': 'foo'}, {'name': 'bar'}]) is None
 871     True
 872
 873     >>> find_object_in_list('foo', 'bar', [{'xyz': 4815162342}]) is None
 874     True
 875
 876     >>> find_object_in_list('foo', 'bar', []) is None
 877     True
 878
 879     :param key: The name of the key.
 880     :param value: The value to search for.
 881     :param iterable: The list to process.
 882     :return: Returns the found object or None.
 883     """
 884     for obj in iterable:
 885         if key in obj and obj[key] == value:
 886             return obj
 887     return None