pkg/scheduler.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2013.  Cumulus Networks, Inc.
   4 # Author: Roopa Prabhu, roopa@cumulusnetworks.com
   5 #
   6 # ifaceScheduler --
   7 #    interface scheduler
   8 #
   9
  10 from statemanager import *
  11 from iface import *
  12 from graph import *
  13 from collections import deque
  14 from collections import OrderedDict
  15 import logging
  16 import traceback
  17 import sys
  18 from graph import *
  19 from collections import deque
  20 from threading import *
  21 from ifupdownbase import *
  22
  23 class ifaceSchedulerFlags():
  24     INORDER = 0x1
  25     POSTORDER = 0x2
  26
  27 class ifaceScheduler():
  28     """ scheduler functions to schedule configuration of interfaces.
  29
  30     supports scheduling of interfaces serially in plain interface list
  31     or dependency graph format.
  32
  33     Algo:
  34         - run topological sort on the iface objects
  35         - In the sorted iface object list, pick up interfaces with no parents
  36         and run ops on them and their children.
  37         - If operation is up and user gave interface list (ie not all)
  38         option, also see if there were upper-devices and run ops on them.
  39         - if operation is down, dont down the interface if it still
  40         has upperifaces present. The down operation is executed when the
  41         last upperiface goes away. If force option is set, this rule does not
  42         apply.
  43         - run ops calls addon modules run operation passing the iface
  44         object and op to each module.
  45         - ops are [pre-up, up, post-up, pre-down, down,
  46                    post-down, query-running, query-check]
  47     """
  48
  49     _STATE_CHECK = True
  50
  51     @classmethod
  52     def run_iface_op(cls, ifupdownobj, ifaceobj, op, cenv=None):
  53         """ Runs sub operation on an interface """
  54         ifacename = ifaceobj.name
  55
  56         if (cls._STATE_CHECK and
  57             (ifaceobj.state >= ifaceState.from_str(op)) and
  58             (ifaceobj.status == ifaceStatus.SUCCESS)):
  59             ifupdownobj.logger.debug('%s: already in state %s' %(ifacename, op))
  60             return
  61         if not ifupdownobj.ADDONS_ENABLE: return
  62         if op == 'query-checkcurr':
  63             query_ifaceobj=ifupdownobj.create_n_save_ifaceobjcurr(ifaceobj)
  64         for mname in ifupdownobj.module_ops.get(op):
  65             m = ifupdownobj.modules.get(mname)
  66             err = 0
  67             try:
  68                 if hasattr(m, 'run'):
  69                     msg = ('%s: %s : running module %s' %(ifacename, op, mname))
  70                     if op == 'query-checkcurr':
  71                         # Dont check curr if the interface object was
  72                         # auto generated
  73                         if (ifaceobj.priv_flags & ifupdownobj.NOCONFIG):
  74                             continue
  75                         ifupdownobj.logger.debug(msg)
  76                         m.run(ifaceobj, op, query_ifaceobj)
  77                     else:
  78                         ifupdownobj.logger.debug(msg)
  79                         m.run(ifaceobj, op)
  80             except Exception, e:
  81                 err = 1
  82                 ifupdownobj.log_error(str(e))
  83             finally:
  84                 if err:
  85                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  86                                                 ifaceStatus.ERROR)
  87                 else:
  88                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  89                                                 ifaceStatus.SUCCESS)
  90
  91         if ifupdownobj.COMPAT_EXEC_SCRIPTS:
  92             # execute /etc/network/ scripts
  93             for mname in ifupdownobj.script_ops.get(op, []):
  94                 ifupdownobj.logger.debug('%s: %s : running script %s'
  95                     %(ifacename, op, mname))
  96                 try:
  97                     ifupdownobj.exec_command(mname, cmdenv=cenv)
  98                 except Exception, e:
  99                     ifupdownobj.log_error(str(e))
 100
 101     @classmethod
 102     def run_iface_list_ops(cls, ifupdownobj, ifaceobjs, ops):
 103         """ Runs all operations on a list of interface
 104             configurations for the same interface
 105         """
 106         # minor optimization. If operation is 'down', proceed only
 107         # if interface exists in the system
 108         ifacename = ifaceobjs[0].name
 109         if ('down' in ops[0] and
 110                 not ifupdownobj.link_exists(ifacename)):
 111             ifupdownobj.logger.debug('%s: does not exist' %ifacename)
 112             # run posthook before you get out of here, so that
 113             # appropriate cleanup is done
 114             posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 115             if posthookfunc:
 116                 for ifaceobj in ifaceobjs:
 117                     ifaceobj.status = ifaceStatus.SUCCESS
 118                     posthookfunc(ifupdownobj, ifaceobj, 'down')
 119             return
 120         for op in ops:
 121             # first run ifupdownobj handlers. This is good enough
 122             # for the first object in the list
 123             handler = ifupdownobj.ops_handlers.get(op)
 124             if handler:
 125                 if (not ifaceobjs[0].addr_method or
 126                     (ifaceobjs[0].addr_method and
 127                     ifaceobjs[0].addr_method != 'manual')):
 128                     handler(ifupdownobj, ifaceobjs[0])
 129             for ifaceobj in ifaceobjs:
 130                 cls.run_iface_op(ifupdownobj, ifaceobj, op,
 131                     cenv=ifupdownobj.generate_running_env(ifaceobj, op)
 132                         if ifupdownobj.COMPAT_EXEC_SCRIPTS else None)
 133                 posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 134                 if posthookfunc:
 135                     posthookfunc(ifupdownobj, ifaceobj, op)
 136
 137     @classmethod
 138     def _check_upperifaces(cls, ifupdownobj, ifaceobj, ops, parent,
 139                            followdependents=False):
 140         """ Check if upperifaces are hanging off us and help caller decide
 141         if he can proceed with the ops on this device
 142
 143         Returns True or False indicating the caller to proceed with the
 144         operation.
 145         """
 146         # proceed only for down operation
 147         if 'down' not in ops[0]:
 148             return True
 149
 150         if (ifupdownobj.FORCE or
 151                 not ifupdownobj.ADDONS_ENABLE or
 152                 (not ifupdownobj.is_ifaceobj_noconfig(ifaceobj) and
 153                 ifupdownobj.config.get('warn_on_ifdown', '0') == '0' and
 154                 not ifupdownobj.ALL)):
 155             return True
 156
 157         ulist = ifaceobj.upperifaces
 158         if not ulist:
 159             return True
 160
 161         # Get the list of upper ifaces other than the parent
 162         tmpulist = ([u for u in ulist if u != parent] if parent
 163                     else ulist)
 164         if not tmpulist:
 165             return True
 166         # XXX: This is expensive. Find a cheaper way to do this.
 167         # if any of the upperdevs are present,
 168         # return false to the caller to skip this interface
 169         for u in tmpulist:
 170             if ifupdownobj.link_exists(u):
 171                 if not ifupdownobj.ALL:
 172                     if ifupdownobj.is_ifaceobj_noconfig(ifaceobj):
 173                         ifupdownobj.logger.info('%s: skipping interface down,'
 174                             %ifaceobj.name + ' upperiface %s still around ' %u)
 175                     else:
 176                         ifupdownobj.logger.warn('%s: skipping interface down,'
 177                             %ifaceobj.name + ' upperiface %s still around ' %u)
 178                 return False
 179         return True
 180
 181     @classmethod
 182     def run_iface_graph(cls, ifupdownobj, ifacename, ops, parent=None,
 183                         order=ifaceSchedulerFlags.POSTORDER,
 184                         followdependents=True):
 185         """ runs interface by traversing all nodes rooted at itself """
 186
 187         # Each ifacename can have a list of iface objects
 188         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 189         if not ifaceobjs:
 190             raise Exception('%s: not found' %ifacename)
 191
 192         for ifaceobj in ifaceobjs:
 193             if not cls._check_upperifaces(ifupdownobj, ifaceobj,
 194                                           ops, parent, followdependents):
 195                 return
 196
 197         # If inorder, run the iface first and then its dependents
 198         if order == ifaceSchedulerFlags.INORDER:
 199             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 200
 201         for ifaceobj in ifaceobjs:
 202             # Run lowerifaces or dependents
 203             dlist = ifaceobj.lowerifaces
 204             if dlist:
 205                 ifupdownobj.logger.debug('%s: found dependents %s'
 206                             %(ifacename, str(dlist)))
 207                 try:
 208                     if not followdependents:
 209                         # XXX: this is yet another extra step,
 210                         # but is needed for interfaces that are
 211                         # implicit dependents. even though we are asked to
 212                         # not follow dependents, we must follow the ones
 213                         # that dont have user given config. Because we own them
 214                         new_dlist = [d for d in dlist
 215                                     if ifupdownobj.is_iface_noconfig(d)]
 216                         if new_dlist:
 217                             cls.run_iface_list(ifupdownobj, new_dlist, ops,
 218                                            ifacename, order, followdependents,
 219                                            continueonfailure=False)
 220                     else:
 221                         cls.run_iface_list(ifupdownobj, dlist, ops,
 222                                             ifacename, order,
 223                                             followdependents,
 224                                             continueonfailure=False)
 225                 except Exception, e:
 226                     if (ifupdownobj.ignore_error(str(e))):
 227                         pass
 228                     else:
 229                         # Dont bring the iface up if children did not come up
 230                         ifaceobj.set_state_n_status(ifaceState.NEW,
 231                                                 ifaceStatus.ERROR)
 232                         raise
 233         if order == ifaceSchedulerFlags.POSTORDER:
 234             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 235
 236     @classmethod
 237     def run_iface_list(cls, ifupdownobj, ifacenames,
 238                        ops, parent=None, order=ifaceSchedulerFlags.POSTORDER,
 239                        followdependents=True, continueonfailure=True):
 240         """ Runs interface list """
 241
 242         for ifacename in ifacenames:
 243             try:
 244               cls.run_iface_graph(ifupdownobj, ifacename, ops, parent,
 245                       order, followdependents)
 246             except Exception, e:
 247                 if continueonfailure:
 248                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 249                         traceback.print_tb(sys.exc_info()[2])
 250                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 251                     pass
 252                 else:
 253                     if (ifupdownobj.ignore_error(str(e))):
 254                         pass
 255                     else:
 256                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 257
 258     @classmethod
 259     def run_iface_graph_upper(cls, ifupdownobj, ifacename, ops, parent=None,
 260                         followdependents=True, skip_root=False):
 261         """ runs interface by traversing all nodes rooted at itself """
 262
 263         # Each ifacename can have a list of iface objects
 264         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 265         if not ifaceobjs:
 266             raise Exception('%s: not found' %ifacename)
 267
 268         if not skip_root:
 269             # run the iface first and then its upperifaces
 270             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 271         for ifaceobj in ifaceobjs:
 272             # Run upperifaces
 273             ulist = ifaceobj.upperifaces
 274             if ulist:
 275                 ifupdownobj.logger.debug('%s: found upperifaces %s'
 276                                             %(ifacename, str(ulist)))
 277                 try:
 278                     cls.run_iface_list_upper(ifupdownobj, ulist, ops,
 279                                             ifacename,
 280                                             followdependents,
 281                                             continueonfailure=True)
 282                 except Exception, e:
 283                     if (ifupdownobj.ignore_error(str(e))):
 284                         pass
 285                     else:
 286                         raise
 287
 288     @classmethod
 289     def run_iface_list_upper(cls, ifupdownobj, ifacenames,
 290                        ops, parent=None, followdependents=True,
 291                        continueonfailure=True, skip_root=False):
 292         """ Runs interface list """
 293
 294         for ifacename in ifacenames:
 295             try:
 296               cls.run_iface_graph_upper(ifupdownobj, ifacename, ops, parent,
 297                       followdependents, skip_root)
 298             except Exception, e:
 299                 if continueonfailure:
 300                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 301                         traceback.print_tb(sys.exc_info()[2])
 302                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 303                     pass
 304                 else:
 305                     if (ifupdownobj.ignore_error(str(e))):
 306                         pass
 307                     else:
 308                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 309
 310     @classmethod
 311     def sched_ifaces(cls, ifupdownobj, ifacenames, ops,
 312                 dependency_graph=None, indegrees=None,
 313                 order=ifaceSchedulerFlags.POSTORDER,
 314                 followdependents=True):
 315         """ Runs iface dependeny graph by visiting all the nodes
 316
 317         Parameters:
 318         -----------
 319         ifupdownobj : ifupdown object (used for getting and updating iface
 320                                         object state)
 321         dependency_graph : dependency graph in adjacency list
 322                             format (contains more than one dependency graph)
 323         ops : list of operations to perform eg ['pre-up', 'up', 'post-up']
 324
 325         indegrees : indegree array if present is used to determine roots
 326                     of the graphs in the dependency_graph
 327         """
 328
 329         if not ifupdownobj.ALL or not followdependents or len(ifacenames) == 1:
 330             # If there is any interface that does exist, maybe it is a
 331             # logical interface and we have to followupperifaces
 332             followupperifaces = (True if
 333                                     [i for i in ifacenames
 334                                         if not ifupdownobj.link_exists(i)]
 335                                         else False)
 336             cls.run_iface_list(ifupdownobj, ifacenames, ops,
 337                                   parent=None,order=order,
 338                                   followdependents=followdependents)
 339             if (not ifupdownobj.ALL and
 340                     (followdependents or followupperifaces) and
 341                     'up' in ops[0]):
 342                 # If user had given a set of interfaces to bring up
 343                 # try and execute 'up' on the upperifaces
 344                 ifupdownobj.logger.info('running upperifaces if available')
 345                 cls._STATE_CHECK = False
 346                 cls.run_iface_list_upper(ifupdownobj, ifacenames, ops,
 347                                          skip_root=True)
 348                 cls._STATE_CHECK = True
 349             return
 350
 351         if ifupdownobj.config.get('skip_ifacesort', '0') == '1':
 352             # This is a backdoor to skip sorting of interfaces, if required
 353             cls.run_iface_list(ifupdownobj, ifacenames, ops,
 354                                   parent=None,order=order,
 355                                   followdependents=followdependents)
 356             return
 357
 358         run_queue = []
 359         # Get a sorted list of all interfaces
 360         if not indegrees:
 361             indegrees = OrderedDict()
 362             for ifacename in dependency_graph.keys():
 363                 indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)
 364         sorted_ifacenames = graph.topological_sort_graphs_all(dependency_graph,
 365                                                           indegrees)
 366         ifupdownobj.logger.debug('sorted ifacenames %s : '
 367                                  %str(sorted_ifacenames))
 368
 369         # From the sorted list, pick interfaces that user asked
 370         # and those that dont have any dependents first
 371         [run_queue.append(ifacename)
 372                     for ifacename in sorted_ifacenames
 373                         if ifacename in ifacenames and
 374                         not indegrees.get(ifacename)]
 375
 376         ifupdownobj.logger.debug('graph roots (interfaces that dont have '
 377                                  'dependents):' + ' %s' %str(run_queue))
 378         cls.run_iface_list(ifupdownobj, run_queue, ops,
 379                                   parent=None,order=order,
 380                                   followdependents=followdependents)