pkg/scheduler.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2013.  Cumulus Networks, Inc.
   4 # Author: Roopa Prabhu, roopa@cumulusnetworks.com
   5 #
   6 # ifaceScheduler --
   7 #    interface scheduler
   8 #
   9
  10 from statemanager import *
  11 from iface import *
  12 from graph import *
  13 from collections import deque
  14 from collections import OrderedDict
  15 import logging
  16 import traceback
  17 import sys
  18 from graph import *
  19 from collections import deque
  20 from threading import *
  21 from ifupdownbase import *
  22
  23 class ifaceSchedulerFlags():
  24     INORDER = 0x1
  25     POSTORDER = 0x2
  26
  27 class ifaceScheduler():
  28     """ scheduler functions to schedule configuration of interfaces.
  29
  30     supports scheduling of interfaces serially in plain interface list
  31     or dependency graph format.
  32
  33     Algo:
  34         - run topological sort on the iface objects
  35         - In the sorted iface object list, pick up interfaces with no parents
  36         and run ops on them and their children.
  37         - If operation is up and user gave interface list (ie not all)
  38         option, also see if there were upper-devices and run ops on them.
  39         - if operation is down, dont down the interface if it still
  40         has upperifaces present. The down operation is executed when the
  41         last upperiface goes away. If force option is set, this rule does not
  42         apply.
  43         - run ops calls addon modules run operation passing the iface
  44         object and op to each module.
  45         - ops are [pre-up, up, post-up, pre-down, down,
  46                    post-down, query-running, query-check]
  47     """
  48
  49     _STATE_CHECK = True
  50
  51     _SCHED_RETVAL = True
  52
  53     @classmethod
  54     def run_iface_op(cls, ifupdownobj, ifaceobj, op, cenv=None):
  55         """ Runs sub operation on an interface """
  56         ifacename = ifaceobj.name
  57
  58         if (cls._STATE_CHECK and
  59             (ifaceobj.state >= ifaceState.from_str(op))):
  60             ifupdownobj.logger.debug('%s: already in state %s' %(ifacename, op))
  61             return
  62         if not ifupdownobj.ADDONS_ENABLE: return
  63         if op == 'query-checkcurr':
  64             query_ifaceobj=ifupdownobj.create_n_save_ifaceobjcurr(ifaceobj)
  65         for mname in ifupdownobj.module_ops.get(op):
  66             m = ifupdownobj.modules.get(mname)
  67             err = 0
  68             try:
  69                 if hasattr(m, 'run'):
  70                     msg = ('%s: %s : running module %s' %(ifacename, op, mname))
  71                     if op == 'query-checkcurr':
  72                         # Dont check curr if the interface object was
  73                         # auto generated
  74                         if (ifaceobj.priv_flags & ifupdownobj.NOCONFIG):
  75                             continue
  76                         ifupdownobj.logger.debug(msg)
  77                         m.run(ifaceobj, op, query_ifaceobj)
  78                     else:
  79                         ifupdownobj.logger.debug(msg)
  80                         m.run(ifaceobj, op)
  81             except Exception, e:
  82                 err = 1
  83                 ifupdownobj.log_error(str(e))
  84                 err = 0  # error can be ignored by log_error, in which case
  85                          # reset err flag
  86             finally:
  87                 if err or ifaceobj.status == ifaceStatus.ERROR:
  88                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  89                                                 ifaceStatus.ERROR)
  90                     if 'up' in  op or 'down' in op:
  91                         cls._SCHED_RETVAL = False
  92                 else:
  93                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  94                                                 ifaceStatus.SUCCESS)
  95
  96         if ifupdownobj.COMPAT_EXEC_SCRIPTS:
  97             # execute /etc/network/ scripts
  98             for mname in ifupdownobj.script_ops.get(op, []):
  99                 ifupdownobj.logger.debug('%s: %s : running script %s'
 100                     %(ifacename, op, mname))
 101                 try:
 102                     ifupdownobj.exec_command(mname, cmdenv=cenv)
 103                 except Exception, e:
 104                     ifupdownobj.log_error(str(e))
 105
 106     @classmethod
 107     def run_iface_list_ops(cls, ifupdownobj, ifaceobjs, ops):
 108         """ Runs all operations on a list of interface
 109             configurations for the same interface
 110         """
 111         # minor optimization. If operation is 'down', proceed only
 112         # if interface exists in the system
 113         ifacename = ifaceobjs[0].name
 114         if ('down' in ops[0] and
 115                 not ifupdownobj.link_exists(ifacename)):
 116             ifupdownobj.logger.debug('%s: does not exist' %ifacename)
 117             # run posthook before you get out of here, so that
 118             # appropriate cleanup is done
 119             posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 120             if posthookfunc:
 121                 for ifaceobj in ifaceobjs:
 122                     ifaceobj.status = ifaceStatus.SUCCESS
 123                     posthookfunc(ifupdownobj, ifaceobj, 'down')
 124             return
 125         for op in ops:
 126             # first run ifupdownobj handlers. This is good enough
 127             # for the first object in the list
 128             handler = ifupdownobj.ops_handlers.get(op)
 129             if handler:
 130                 if (not ifaceobjs[0].addr_method or
 131                     (ifaceobjs[0].addr_method and
 132                     ifaceobjs[0].addr_method != 'manual')):
 133                     handler(ifupdownobj, ifaceobjs[0])
 134             for ifaceobj in ifaceobjs:
 135                 cls.run_iface_op(ifupdownobj, ifaceobj, op,
 136                     cenv=ifupdownobj.generate_running_env(ifaceobj, op)
 137                         if ifupdownobj.COMPAT_EXEC_SCRIPTS else None)
 138                 posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 139                 if posthookfunc:
 140                     posthookfunc(ifupdownobj, ifaceobj, op)
 141
 142     @classmethod
 143     def _check_upperifaces(cls, ifupdownobj, ifaceobj, ops, parent,
 144                            followdependents=False):
 145         """ Check if upperifaces are hanging off us and help caller decide
 146         if he can proceed with the ops on this device
 147
 148         Returns True or False indicating the caller to proceed with the
 149         operation.
 150         """
 151         # proceed only for down operation
 152         if 'down' not in ops[0]:
 153             return True
 154
 155         if (ifupdownobj.FORCE or
 156                 not ifupdownobj.ADDONS_ENABLE or
 157                 (not ifupdownobj.is_ifaceobj_noconfig(ifaceobj) and
 158                 ifupdownobj.config.get('warn_on_ifdown', '0') == '0' and
 159                 not ifupdownobj.ALL)):
 160             return True
 161
 162         ulist = ifaceobj.upperifaces
 163         if not ulist:
 164             return True
 165
 166         # Get the list of upper ifaces other than the parent
 167         tmpulist = ([u for u in ulist if u != parent] if parent
 168                     else ulist)
 169         if not tmpulist:
 170             return True
 171         # XXX: This is expensive. Find a cheaper way to do this.
 172         # if any of the upperdevs are present,
 173         # return false to the caller to skip this interface
 174         for u in tmpulist:
 175             if ifupdownobj.link_exists(u):
 176                 if not ifupdownobj.ALL:
 177                     if ifupdownobj.is_ifaceobj_noconfig(ifaceobj):
 178                         ifupdownobj.logger.info('%s: skipping interface down,'
 179                             %ifaceobj.name + ' upperiface %s still around ' %u)
 180                     else:
 181                         ifupdownobj.logger.warn('%s: skipping interface down,'
 182                             %ifaceobj.name + ' upperiface %s still around ' %u)
 183                 return False
 184         return True
 185
 186     @classmethod
 187     def run_iface_graph(cls, ifupdownobj, ifacename, ops, parent=None,
 188                         order=ifaceSchedulerFlags.POSTORDER,
 189                         followdependents=True):
 190         """ runs interface by traversing all nodes rooted at itself """
 191
 192         # Each ifacename can have a list of iface objects
 193         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 194         if not ifaceobjs:
 195             raise Exception('%s: not found' %ifacename)
 196
 197         for ifaceobj in ifaceobjs:
 198             if not cls._check_upperifaces(ifupdownobj, ifaceobj,
 199                                           ops, parent, followdependents):
 200                 return
 201
 202         # If inorder, run the iface first and then its dependents
 203         if order == ifaceSchedulerFlags.INORDER:
 204             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 205
 206         for ifaceobj in ifaceobjs:
 207             # Run lowerifaces or dependents
 208             dlist = ifaceobj.lowerifaces
 209             if dlist:
 210                 ifupdownobj.logger.debug('%s: found dependents %s'
 211                             %(ifacename, str(dlist)))
 212                 try:
 213                     if not followdependents:
 214                         # XXX: this is yet another extra step,
 215                         # but is needed for interfaces that are
 216                         # implicit dependents. even though we are asked to
 217                         # not follow dependents, we must follow the ones
 218                         # that dont have user given config. Because we own them
 219                         new_dlist = [d for d in dlist
 220                                     if ifupdownobj.is_iface_noconfig(d)]
 221                         if new_dlist:
 222                             cls.run_iface_list(ifupdownobj, new_dlist, ops,
 223                                            ifacename, order, followdependents,
 224                                            continueonfailure=False)
 225                     else:
 226                         cls.run_iface_list(ifupdownobj, dlist, ops,
 227                                             ifacename, order,
 228                                             followdependents,
 229                                             continueonfailure=False)
 230                 except Exception, e:
 231                     if (ifupdownobj.ignore_error(str(e))):
 232                         pass
 233                     else:
 234                         # Dont bring the iface up if children did not come up
 235                         ifaceobj.set_state_n_status(ifaceState.NEW,
 236                                                 ifaceStatus.ERROR)
 237                         raise
 238         if order == ifaceSchedulerFlags.POSTORDER:
 239             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 240
 241     @classmethod
 242     def run_iface_list(cls, ifupdownobj, ifacenames,
 243                        ops, parent=None, order=ifaceSchedulerFlags.POSTORDER,
 244                        followdependents=True, continueonfailure=True):
 245         """ Runs interface list """
 246
 247         for ifacename in ifacenames:
 248             try:
 249               cls.run_iface_graph(ifupdownobj, ifacename, ops, parent,
 250                       order, followdependents)
 251             except Exception, e:
 252                 if continueonfailure:
 253                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 254                         traceback.print_tb(sys.exc_info()[2])
 255                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 256                     pass
 257                 else:
 258                     if (ifupdownobj.ignore_error(str(e))):
 259                         pass
 260                     else:
 261                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 262
 263     @classmethod
 264     def run_iface_graph_upper(cls, ifupdownobj, ifacename, ops, parent=None,
 265                         followdependents=True, skip_root=False):
 266         """ runs interface by traversing all nodes rooted at itself """
 267
 268         # Each ifacename can have a list of iface objects
 269         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 270         if not ifaceobjs:
 271             raise Exception('%s: not found' %ifacename)
 272
 273         if not skip_root:
 274             # run the iface first and then its upperifaces
 275             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 276         for ifaceobj in ifaceobjs:
 277             # Run upperifaces
 278             ulist = ifaceobj.upperifaces
 279             if ulist:
 280                 ifupdownobj.logger.debug('%s: found upperifaces %s'
 281                                             %(ifacename, str(ulist)))
 282                 try:
 283                     cls.run_iface_list_upper(ifupdownobj, ulist, ops,
 284                                             ifacename,
 285                                             followdependents,
 286                                             continueonfailure=True)
 287                 except Exception, e:
 288                     if (ifupdownobj.ignore_error(str(e))):
 289                         pass
 290                     else:
 291                         raise
 292
 293     @classmethod
 294     def run_iface_list_upper(cls, ifupdownobj, ifacenames,
 295                        ops, parent=None, followdependents=True,
 296                        continueonfailure=True, skip_root=False):
 297         """ Runs interface list """
 298
 299         for ifacename in ifacenames:
 300             try:
 301               cls.run_iface_graph_upper(ifupdownobj, ifacename, ops, parent,
 302                       followdependents, skip_root)
 303             except Exception, e:
 304                 if continueonfailure:
 305                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 306                         traceback.print_tb(sys.exc_info()[2])
 307                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 308                     pass
 309                 else:
 310                     if (ifupdownobj.ignore_error(str(e))):
 311                         pass
 312                     else:
 313                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 314
 315     @classmethod
 316     def get_sorted_iface_list(cls, ifupdownobj, ifacenames, ops,
 317                               dependency_graph, indegrees=None):
 318         if len(ifacenames) == 1:
 319             return ifacenames
 320         # Get a sorted list of all interfaces
 321         if not indegrees:
 322             indegrees = OrderedDict()
 323             for ifacename in dependency_graph.keys():
 324                 indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)
 325         ifacenames_all_sorted = graph.topological_sort_graphs_all(
 326                                         dependency_graph, indegrees)
 327         # if ALL was set, return all interfaces
 328         if ifupdownobj.ALL:
 329             return ifacenames_all_sorted
 330
 331         # else return ifacenames passed as argument in sorted order
 332         ifacenames_sorted = []
 333         [ifacenames_sorted.append(ifacename)
 334                         for ifacename in ifacenames_all_sorted
 335                             if ifacename in ifacenames]
 336         return ifacenames_sorted
 337
 338     @classmethod
 339     def sched_ifaces(cls, ifupdownobj, ifacenames, ops,
 340                 dependency_graph=None, indegrees=None,
 341                 order=ifaceSchedulerFlags.POSTORDER,
 342                 followdependents=True):
 343         """ Runs iface dependeny graph by visiting all the nodes
 344
 345         Parameters:
 346         -----------
 347         ifupdownobj : ifupdown object (used for getting and updating iface
 348                                         object state)
 349         dependency_graph : dependency graph in adjacency list
 350                             format (contains more than one dependency graph)
 351         ops : list of operations to perform eg ['pre-up', 'up', 'post-up']
 352
 353         indegrees : indegree array if present is used to topologically sort
 354                     the graphs in the dependency_graph
 355         """
 356         #
 357         # Algo:
 358         # if ALL/auto interfaces are specified,
 359         #   - walk the dependency tree in postorder or inorder depending
 360         #     on the operation.
 361         #     (This is to run interfaces correctly in order)
 362         # else:
 363         #   - sort iface list if the ifaces belong to a "class"
 364         #   - else just run iface list in the order they were specified
 365         #
 366         # Run any upperifaces if available
 367         #
 368         followupperifaces = []
 369         run_queue = []
 370         skip_ifacesort = int(ifupdownobj.config.get('skip_ifacesort', '0'))
 371         if not skip_ifacesort and not indegrees:
 372             indegrees = OrderedDict()
 373             for ifacename in dependency_graph.keys():
 374                 indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)
 375
 376         if not ifupdownobj.ALL:
 377             # If there is any interface that does exist, maybe it is a
 378             # logical interface and we have to followupperifaces when it
 379             # comes up, so get that list.
 380             followupperifaces = (True if
 381                                     [i for i in ifacenames
 382                                         if not ifupdownobj.link_exists(i)]
 383                                         else False)
 384             if not skip_ifacesort and ifupdownobj.IFACE_CLASS:
 385                 # sort interfaces only if allow class was specified and
 386                 # not skip_ifacesort
 387                 run_queue = cls.get_sorted_iface_list(ifupdownobj, ifacenames,
 388                                     ops, dependency_graph, indegrees)
 389                 if run_queue and 'up' in ops[0]:
 390                     run_queue.reverse()
 391         else:
 392             # if -a is set, we dont really have to sort. We pick the interfaces
 393             # that have no parents and
 394             if not skip_ifacesort:
 395                 sorted_ifacenames = cls.get_sorted_iface_list(ifupdownobj,
 396                                             ifacenames, ops, dependency_graph,
 397                                             indegrees)
 398                 if sorted_ifacenames:
 399                     # pick interfaces that user asked
 400                     # and those that dont have any dependents first
 401                     [run_queue.append(ifacename)
 402                         for ifacename in sorted_ifacenames
 403                             if ifacename in ifacenames and
 404                             not indegrees.get(ifacename)]
 405                     ifupdownobj.logger.debug('graph roots (interfaces that ' +
 406                             'dont have dependents):' + ' %s' %str(run_queue))
 407                 else:
 408                     ifupdownobj.logger.warn('interface sort returned None')
 409
 410         # If queue not present, just run interfaces that were asked by the user
 411         if not run_queue:
 412             run_queue = list(ifacenames)
 413             if 'down' in ops[0]:
 414                 run_queue.reverse()
 415
 416         # run interface list
 417         ifupdownobj.logger.info('running interfaces: %s' %str(run_queue))
 418         cls.run_iface_list(ifupdownobj, run_queue, ops,
 419                            parent=None, order=order,
 420                            followdependents=followdependents)
 421         if (((not ifupdownobj.ALL and followdependents) or
 422                 followupperifaces) and
 423                 'up' in ops[0]):
 424             # If user had given a set of interfaces to bring up
 425             # try and execute 'up' on the upperifaces
 426             ifupdownobj.logger.info('running upperifaces if available ..')
 427             cls._STATE_CHECK = False
 428             cls.run_iface_list_upper(ifupdownobj, ifacenames, ops,
 429                                      skip_root=True)
 430             cls._STATE_CHECK = True
 431
 432         if not cls._SCHED_RETVAL:
 433             raise Exception()