pkg/scheduler.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright 2013.  Cumulus Networks, Inc.
   4 # Author: Roopa Prabhu, roopa@cumulusnetworks.com
   5 #
   6 # ifaceScheduler --
   7 #    interface scheduler
   8 #
   9
  10 from statemanager import *
  11 from iface import *
  12 from graph import *
  13 from collections import deque
  14 from collections import OrderedDict
  15 import logging
  16 import traceback
  17 import sys
  18 from graph import *
  19 from collections import deque
  20 from threading import *
  21 from ifupdownbase import *
  22
  23 class ifaceSchedulerFlags():
  24     INORDER = 0x1
  25     POSTORDER = 0x2
  26
  27 class ifaceScheduler():
  28     """ scheduler functions to schedule configuration of interfaces.
  29
  30     supports scheduling of interfaces serially in plain interface list
  31     or dependency graph format.
  32
  33     Algo:
  34         - run topological sort on the iface objects
  35         - In the sorted iface object list, pick up interfaces with no parents
  36         and run ops on them and their children.
  37         - If operation is up and user gave interface list (ie not all)
  38         option, also see if there were upper-devices and run ops on them.
  39         - if operation is down, dont down the interface if it still
  40         has upperifaces present. The down operation is executed when the
  41         last upperiface goes away. If force option is set, this rule does not
  42         apply.
  43         - run ops calls addon modules run operation passing the iface
  44         object and op to each module.
  45         - ops are [pre-up, up, post-up, pre-down, down,
  46                    post-down, query-running, query-check]
  47     """
  48
  49     _STATE_CHECK = True
  50
  51     @classmethod
  52     def run_iface_op(cls, ifupdownobj, ifaceobj, op, cenv=None):
  53         """ Runs sub operation on an interface """
  54         ifacename = ifaceobj.name
  55
  56         if (cls._STATE_CHECK and
  57             (ifaceobj.state >= ifaceState.from_str(op)) and
  58             (ifaceobj.status == ifaceStatus.SUCCESS)):
  59             ifupdownobj.logger.debug('%s: already in state %s' %(ifacename, op))
  60             return
  61         if not ifupdownobj.ADDONS_ENABLE: return
  62         if op == 'query-checkcurr':
  63             query_ifaceobj=ifupdownobj.create_n_save_ifaceobjcurr(ifaceobj)
  64         for mname in ifupdownobj.module_ops.get(op):
  65             m = ifupdownobj.modules.get(mname)
  66             err = 0
  67             try:
  68                 if hasattr(m, 'run'):
  69                     msg = ('%s: %s : running module %s' %(ifacename, op, mname))
  70                     if op == 'query-checkcurr':
  71                         # Dont check curr if the interface object was
  72                         # auto generated
  73                         if (ifaceobj.priv_flags & ifupdownobj.NOCONFIG):
  74                             continue
  75                         ifupdownobj.logger.debug(msg)
  76                         m.run(ifaceobj, op, query_ifaceobj)
  77                     else:
  78                         ifupdownobj.logger.debug(msg)
  79                         m.run(ifaceobj, op)
  80             except Exception, e:
  81                 err = 1
  82                 ifupdownobj.log_error(str(e))
  83             finally:
  84                 if err:
  85                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  86                                                 ifaceStatus.ERROR)
  87                 else:
  88                     ifaceobj.set_state_n_status(ifaceState.from_str(op),
  89                                                 ifaceStatus.SUCCESS)
  90
  91         if ifupdownobj.COMPAT_EXEC_SCRIPTS:
  92             # execute /etc/network/ scripts
  93             for mname in ifupdownobj.script_ops.get(op, []):
  94                 ifupdownobj.logger.debug('%s: %s : running script %s'
  95                     %(ifacename, op, mname))
  96                 try:
  97                     ifupdownobj.exec_command(mname, cmdenv=cenv)
  98                 except Exception, e:
  99                     ifupdownobj.log_error(str(e))
 100
 101     @classmethod
 102     def run_iface_list_ops(cls, ifupdownobj, ifaceobjs, ops):
 103         """ Runs all operations on a list of interface
 104             configurations for the same interface
 105         """
 106         # minor optimization. If operation is 'down', proceed only
 107         # if interface exists in the system
 108         ifacename = ifaceobjs[0].name
 109         if ('down' in ops[0] and
 110                 not ifupdownobj.link_exists(ifacename)):
 111             ifupdownobj.logger.debug('%s: does not exist' %ifacename)
 112             # run posthook before you get out of here, so that
 113             # appropriate cleanup is done
 114             posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 115             if posthookfunc:
 116                 for ifaceobj in ifaceobjs:
 117                     ifaceobj.status = ifaceStatus.SUCCESS
 118                     posthookfunc(ifupdownobj, ifaceobj, 'down')
 119             return
 120         for op in ops:
 121             # first run ifupdownobj handlers. This is good enough
 122             # for the first object in the list
 123             handler = ifupdownobj.ops_handlers.get(op)
 124             if handler:
 125                 if (not ifaceobjs[0].addr_method or
 126                     (ifaceobjs[0].addr_method and
 127                     ifaceobjs[0].addr_method != 'manual')):
 128                     handler(ifupdownobj, ifaceobjs[0])
 129             for ifaceobj in ifaceobjs:
 130                 cls.run_iface_op(ifupdownobj, ifaceobj, op,
 131                     cenv=ifupdownobj.generate_running_env(ifaceobj, op)
 132                         if ifupdownobj.COMPAT_EXEC_SCRIPTS else None)
 133                 posthookfunc = ifupdownobj.sched_hooks.get('posthook')
 134                 if posthookfunc:
 135                     posthookfunc(ifupdownobj, ifaceobj, op)
 136
 137     @classmethod
 138     def _check_upperifaces(cls, ifupdownobj, ifaceobj, ops, parent,
 139                            followdependents=False):
 140         """ Check if upperifaces are hanging off us and help caller decide
 141         if he can proceed with the ops on this device
 142
 143         Returns True or False indicating the caller to proceed with the
 144         operation.
 145         """
 146         # proceed only for down operation
 147         if 'down' not in ops[0]:
 148             return True
 149
 150         if (ifupdownobj.FORCE or
 151                 not ifupdownobj.ADDONS_ENABLE or
 152                 (not ifupdownobj.is_ifaceobj_noconfig(ifaceobj) and
 153                 ifupdownobj.config.get('warn_on_ifdown', '0') == '0')):
 154             return True
 155
 156         ulist = ifaceobj.upperifaces
 157         if not ulist:
 158             return True
 159         # Get the list of upper ifaces other than the parent
 160         tmpulist = ([u for u in ulist if u != parent] if parent
 161                     else ulist)
 162         if not tmpulist:
 163             return True
 164         # XXX: This is expensive. Find a cheaper way to do this.
 165         # if any of the upperdevs are present,
 166         # return false to the caller to skip this interface
 167         for u in tmpulist:
 168             if ifupdownobj.link_exists(u):
 169                 if not ifupdownobj.ALL:
 170                     if ifupdownobj.is_ifaceobj_noconfig(ifaceobj):
 171                         ifupdownobj.logger.info('%s: skipping interface down,'
 172                             %ifaceobj.name + ' upperiface %s still around ' %u)
 173                     else:
 174                         ifupdownobj.logger.warn('%s: skipping interface down,'
 175                             %ifaceobj.name + ' upperiface %s still around ' %u)
 176                 return False
 177         return True
 178
 179     @classmethod
 180     def run_iface_graph(cls, ifupdownobj, ifacename, ops, parent=None,
 181                         order=ifaceSchedulerFlags.POSTORDER,
 182                         followdependents=True):
 183         """ runs interface by traversing all nodes rooted at itself """
 184
 185         # Each ifacename can have a list of iface objects
 186         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 187         if not ifaceobjs:
 188             raise Exception('%s: not found' %ifacename)
 189
 190         for ifaceobj in ifaceobjs:
 191             if not cls._check_upperifaces(ifupdownobj, ifaceobj,
 192                                           ops, parent, followdependents):
 193                 return
 194
 195         # If inorder, run the iface first and then its dependents
 196         if order == ifaceSchedulerFlags.INORDER:
 197             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 198
 199         for ifaceobj in ifaceobjs:
 200             # Run lowerifaces or dependents
 201             dlist = ifaceobj.lowerifaces
 202             if dlist:
 203                 ifupdownobj.logger.debug('%s: found dependents %s'
 204                             %(ifacename, str(dlist)))
 205                 try:
 206                     if not followdependents:
 207                         # XXX: this is yet another extra step,
 208                         # but is needed for interfaces that are
 209                         # implicit dependents. even though we are asked to
 210                         # not follow dependents, we must follow the ones
 211                         # that dont have user given config. Because we own them
 212                         new_dlist = [d for d in dlist
 213                                     if ifupdownobj.is_iface_noconfig(d)]
 214                         if new_dlist:
 215                             cls.run_iface_list(ifupdownobj, new_dlist, ops,
 216                                            ifacename, order, followdependents,
 217                                            continueonfailure=False)
 218                     else:
 219                         cls.run_iface_list(ifupdownobj, dlist, ops,
 220                                             ifacename, order,
 221                                             followdependents,
 222                                             continueonfailure=False)
 223                 except Exception, e:
 224                     if (ifupdownobj.ignore_error(str(e))):
 225                         pass
 226                     else:
 227                         # Dont bring the iface up if children did not come up
 228                         ifaceobj.set_state_n_status(ifaceState.NEW,
 229                                                 ifaceStatus.ERROR)
 230                         raise
 231         if order == ifaceSchedulerFlags.POSTORDER:
 232             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 233
 234     @classmethod
 235     def run_iface_list(cls, ifupdownobj, ifacenames,
 236                        ops, parent=None, order=ifaceSchedulerFlags.POSTORDER,
 237                        followdependents=True, continueonfailure=True):
 238         """ Runs interface list """
 239
 240         for ifacename in ifacenames:
 241             try:
 242               cls.run_iface_graph(ifupdownobj, ifacename, ops, parent,
 243                       order, followdependents)
 244             except Exception, e:
 245                 if continueonfailure:
 246                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 247                         traceback.print_tb(sys.exc_info()[2])
 248                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 249                     pass
 250                 else:
 251                     if (ifupdownobj.ignore_error(str(e))):
 252                         pass
 253                     else:
 254                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 255
 256     @classmethod
 257     def run_iface_graph_upper(cls, ifupdownobj, ifacename, ops, parent=None,
 258                         followdependents=True, skip_root=False):
 259         """ runs interface by traversing all nodes rooted at itself """
 260
 261         # Each ifacename can have a list of iface objects
 262         ifaceobjs = ifupdownobj.get_ifaceobjs(ifacename)
 263         if not ifaceobjs:
 264             raise Exception('%s: not found' %ifacename)
 265
 266         if not skip_root:
 267             # run the iface first and then its upperifaces
 268             cls.run_iface_list_ops(ifupdownobj, ifaceobjs, ops)
 269         for ifaceobj in ifaceobjs:
 270             # Run upperifaces
 271             ulist = ifaceobj.upperifaces
 272             if ulist:
 273                 ifupdownobj.logger.debug('%s: found upperifaces %s'
 274                                             %(ifacename, str(ulist)))
 275                 try:
 276                     cls.run_iface_list_upper(ifupdownobj, ulist, ops,
 277                                             ifacename,
 278                                             followdependents,
 279                                             continueonfailure=True)
 280                 except Exception, e:
 281                     if (ifupdownobj.ignore_error(str(e))):
 282                         pass
 283                     else:
 284                         raise
 285
 286     @classmethod
 287     def run_iface_list_upper(cls, ifupdownobj, ifacenames,
 288                        ops, parent=None, followdependents=True,
 289                        continueonfailure=True, skip_root=False):
 290         """ Runs interface list """
 291
 292         for ifacename in ifacenames:
 293             try:
 294               cls.run_iface_graph_upper(ifupdownobj, ifacename, ops, parent,
 295                       followdependents, skip_root)
 296             except Exception, e:
 297                 if continueonfailure:
 298                     if ifupdownobj.logger.isEnabledFor(logging.DEBUG):
 299                         traceback.print_tb(sys.exc_info()[2])
 300                     ifupdownobj.logger.error('%s : %s' %(ifacename, str(e)))
 301                     pass
 302                 else:
 303                     if (ifupdownobj.ignore_error(str(e))):
 304                         pass
 305                     else:
 306                         raise Exception('%s : (%s)' %(ifacename, str(e)))
 307
 308     @classmethod
 309     def sched_ifaces(cls, ifupdownobj, ifacenames, ops,
 310                 dependency_graph=None, indegrees=None,
 311                 order=ifaceSchedulerFlags.POSTORDER,
 312                 followdependents=True):
 313         """ Runs iface dependeny graph by visiting all the nodes
 314
 315         Parameters:
 316         -----------
 317         ifupdownobj : ifupdown object (used for getting and updating iface
 318                                         object state)
 319         dependency_graph : dependency graph in adjacency list
 320                             format (contains more than one dependency graph)
 321         ops : list of operations to perform eg ['pre-up', 'up', 'post-up']
 322
 323         indegrees : indegree array if present is used to determine roots
 324                     of the graphs in the dependency_graph
 325         """
 326
 327         if not ifupdownobj.ALL or not followdependents or len(ifacenames) == 1:
 328             # If there is any interface that does exist, maybe it is a
 329             # logical interface and we have to followupperifaces
 330             followupperifaces = (True if
 331                                     [i for i in ifacenames
 332                                         if not ifupdownobj.link_exists(i)]
 333                                         else False)
 334             cls.run_iface_list(ifupdownobj, ifacenames, ops,
 335                                   parent=None,order=order,
 336                                   followdependents=followdependents)
 337             if (not ifupdownobj.ALL and
 338                     (followdependents or followupperifaces) and
 339                     'up' in ops[0]):
 340                 # If user had given a set of interfaces to bring up
 341                 # try and execute 'up' on the upperifaces
 342                 ifupdownobj.logger.info('running upperifaces if available')
 343                 cls._STATE_CHECK = False
 344                 cls.run_iface_list_upper(ifupdownobj, ifacenames, ops,
 345                                          skip_root=True)
 346                 cls._STATE_CHECK = True
 347             return
 348
 349         if ifupdownobj.config.get('skip_ifacesort', '0') == '1':
 350             # This is a backdoor to skip sorting of interfaces, if required
 351             cls.run_iface_list(ifupdownobj, ifacenames, ops,
 352                                   parent=None,order=order,
 353                                   followdependents=followdependents)
 354             return
 355
 356         run_queue = []
 357         # Get a sorted list of all interfaces
 358         if not indegrees:
 359             indegrees = OrderedDict()
 360             for ifacename in dependency_graph.keys():
 361                 indegrees[ifacename] = ifupdownobj.get_iface_refcnt(ifacename)
 362         sorted_ifacenames = graph.topological_sort_graphs_all(dependency_graph,
 363                                                           indegrees)
 364         ifupdownobj.logger.debug('sorted ifacenames %s : '
 365                                  %str(sorted_ifacenames))
 366
 367         # From the sorted list, pick interfaces that user asked
 368         # and those that dont have any dependents first
 369         [run_queue.append(ifacename)
 370                     for ifacename in sorted_ifacenames
 371                         if ifacename in ifacenames and
 372                         not indegrees.get(ifacename)]
 373
 374         ifupdownobj.logger.debug('graph roots (interfaces that dont have '
 375                                  'dependents):' + ' %s' %str(run_queue))
 376         cls.run_iface_list(ifupdownobj, run_queue, ops,
 377                                   parent=None,order=order,
 378                                   followdependents=followdependents)