addons: vrf: redo iproute2 vrf interface map handling

Ticket: CM-10188, CM-10061
Reviewed By: dsa, nikhil, julien
Testing Done: Tested static routes with vrf names for tables

This patch does the following:
- if a single vrf device is present in the config,
builds the vrf map by reading vrf interfaces from the kernel (with
existing link cache. Builds a shadow vrf only attribute cache)
- reads existing table map and adjusts it if required
- main change is the iproute2 map file on disk is updated
immediately on vrf creation, so that static routes used along with the
vrf slaves can use the vrf name for the table. This also helps dhclient dns
hook script which may use mgmt table name directly.
- cleans up default routes on down

Signed-off-by: Roopa Prabhu <roopa@cumulusnetworks.com>

diff --git a/addons/vrf.py b/addons/vrf.py
index ff4a348a760ae5d8a51e038f5c2fa9c124d76e20..bb25427326bb9e6f9a327fbca3e5dee54106ad30 100644 (file)
--- a/addons/vrf.py
+++ b/addons/vrf.py
@@ -81,46 +81,83 @@ class vrf(moduleBase):
         #self.logger.info("vrf: ip -6 rule cache")
         #self.logger.info(self.ip6_rule_cache)
 
+        self._iproute2_vrf_map_initialized = False
+        self.iproute2_vrf_map = {}
+        self.iproute2_vrf_map_fd = None
+        self.iproute2_vrf_map_sync_to_disk = False
+
+        self.vrf_table_id_start = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-start')
+        if not self.vrf_table_id_start:
+            self.vrf_table_id_start = self.VRF_TABLE_START
+        self.vrf_table_id_end = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-end')
+        if not self.vrf_table_id_end:
+            self.vrf_table_id_end = self.VRF_TABLE_END
+        self.vrf_max_count = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-max-count')
+
+        self.vrf_fix_local_table = True
+        self.vrf_count = 0
+        self.vrf_cgroup_create = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-cgroup-create')
+        if not self.vrf_cgroup_create:
+            self.vrf_cgroup_create = False
+        elif self.vrf_cgroup_create == 'yes':
+            self.vrf_cgroup_create = True
+        else:
+            self.vrf_cgroup_create = False
+        self.vrf_mgmt_devname = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-mgmt-devname')
+
+    def _iproute2_vrf_map_initialize(self):
+        if self._iproute2_vrf_map_initialized:
+            return
+
         # XXX: check for vrf reserved overlap in /etc/iproute2/rt_tables
         self.iproute2_vrf_map = {}
+        iproute2_vrf_map_force_rewrite = False
         # read or create /etc/iproute2/rt_tables.d/ifupdown2.vrf_map
         if os.path.exists(self.iproute2_vrf_filename):
-            self.vrf_map_fd = open(self.iproute2_vrf_filename, 'a+')
-            lines = self.vrf_map_fd.readlines()
+            vrf_map_fd = open(self.iproute2_vrf_filename, 'r+')
+            lines = vrf_map_fd.readlines()
             for l in lines:
                 l = l.strip()
                 if l[0] == '#':
                     continue
                 try:
                     (table, vrf_name) = l.strip().split()
-                    self.iproute2_vrf_map[table] = vrf_name
+                    if self.iproute2_vrf_map.get(int(table)):
+                        # looks like the existing file has
+                        # duplicate entries, force rewrite of the
+                        # file
+                        iproute2_vrf_map_force_rewrite = True
+                        continue
+                    self.iproute2_vrf_map[int(table)] = vrf_name
                 except Exception, e:
                     self.logger.info('vrf: iproute2_vrf_map: unable to parse %s'
                                      %l)
                     pass
-        #self.logger.info("vrf: dumping iproute2_vrf_map")
-        #self.logger.info(self.iproute2_vrf_map)
 
-        # purge vrf table entries that are not around
-        iproute2_vrf_map_pruned = {}
-        for t, v in self.iproute2_vrf_map.iteritems():
-            if os.path.exists('/sys/class/net/%s' %v):
-                iproute2_vrf_map_pruned[int(t)] = v
-            else:
-                try:
-                    # cleanup rules
-                    self._del_vrf_rules(v, t)
-                except Exception:
-                    pass
-        self.iproute2_vrf_map = iproute2_vrf_map_pruned
+        vrfs = self.ipcmd.link_get_vrfs()
+        running_vrf_map = {}
+        if vrfs:
+            for v, lattrs in vrfs.iteritems():
+                table = lattrs.get('table', None)
+                if table:
+                   running_vrf_map[int(table)] = v
+
+        if running_vrf_map and (running_vrf_map != self.iproute2_vrf_map):
+            self.iproute2_vrf_map = running_vrf_map
+            iproute2_vrf_map_force_rewrite = True
+
+        self.iproute2_vrf_map_fd = None
+        if iproute2_vrf_map_force_rewrite:
+            # reopen the file and rewrite the map
+            self._iproute2_vrf_map_open(True, False)
+        else:
+            self._iproute2_vrf_map_open(False, True)
 
-        self.vrf_table_id_start = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-start')
-        if not self.vrf_table_id_start:
-            self.vrf_table_id_start = self.VRF_TABLE_START
-        self.vrf_table_id_end = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-end')
-        if not self.vrf_table_id_end:
-            self.vrf_table_id_end = self.VRF_TABLE_END
-        self.vrf_max_count = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-max-count')
+        self.iproute2_vrf_map_sync_to_disk = False
+        atexit.register(self._iproute2_vrf_map_sync_to_disk)
+
+        self.logger.info("vrf: dumping iproute2_vrf_map")
+        self.logger.info(self.iproute2_vrf_map)
 
         last_used_vrf_table = None
         for t in range(self.vrf_table_id_start,
@@ -129,31 +166,40 @@ class vrf(moduleBase):
                 break
             last_used_vrf_table = t
         self.last_used_vrf_table = last_used_vrf_table
+        self._iproute2_vrf_map_initialized = True
 
-        self.iproute2_write_vrf_map = False
-        atexit.register(self.iproute2_vrf_map_write)
-        self.vrf_fix_local_table = True
-        self.vrf_count = 0
-        self.vrf_cgroup_create = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-cgroup-create')
-        if not self.vrf_cgroup_create:
-            self.vrf_cgroup_create = False
-        elif self.vrf_cgroup_create == 'yes':
-            self.vrf_cgroup_create = True
-        else:
-            self.vrf_cgroup_create = False
-
-        self.vrf_mgmt_devname = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-mgmt-devname')
-
-    def iproute2_vrf_map_write(self):
-        if not self.iproute2_write_vrf_map:
+    def _iproute2_vrf_map_sync_to_disk(self):
+        if not self.iproute2_vrf_map_sync_to_disk:
             return
-        self.logger.info('vrf: writing table map to %s'
+        self.logger.info('vrf: syncing table map to %s'
                          %self.iproute2_vrf_filename)
         with open(self.iproute2_vrf_filename, 'w') as f:
             f.write(self.iproute2_vrf_filehdr %(self.vrf_table_id_start,
                     self.vrf_table_id_end))
             for t, v in self.iproute2_vrf_map.iteritems():
                 f.write('%s %s\n' %(t, v))
+            f.flush()
+
+    def _iproute2_vrf_map_open(self, sync_vrfs=False, append=False):
+        self.logger.info('vrf: syncing table map to %s'
+                         %self.iproute2_vrf_filename)
+        fmode = 'a+' if append else 'w'
+        try:
+            self.iproute2_vrf_map_fd = open(self.iproute2_vrf_filename,
+                                         '%s' %fmode)
+        except Exception, e:
+            self.log_warn('vrf: error opening %s (%s)'
+                          %(self.iproute2_vrf_filename, str(e)))
+            return
+
+        if not append:
+            # write file header
+            self.iproute2_vrf_map_fd.write(self.iproute2_vrf_filehdr
+                                           %(self.vrf_table_id_start,
+                                             self.vrf_table_id_end))
+            for t, v in self.iproute2_vrf_map.iteritems():
+                self.iproute2_vrf_map_fd.write('%s %s\n' %(t, v))
+            self.iproute2_vrf_map_fd.flush()
 
     def _is_vrf(self, ifaceobj):
         if ifaceobj.get_attr_value_first('vrf-table'):
@@ -197,13 +243,19 @@ class vrf(moduleBase):
         return None
 
     def _iproute2_vrf_table_entry_add(self, vrf_dev_name, table_id):
-        self.iproute2_vrf_map[int(table_id)] = vrf_dev_name
-        self.iproute2_write_vrf_map = True
+        old_vrf_name = self.iproute2_vrf_map.get(int(table_id))
+        if not old_vrf_name or (old_vrf_name != vrf_dev_name):
+            self.iproute2_vrf_map[int(table_id)] = vrf_dev_name
+            if self.iproute2_vrf_map_fd:
+                self.iproute2_vrf_map_fd.write('%s %s\n'
+                                               %(table_id, vrf_dev_name))
+                self.iproute2_vrf_map_fd.flush()
 
     def _iproute2_vrf_table_entry_del(self, table_id):
         try:
+            # with any del of vrf map, we need to force sync to disk
+            self.iproute2_vrf_map_sync_to_disk = True
             del self.iproute2_vrf_map[int(table_id)]
-            self.iproute2_write_vrf_map = True
         except Exception, e:
             self.logger.info('vrf: iproute2 vrf map del failed for %d (%s)'
                              %(table_id, str(e)))
@@ -481,7 +533,7 @@ class vrf(moduleBase):
 
         return vrf_table
 
-    def _add_vrf_default_route(self, ifaceobj,  vrf_table):
+    def _add_del_vrf_default_route(self, ifaceobj,  vrf_table, add=True):
         vrf_default_route = ifaceobj.get_attr_value_first('vrf-default-route')
         if not vrf_default_route:
             vrf_default_route = policymanager.policymanager_api.get_attr_default(
@@ -491,19 +543,33 @@ class vrf(moduleBase):
             return
         if str(vrf_default_route).lower() == "yes":
             try:
-                self.exec_command('ip route add table %s unreachable default'
-                                  ' metric %d' %(vrf_table, 240))
+                if add:
+                    self.exec_command('ip route add table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
+                else:
+                    self.exec_command('ip route del table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
             except OSError, e:
-                if e.errno != 17:
+                if add and e.errno != 17:
                     raise
+                else:
+                    self.logger.info('%s: error deleting default route (%s)'
+                                     %(ifaceobj.name, str(e)))
                 pass
 
             try:
-                self.exec_command('ip -6 route add table %s unreachable '
-                                  'default metric %d' %(vrf_table, 240))
+                if add:
+                    self.exec_command('ip -6 route add table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
+                else:
+                    self.exec_command('ip -6 route del table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
             except OSError, e:
-                if e.errno != 17:
+                if add and e.errno != 17:
                     raise
+                else:
+                    self.logger.info('%s: error deleting default route (%s)'
+                                     %(ifaceobj.name, str(e)))
                 pass
 
     def _up_vrf_dev(self, ifaceobj, vrf_table, add_slaves=True,
@@ -521,7 +587,7 @@ class vrf(moduleBase):
             self._create_cgroup(ifaceobj)
             if add_slaves:
                 self._add_vrf_slaves(ifaceobj, ifaceobj_getfunc)
-            self._add_vrf_default_route(ifaceobj, vrf_table)
+            self._add_del_vrf_default_route(ifaceobj, vrf_table)
             self._set_vrf_dev_processed_flag(ifaceobj)
             rtnetlink_api.rtnl_api.link_set(ifaceobj.name, "up")
         except Exception, e:
@@ -596,6 +662,7 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 # This is a vrf device
                 if self.vrf_count == self.vrf_max_count:
                     self.log_error('%s: max vrf count %d hit...not '
@@ -605,6 +672,7 @@ class vrf(moduleBase):
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     # This is a vrf slave
                     self._up_vrf_slave(ifaceobj.name, vrf, ifaceobj,
                                        ifaceobj_getfunc)
@@ -663,6 +731,7 @@ class vrf(moduleBase):
 
         try:
             self._iproute2_vrf_table_entry_del(vrf_table)
+            self._add_del_vrf_default_route(ifaceobj, vrf_table, False)
             self._delete_cgroup(ifaceobj)
         except Exception, e:
             self.logger.info('%s: %s' %(ifaceobj.name, str(e)))
@@ -681,10 +750,12 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 self._down_vrf_dev(ifaceobj, vrf_table, ifaceobj_getfunc)
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     self._down_vrf_slave(ifaceobj.name, ifaceobj, None)
         except Exception, e:
             self.log_warn(str(e))
@@ -729,10 +800,12 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 self._query_check_vrf_dev(ifaceobj, ifaceobjcurr, vrf_table)
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     self._query_check_vrf_slave(ifaceobj, ifaceobjcurr, vrf)
         except Exception, e:
             self.log_warn(str(e))

diff --git a/ifupdown/ifupdownmain.py b/ifupdown/ifupdownmain.py
index cb5ac7a876e4f155a8c51f4ba87e66d82714ad0b..0f6a019f74167ba3a377d5a77867370fa7e2ba8c 100644 (file)
--- a/ifupdown/ifupdownmain.py
+++ b/ifupdown/ifupdownmain.py
@@ -1440,6 +1440,8 @@ class ifupdownMain(ifupdownBase):
         else:
             # oldconfig not available, continue with 'up' with new config
             op = 'up'
+            new_ifaceobjdict = self.ifaceobjdict
+            new_dependency_graph = self.dependency_graph
 
         if op == 'reload' and ifacenames:
             ifacenames = self.ifaceobjdict.keys()

diff --git a/ifupdownaddons/cache.py b/ifupdownaddons/cache.py
index 61773b5aef1ee67eedcefed24fea42745aff3ad1..311082e08181b2809759c1cf5d4f3c2a7cc930d0 100644 (file)
--- a/ifupdownaddons/cache.py
+++ b/ifupdownaddons/cache.py
@@ -23,6 +23,8 @@ class linkCache():
                                         <ports> : {
                                                   } """
     links = {}
+    vrfs = {}
+
     @classmethod
     def get_attr(cls, mapList):
         return reduce(lambda d, k: d[k], mapList, linkCache.links)

diff --git a/ifupdownaddons/iproute2.py b/ifupdownaddons/iproute2.py
index add27822563b8e14518b9cdf1155353c78cb7ba2..9d842f98f7f6dbd295830ab26d46cc892316c3df 100644 (file)
--- a/ifupdownaddons/iproute2.py
+++ b/ifupdownaddons/iproute2.py
@@ -24,10 +24,16 @@ class iproute2(utilsBase):
 
     def __init__(self, *args, **kargs):
         utilsBase.__init__(self, *args, **kargs)
-        if self.CACHE and not iproute2._cache_fill_done:
+        if self.CACHE:
+            self._fill_cache()
+
+    def _fill_cache(self):
+        if not iproute2._cache_fill_done:
             self._link_fill()
             self._addr_fill()
             iproute2._cache_fill_done = True
+            return True
+        return False
         
     def _link_fill(self, ifacename=None, refresh=False):
         """ fills cache with link information
@@ -99,6 +105,7 @@ class iproute2(utilsBase):
                     vattrs = {'table' : citems[i+2]}
                     linkattrs['linkinfo'] = vattrs
                     linkattrs['kind'] = 'vrf'
+                    linkCache.vrfs[ifname] = vattrs
                     break
                 elif citems[i] == 'vrf_slave':
                     linkattrs['kind'] = 'vrf_slave'
@@ -173,10 +180,8 @@ class iproute2(utilsBase):
             if self.DRYRUN:
                 return False
             if self.CACHE:
-                if not iproute2._cache_fill_done: 
-                    self._link_fill()
-                    self._addr_fill()
-                    iproute2._cache_fill_done = True
+                if self._fill_cache():
+                    # if we filled the cache, return new data
                     return linkCache.get_attr(attrlist)
                 if not refresh:
                     return linkCache.get_attr(attrlist)
@@ -847,3 +852,7 @@ class iproute2(utilsBase):
             return os.path.basename(upper[0])[6:]
         except:
             return None
+
+    def link_get_vrfs(self):
+        self._fill_cache()
+        return linkCache.vrfs