addons: vrf: redo iproute2 vrf interface map handling

Ticket: CM-10188, CM-10061
Reviewed By: dsa, nikhil, julien
Testing Done: Tested static routes with vrf names for tables

This patch does the following:
- if a single vrf device is present in the config,
builds the vrf map by reading vrf interfaces from the kernel (with
existing link cache. Builds a shadow vrf only attribute cache)
- reads existing table map and adjusts it if required
- main change is the iproute2 map file on disk is updated
immediately on vrf creation, so that static routes used along with the
vrf slaves can use the vrf name for the table. This also helps dhclient dns
hook script which may use mgmt table name directly.
- cleans up default routes on down

Signed-off-by: Roopa Prabhu <roopa@cumulusnetworks.com>

diff --git a/addons/vrf.py b/addons/vrf.py
index 8d1b3cf7d351c767c0371bd32ecb5ebc1943c9ad..338a06d34260376370277fbdb59dcd150da76f3e 100644 (file)
--- a/addons/vrf.py
+++ b/addons/vrf.py
@@ -81,46 +81,83 @@ class vrf(moduleBase):
         #self.logger.info("vrf: ip -6 rule cache")
         #self.logger.info(self.ip6_rule_cache)
 
+        self._iproute2_vrf_map_initialized = False
+        self.iproute2_vrf_map = {}
+        self.iproute2_vrf_map_fd = None
+        self.iproute2_vrf_map_sync_to_disk = False
+
+        self.vrf_table_id_start = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-start')
+        if not self.vrf_table_id_start:
+            self.vrf_table_id_start = self.VRF_TABLE_START
+        self.vrf_table_id_end = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-end')
+        if not self.vrf_table_id_end:
+            self.vrf_table_id_end = self.VRF_TABLE_END
+        self.vrf_max_count = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-max-count')
+
+        self.vrf_fix_local_table = True
+        self.vrf_count = 0
+        self.vrf_cgroup_create = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-cgroup-create')
+        if not self.vrf_cgroup_create:
+            self.vrf_cgroup_create = False
+        elif self.vrf_cgroup_create == 'yes':
+            self.vrf_cgroup_create = True
+        else:
+            self.vrf_cgroup_create = False
+        self.vrf_mgmt_devname = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-mgmt-devname')
+
+    def _iproute2_vrf_map_initialize(self):
+        if self._iproute2_vrf_map_initialized:
+            return
+
         # XXX: check for vrf reserved overlap in /etc/iproute2/rt_tables
         self.iproute2_vrf_map = {}
+        iproute2_vrf_map_force_rewrite = False
         # read or create /etc/iproute2/rt_tables.d/ifupdown2.vrf_map
         if os.path.exists(self.iproute2_vrf_filename):
-            self.vrf_map_fd = open(self.iproute2_vrf_filename, 'a+')
-            lines = self.vrf_map_fd.readlines()
+            vrf_map_fd = open(self.iproute2_vrf_filename, 'r+')
+            lines = vrf_map_fd.readlines()
             for l in lines:
                 l = l.strip()
                 if l[0] == '#':
                     continue
                 try:
                     (table, vrf_name) = l.strip().split()
-                    self.iproute2_vrf_map[table] = vrf_name
+                    if self.iproute2_vrf_map.get(int(table)):
+                        # looks like the existing file has
+                        # duplicate entries, force rewrite of the
+                        # file
+                        iproute2_vrf_map_force_rewrite = True
+                        continue
+                    self.iproute2_vrf_map[int(table)] = vrf_name
                 except Exception, e:
                     self.logger.info('vrf: iproute2_vrf_map: unable to parse %s'
                                      %l)
                     pass
-        #self.logger.info("vrf: dumping iproute2_vrf_map")
-        #self.logger.info(self.iproute2_vrf_map)
 
-        # purge vrf table entries that are not around
-        iproute2_vrf_map_pruned = {}
-        for t, v in self.iproute2_vrf_map.iteritems():
-            if os.path.exists('/sys/class/net/%s' %v):
-                iproute2_vrf_map_pruned[int(t)] = v
-            else:
-                try:
-                    # cleanup rules
-                    self._del_vrf_rules(v, t)
-                except Exception:
-                    pass
-        self.iproute2_vrf_map = iproute2_vrf_map_pruned
+        vrfs = self.ipcmd.link_get_vrfs()
+        running_vrf_map = {}
+        if vrfs:
+            for v, lattrs in vrfs.iteritems():
+                table = lattrs.get('table', None)
+                if table:
+                   running_vrf_map[int(table)] = v
+
+        if running_vrf_map and (running_vrf_map != self.iproute2_vrf_map):
+            self.iproute2_vrf_map = running_vrf_map
+            iproute2_vrf_map_force_rewrite = True
+
+        self.iproute2_vrf_map_fd = None
+        if iproute2_vrf_map_force_rewrite:
+            # reopen the file and rewrite the map
+            self._iproute2_vrf_map_open(True, False)
+        else:
+            self._iproute2_vrf_map_open(False, True)
 
-        self.vrf_table_id_start = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-start')
-        if not self.vrf_table_id_start:
-            self.vrf_table_id_start = self.VRF_TABLE_START
-        self.vrf_table_id_end = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-table-id-end')
-        if not self.vrf_table_id_end:
-            self.vrf_table_id_end = self.VRF_TABLE_END
-        self.vrf_max_count = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-max-count')
+        self.iproute2_vrf_map_sync_to_disk = False
+        atexit.register(self._iproute2_vrf_map_sync_to_disk)
+
+        self.logger.info("vrf: dumping iproute2_vrf_map")
+        self.logger.info(self.iproute2_vrf_map)
 
         last_used_vrf_table = None
         for t in range(self.vrf_table_id_start,
@@ -129,31 +166,40 @@ class vrf(moduleBase):
                 break
             last_used_vrf_table = t
         self.last_used_vrf_table = last_used_vrf_table
+        self._iproute2_vrf_map_initialized = True
 
-        self.iproute2_write_vrf_map = False
-        atexit.register(self.iproute2_vrf_map_write)
-        self.vrf_fix_local_table = True
-        self.vrf_count = 0
-        self.vrf_cgroup_create = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-cgroup-create')
-        if not self.vrf_cgroup_create:
-            self.vrf_cgroup_create = False
-        elif self.vrf_cgroup_create == 'yes':
-            self.vrf_cgroup_create = True
-        else:
-            self.vrf_cgroup_create = False
-
-        self.vrf_mgmt_devname = policymanager.policymanager_api.get_module_globals(module_name=self.__class__.__name__, attr='vrf-mgmt-devname')
-
-    def iproute2_vrf_map_write(self):
-        if not self.iproute2_write_vrf_map:
+    def _iproute2_vrf_map_sync_to_disk(self):
+        if not self.iproute2_vrf_map_sync_to_disk:
             return
-        self.logger.info('vrf: writing table map to %s'
+        self.logger.info('vrf: syncing table map to %s'
                          %self.iproute2_vrf_filename)
         with open(self.iproute2_vrf_filename, 'w') as f:
             f.write(self.iproute2_vrf_filehdr %(self.vrf_table_id_start,
                     self.vrf_table_id_end))
             for t, v in self.iproute2_vrf_map.iteritems():
                 f.write('%s %s\n' %(t, v))
+            f.flush()
+
+    def _iproute2_vrf_map_open(self, sync_vrfs=False, append=False):
+        self.logger.info('vrf: syncing table map to %s'
+                         %self.iproute2_vrf_filename)
+        fmode = 'a+' if append else 'w'
+        try:
+            self.iproute2_vrf_map_fd = open(self.iproute2_vrf_filename,
+                                         '%s' %fmode)
+        except Exception, e:
+            self.log_warn('vrf: error opening %s (%s)'
+                          %(self.iproute2_vrf_filename, str(e)))
+            return
+
+        if not append:
+            # write file header
+            self.iproute2_vrf_map_fd.write(self.iproute2_vrf_filehdr
+                                           %(self.vrf_table_id_start,
+                                             self.vrf_table_id_end))
+            for t, v in self.iproute2_vrf_map.iteritems():
+                self.iproute2_vrf_map_fd.write('%s %s\n' %(t, v))
+            self.iproute2_vrf_map_fd.flush()
 
     def _is_vrf(self, ifaceobj):
         if ifaceobj.get_attr_value_first('vrf-table'):
@@ -197,13 +243,19 @@ class vrf(moduleBase):
         return None
 
     def _iproute2_vrf_table_entry_add(self, vrf_dev_name, table_id):
-        self.iproute2_vrf_map[int(table_id)] = vrf_dev_name
-        self.iproute2_write_vrf_map = True
+        old_vrf_name = self.iproute2_vrf_map.get(int(table_id))
+        if not old_vrf_name or (old_vrf_name != vrf_dev_name):
+            self.iproute2_vrf_map[int(table_id)] = vrf_dev_name
+            if self.iproute2_vrf_map_fd:
+                self.iproute2_vrf_map_fd.write('%s %s\n'
+                                               %(table_id, vrf_dev_name))
+                self.iproute2_vrf_map_fd.flush()
 
     def _iproute2_vrf_table_entry_del(self, table_id):
         try:
+            # with any del of vrf map, we need to force sync to disk
+            self.iproute2_vrf_map_sync_to_disk = True
             del self.iproute2_vrf_map[int(table_id)]
-            self.iproute2_write_vrf_map = True
         except Exception, e:
             self.logger.info('vrf: iproute2 vrf map del failed for %d (%s)'
                              %(table_id, str(e)))
@@ -481,7 +533,7 @@ class vrf(moduleBase):
 
         return vrf_table
 
-    def _add_vrf_default_route(self, ifaceobj,  vrf_table):
+    def _add_del_vrf_default_route(self, ifaceobj,  vrf_table, add=True):
         vrf_default_route = ifaceobj.get_attr_value_first('vrf-default-route')
         if not vrf_default_route:
             vrf_default_route = policymanager.policymanager_api.get_attr_default(
@@ -491,19 +543,33 @@ class vrf(moduleBase):
             return
         if str(vrf_default_route).lower() == "yes":
             try:
-                self.exec_command('ip route add table %s unreachable default'
-                                  ' metric %d' %(vrf_table, 240))
+                if add:
+                    self.exec_command('ip route add table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
+                else:
+                    self.exec_command('ip route del table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
             except OSError, e:
-                if e.errno != 17:
+                if add and e.errno != 17:
                     raise
+                else:
+                    self.logger.info('%s: error deleting default route (%s)'
+                                     %(ifaceobj.name, str(e)))
                 pass
 
             try:
-                self.exec_command('ip -6 route add table %s unreachable '
-                                  'default metric %d' %(vrf_table, 240))
+                if add:
+                    self.exec_command('ip -6 route add table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
+                else:
+                    self.exec_command('ip -6 route del table %s unreachable '
+                                      'default metric %d' %(vrf_table, 240))
             except OSError, e:
-                if e.errno != 17:
+                if add and e.errno != 17:
                     raise
+                else:
+                    self.logger.info('%s: error deleting default route (%s)'
+                                     %(ifaceobj.name, str(e)))
                 pass
 
     def _up_vrf_dev(self, ifaceobj, vrf_table, add_slaves=True,
@@ -521,7 +587,7 @@ class vrf(moduleBase):
             self._create_cgroup(ifaceobj)
             if add_slaves:
                 self._add_vrf_slaves(ifaceobj, ifaceobj_getfunc)
-            self._add_vrf_default_route(ifaceobj, vrf_table)
+            self._add_del_vrf_default_route(ifaceobj, vrf_table)
             self._set_vrf_dev_processed_flag(ifaceobj)
             rtnetlink_api.rtnl_api.link_set(ifaceobj.name, "up")
         except Exception, e:
@@ -596,6 +662,7 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 # This is a vrf device
                 if self.vrf_count == self.vrf_max_count:
                     self.log_error('%s: max vrf count %d hit...not '
@@ -605,6 +672,7 @@ class vrf(moduleBase):
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     # This is a vrf slave
                     self._up_vrf_slave(ifaceobj.name, vrf, ifaceobj,
                                        ifaceobj_getfunc)
@@ -663,6 +731,7 @@ class vrf(moduleBase):
 
         try:
             self._iproute2_vrf_table_entry_del(vrf_table)
+            self._add_del_vrf_default_route(ifaceobj, vrf_table, False)
             self._delete_cgroup(ifaceobj)
         except Exception, e:
             self.logger.info('%s: %s' %(ifaceobj.name, str(e)))
@@ -681,10 +750,12 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 self._down_vrf_dev(ifaceobj, vrf_table, ifaceobj_getfunc)
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     self._down_vrf_slave(ifaceobj.name, ifaceobj, None)
         except Exception, e:
             self.log_warn(str(e))
@@ -729,10 +800,12 @@ class vrf(moduleBase):
         try:
             vrf_table = ifaceobj.get_attr_value_first('vrf-table')
             if vrf_table:
+                self._iproute2_vrf_map_initialize()
                 self._query_check_vrf_dev(ifaceobj, ifaceobjcurr, vrf_table)
             else:
                 vrf = ifaceobj.get_attr_value_first('vrf')
                 if vrf:
+                    self._iproute2_vrf_map_initialize()
                     self._query_check_vrf_slave(ifaceobj, ifaceobjcurr, vrf)
         except Exception, e:
             self.log_warn(str(e))

diff --git a/ifupdown/ifupdownmain.py b/ifupdown/ifupdownmain.py
index cb5ac7a876e4f155a8c51f4ba87e66d82714ad0b..0f6a019f74167ba3a377d5a77867370fa7e2ba8c 100644 (file)
--- a/ifupdown/ifupdownmain.py
+++ b/ifupdown/ifupdownmain.py
@@ -1440,6 +1440,8 @@ class ifupdownMain(ifupdownBase):
         else:
             # oldconfig not available, continue with 'up' with new config
             op = 'up'
+            new_ifaceobjdict = self.ifaceobjdict
+            new_dependency_graph = self.dependency_graph
 
         if op == 'reload' and ifacenames:
             ifacenames = self.ifaceobjdict.keys()

diff --git a/ifupdownaddons/cache.py b/ifupdownaddons/cache.py
index 61773b5aef1ee67eedcefed24fea42745aff3ad1..311082e08181b2809759c1cf5d4f3c2a7cc930d0 100644 (file)
--- a/ifupdownaddons/cache.py
+++ b/ifupdownaddons/cache.py
@@ -23,6 +23,8 @@ class linkCache():
                                         <ports> : {
                                                   } """
     links = {}
+    vrfs = {}
+
     @classmethod
     def get_attr(cls, mapList):
         return reduce(lambda d, k: d[k], mapList, linkCache.links)

diff --git a/ifupdownaddons/iproute2.py b/ifupdownaddons/iproute2.py
index 15888a4c59f7a233bcfe7edc445b90f3a563dcd8..fee21b463547ac012665c0761177c1c2096329b5 100644 (file)
--- a/ifupdownaddons/iproute2.py
+++ b/ifupdownaddons/iproute2.py
@@ -24,10 +24,16 @@ class iproute2(utilsBase):
 
     def __init__(self, *args, **kargs):
         utilsBase.__init__(self, *args, **kargs)
-        if self.CACHE and not iproute2._cache_fill_done:
+        if self.CACHE:
+            self._fill_cache()
+
+    def _fill_cache(self):
+        if not iproute2._cache_fill_done:
             self._link_fill()
             self._addr_fill()
             iproute2._cache_fill_done = True
+            return True
+        return False
         
     def _link_fill(self, ifacename=None, refresh=False):
         """ fills cache with link information
@@ -99,6 +105,7 @@ class iproute2(utilsBase):
                     vattrs = {'table' : citems[i+2]}
                     linkattrs['linkinfo'] = vattrs
                     linkattrs['kind'] = 'vrf'
+                    linkCache.vrfs[ifname] = vattrs
                     break
                 elif citems[i] == 'vrf_slave':
                     linkattrs['kind'] = 'vrf_slave'
@@ -173,10 +180,8 @@ class iproute2(utilsBase):
             if self.DRYRUN:
                 return False
             if self.CACHE:
-                if not iproute2._cache_fill_done: 
-                    self._link_fill()
-                    self._addr_fill()
-                    iproute2._cache_fill_done = True
+                if self._fill_cache():
+                    # if we filled the cache, return new data
                     return linkCache.get_attr(attrlist)
                 if not refresh:
                     return linkCache.get_attr(attrlist)
@@ -847,3 +852,7 @@ class iproute2(utilsBase):
             return os.path.basename(upper[0])[6:]
         except:
             return None
+
+    def link_get_vrfs(self):
+        self._fill_cache()
+        return linkCache.vrfs