ifupdown2/addons/addressvirtual.py

   1 #!/usr/bin/env python3
   2 #
   3 # Copyright 2014-2017 Cumulus Networks, Inc. All rights reserved.
   4 # Author: Roopa Prabhu, roopa@cumulusnetworks.com
   5 #
   6
   7 import os
   8 import glob
   9 import ipaddress
  10 import subprocess
  11
  12 from collections import deque
  13
  14 try:
  15     from ifupdown2.lib.addon import AddonWithIpBlackList
  16     from ifupdown2.ifupdown.iface import *
  17     from ifupdown2.ifupdown.utils import utils
  18
  19     from ifupdown2.nlmanager.nlpacket import Link
  20
  21     from ifupdown2.ifupdownaddons.modulebase import moduleBase
  22
  23     import ifupdown2.nlmanager.ipnetwork as ipnetwork
  24
  25     import ifupdown2.ifupdown.statemanager as statemanager
  26     import ifupdown2.ifupdown.policymanager as policymanager
  27     import ifupdown2.ifupdown.ifupdownflags as ifupdownflags
  28     import ifupdown2.ifupdown.ifupdownconfig as ifupdownconfig
  29 except (ImportError, ModuleNotFoundError):
  30     from lib.addon import AddonWithIpBlackList
  31     from ifupdown.iface import *
  32     from ifupdown.utils import utils
  33
  34     from nlmanager.nlpacket import Link
  35
  36     from ifupdownaddons.modulebase import moduleBase
  37
  38     import nlmanager.ipnetwork as ipnetwork
  39
  40     import ifupdown.statemanager as statemanager
  41     import ifupdown.policymanager as policymanager
  42     import ifupdown.ifupdownflags as ifupdownflags
  43     import ifupdown.ifupdownconfig as ifupdownconfig
  44
  45
  46 class addressvirtual(AddonWithIpBlackList, moduleBase):
  47     """  ifupdown2 addon module to configure virtual addresses """
  48
  49     _modinfo = {
  50         "mhelp": "address module configures virtual addresses for interfaces. "
  51                  "It creates a macvlan interface for every mac ip address-virtual line",
  52         "attrs": {
  53             "address-virtual": {
  54                 "help": "bridge router virtual mac and ips",
  55                 "multivalue": True,
  56                 "validvals": ["<mac-ip/prefixlen-list>", ],
  57                 "example": ["address-virtual 00:11:22:33:44:01 11.0.1.1/24 11.0.1.2/24"]
  58             },
  59             "address-virtual-ipv6-addrgen": {
  60                 "help": "enable disable ipv6 link addrgenmode",
  61                 "validvals": ["on", "off"],
  62                 "default": "on",
  63                 "example": [
  64                     "address-virtual-ipv6-addrgen on",
  65                     "address-virtual-ipv6-addrgen off"
  66                 ]
  67             },
  68             "vrrp": {
  69                 "help": "VRRP support",
  70                 "multivalue": True,
  71                 "example": [
  72                     "vrrp 1 10.0.0.15/24 2001:0db8::0370:7334/64",
  73                     "vrrp 42 10.0.0.42/24"
  74                 ]
  75             }
  76         }
  77     }
  78
  79     DEFAULT_IP_METRIC = 1024
  80     ADDR_METRIC_SUPPORT = None
  81
  82     def __init__(self, *args, **kargs):
  83         AddonWithIpBlackList.__init__(self)
  84         moduleBase.__init__(self, *args, **kargs)
  85         self._bridge_fdb_query_cache = {}
  86         self.addressvirtual_with_route_metric = utils.get_boolean_from_string(
  87             policymanager.policymanager_api.get_module_globals(
  88                 module_name=self.__class__.__name__,
  89                 attr='addressvirtual_with_route_metric'
  90             ),
  91             default=True
  92         )
  93
  94         self.address_virtual_ipv6_addrgen_value_dict = {'on': 0, 'yes': 0, '0': 0, 'off': 1, 'no': 1, '1': 1}
  95
  96         if addressvirtual.ADDR_METRIC_SUPPORT is None:
  97             try:
  98                 cmd = [utils.ip_cmd, 'addr', 'help']
  99                 self.logger.info('executing %s addr help' % utils.ip_cmd)
 100
 101                 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 102                 stdout, stderr = process.communicate()
 103                 addressvirtual.ADDR_METRIC_SUPPORT = '[ metric METRIC ]' in stderr.decode() or ''
 104                 self.logger.info('address metric support: %s' % ('OK' if addressvirtual.ADDR_METRIC_SUPPORT else 'KO'))
 105             except Exception:
 106                 addressvirtual.ADDR_METRIC_SUPPORT = False
 107                 self.logger.info('address metric support: KO')
 108
 109     @classmethod
 110     def get_addr_metric_support(cls):
 111         return cls.ADDR_METRIC_SUPPORT
 112
 113     @classmethod
 114     def get_default_ip_metric(cls):
 115         return cls.DEFAULT_IP_METRIC
 116
 117     def get_dependent_ifacenames(self, ifaceobj, ifacenames_all=None, old_ifaceobjs=False):
 118         if ifaceobj.get_attr_value('address-virtual') or ifaceobj.get_attr_value("vrrp"):
 119             ifaceobj.link_privflags |= ifaceLinkPrivFlags.ADDRESS_VIRTUAL_SLAVE
 120
 121     def _get_macvlan_prefix(self, ifaceobj):
 122         return '%s-v' %ifaceobj.name[0:13].replace('.', '-')
 123
 124     def get_vrrp_prefix(self, ifname, family):
 125         return "vrrp%s-%s-" % (family, self.cache.get_ifindex(ifname))
 126
 127     def _add_addresses_to_bridge(self, ifaceobj, hwaddress):
 128         # XXX: batch the addresses
 129         if ifaceobj.link_kind & ifaceLinkKind.VLAN:
 130             bridgename = ifaceobj.lowerifaces[0]
 131             vlan = self._get_vlan_id(ifaceobj)
 132             if self.cache.bridge_is_vlan_aware(bridgename):
 133                 [self.iproute2.bridge_fdb_add(bridgename, addr,
 134                     vlan) for addr in hwaddress]
 135         elif self.cache.link_is_bridge(ifaceobj.name):
 136             [self.iproute2.bridge_fdb_add(ifaceobj.name, addr)
 137                     for addr in hwaddress]
 138
 139     def _remove_addresses_from_bridge(self, ifaceobj, hwaddress):
 140         # XXX: batch the addresses
 141         if ifaceobj.link_kind & ifaceLinkKind.VLAN:
 142             bridgename = ifaceobj.lowerifaces[0]
 143             vlan = self._get_vlan_id(ifaceobj)
 144             if self.cache.bridge_is_vlan_aware(bridgename):
 145                 for addr in hwaddress:
 146                     try:
 147                         self.iproute2.bridge_fdb_del(bridgename, addr, vlan)
 148                     except Exception as e:
 149                         self.logger.debug("%s: %s" %(ifaceobj.name, str(e)))
 150                         pass
 151         elif self.cache.link_is_bridge(ifaceobj.name):
 152             for addr in hwaddress:
 153                 try:
 154                     self.iproute2.bridge_fdb_del(ifaceobj.name, addr)
 155                 except Exception as e:
 156                     self.logger.debug("%s: %s" %(ifaceobj.name, str(e)))
 157                     pass
 158
 159     def _get_bridge_fdbs(self, bridgename, vlan):
 160         fdbs = self._bridge_fdb_query_cache.get(bridgename)
 161         if not fdbs:
 162            fdbs = self.iproute2.bridge_fdb_show_dev(bridgename)
 163            if not fdbs:
 164               return
 165            self._bridge_fdb_query_cache[bridgename] = fdbs
 166         return fdbs.get(vlan)
 167
 168     def _check_addresses_in_bridge(self, ifaceobj, hwaddress):
 169         """ If the device is a bridge, make sure the addresses
 170         are in the bridge """
 171         if ifaceobj.link_kind & ifaceLinkKind.VLAN:
 172             bridgename = ifaceobj.lowerifaces[0]
 173             vlan = self._get_vlan_id(ifaceobj)
 174             if self.cache.bridge_is_vlan_aware(bridgename):
 175                 fdb_addrs = self._get_bridge_fdbs(bridgename, str(vlan))
 176                 if not fdb_addrs:
 177                    return False
 178                 hwaddress_int = utils.mac_str_to_int(hwaddress)
 179                 for mac in fdb_addrs:
 180                     if utils.mac_str_to_int(mac) == hwaddress_int:
 181                         return True
 182                 return False
 183         return True
 184
 185     def _fix_connected_route(self, ifaceobj, vifacename, addr):
 186         #
 187         # XXX: Hack to make sure the primary address
 188         # is the first in the routing table.
 189         #
 190         # We use `ip route get` on the vrr network to see which
 191         # device the kernel returns. if it is the mac vlan device,
 192         # flap the macvlan device to adjust the routing table entry.
 193         #
 194         # flapping the macvlan device makes sure the macvlan
 195         # connected route goes through delete + add, hence adjusting
 196         # the order in the routing table.
 197         #
 198         try:
 199             self.logger.info('%s: checking route entry ...' %ifaceobj.name)
 200
 201             # here we need to convert the ip address using the standard IPNetwork
 202             # object from the ipaddress not the custom IPNetwork object from
 203             # python3-nlmanager, because the standard IPNetwork will automatically
 204             # convert our ip address with prefixlen:
 205             # >>> ipaddress.ip_network("10.10.10.242/10", False)
 206             # IPv4Network('10.0.0.0/10')
 207             ip = ipaddress.ip_network(addr, False)
 208
 209             # we don't support ip6 route fix yet
 210             if ip.version == 6:
 211                 return
 212
 213             if ifaceobj.link_privflags & ifaceLinkPrivFlags.VRF_SLAVE:
 214                 vrf_master = self.cache.get_master(ifaceobj.name)
 215             else:
 216                 vrf_master = None
 217
 218             dev = self.iproute2.ip_route_get_dev(ip.with_prefixlen, vrf_master=vrf_master)
 219
 220             if dev and dev != ifaceobj.name:
 221                 self.logger.info('%s: preferred routing entry ' %ifaceobj.name +
 222                                  'seems to be of the macvlan dev %s'
 223                                  %vifacename +
 224                                  ' .. flapping macvlan dev to fix entry.')
 225                 self.iproute2.link_down(vifacename)
 226                 self.iproute2.link_up(vifacename)
 227         except Exception as e:
 228             self.logger.debug('%s: fixing route entry failed (%s)'
 229                               % (ifaceobj.name, str(e)))
 230             pass
 231
 232     def _get_macs_from_old_config(self, ifaceobj=None):
 233         """ This method returns a list of the mac addresses
 234         in the address-virtual attribute for the bridge. """
 235         maclist = []
 236         saved_ifaceobjs = statemanager.statemanager_api.get_ifaceobjs(ifaceobj.name)
 237         if not saved_ifaceobjs:
 238             return maclist
 239         # we need the old saved configs from the statemanager
 240         for oldifaceobj in saved_ifaceobjs:
 241             if not oldifaceobj.get_attr_value('address-virtual'):
 242                 continue
 243             for av in oldifaceobj.get_attr_value('address-virtual'):
 244                 macip = av.split()
 245                 if len(macip) < 2:
 246                     self.logger.debug("%s: incorrect old address-virtual attrs '%s'"
 247                                       %(oldifaceobj.name,  av))
 248                     continue
 249                 maclist.append(macip[0])
 250         return maclist
 251
 252     def get_addressvirtual_ipv6_addrgen_user_conf(self, ifaceobj):
 253         ipv6_addrgen = ifaceobj.get_attr_value_first('address-virtual-ipv6-addrgen')
 254
 255         if ipv6_addrgen:
 256             # IFLA_INET6_ADDR_GEN_MODE values:
 257             # 0 = eui64
 258             # 1 = none
 259             ipv6_addrgen_nl = self.address_virtual_ipv6_addrgen_value_dict.get(ipv6_addrgen.lower(), None)
 260
 261             if ipv6_addrgen_nl is None:
 262                 self.logger.warning('%s: invalid value "%s" for attribute address-virtual-ipv6-addrgen' % (ifaceobj.name, ipv6_addrgen))
 263             else:
 264                 return True, ipv6_addrgen_nl
 265
 266         else:
 267             # if user didn't configure ipv6-addrgen, should we reset to default?
 268             ipv6_addrgen_nl = self.address_virtual_ipv6_addrgen_value_dict.get(
 269                 self.get_attr_default_value('address-virtual-ipv6-addrgen'),
 270                 None
 271             )
 272             if ipv6_addrgen_nl is not None:
 273                 return True, ipv6_addrgen_nl
 274
 275         return False, None
 276
 277     def _remove_running_address_config(self, ifaceobj):
 278         if not self.cache.link_exists(ifaceobj.name):
 279             return
 280         hwaddress = []
 281
 282         for macvlan_prefix in [
 283             self._get_macvlan_prefix(ifaceobj),
 284             self.get_vrrp_prefix(ifaceobj.name, "4"),
 285             self.get_vrrp_prefix(ifaceobj.name, "6")
 286         ]:
 287             for macvlan_ifacename in glob.glob("/sys/class/net/%s*" % macvlan_prefix):
 288                 macvlan_ifacename = os.path.basename(macvlan_ifacename)
 289                 if not self.cache.link_exists(macvlan_ifacename) or self.cache.get_link_kind(macvlan_ifacename) != "macvlan":
 290                     continue
 291                 hwaddress.append(self.cache.get_link_address(macvlan_ifacename))
 292                 self.netlink.link_del(os.path.basename(macvlan_ifacename))
 293                 # XXX: Also delete any fdb addresses. This requires, checking mac address
 294                 # on individual macvlan interfaces and deleting the vlan from that.
 295
 296         if any(hwaddress):
 297             self._remove_addresses_from_bridge(ifaceobj, hwaddress)
 298
 299     def _remove_address_config(self, ifaceobj, address_virtual_list=None):
 300         if not address_virtual_list:
 301             self._remove_running_address_config(ifaceobj)
 302             return
 303
 304         if not self.cache.link_exists(ifaceobj.name):
 305             return
 306         hwaddress = []
 307         av_idx = 0
 308         macvlan_prefix = self._get_macvlan_prefix(ifaceobj)
 309         for av in address_virtual_list:
 310             av_attrs = av.split()
 311
 312             # Delete the macvlan device on this device
 313             macvlan_ifacename = '%s%d' %(macvlan_prefix, av_idx)
 314             self.netlink.link_del(os.path.basename(macvlan_ifacename))
 315             if av_attrs[0] != 'None':
 316                 hwaddress.append(av_attrs[0])
 317             av_idx += 1
 318         self._remove_addresses_from_bridge(ifaceobj, hwaddress)
 319
 320     def check_mac_address(self, ifaceobj, mac):
 321         if mac == 'none':
 322             return True
 323         try:
 324             if int(mac.split(":")[0], 16) & 1 :
 325                 self.log_error("%s: Multicast bit is set in the virtual mac address '%s'"
 326                                % (ifaceobj.name, mac), ifaceobj=ifaceobj)
 327                 return False
 328             return True
 329         except ValueError:
 330             return False
 331
 332     def _fixup_vrf_enslavements(self, ifaceobj, ifaceobj_getfunc=None):
 333         """ This function fixes up address virtual interfaces
 334         (macvlans) on vrf slaves. Since this fixup is an overhead,
 335         this must be called only in cases when ifupdown2 is
 336         called on the vrf device or its slave and not when
 337         ifupdown2 is called for all devices. When all
 338         interfaces are brought up, the expectation is that
 339         the normal path will fix up a vrf device or its slaves"""
 340
 341         if not ifaceobj_getfunc:
 342             return
 343         if ((ifaceobj.link_kind & ifaceLinkKind.VRF) and
 344             self.cache.link_exists(ifaceobj.name)):
 345             # if I am a vrf device and I have slaves
 346             # that have address virtual config,
 347             # enslave the slaves 'address virtual
 348             # interfaces (macvlans)' to myself:
 349             running_slaves = self.sysfs.link_get_lowers(ifaceobj.name)
 350             if running_slaves:
 351                 # pick up any existing slaves of a vrf device and
 352                 # look for their upperdevices and enslave them to the
 353                 # vrf device:
 354                 for s in running_slaves:
 355                     sobjs = ifaceobj_getfunc(s)
 356                     if (sobjs and
 357                         (sobjs[0].link_privflags & ifaceLinkPrivFlags.ADDRESS_VIRTUAL_SLAVE)):
 358                         # enslave all its upper devices to
 359                         # the vrf device
 360                         upperdevs = self.sysfs.link_get_uppers(sobjs[0].name)
 361                         if not upperdevs:
 362                             continue
 363                         for u in upperdevs:
 364                             # skip vrf device which
 365                             # will also show up in the
 366                             # upper device list
 367                             if u == ifaceobj.name:
 368                                 continue
 369                             self.netlink.link_set_master(u, ifaceobj.name)
 370                             self.netlink.link_up(u)
 371         elif ((ifaceobj.link_privflags & ifaceLinkPrivFlags.ADDRESS_VIRTUAL_SLAVE) and
 372               (ifaceobj.link_privflags & ifaceLinkPrivFlags.VRF_SLAVE) and
 373               self.cache.link_exists(ifaceobj.name)):
 374             # If I am a vrf slave and I have 'address virtual'
 375             # config, make sure my addrress virtual interfaces
 376             # (macvlans) are also enslaved to the vrf device
 377             vrfname = ifaceobj.get_attr_value_first('vrf')
 378             if not vrfname or not self.cache.link_exists(vrfname):
 379                 return
 380             running_uppers = self.sysfs.link_get_uppers(ifaceobj.name)
 381             if not running_uppers:
 382                 return
 383             macvlan_prefix = self._get_macvlan_prefix(ifaceobj)
 384             if not macvlan_prefix:
 385                 return
 386             for u in running_uppers:
 387                 if u == vrfname:
 388                     continue
 389                 if u.startswith(macvlan_prefix):
 390                     self.netlink.link_set_master(u, vrfname)
 391                     self.netlink.link_up(u)
 392
 393     def sync_macvlan_forwarding_state(self, ifname, macvlan_ifname):
 394         try:
 395             self.write_file(
 396                 "/proc/sys/net/ipv4/conf/%s/forwarding" % macvlan_ifname,
 397                 self.read_file_oneline("/proc/sys/net/ipv4/conf/%s/forwarding" % ifname)
 398             )
 399         except Exception as e:
 400             self.logger.info("%s: syncing macvlan forwarding with lower device forwarding state failed: %s" % (ifname, str(e)))
 401
 402     def create_macvlan_and_apply_config(self, ifaceobj, intf_config_list, vrrp=False):
 403
 404         """
 405         intf_config_list = [
 406             {
 407                 "ifname": "macvlan_ifname",
 408                 "hwaddress": "macvlan_hwaddress",
 409                 "ips": [str(IPNetwork), ]
 410             },
 411         ]
 412         """
 413         hw_address_list = []
 414
 415         if not intf_config_list:
 416             return hw_address_list
 417
 418         user_configured_ipv6_addrgenmode, ipv6_addrgen_user_value = self.get_addressvirtual_ipv6_addrgen_user_conf(ifaceobj)
 419         purge_existing = False if ifupdownflags.flags.PERFMODE else True
 420         ifname = ifaceobj.name
 421
 422         update_mtu = lower_iface_mtu = lower_iface_mtu_str = None
 423         if ifupdownconfig.config.get("adjust_logical_dev_mtu", "1") != "0" and ifaceobj.lowerifaces and intf_config_list:
 424             update_mtu = True
 425
 426         if update_mtu:
 427             lower_iface_mtu = self.cache.get_link_mtu(ifaceobj.name)
 428             lower_iface_mtu_str = str(lower_iface_mtu)
 429
 430         self.iproute2.batch_start()  # TODO: make sure we only do 1 ip link set down and set up (only one flap in the batch)
 431
 432         for intf_config_dict in intf_config_list:
 433             link_created = False
 434             macvlan_ifname = intf_config_dict.get("ifname")
 435             macvlan_hwaddr = intf_config_dict.get("hwaddress")
 436             macvlan_mode = intf_config_dict.get("mode")
 437             ips = intf_config_dict.get("ips")
 438
 439             if len(macvlan_ifname) > 15:
 440                 self.logger.error("%s: macvlan name will exceed the 15 chars limitation - please rename the underlying interface (%s)" % (macvlan_ifname, ifname))
 441                 ifaceobj.set_status(ifaceStatus.ERROR)
 442                 continue
 443
 444             is_ip6 = False
 445             for ip in ips:
 446                 self.ip_blacklist_check(ifname, ip)
 447                 ip_network_obj = ipnetwork.IPNetwork(ip)
 448                 is_ip6 |= ip_network_obj.version == 6
 449
 450             if not self.cache.link_exists(macvlan_ifname):
 451                 # When creating VRRP macvlan with bridge mode, the kernel
 452                 # return an error: 'Invalid argument' (22)
 453                 # so for now we should only use the iproute2 API.
 454                 # try:
 455                 #    self.netlink.link_add_macvlan(ifname, macvlan_ifname)
 456                 # except Exception:
 457                 self.iproute2.link_add_macvlan(ifname, macvlan_ifname, macvlan_mode)
 458                 self.sync_macvlan_forwarding_state(ifname, macvlan_ifname)
 459                 link_created = True
 460
 461             # first thing we need to handle vrf enslavement
 462             if ifaceobj.link_privflags & ifaceLinkPrivFlags.VRF_SLAVE:
 463                 vrf_ifname = self.cache.get_master(ifaceobj.name)
 464                 if vrf_ifname:
 465                     self.iproute2.link_set_master(macvlan_ifname, vrf_ifname)
 466
 467             # If we are dealing with a VRRP macvlan we need to set protodown on
 468             # and set addrgenmode appropriately. For IPv4, a VRRP user only
 469             # needs the VIP (which is explicitly configured) so addrgenmode
 470             # should be NONE. For IPv6, a unique link-local address is needed
 471             # as the SIP for vrrp6 hellos, so addrgenmode should be RANDOM.
 472             if vrrp:
 473                 try:
 474                     v6_ag_mode = Link.IN6_ADDR_GEN_MODE_RANDOM if is_ip6 else Link.IN6_ADDR_GEN_MODE_NONE
 475                     self.iproute2.link_set_ipv6_addrgen(
 476                         macvlan_ifname,
 477                         v6_ag_mode,
 478                         link_created
 479                     )
 480                 except Exception as e:
 481                     self.logger.warning("%s: %s: ip link set dev %s addrgenmode none: "
 482                                      "operation not supported: %s" % (ifname, macvlan_ifname, macvlan_ifname, str(e)))
 483                 try:
 484                     if link_created:
 485                         self.netlink.link_set_protodown_on(macvlan_ifname)
 486                 except Exception as e:
 487                     self.logger.warning("%s: %s: ip link set dev %s protodown on: operation not supported: %s" % (ifname, macvlan_ifname, macvlan_ifname, str(e)))
 488             elif user_configured_ipv6_addrgenmode:
 489                 self.iproute2.link_set_ipv6_addrgen(macvlan_ifname, ipv6_addrgen_user_value, link_created)
 490
 491             if macvlan_hwaddr:
 492                 self.iproute2.link_set_address_and_keep_down(
 493                     macvlan_ifname,
 494                     macvlan_hwaddr,
 495                     keep_down=ifaceobj.link_privflags & ifaceLinkPrivFlags.KEEP_LINK_DOWN
 496                 )
 497                 hw_address_list.append(macvlan_hwaddr)
 498
 499             if self.addressvirtual_with_route_metric and self.get_addr_metric_support():
 500                 metric = self.get_default_ip_metric()
 501             else:
 502                 metric = None
 503
 504             self.iproute2.add_addresses(
 505                 ifaceobj,
 506                 macvlan_ifname,
 507                 ips,
 508                 purge_existing,
 509                 metric=metric
 510             )
 511
 512             if ifaceobj.link_privflags & ifaceLinkPrivFlags.KEEP_LINK_DOWN:
 513                 self.logger.info("%s: keeping macvlan down - link-down yes on lower device %s" % (macvlan_ifname, ifname))
 514                 self.netlink.link_down(macvlan_ifname)
 515
 516             # If link existed before, flap the link
 517             if not link_created:
 518
 519                 if not self.addressvirtual_with_route_metric or not self.get_addr_metric_support():
 520                     # if the system doesn't support ip addr set METRIC
 521                     # we need to do manually check the ordering of the ip4 routes
 522                     self._fix_connected_route(ifaceobj, macvlan_ifname, ips[0])
 523
 524                 if update_mtu:
 525                     update_mtu = False
 526
 527                     try:
 528                         self.sysfs.link_set_mtu(macvlan_ifname, mtu_str=lower_iface_mtu_str, mtu_int=lower_iface_mtu)
 529                     except Exception as e:
 530                         self.logger.info('%s: failed to set mtu %s: %s' % (macvlan_ifname, lower_iface_mtu, e))
 531
 532                 # set macvlan device to up in anycase.
 533                 # since we auto create them here..we are responsible
 534                 # to bring them up here in the case they were brought down
 535                 # by some other entity in the system.
 536                 if not ifaceobj.link_privflags & ifaceLinkPrivFlags.KEEP_LINK_DOWN:
 537                     self.netlink.link_up(macvlan_ifname)
 538             else:
 539                 try:
 540                     if not self.addressvirtual_with_route_metric or not self.get_addr_metric_support():
 541                         # if the system doesn't support ip addr set METRIC
 542                         # we need to do manually check the ordering of the ip6 routes
 543                         self.iproute2.fix_ipv6_route_metric(ifaceobj, macvlan_ifname, ips)
 544                 except Exception as e:
 545                     self.logger.debug('fix_vrf_slave_ipv6_route_metric: failed: %s' % e)
 546
 547             # Disable IPv6 duplicate address detection on VRR interfaces
 548             sysctl_prefix = "net.ipv6.conf.%s" % macvlan_ifname
 549
 550             try:
 551                 syskey = "%s.%s" % (sysctl_prefix, "enhanced_dad")
 552                 if self.sysctl_get(syskey) != "0":
 553                     self.sysctl_set(syskey, "0")
 554             except Exception as e:
 555                 self.logger.info("sysctl failure: operation not supported: %s" % str(e))
 556
 557             for key, sysval in {
 558                 "accept_dad": "0",
 559                 "dad_transmits": "0"
 560             }.items():
 561                 syskey = "%s.%s" % (sysctl_prefix, key)
 562                 if self.sysctl_get(syskey) != sysval:
 563                     self.sysctl_set(syskey, sysval)
 564
 565         self.iproute2.batch_commit()
 566         return hw_address_list
 567
 568     def _up(self, ifaceobj, ifaceobj_getfunc=None):
 569         if not ifupdownflags.flags.ALL:
 570             self._fixup_vrf_enslavements(ifaceobj, ifaceobj_getfunc)
 571
 572         address_virtual_list = ifaceobj.get_attr_value('address-virtual')
 573         vrr_config_list = ifaceobj.get_attr_value("vrrp")
 574
 575         if not address_virtual_list and not vrr_config_list:
 576             # XXX: address virtual is not present. In which case,
 577             # delete stale macvlan devices.
 578             self._remove_running_address_config(ifaceobj)
 579             return
 580
 581         if ifaceobj.upperifaces and not ifaceobj.link_privflags & ifaceLinkPrivFlags.VRF_SLAVE:
 582             self.log_error("%s: invalid placement of address-virtual/vrrp lines "
 583                            "(must be configured under an interface "
 584                            "with no upper interfaces or parent interfaces)"
 585                            % ifaceobj.name, ifaceobj)
 586
 587         if not self.cache.link_exists(ifaceobj.name):
 588             return
 589
 590         addr_virtual_macs = self.create_macvlan_and_apply_config(
 591             ifaceobj,
 592             self.translate_addrvirtual_user_config_to_list(
 593                 ifaceobj,
 594                 address_virtual_list
 595             )
 596         )
 597
 598         vrr_macs = self.create_macvlan_and_apply_config(
 599             ifaceobj,
 600             self.translate_vrr_user_config_to_list(
 601                 ifaceobj,
 602                 vrr_config_list
 603             ),
 604             vrrp=True
 605         )
 606
 607         hw_address_list = addr_virtual_macs + vrr_macs
 608
 609         # check the statemanager for old configs.
 610         # We need to remove only the previously configured FDB entries
 611         oldmacs = self._get_macs_from_old_config(ifaceobj)
 612         # get a list of fdbs in old that are not in new config meaning they should
 613         # be removed since they are gone from the config
 614         removed_macs = [mac for mac in oldmacs if mac.lower() not in hw_address_list]
 615         self._remove_addresses_from_bridge(ifaceobj, removed_macs)
 616         # if ifaceobj is a bridge and bridge is a vlan aware bridge
 617         # add the vid to the bridge
 618         self._add_addresses_to_bridge(ifaceobj, hw_address_list)
 619
 620     def translate_vrr_user_config_to_list(self, ifaceobj, vrr_config_list, ifquery=False):
 621         """
 622         If (IPv4 addresses provided):
 623             00:00:5e:00:01:<V>
 624         else if (IPv6 addresses provided):
 625             00:00:5e:00:02:<V>
 626
 627         vrrp 1 10.0.0.15/24
 628         vrrp 1 2001:0db8::0370:7334/64
 629
 630         # Translate:
 631         #       vrrp 255 10.0.0.15/24 10.0.0.2/1
 632         # To:
 633         # [
 634         #   {
 635         #        "ifname": "macvlan_ifname",
 636         #        "hwaddress": "macvlan_hwaddress",
 637         #        "mode": "macvlan_mode",
 638         #        "ips": [str(IPNetwork), ]
 639         #    },
 640         # ]
 641         """
 642         ifname = ifaceobj.name
 643         user_config_list = []
 644
 645         for index, config in enumerate(vrr_config_list or []):
 646             vrrp_id, ip_addrs = config.split(" ", 1)
 647             hex_id = '%02x' % int(vrrp_id)
 648             ip4 = []
 649             ip6 = []
 650
 651             for ip_addr in ip_addrs.split():
 652                 ip_network_obj = ipnetwork.IPNetwork(ip_addr)
 653                 is_ip6 = ip_network_obj.version == 6
 654
 655                 if is_ip6:
 656                     ip6.append(ip_network_obj)
 657                 else:
 658                     ip4.append(ip_network_obj)
 659
 660             macvlan_ip4_ifname = "%s%s" % (self.get_vrrp_prefix(ifname, "4"), vrrp_id)
 661             macvlan_ip6_ifname = "%s%s" % (self.get_vrrp_prefix(ifname, "6"), vrrp_id)
 662
 663             if ip4 or ifquery:
 664                 merged_with_existing_obj = False
 665                 macvlan_ip4_mac = "00:00:5e:00:01:%s" % hex_id
 666                 macvlan_ip4_mac_int = utils.mac_str_to_int(macvlan_ip4_mac)
 667                 # if the vrr config is defined in different lines for the same ID
 668                 # we need to save the ip4 and ip6 in the objects we previously
 669                 # created, example:
 670                 # vrrp 255 10.0.0.15/24 10.0.0.2/15
 671                 # vrrp 255 fe80::a00:27ff:fe04:42/64
 672                 for obj in user_config_list:
 673                     if obj.get("hwaddress_int") == macvlan_ip4_mac_int:
 674                         obj["ips"] += ip4
 675                         merged_with_existing_obj = True
 676
 677                 if not merged_with_existing_obj:
 678                     # if ip4 config wasn't merge with an existing object
 679                     # we need to insert it in our list
 680                     user_config_list.append({
 681                         "ifname": macvlan_ip4_ifname,
 682                         "hwaddress": macvlan_ip4_mac,
 683                         "hwaddress_int": macvlan_ip4_mac_int,
 684                         "mode": "bridge",
 685                         "ips": ip4,
 686                         "id": vrrp_id
 687                     })
 688             elif not ip4 and not ifquery and self.cache.link_exists(macvlan_ip4_ifname):
 689                 # special check to see if all ipv4 were removed from the vrrp
 690                 # configuration, if so we need to remove the associated macvlan
 691                 self.netlink.link_del(macvlan_ip4_ifname)
 692
 693             if ip6 or ifquery:
 694                 merged_with_existing_obj = False
 695                 macvlan_ip6_mac = "00:00:5e:00:02:%s" % hex_id
 696                 macvlan_ip6_mac_int = utils.mac_str_to_int(macvlan_ip6_mac)
 697                 # if the vrr config is defined in different lines for the same ID
 698                 # we need to save the ip4 and ip6 in the objects we previously
 699                 # created, example:
 700                 # vrrp 255 10.0.0.15/24 10.0.0.2/15
 701                 # vrrp 255 fe80::a00:27ff:fe04:42/64
 702
 703                 for obj in user_config_list:
 704                     if obj.get("hwaddress_int") == macvlan_ip6_mac_int:
 705                         obj["ips"] += ip6
 706                         merged_with_existing_obj = True
 707
 708                 if not merged_with_existing_obj:
 709                     # if ip6 config wasn't merge with an existing object
 710                     # we need to insert it in our list
 711                     user_config_list.append({
 712                         "ifname": macvlan_ip6_ifname,
 713                         "hwaddress": macvlan_ip6_mac,
 714                         "hwaddress_int": macvlan_ip6_mac_int,
 715                         "mode": "bridge",
 716                         "ips": ip6,
 717                         "id": vrrp_id
 718                     })
 719             elif not ip6 and not ifquery and self.cache.link_exists(macvlan_ip6_ifname):
 720                 # special check to see if all ipv6 were removed from the vrrp
 721                 # configuration, if so we need to remove the associated macvlan
 722                 self.netlink.link_del(macvlan_ip6_ifname)
 723
 724         if not ifquery:
 725             # check if vrrp attribute was removed/re-assigned
 726             old_vrr_ids = set()
 727
 728             try:
 729                 for old_ifaceobj in statemanager.statemanager_api.get_ifaceobjs(ifname) or []:
 730                     for vrr_config in old_ifaceobj.get_attr_value("vrrp") or []:
 731                         try:
 732                             old_vrr_ids.add(vrr_config.split()[0])
 733                         except Exception:
 734                             continue
 735
 736                 if old_vrr_ids:
 737
 738                     for config in user_config_list:
 739                         try:
 740                             old_vrr_ids.remove(config["id"])
 741                         except KeyError:
 742                             pass
 743
 744                     for id_to_remove in old_vrr_ids:
 745                         macvlan_ip4_ifname = "%s%s" % (self.get_vrrp_prefix(ifname, "4"), id_to_remove)
 746                         macvlan_ip6_ifname = "%s%s" % (self.get_vrrp_prefix(ifname, "6"), id_to_remove)
 747
 748                         if self.cache.link_exists(macvlan_ip4_ifname):
 749                             self.netlink.link_del(macvlan_ip4_ifname)
 750
 751                         if self.cache.link_exists(macvlan_ip6_ifname):
 752                             self.netlink.link_del(macvlan_ip6_ifname)
 753
 754             except Exception as e:
 755                 self.logger.debug("%s: vrrp: failure while removing unused macvlan(s)" % ifname)
 756
 757         return user_config_list
 758
 759     def translate_addrvirtual_user_config_to_list(self, ifaceobj, address_virtual_list):
 760         """
 761         # Translate:
 762         #       address-virtual 00:11:22:33:44:01 2001:0db8::0370:7334/64 11.0.1.1/24 11.0.1.2/24
 763         # To:
 764         # [
 765         #   {
 766         #        "ifname": "macvlan_ifname",
 767         #        "hwaddress": "macvlan_hwaddress",
 768         #        "ips": [str(IPNetwork), ]
 769         #    },
 770         # ]
 771         """
 772         user_config_list = []
 773
 774         if not address_virtual_list:
 775             return user_config_list
 776
 777         macvlan_prefix = self._get_macvlan_prefix(ifaceobj)
 778
 779         for index, addr_virtual in enumerate(address_virtual_list):
 780             av_attrs = addr_virtual.split()
 781             mac = av_attrs[0]
 782             if mac:
 783                 mac = mac.lower()
 784
 785             if not self.check_mac_address(ifaceobj, mac):
 786                 continue
 787
 788             config = {
 789                 "ifname": "%s%d" % (macvlan_prefix, index),
 790                 "mode": "private"
 791             }
 792
 793             if mac != "none":
 794                 config["hwaddress"] = mac
 795                 config["hwaddress_int"] = utils.mac_str_to_int(mac)
 796
 797             ip_network_obj_list = []
 798             for ip in av_attrs[1:]:
 799                 ip_network_obj_list.append(ipnetwork.IPNetwork(ip))
 800
 801             config["ips"] = ip_network_obj_list
 802             user_config_list.append(config)
 803
 804         return user_config_list
 805
 806     def _down(self, ifaceobj, ifaceobj_getfunc=None):
 807         try:
 808             self._remove_address_config(ifaceobj,
 809                          ifaceobj.get_attr_value('address-virtual'))
 810
 811             #### VRR
 812             hwaddress = []
 813             for vrr_prefix in [self.get_vrrp_prefix(ifaceobj.name, "4"), self.get_vrrp_prefix(ifaceobj.name, "6")]:
 814                 for macvlan_ifacename in glob.glob("/sys/class/net/%s*" % vrr_prefix):
 815                     macvlan_ifacename = os.path.basename(macvlan_ifacename)
 816                     if not self.cache.link_exists(macvlan_ifacename):
 817                         continue
 818                     hwaddress.append(self.cache.get_link_address(macvlan_ifacename))
 819                     self.netlink.link_del(macvlan_ifacename)
 820                     # XXX: Also delete any fdb addresses. This requires, checking mac address
 821                     # on individual macvlan interfaces and deleting the vlan from that.
 822             if any(hwaddress):
 823                 self._remove_addresses_from_bridge(ifaceobj, hwaddress)
 824         except Exception as e:
 825             import traceback
 826             traceback.print_exc()
 827             self.log_warn(str(e))
 828
 829     def _query_check(self, ifaceobj, ifaceobjcurr):
 830
 831         if not self.cache.link_exists(ifaceobj.name):
 832             return
 833
 834         user_config_address_virtual_ipv6_addr = ifaceobj.get_attr_value_first('address-virtual-ipv6-addrgen')
 835         if user_config_address_virtual_ipv6_addr and user_config_address_virtual_ipv6_addr not in utils._string_values:
 836             ifaceobjcurr.update_config_with_status('address-virtual-ipv6-addrgen', user_config_address_virtual_ipv6_addr, 1)
 837             user_config_address_virtual_ipv6_addr = None
 838
 839         address_virtual_list = ifaceobj.get_attr_value('address-virtual')
 840
 841         macvlans_running_ipv6_addr_virtual = self.query_check_macvlan_config(
 842             ifaceobj,
 843             ifaceobjcurr,
 844             "address-virtual",
 845             user_config_address_virtual_ipv6_addr,
 846             virtual_addr_list_raw=address_virtual_list,
 847             macvlan_config_list=self.translate_addrvirtual_user_config_to_list(
 848                 ifaceobj,
 849                 address_virtual_list
 850             )
 851         )
 852
 853         vrr_config_list = ifaceobj.get_attr_value("vrrp")
 854
 855         macvlans_running_ipv6_addr_vrr = self.query_check_macvlan_config(
 856             ifaceobj,
 857             ifaceobjcurr,
 858             "vrrp",
 859             user_config_address_virtual_ipv6_addr,
 860             virtual_addr_list_raw=vrr_config_list,
 861             macvlan_config_list=self.translate_vrr_user_config_to_list(
 862                 ifaceobj,
 863                 vrr_config_list,
 864                 ifquery=True
 865             )
 866         )
 867
 868         macvlans_running_ipv6_addr = macvlans_running_ipv6_addr_virtual + macvlans_running_ipv6_addr_vrr
 869         if user_config_address_virtual_ipv6_addr:
 870             bool_user_ipv6_addrgen = utils.get_boolean_from_string(user_config_address_virtual_ipv6_addr)
 871             for running_ipv6_addrgen in macvlans_running_ipv6_addr:
 872                 if (not bool_user_ipv6_addrgen) != running_ipv6_addrgen:
 873                     ifaceobjcurr.update_config_with_status('address-virtual-ipv6-addrgen', user_config_address_virtual_ipv6_addr, 1)
 874                     return
 875             ifaceobjcurr.update_config_with_status('address-virtual-ipv6-addrgen', user_config_address_virtual_ipv6_addr, 0)
 876
 877     @staticmethod
 878     def compare_user_config_vs_running_state(running_addrs, user_addrs):
 879         ip4 = []
 880         ip6 = []
 881
 882         for ip in user_addrs or []:
 883             if ip.version == 6:
 884                 ip6.append(ip)
 885             else:
 886                 ip4.append(ip)
 887
 888         running_ipobj = []
 889         for ip in running_addrs or []:
 890             running_ipobj.append(ip)
 891
 892         return running_ipobj == (ip4 + ip6)
 893
 894     def query_check_macvlan_config(self, ifaceobj, ifaceobjcurr, attr_name, user_config_address_virtual_ipv6_addr, virtual_addr_list_raw, macvlan_config_list):
 895         """
 896         macvlan_config_list = [
 897             {
 898                 "ifname": "macvlan_ifname",
 899                 "hwaddress": "macvlan_hwaddress",
 900                 "ips": [str(IPNetwork), ]
 901             },
 902         ]
 903         """
 904         is_vrr = attr_name == "vrrp"
 905         macvlans_running_ipv6_addr = []
 906
 907         if not virtual_addr_list_raw:
 908             return macvlans_running_ipv6_addr
 909
 910         macvlan_config_queue = deque(macvlan_config_list)
 911
 912         while macvlan_config_queue:
 913
 914             ip4_config = None
 915             ip6_config = None
 916
 917             config = macvlan_config_queue.popleft()
 918
 919             if is_vrr:
 920                 ip4_config = config
 921                 ip6_config = macvlan_config_queue.popleft()
 922
 923             macvlan_ifacename = config.get("ifname")
 924
 925             if not self.cache.link_exists(macvlan_ifacename):
 926                 ifaceobjcurr.update_config_with_status(attr_name, "", 1)
 927                 continue
 928
 929             macvlan_hwaddress = config.get("hwaddress")
 930             macvlan_hwaddress_int = config.get("hwaddress_int")
 931
 932             if user_config_address_virtual_ipv6_addr:
 933                 macvlans_running_ipv6_addr.append(self.cache.get_link_ipv6_addrgen_mode(macvlan_ifacename))
 934
 935             # Check mac and ip address
 936             rhwaddress = ip4_macvlan_hwaddress = self.cache.get_link_address(macvlan_ifacename)
 937             raddrs = ip4_running_addrs = self.cache.get_managed_ip_addresses(
 938                 ifname=macvlan_ifacename,
 939                 ifaceobj_list=[ifaceobj],
 940                 with_address_virtual=True
 941             )
 942
 943             if not is_vrr:
 944                 ips = config.get("ips")
 945
 946                 if not rhwaddress:
 947                     ifaceobjcurr.update_config_with_status(attr_name, "", 1)
 948                     continue
 949
 950                 try:
 951                     if utils.mac_str_to_int(rhwaddress) == macvlan_hwaddress_int \
 952                             and self.compare_user_config_vs_running_state(raddrs, ips) \
 953                             and self._check_addresses_in_bridge(ifaceobj, macvlan_hwaddress):
 954                         ifaceobjcurr.update_config_with_status(
 955                             attr_name,
 956                             " ".join(virtual_addr_list_raw),
 957                             0
 958                         )
 959                     else:
 960                         if raddrs:
 961                             address_virtual_value = "%s %s" % (rhwaddress, " ".join(raddrs))
 962                         else:
 963                             address_virtual_value = rhwaddress
 964                         ifaceobjcurr.update_config_with_status(attr_name, address_virtual_value, 1)
 965                 except Exception as e:
 966                     self.logger.debug("addressvirtual: %s" % str(e))
 967                     if raddrs:
 968                         address_virtual_value = "%s %s" % (rhwaddress, " ".join(raddrs))
 969                     else:
 970                         address_virtual_value = rhwaddress
 971
 972                     ifaceobjcurr.update_config_with_status(attr_name, address_virtual_value, 1)
 973             else:
 974                 # VRRP
 975
 976                 ok = False
 977                 # check macvlan ip4 hwaddress (only if ip4 were provided by the user)
 978                 if not ip4_config.get("ips") or ip4_macvlan_hwaddress == ip4_config.get("hwaddress"):
 979                     ip6_macvlan_ifname = ip6_config.get("ifname")
 980                     ip6_macvlan_hwaddress = ip6_config.get("hwaddress")
 981
 982                     # check macvlan ip6 hwaddress (only if ip6 were provided by the user)
 983                     if not ip6_config.get("ips") or self.cache.get_link_address_raw(ip6_macvlan_ifname) == ip6_config.get("hwaddress_int"):
 984
 985                         # check all ip4
 986                         if self.compare_user_config_vs_running_state(
 987                                 ip4_running_addrs,
 988                                 ip4_config.get("ips")
 989                         ) and self._check_addresses_in_bridge(ifaceobj, ip4_macvlan_hwaddress):
 990                             ip6_running_addrs = self.cache.get_managed_ip_addresses(
 991                                 ifname=ip6_macvlan_ifname,
 992                                 ifaceobj_list=[ifaceobj],
 993                                 with_address_virtual=True
 994                             )
 995
 996                             # check all ip6
 997                             if self.compare_user_config_vs_running_state(
 998                                     ip6_running_addrs,
 999                                     ip6_config.get("ips")
1000                             ) and self._check_addresses_in_bridge(ifaceobj, ip6_macvlan_hwaddress):
1001                                 ifaceobjcurr.update_config_with_status(
1002                                     attr_name,
1003                                     "%s %s" % (ip4_config.get("id"), " ".join(ip4_config.get("ips") + ip6_config.get("ips"))),
1004                                     0
1005                                 )
1006                                 ok = True
1007
1008                 if not ok:
1009                     ifaceobjcurr.update_config_with_status(
1010                         attr_name,
1011                         "%s %s" % (ip4_config.get("id"), " ".join(ip4_config.get("ips") + ip6_config.get("ips"))),
1012                         1
1013                     )
1014
1015         return macvlans_running_ipv6_addr
1016
1017     def _query_running(self, ifaceobjrunning, ifaceobj_getfunc=None):
1018         macvlan_prefix = self._get_macvlan_prefix(ifaceobjrunning)
1019         address_virtuals = glob.glob("/sys/class/net/%s*" %macvlan_prefix)
1020         macvlans_ipv6_addrgen_list = []
1021         for av in address_virtuals:
1022             macvlan_ifacename = os.path.basename(av)
1023             rhwaddress = self.cache.get_link_address(macvlan_ifacename)
1024             raddress = self.cache.get_managed_ip_addresses(
1025                 ifname=ifaceobjrunning.name,
1026                 ifaceobj_list=ifaceobj_getfunc(ifaceobjrunning.name) or [],
1027                 with_address_virtual=True
1028             )
1029
1030             raddress = list(set(raddress))
1031
1032             if not raddress:
1033                 self.logger.warning('%s: no running addresses'
1034                                  %ifaceobjrunning.name)
1035                 raddress = []
1036
1037             ifaceobjrunning.update_config('address-virtual', '%s %s' %(rhwaddress, ' '.join([str(a) for a in raddress])))
1038
1039             macvlans_ipv6_addrgen_list.append((macvlan_ifacename, self.cache.get_link_ipv6_addrgen_mode(macvlan_ifacename)))
1040
1041         macvlan_count = len(address_virtuals)
1042         if not macvlan_count:
1043             return
1044         ipv6_addrgen = macvlans_ipv6_addrgen_list[0][1]
1045
1046         for macvlan_ifname, macvlan_ipv6_addrgen in macvlans_ipv6_addrgen_list:
1047             if macvlan_ipv6_addrgen != ipv6_addrgen:
1048                 # one macvlan has a different ipv6-addrgen configuration
1049                 # we simply return, ifquery-running will print the macvlan
1050                 # stanzas with the ipv6-addrgen on/off attribute
1051                 return
1052         ifaceobjrunning.update_config('address-virtual-ipv6-addrgen', 'off' if ipv6_addrgen else 'on')
1053
1054     _run_ops = {
1055         'up': _up,
1056         'down': _down,
1057         'query-checkcurr': _query_check,
1058         'query-running': _query_running
1059     }
1060
1061     def get_ops(self):
1062         """ returns list of ops supported by this module """
1063         return list(self._run_ops.keys())
1064
1065
1066     def run(self, ifaceobj, operation, query_ifaceobj=None,
1067             ifaceobj_getfunc=None, **extra_args):
1068         """ run vlan configuration on the interface object passed as argument
1069
1070         Args:
1071             **ifaceobj** (object): iface object
1072
1073             **operation** (str): any of 'pre-up', 'post-down', 'query-checkcurr',
1074                                  'query-running'
1075         Kwargs:
1076             **query_ifaceobj** (object): query check ifaceobject. This is only
1077                 valid when op is 'query-checkcurr'. It is an object same as
1078                 ifaceobj, but contains running attribute values and its config
1079                 status. The modules can use it to return queried running state
1080                 of interfaces. status is success if the running state is same
1081                 as user required state in ifaceobj. error otherwise.
1082         """
1083         if ifaceobj.type == ifaceType.BRIDGE_VLAN:
1084             return
1085         op_handler = self._run_ops.get(operation)
1086         if not op_handler:
1087             return
1088
1089         if operation == 'query-checkcurr':
1090             op_handler(self, ifaceobj, query_ifaceobj)
1091         else:
1092             op_handler(self, ifaceobj, ifaceobj_getfunc=ifaceobj_getfunc)