tests/topotests/lib/topotest.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # topotest.py
   5 # Library of helper functions for NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2016 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 import json
  26 import os
  27 import errno
  28 import re
  29 import sys
  30 import glob
  31 import StringIO
  32 import subprocess
  33 import tempfile
  34 import platform
  35 import difflib
  36 import time
  37
  38 from lib.topolog import logger
  39
  40 from mininet.topo import Topo
  41 from mininet.net import Mininet
  42 from mininet.node import Node, OVSSwitch, Host
  43 from mininet.log import setLogLevel, info
  44 from mininet.cli import CLI
  45 from mininet.link import Intf
  46
  47 class json_cmp_result(object):
  48     "json_cmp result class for better assertion messages"
  49
  50     def __init__(self):
  51         self.errors = []
  52
  53     def add_error(self, error):
  54         "Append error message to the result"
  55         for line in error.splitlines():
  56             self.errors.append(line)
  57
  58     def has_errors(self):
  59         "Returns True if there were errors, otherwise False."
  60         return len(self.errors) > 0
  61
  62 def json_diff(d1, d2):
  63     """
  64     Returns a string with the difference between JSON data.
  65     """
  66     json_format_opts = {
  67         'indent': 4,
  68         'sort_keys': True,
  69     }
  70     dstr1 = json.dumps(d1, **json_format_opts)
  71     dstr2 = json.dumps(d2, **json_format_opts)
  72     return difflines(dstr2, dstr1, title1='Expected value', title2='Current value', n=0)
  73
  74 def json_cmp(d1, d2):
  75     """
  76     JSON compare function. Receives two parameters:
  77     * `d1`: json value
  78     * `d2`: json subset which we expect
  79
  80     Returns `None` when all keys that `d1` has matches `d2`,
  81     otherwise a string containing what failed.
  82
  83     Note: key absence can be tested by adding a key with value `None`.
  84     """
  85     squeue = [(d1, d2, 'json')]
  86     result = json_cmp_result()
  87     for s in squeue:
  88         nd1, nd2, parent = s
  89         s1, s2 = set(nd1), set(nd2)
  90
  91         # Expect all required fields to exist.
  92         s2_req = set([key for key in nd2 if nd2[key] is not None])
  93         diff = s2_req - s1
  94         if diff != set({}):
  95             result.add_error('expected key(s) {} in {} (have {})'.format(
  96                 str(list(diff)), parent, str(list(s1))))
  97
  98         for key in s2.intersection(s1):
  99             # Test for non existence of key in d2
 100             if nd2[key] is None:
 101                 result.add_error('"{}" should not exist in {} (have {})'.format(
 102                     key, parent, str(s1)))
 103                 continue
 104             # If nd1 key is a dict, we have to recurse in it later.
 105             if isinstance(nd2[key], type({})):
 106                 if not isinstance(nd1[key], type({})):
 107                     result.add_error(
 108                         '{}["{}"] has different type than expected '.format(parent, key) +
 109                         '(have {}, expected {})'.format(type(nd1[key]), type(nd2[key])))
 110                     continue
 111                 nparent = '{}["{}"]'.format(parent, key)
 112                 squeue.append((nd1[key], nd2[key], nparent))
 113                 continue
 114             # Check list items
 115             if isinstance(nd2[key], type([])):
 116                 if not isinstance(nd1[key], type([])):
 117                     result.add_error(
 118                         '{}["{}"] has different type than expected '.format(parent, key) +
 119                         '(have {}, expected {})'.format(type(nd1[key]), type(nd2[key])))
 120                     continue
 121                 # Check list size
 122                 if len(nd2[key]) > len(nd1[key]):
 123                     result.add_error(
 124                         '{}["{}"] too few items '.format(parent, key) +
 125                         '(have {}, expected {}:\n {})'.format(
 126                             len(nd1[key]), len(nd2[key]),
 127                             json_diff(nd1[key], nd2[key])))
 128                     continue
 129
 130                 # List all unmatched items errors
 131                 unmatched = []
 132                 for expected in nd2[key]:
 133                     matched = False
 134                     for value in nd1[key]:
 135                         if json_cmp({'json': value}, {'json': expected}) is None:
 136                             matched = True
 137                             break
 138
 139                     if matched:
 140                         break
 141                     if not matched:
 142                         unmatched.append(expected)
 143
 144                 # If there are unmatched items, error out.
 145                 if unmatched:
 146                     result.add_error(
 147                         '{}["{}"] value is different (\n{})'.format(
 148                             parent, key, json_diff(nd1[key], nd2[key])))
 149                 continue
 150
 151             # Compare JSON values
 152             if nd1[key] != nd2[key]:
 153                 result.add_error(
 154                     '{}["{}"] value is different (\n{})'.format(
 155                         parent, key, json_diff(nd1[key], nd2[key])))
 156                 continue
 157
 158     if result.has_errors():
 159         return result
 160
 161     return None
 162
 163 def run_and_expect(func, what, count=20, wait=3):
 164     """
 165     Run `func` and compare the result with `what`. Do it for `count` times
 166     waiting `wait` seconds between tries. By default it tries 20 times with
 167     3 seconds delay between tries.
 168
 169     Returns (True, func-return) on success or
 170     (False, func-return) on failure.
 171     """
 172     while count > 0:
 173         result = func()
 174         if result != what:
 175             time.sleep(wait)
 176             count -= 1
 177             continue
 178         return (True, result)
 179     return (False, result)
 180
 181
 182 def int2dpid(dpid):
 183     "Converting Integer to DPID"
 184
 185     try:
 186         dpid = hex(dpid)[2:]
 187         dpid = '0'*(16-len(dpid))+dpid
 188         return dpid
 189     except IndexError:
 190         raise Exception('Unable to derive default datapath ID - '
 191                         'please either specify a dpid or use a '
 192                         'canonical switch name such as s23.')
 193
 194 def pid_exists(pid):
 195     "Check whether pid exists in the current process table."
 196
 197     if pid <= 0:
 198         return False
 199     try:
 200         os.kill(pid, 0)
 201     except OSError as err:
 202         if err.errno == errno.ESRCH:
 203             # ESRCH == No such process
 204             return False
 205         elif err.errno == errno.EPERM:
 206             # EPERM clearly means there's a process to deny access to
 207             return True
 208         else:
 209             # According to "man 2 kill" possible error values are
 210             # (EINVAL, EPERM, ESRCH)
 211             raise
 212     else:
 213         return True
 214
 215 def get_textdiff(text1, text2, title1="", title2="", **opts):
 216     "Returns empty string if same or formatted diff"
 217
 218     diff = '\n'.join(difflib.unified_diff(text1, text2,
 219            fromfile=title1, tofile=title2, **opts))
 220     # Clean up line endings
 221     diff = os.linesep.join([s for s in diff.splitlines() if s])
 222     return diff
 223
 224 def difflines(text1, text2, title1='', title2='', **opts):
 225     "Wrapper for get_textdiff to avoid string transformations."
 226     text1 = ('\n'.join(text1.rstrip().splitlines()) + '\n').splitlines(1)
 227     text2 = ('\n'.join(text2.rstrip().splitlines()) + '\n').splitlines(1)
 228     return get_textdiff(text1, text2, title1, title2, **opts)
 229
 230 def get_file(content):
 231     """
 232     Generates a temporary file in '/tmp' with `content` and returns the file name.
 233     """
 234     fde = tempfile.NamedTemporaryFile(mode='w', delete=False)
 235     fname = fde.name
 236     fde.write(content)
 237     fde.close()
 238     return fname
 239
 240 def normalize_text(text):
 241     """
 242     Strips formating spaces/tabs and carriage returns.
 243     """
 244     text = re.sub(r'[ \t]+', ' ', text)
 245     text = re.sub(r'\r', '', text)
 246     return text
 247
 248 def version_cmp(v1, v2):
 249     """
 250     Compare two version strings and returns:
 251
 252     * `-1`: if `v1` is less than `v2`
 253     * `0`: if `v1` is equal to `v2`
 254     * `1`: if `v1` is greater than `v2`
 255
 256     Raises `ValueError` if versions are not well formated.
 257     """
 258     vregex = r'(?P<whole>\d+(\.(\d+))*)'
 259     v1m = re.match(vregex, v1)
 260     v2m = re.match(vregex, v2)
 261     if v1m is None or v2m is None:
 262         raise ValueError("got a invalid version string")
 263
 264     # Split values
 265     v1g = v1m.group('whole').split('.')
 266     v2g = v2m.group('whole').split('.')
 267
 268     # Get the longest version string
 269     vnum = len(v1g)
 270     if len(v2g) > vnum:
 271         vnum = len(v2g)
 272
 273     # Reverse list because we are going to pop the tail
 274     v1g.reverse()
 275     v2g.reverse()
 276     for _ in range(vnum):
 277         try:
 278             v1n = int(v1g.pop())
 279         except IndexError:
 280             while v2g:
 281                 v2n = int(v2g.pop())
 282                 if v2n > 0:
 283                     return -1
 284             break
 285
 286         try:
 287             v2n = int(v2g.pop())
 288         except IndexError:
 289             if v1n > 0:
 290                 return 1
 291             while v1g:
 292                 v1n = int(v1g.pop())
 293                 if v1n > 0:
 294                     return 1
 295             break
 296
 297         if v1n > v2n:
 298             return 1
 299         if v1n < v2n:
 300             return -1
 301     return 0
 302
 303 def ip4_route(node):
 304     """
 305     Gets a structured return of the command 'ip route'. It can be used in
 306     conjuction with json_cmp() to provide accurate assert explanations.
 307
 308     Return example:
 309     {
 310         '10.0.1.0/24': {
 311             'dev': 'eth0',
 312             'via': '172.16.0.1',
 313             'proto': '188',
 314         },
 315         '10.0.2.0/24': {
 316             'dev': 'eth1',
 317             'proto': 'kernel',
 318         }
 319     }
 320     """
 321     output = normalize_text(node.run('ip route')).splitlines()
 322     result = {}
 323     for line in output:
 324         columns = line.split(' ')
 325         route = result[columns[0]] = {}
 326         prev = None
 327         for column in columns:
 328             if prev == 'dev':
 329                 route['dev'] = column
 330             if prev == 'via':
 331                 route['via'] = column
 332             if prev == 'proto':
 333                 route['proto'] = column
 334             if prev == 'metric':
 335                 route['metric'] = column
 336             if prev == 'scope':
 337                 route['scope'] = column
 338             prev = column
 339
 340     return result
 341
 342 def ip6_route(node):
 343     """
 344     Gets a structured return of the command 'ip -6 route'. It can be used in
 345     conjuction with json_cmp() to provide accurate assert explanations.
 346
 347     Return example:
 348     {
 349         '2001:db8:1::/64': {
 350             'dev': 'eth0',
 351             'proto': '188',
 352         },
 353         '2001:db8:2::/64': {
 354             'dev': 'eth1',
 355             'proto': 'kernel',
 356         }
 357     }
 358     """
 359     output = normalize_text(node.run('ip -6 route')).splitlines()
 360     result = {}
 361     for line in output:
 362         columns = line.split(' ')
 363         route = result[columns[0]] = {}
 364         prev = None
 365         for column in columns:
 366             if prev == 'dev':
 367                 route['dev'] = column
 368             if prev == 'via':
 369                 route['via'] = column
 370             if prev == 'proto':
 371                 route['proto'] = column
 372             if prev == 'metric':
 373                 route['metric'] = column
 374             if prev == 'pref':
 375                 route['pref'] = column
 376             prev = column
 377
 378     return result
 379
 380 def sleep(amount, reason=None):
 381     """
 382     Sleep wrapper that registers in the log the amount of sleep
 383     """
 384     if reason is None:
 385         logger.info('Sleeping for {} seconds'.format(amount))
 386     else:
 387         logger.info(reason + ' ({} seconds)'.format(amount))
 388
 389     time.sleep(amount)
 390
 391 def checkAddressSanitizerError(output, router, component):
 392     "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise"
 393
 394     addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', output)
 395     if addressSantizerError:
 396         sys.stderr.write("%s: %s triggered an exception by AddressSanitizer\n" % (router, component))
 397         # Sanitizer Error found in log
 398         pidMark = addressSantizerError.group(1)
 399         addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), output, re.DOTALL)
 400         if addressSantizerLog:
 401             callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_back.f_globals['__file__'])
 402             callingProc = sys._getframe(2).f_code.co_name
 403             with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile:
 404                 sys.stderr.write('\n'.join(addressSantizerLog.group(1).splitlines()) + '\n')
 405                 addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2))
 406                 addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router))
 407                 addrSanFile.write('    '+ '\n    '.join(addressSantizerLog.group(1).splitlines()) + '\n')
 408                 addrSanFile.write("\n---------------\n")
 409         return True
 410     return False
 411
 412 def addRouter(topo, name):
 413     "Adding a FRRouter (or Quagga) to Topology"
 414
 415     MyPrivateDirs = ['/etc/frr',
 416                          '/etc/quagga',
 417                          '/var/run/frr',
 418                          '/var/run/quagga',
 419                          '/var/log']
 420     return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs)
 421
 422 def set_sysctl(node, sysctl, value):
 423     "Set a sysctl value and return None on success or an error string"
 424     valuestr = '{}'.format(value)
 425     command = "sysctl {0}={1}".format(sysctl, valuestr)
 426     cmdret = node.cmd(command)
 427
 428     matches = re.search(r'([^ ]+) = ([^\s]+)', cmdret)
 429     if matches is None:
 430         return cmdret
 431     if matches.group(1) != sysctl:
 432         return cmdret
 433     if matches.group(2) != valuestr:
 434         return cmdret
 435
 436     return None
 437
 438 def assert_sysctl(node, sysctl, value):
 439     "Set and assert that the sysctl is set with the specified value."
 440     assert set_sysctl(node, sysctl, value) is None
 441
 442 class LinuxRouter(Node):
 443     "A Node with IPv4/IPv6 forwarding enabled."
 444
 445     def config(self, **params):
 446         super(LinuxRouter, self).config(**params)
 447         # Enable forwarding on the router
 448         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 449         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 450     def terminate(self):
 451         """
 452         Terminate generic LinuxRouter Mininet instance
 453         """
 454         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 455         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 456         super(LinuxRouter, self).terminate()
 457
 458 class Router(Node):
 459     "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine"
 460
 461     def __init__(self, name, **params):
 462         super(Router, self).__init__(name, **params)
 463         self.logdir = params.get('logdir', '/tmp')
 464         self.daemondir = None
 465         self.routertype = 'frr'
 466         self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0,
 467                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0,
 468                         'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0}
 469
 470     def _config_frr(self, **params):
 471         "Configure FRR binaries"
 472         self.daemondir = params.get('frrdir')
 473         if self.daemondir is None:
 474             self.daemondir = '/usr/lib/frr'
 475
 476         zebra_path = os.path.join(self.daemondir, 'zebra')
 477         if not os.path.isfile(zebra_path):
 478             raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path))
 479
 480     def _config_quagga(self, **params):
 481         "Configure Quagga binaries"
 482         self.daemondir = params.get('quaggadir')
 483         if self.daemondir is None:
 484             self.daemondir = '/usr/lib/quagga'
 485
 486         zebra_path = os.path.join(self.daemondir, 'zebra')
 487         if not os.path.isfile(zebra_path):
 488             raise Exception("Quagga zebra binary doesn't exist at {}".format(zebra_path))
 489
 490     # pylint: disable=W0221
 491     # Some params are only meaningful for the parent class.
 492     def config(self, **params):
 493         super(Router, self).config(**params)
 494
 495         # User did not specify the daemons directory, try to autodetect it.
 496         self.daemondir = params.get('daemondir')
 497         if self.daemondir is None:
 498             self.routertype = params.get('routertype', 'frr')
 499             if self.routertype == 'quagga':
 500                 self._config_quagga(**params)
 501             else:
 502                 self._config_frr(**params)
 503         else:
 504             # Test the provided path
 505             zpath = os.path.join(self.daemondir, 'zebra')
 506             if not os.path.isfile(zpath):
 507                 raise Exception('No zebra binary found in {}'.format(zpath))
 508             # Allow user to specify routertype when the path was specified.
 509             if params.get('routertype') is not None:
 510                 self.routertype = self.params.get('routertype')
 511
 512         # Enable forwarding on the router
 513         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 514         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 515         # Enable coredumps
 516         assert_sysctl(self, 'kernel.core_uses_pid', 1)
 517         assert_sysctl(self, 'fs.suid_dumpable', 2)
 518         corefile = '{}/{}_%e_core-sig_%s-pid_%p.dmp'.format(self.logdir, self.name)
 519         assert_sysctl(self, 'kernel.core_pattern', corefile)
 520         self.cmd('ulimit -c unlimited')
 521         # Set ownership of config files
 522         self.cmd('chown {0}:{0}vty /etc/{0}'.format(self.routertype))
 523
 524     def terminate(self):
 525         # Delete Running Quagga or FRR Daemons
 526         self.stopRouter()
 527         # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 528         # for d in StringIO.StringIO(rundaemons):
 529         #     self.cmd('kill -7 `cat %s`' % d.rstrip())
 530         #     self.waitOutput()
 531         # Disable forwarding
 532         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 533         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 534         super(Router, self).terminate()
 535     def stopRouter(self):
 536         # Stop Running Quagga or FRR Daemons
 537         rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 538         if rundaemons is not None:
 539             for d in StringIO.StringIO(rundaemons):
 540                 daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 541                 if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 542                     self.cmd('kill -TERM %s' % daemonpid)
 543                     self.waitOutput()
 544             sleep(2, 'waiting for router "{}" daemons to finish'.format(
 545                 self.name))
 546             # 2nd round of kill if daemons didn't exist
 547             for d in StringIO.StringIO(rundaemons):
 548                 daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 549                 if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 550                     self.cmd('kill -7 %s' % daemonpid)
 551                     self.waitOutput()
 552     def removeIPs(self):
 553         for interface in self.intfNames():
 554             self.cmd('ip address flush', interface)
 555     def loadConf(self, daemon, source=None):
 556         # print "Daemons before:", self.daemons
 557         if daemon in self.daemons.keys():
 558             self.daemons[daemon] = 1
 559             if source is None:
 560                 self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon))
 561                 self.waitOutput()
 562             else:
 563                 self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon))
 564                 self.waitOutput()
 565             self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon))
 566             self.waitOutput()
 567             self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon))
 568             self.waitOutput()
 569         else:
 570             logger.info('No daemon {} known'.format(daemon))
 571         # print "Daemons after:", self.daemons
 572     def startRouter(self):
 573         # Disable integrated-vtysh-config
 574         self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype)
 575         self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype))
 576         # TODO remove the following lines after all tests are migrated to Topogen.
 577         # Try to find relevant old logfiles in /tmp and delete them
 578         map(os.remove, glob.glob("/tmp/*%s*.log" % self.name))
 579         # Remove old core files
 580         map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name))
 581         # Remove IP addresses from OS first - we have them in zebra.conf
 582         self.removeIPs()
 583         # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher
 584         # No error - but return message and skip all the tests
 585         if self.daemons['ldpd'] == 1:
 586             ldpd_path = os.path.join(self.daemondir, 'ldpd')
 587             if not os.path.isfile(ldpd_path):
 588                 logger.info("LDP Test, but no ldpd compiled or installed")
 589                 return "LDP Test, but no ldpd compiled or installed"
 590
 591             if version_cmp(platform.release(), '4.5') < 0:
 592                 logger.info("LDP Test need Linux Kernel 4.5 minimum")
 593                 return "LDP Test need Linux Kernel 4.5 minimum"
 594
 595             # Check if required kernel modules are available with a dryrun modprobe
 596             # Silent accept of modprobe command assumes ok status
 597             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 598                 logger.info("LDP Test needs mpls-router kernel module")
 599                 return "LDP Test needs mpls-router kernel module"
 600             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 601                 logger.info("LDP Test needs mpls-iptunnel kernel module")
 602                 return "LDP Test needs mpls-router kernel module"
 603
 604             self.cmd('/sbin/modprobe mpls-router')
 605             self.cmd('/sbin/modprobe mpls-iptunnel')
 606             self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels')
 607
 608         if self.daemons['eigrpd'] == 1:
 609             eigrpd_path = os.path.join(self.daemondir, 'eigrpd')
 610             if not os.path.isfile(eigrpd_path):
 611                 logger.info("EIGRP Test, but no eigrpd compiled or installed")
 612                 return "EIGRP Test, but no eigrpd compiled or installed"
 613
 614         # Init done - now restarting daemons
 615         self.restartRouter()
 616         return ""
 617     def restartRouter(self):
 618         # Starts actuall daemons without init (ie restart)
 619         # Start Zebra first
 620         if self.daemons['zebra'] == 1:
 621             zebra_path = os.path.join(self.daemondir, 'zebra')
 622             self.cmd('{0} > {1}/{2}-zebra.out 2> {1}/{2}-zebra.err &'.format(
 623                 zebra_path, self.logdir, self.name
 624             ))
 625             self.waitOutput()
 626             logger.debug('{}: {} zebra started'.format(self, self.routertype))
 627             time.sleep(1)
 628         # Fix Link-Local Addresses
 629         # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
 630         self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done')
 631         # Now start all the other daemons
 632         for daemon in self.daemons:
 633             # Skip disabled daemons and zebra
 634             if self.daemons[daemon] == 0 or daemon == 'zebra':
 635                 continue
 636
 637             daemon_path = os.path.join(self.daemondir, daemon)
 638             self.cmd('{0} > {1}/{2}-{3}.out 2> {1}/{2}-{3}.err &'.format(
 639                 daemon_path, self.logdir, self.name, daemon
 640             ))
 641             self.waitOutput()
 642             logger.debug('{}: {} {} started'.format(self, self.routertype, daemon))
 643     def getStdErr(self, daemon):
 644         return self.getLog('err', daemon)
 645     def getStdOut(self, daemon):
 646         return self.getLog('out', daemon)
 647     def getLog(self, log, daemon):
 648         return self.cmd('cat {}/{}-{}.{}'.format(self.logdir, self.name, daemon, log))
 649     def checkRouterRunning(self):
 650         "Check if router daemons are running and collect crashinfo they don't run"
 651
 652         global fatal_error
 653
 654         daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"')
 655         # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found
 656         if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"):
 657             return "%s: vtysh killed by AddressSanitizer" % (self.name)
 658
 659         for daemon in self.daemons:
 660             if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning):
 661                 sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon))
 662                 # Look for core file
 663                 corefiles = glob.glob('{}/{}_{}_core*.dmp'.format(
 664                     self.logdir, self.name, daemon))
 665                 if (len(corefiles) > 0):
 666                     daemon_path = os.path.join(self.daemondir, daemon)
 667                     backtrace = subprocess.check_output([
 668                         "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0])
 669                     ], shell=True)
 670                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
 671                     sys.stderr.write("%s\n" % backtrace)
 672                 else:
 673                     # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
 674                     if os.path.isfile('{}/{}-{}.log'.format(self.logdir, self.name, daemon)):
 675                         log_tail = subprocess.check_output([
 676                             "tail -n20 {}/{}-{}.log 2> /dev/null".format(
 677                                 self.logdir, self.name, daemon)
 678                             ], shell=True)
 679                         sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon))
 680                         sys.stderr.write("%s\n" % log_tail)
 681
 682                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
 683                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
 684                     return "%s: Daemon %s not running - killed by AddressSanitizer" % (self.name, daemon)
 685
 686                 return "%s: Daemon %s not running" % (self.name, daemon)
 687         return ""
 688     def get_ipv6_linklocal(self):
 689         "Get LinkLocal Addresses from interfaces"
 690
 691         linklocal = []
 692
 693         ifaces = self.cmd('ip -6 address')
 694         # Fix newlines (make them all the same)
 695         ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines()
 696         interface=""
 697         ll_per_if_count=0
 698         for line in ifaces:
 699             m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line)
 700             if m:
 701                 interface = m.group(1)
 702                 ll_per_if_count = 0
 703             m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line)
 704             if m:
 705                 local = m.group(1)
 706                 ll_per_if_count += 1
 707                 if (ll_per_if_count > 1):
 708                     linklocal += [["%s-%s" % (interface, ll_per_if_count), local]]
 709                 else:
 710                     linklocal += [[interface, local]]
 711         return linklocal
 712     def daemon_available(self, daemon):
 713         "Check if specified daemon is installed (and for ldp if kernel supports MPLS)"
 714
 715         daemon_path = os.path.join(self.daemondir, daemon)
 716         if not os.path.isfile(daemon_path):
 717             return False
 718         if (daemon == 'ldpd'):
 719             if version_cmp(platform.release(), '4.5') < 0:
 720                 return False
 721             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 722                 return False
 723             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 724                 return False
 725
 726         return True
 727     def get_routertype(self):
 728         "Return the type of Router (frr or quagga)"
 729
 730         return self.routertype
 731     def report_memory_leaks(self, filename_prefix, testscript):
 732         "Report Memory Leaks to file prefixed with given string"
 733
 734         leakfound = False
 735         filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt"
 736         for daemon in self.daemons:
 737             if (self.daemons[daemon] == 1):
 738                 log = self.getStdErr(daemon)
 739                 if "memstats" in log:
 740                     # Found memory leak
 741                     logger.info('\nRouter {} {} StdErr Log:\n{}'.format(
 742                         self.name, daemon, log))
 743                     if not leakfound:
 744                         leakfound = True
 745                         # Check if file already exists
 746                         fileexists = os.path.isfile(filename)
 747                         leakfile = open(filename, "a")
 748                         if not fileexists:
 749                             # New file - add header
 750                             leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript)
 751                         leakfile.write("## Router %s\n" % self.name)
 752                     leakfile.write("### Process %s\n" % daemon)
 753                     log = re.sub("core_handler: ", "", log)
 754                     log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log)
 755                     log = re.sub("memstats:  ", "    ", log)
 756                     leakfile.write(log)
 757                     leakfile.write("\n")
 758         if leakfound:
 759             leakfile.close()
 760
 761
 762 class LegacySwitch(OVSSwitch):
 763     "A Legacy Switch without OpenFlow"
 764
 765     def __init__(self, name, **params):
 766         OVSSwitch.__init__(self, name, failMode='standalone', **params)
 767         self.switchIP = None