tests/topotests/lib/topotest.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # topotest.py
   5 # Library of helper functions for NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2016 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 import json
  26 import os
  27 import errno
  28 import re
  29 import sys
  30 import glob
  31 import StringIO
  32 import subprocess
  33 import tempfile
  34 import platform
  35 import difflib
  36 import time
  37
  38 from lib.topolog import logger
  39
  40 from mininet.topo import Topo
  41 from mininet.net import Mininet
  42 from mininet.node import Node, OVSSwitch, Host
  43 from mininet.log import setLogLevel, info
  44 from mininet.cli import CLI
  45 from mininet.link import Intf
  46
  47 class json_cmp_result(object):
  48     "json_cmp result class for better assertion messages"
  49
  50     def __init__(self):
  51         self.errors = []
  52
  53     def add_error(self, error):
  54         "Append error message to the result"
  55         for line in error.splitlines():
  56             self.errors.append(line)
  57
  58     def has_errors(self):
  59         "Returns True if there were errors, otherwise False."
  60         return len(self.errors) > 0
  61
  62 def json_diff(d1, d2):
  63     """
  64     Returns a string with the difference between JSON data.
  65     """
  66     json_format_opts = {
  67         'indent': 4,
  68         'sort_keys': True,
  69     }
  70     dstr1 = json.dumps(d1, **json_format_opts)
  71     dstr2 = json.dumps(d2, **json_format_opts)
  72     return difflines(dstr2, dstr1, title1='Expected value', title2='Current value', n=0)
  73
  74 def json_cmp(d1, d2):
  75     """
  76     JSON compare function. Receives two parameters:
  77     * `d1`: json value
  78     * `d2`: json subset which we expect
  79
  80     Returns `None` when all keys that `d1` has matches `d2`,
  81     otherwise a string containing what failed.
  82
  83     Note: key absence can be tested by adding a key with value `None`.
  84     """
  85     squeue = [(d1, d2, 'json')]
  86     result = json_cmp_result()
  87     for s in squeue:
  88         nd1, nd2, parent = s
  89         s1, s2 = set(nd1), set(nd2)
  90
  91         # Expect all required fields to exist.
  92         s2_req = set([key for key in nd2 if nd2[key] is not None])
  93         diff = s2_req - s1
  94         if diff != set({}):
  95             result.add_error('expected key(s) {} in {} (have {}):\n{}'.format(
  96                 str(list(diff)), parent, str(list(s1)), json_diff(nd1, nd2)))
  97
  98         for key in s2.intersection(s1):
  99             # Test for non existence of key in d2
 100             if nd2[key] is None:
 101                 result.add_error('"{}" should not exist in {} (have {}):\n{}'.format(
 102                     key, parent, str(s1), json_diff(nd1[key], nd2[key])))
 103                 continue
 104             # If nd1 key is a dict, we have to recurse in it later.
 105             if isinstance(nd2[key], type({})):
 106                 if not isinstance(nd1[key], type({})):
 107                     result.add_error(
 108                         '{}["{}"] has different type than expected '.format(parent, key) +
 109                         '(have {}, expected {}):\n{}'.format(
 110                             type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key])))
 111                     continue
 112                 nparent = '{}["{}"]'.format(parent, key)
 113                 squeue.append((nd1[key], nd2[key], nparent))
 114                 continue
 115             # Check list items
 116             if isinstance(nd2[key], type([])):
 117                 if not isinstance(nd1[key], type([])):
 118                     result.add_error(
 119                         '{}["{}"] has different type than expected '.format(parent, key) +
 120                         '(have {}, expected {}):\n{}'.format(
 121                             type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key])))
 122                     continue
 123                 # Check list size
 124                 if len(nd2[key]) > len(nd1[key]):
 125                     result.add_error(
 126                         '{}["{}"] too few items '.format(parent, key) +
 127                         '(have {}, expected {}:\n {})'.format(
 128                             len(nd1[key]), len(nd2[key]),
 129                             json_diff(nd1[key], nd2[key])))
 130                     continue
 131
 132                 # List all unmatched items errors
 133                 unmatched = []
 134                 for expected in nd2[key]:
 135                     matched = False
 136                     for value in nd1[key]:
 137                         if json_cmp({'json': value}, {'json': expected}) is None:
 138                             matched = True
 139                             break
 140
 141                     if matched:
 142                         break
 143                     if not matched:
 144                         unmatched.append(expected)
 145
 146                 # If there are unmatched items, error out.
 147                 if unmatched:
 148                     result.add_error(
 149                         '{}["{}"] value is different (\n{})'.format(
 150                             parent, key, json_diff(nd1[key], nd2[key])))
 151                 continue
 152
 153             # Compare JSON values
 154             if nd1[key] != nd2[key]:
 155                 result.add_error(
 156                     '{}["{}"] value is different (\n{})'.format(
 157                         parent, key, json_diff(nd1[key], nd2[key])))
 158                 continue
 159
 160     if result.has_errors():
 161         return result
 162
 163     return None
 164
 165 def run_and_expect(func, what, count=20, wait=3):
 166     """
 167     Run `func` and compare the result with `what`. Do it for `count` times
 168     waiting `wait` seconds between tries. By default it tries 20 times with
 169     3 seconds delay between tries.
 170
 171     Returns (True, func-return) on success or
 172     (False, func-return) on failure.
 173     """
 174     while count > 0:
 175         result = func()
 176         if result != what:
 177             time.sleep(wait)
 178             count -= 1
 179             continue
 180         return (True, result)
 181     return (False, result)
 182
 183
 184 def int2dpid(dpid):
 185     "Converting Integer to DPID"
 186
 187     try:
 188         dpid = hex(dpid)[2:]
 189         dpid = '0'*(16-len(dpid))+dpid
 190         return dpid
 191     except IndexError:
 192         raise Exception('Unable to derive default datapath ID - '
 193                         'please either specify a dpid or use a '
 194                         'canonical switch name such as s23.')
 195
 196 def pid_exists(pid):
 197     "Check whether pid exists in the current process table."
 198
 199     if pid <= 0:
 200         return False
 201     try:
 202         os.kill(pid, 0)
 203     except OSError as err:
 204         if err.errno == errno.ESRCH:
 205             # ESRCH == No such process
 206             return False
 207         elif err.errno == errno.EPERM:
 208             # EPERM clearly means there's a process to deny access to
 209             return True
 210         else:
 211             # According to "man 2 kill" possible error values are
 212             # (EINVAL, EPERM, ESRCH)
 213             raise
 214     else:
 215         return True
 216
 217 def get_textdiff(text1, text2, title1="", title2="", **opts):
 218     "Returns empty string if same or formatted diff"
 219
 220     diff = '\n'.join(difflib.unified_diff(text1, text2,
 221            fromfile=title1, tofile=title2, **opts))
 222     # Clean up line endings
 223     diff = os.linesep.join([s for s in diff.splitlines() if s])
 224     return diff
 225
 226 def difflines(text1, text2, title1='', title2='', **opts):
 227     "Wrapper for get_textdiff to avoid string transformations."
 228     text1 = ('\n'.join(text1.rstrip().splitlines()) + '\n').splitlines(1)
 229     text2 = ('\n'.join(text2.rstrip().splitlines()) + '\n').splitlines(1)
 230     return get_textdiff(text1, text2, title1, title2, **opts)
 231
 232 def get_file(content):
 233     """
 234     Generates a temporary file in '/tmp' with `content` and returns the file name.
 235     """
 236     fde = tempfile.NamedTemporaryFile(mode='w', delete=False)
 237     fname = fde.name
 238     fde.write(content)
 239     fde.close()
 240     return fname
 241
 242 def normalize_text(text):
 243     """
 244     Strips formating spaces/tabs and carriage returns.
 245     """
 246     text = re.sub(r'[ \t]+', ' ', text)
 247     text = re.sub(r'\r', '', text)
 248     return text
 249
 250 def version_cmp(v1, v2):
 251     """
 252     Compare two version strings and returns:
 253
 254     * `-1`: if `v1` is less than `v2`
 255     * `0`: if `v1` is equal to `v2`
 256     * `1`: if `v1` is greater than `v2`
 257
 258     Raises `ValueError` if versions are not well formated.
 259     """
 260     vregex = r'(?P<whole>\d+(\.(\d+))*)'
 261     v1m = re.match(vregex, v1)
 262     v2m = re.match(vregex, v2)
 263     if v1m is None or v2m is None:
 264         raise ValueError("got a invalid version string")
 265
 266     # Split values
 267     v1g = v1m.group('whole').split('.')
 268     v2g = v2m.group('whole').split('.')
 269
 270     # Get the longest version string
 271     vnum = len(v1g)
 272     if len(v2g) > vnum:
 273         vnum = len(v2g)
 274
 275     # Reverse list because we are going to pop the tail
 276     v1g.reverse()
 277     v2g.reverse()
 278     for _ in range(vnum):
 279         try:
 280             v1n = int(v1g.pop())
 281         except IndexError:
 282             while v2g:
 283                 v2n = int(v2g.pop())
 284                 if v2n > 0:
 285                     return -1
 286             break
 287
 288         try:
 289             v2n = int(v2g.pop())
 290         except IndexError:
 291             if v1n > 0:
 292                 return 1
 293             while v1g:
 294                 v1n = int(v1g.pop())
 295                 if v1n > 0:
 296                     return 1
 297             break
 298
 299         if v1n > v2n:
 300             return 1
 301         if v1n < v2n:
 302             return -1
 303     return 0
 304
 305 def ip4_route(node):
 306     """
 307     Gets a structured return of the command 'ip route'. It can be used in
 308     conjuction with json_cmp() to provide accurate assert explanations.
 309
 310     Return example:
 311     {
 312         '10.0.1.0/24': {
 313             'dev': 'eth0',
 314             'via': '172.16.0.1',
 315             'proto': '188',
 316         },
 317         '10.0.2.0/24': {
 318             'dev': 'eth1',
 319             'proto': 'kernel',
 320         }
 321     }
 322     """
 323     output = normalize_text(node.run('ip route')).splitlines()
 324     result = {}
 325     for line in output:
 326         columns = line.split(' ')
 327         route = result[columns[0]] = {}
 328         prev = None
 329         for column in columns:
 330             if prev == 'dev':
 331                 route['dev'] = column
 332             if prev == 'via':
 333                 route['via'] = column
 334             if prev == 'proto':
 335                 route['proto'] = column
 336             if prev == 'metric':
 337                 route['metric'] = column
 338             if prev == 'scope':
 339                 route['scope'] = column
 340             prev = column
 341
 342     return result
 343
 344 def ip6_route(node):
 345     """
 346     Gets a structured return of the command 'ip -6 route'. It can be used in
 347     conjuction with json_cmp() to provide accurate assert explanations.
 348
 349     Return example:
 350     {
 351         '2001:db8:1::/64': {
 352             'dev': 'eth0',
 353             'proto': '188',
 354         },
 355         '2001:db8:2::/64': {
 356             'dev': 'eth1',
 357             'proto': 'kernel',
 358         }
 359     }
 360     """
 361     output = normalize_text(node.run('ip -6 route')).splitlines()
 362     result = {}
 363     for line in output:
 364         columns = line.split(' ')
 365         route = result[columns[0]] = {}
 366         prev = None
 367         for column in columns:
 368             if prev == 'dev':
 369                 route['dev'] = column
 370             if prev == 'via':
 371                 route['via'] = column
 372             if prev == 'proto':
 373                 route['proto'] = column
 374             if prev == 'metric':
 375                 route['metric'] = column
 376             if prev == 'pref':
 377                 route['pref'] = column
 378             prev = column
 379
 380     return result
 381
 382 def sleep(amount, reason=None):
 383     """
 384     Sleep wrapper that registers in the log the amount of sleep
 385     """
 386     if reason is None:
 387         logger.info('Sleeping for {} seconds'.format(amount))
 388     else:
 389         logger.info(reason + ' ({} seconds)'.format(amount))
 390
 391     time.sleep(amount)
 392
 393 def checkAddressSanitizerError(output, router, component):
 394     "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise"
 395
 396     addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', output)
 397     if addressSantizerError:
 398         sys.stderr.write("%s: %s triggered an exception by AddressSanitizer\n" % (router, component))
 399         # Sanitizer Error found in log
 400         pidMark = addressSantizerError.group(1)
 401         addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), output, re.DOTALL)
 402         if addressSantizerLog:
 403             callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_back.f_globals['__file__'])
 404             callingProc = sys._getframe(2).f_code.co_name
 405             with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile:
 406                 sys.stderr.write('\n'.join(addressSantizerLog.group(1).splitlines()) + '\n')
 407                 addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2))
 408                 addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router))
 409                 addrSanFile.write('    '+ '\n    '.join(addressSantizerLog.group(1).splitlines()) + '\n')
 410                 addrSanFile.write("\n---------------\n")
 411         return True
 412     return False
 413
 414 def addRouter(topo, name):
 415     "Adding a FRRouter (or Quagga) to Topology"
 416
 417     MyPrivateDirs = ['/etc/frr',
 418                          '/etc/quagga',
 419                          '/var/run/frr',
 420                          '/var/run/quagga',
 421                          '/var/log']
 422     return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs)
 423
 424 def set_sysctl(node, sysctl, value):
 425     "Set a sysctl value and return None on success or an error string"
 426     valuestr = '{}'.format(value)
 427     command = "sysctl {0}={1}".format(sysctl, valuestr)
 428     cmdret = node.cmd(command)
 429
 430     matches = re.search(r'([^ ]+) = ([^\s]+)', cmdret)
 431     if matches is None:
 432         return cmdret
 433     if matches.group(1) != sysctl:
 434         return cmdret
 435     if matches.group(2) != valuestr:
 436         return cmdret
 437
 438     return None
 439
 440 def assert_sysctl(node, sysctl, value):
 441     "Set and assert that the sysctl is set with the specified value."
 442     assert set_sysctl(node, sysctl, value) is None
 443
 444 class LinuxRouter(Node):
 445     "A Node with IPv4/IPv6 forwarding enabled."
 446
 447     def config(self, **params):
 448         super(LinuxRouter, self).config(**params)
 449         # Enable forwarding on the router
 450         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 451         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 452     def terminate(self):
 453         """
 454         Terminate generic LinuxRouter Mininet instance
 455         """
 456         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 457         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 458         super(LinuxRouter, self).terminate()
 459
 460 class Router(Node):
 461     "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine"
 462
 463     def __init__(self, name, **params):
 464         super(Router, self).__init__(name, **params)
 465         self.logdir = params.get('logdir', '/tmp')
 466         self.daemondir = None
 467         self.routertype = 'frr'
 468         self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0,
 469                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0,
 470                         'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0}
 471
 472     def _config_frr(self, **params):
 473         "Configure FRR binaries"
 474         self.daemondir = params.get('frrdir')
 475         if self.daemondir is None:
 476             self.daemondir = '/usr/lib/frr'
 477
 478         zebra_path = os.path.join(self.daemondir, 'zebra')
 479         if not os.path.isfile(zebra_path):
 480             raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path))
 481
 482     def _config_quagga(self, **params):
 483         "Configure Quagga binaries"
 484         self.daemondir = params.get('quaggadir')
 485         if self.daemondir is None:
 486             self.daemondir = '/usr/lib/quagga'
 487
 488         zebra_path = os.path.join(self.daemondir, 'zebra')
 489         if not os.path.isfile(zebra_path):
 490             raise Exception("Quagga zebra binary doesn't exist at {}".format(zebra_path))
 491
 492     # pylint: disable=W0221
 493     # Some params are only meaningful for the parent class.
 494     def config(self, **params):
 495         super(Router, self).config(**params)
 496
 497         # User did not specify the daemons directory, try to autodetect it.
 498         self.daemondir = params.get('daemondir')
 499         if self.daemondir is None:
 500             self.routertype = params.get('routertype', 'frr')
 501             if self.routertype == 'quagga':
 502                 self._config_quagga(**params)
 503             else:
 504                 self._config_frr(**params)
 505         else:
 506             # Test the provided path
 507             zpath = os.path.join(self.daemondir, 'zebra')
 508             if not os.path.isfile(zpath):
 509                 raise Exception('No zebra binary found in {}'.format(zpath))
 510             # Allow user to specify routertype when the path was specified.
 511             if params.get('routertype') is not None:
 512                 self.routertype = self.params.get('routertype')
 513
 514         # Enable forwarding on the router
 515         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 516         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 517         # Enable coredumps
 518         assert_sysctl(self, 'kernel.core_uses_pid', 1)
 519         assert_sysctl(self, 'fs.suid_dumpable', 2)
 520         corefile = '{}/{}_%e_core-sig_%s-pid_%p.dmp'.format(self.logdir, self.name)
 521         assert_sysctl(self, 'kernel.core_pattern', corefile)
 522         self.cmd('ulimit -c unlimited')
 523         # Set ownership of config files
 524         self.cmd('chown {0}:{0}vty /etc/{0}'.format(self.routertype))
 525
 526     def terminate(self):
 527         # Delete Running Quagga or FRR Daemons
 528         self.stopRouter()
 529         # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 530         # for d in StringIO.StringIO(rundaemons):
 531         #     self.cmd('kill -7 `cat %s`' % d.rstrip())
 532         #     self.waitOutput()
 533         # Disable forwarding
 534         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 535         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 536         super(Router, self).terminate()
 537     def stopRouter(self, wait=True):
 538         # Stop Running Quagga or FRR Daemons
 539         rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 540         if rundaemons is not None:
 541             numRunning = 0
 542             for d in StringIO.StringIO(rundaemons):
 543                 daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 544                 if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 545                     logger.info('{}: stopping {}'.format(
 546                         self.name,
 547                         os.path.basename(d.rstrip().rsplit(".", 1)[0])
 548                     ))
 549                     self.cmd('kill -TERM %s' % daemonpid)
 550                     self.waitOutput()
 551                     if pid_exists(int(daemonpid)):
 552                         numRunning += 1
 553             if wait and numRunning > 0:
 554                 sleep(2, '{}: waiting for daemons stopping'.format(self.name))
 555                 # 2nd round of kill if daemons didn't exit
 556                 for d in StringIO.StringIO(rundaemons):
 557                     daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 558                     if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 559                         logger.info('{}: killing {}'.format(
 560                             self.name,
 561                             os.path.basename(d.rstrip().rsplit(".", 1)[0])
 562                         ))
 563                         self.cmd('kill -7 %s' % daemonpid)
 564                         self.waitOutput()
 565     def removeIPs(self):
 566         for interface in self.intfNames():
 567             self.cmd('ip address flush', interface)
 568     def loadConf(self, daemon, source=None):
 569         # print "Daemons before:", self.daemons
 570         if daemon in self.daemons.keys():
 571             self.daemons[daemon] = 1
 572             if source is None:
 573                 self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon))
 574                 self.waitOutput()
 575             else:
 576                 self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon))
 577                 self.waitOutput()
 578             self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon))
 579             self.waitOutput()
 580             self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon))
 581             self.waitOutput()
 582         else:
 583             logger.info('No daemon {} known'.format(daemon))
 584         # print "Daemons after:", self.daemons
 585     def startRouter(self):
 586         # Disable integrated-vtysh-config
 587         self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype)
 588         self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype))
 589         # TODO remove the following lines after all tests are migrated to Topogen.
 590         # Try to find relevant old logfiles in /tmp and delete them
 591         map(os.remove, glob.glob("/tmp/*%s*.log" % self.name))
 592         # Remove old core files
 593         map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name))
 594         # Remove IP addresses from OS first - we have them in zebra.conf
 595         self.removeIPs()
 596         # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher
 597         # No error - but return message and skip all the tests
 598         if self.daemons['ldpd'] == 1:
 599             ldpd_path = os.path.join(self.daemondir, 'ldpd')
 600             if not os.path.isfile(ldpd_path):
 601                 logger.info("LDP Test, but no ldpd compiled or installed")
 602                 return "LDP Test, but no ldpd compiled or installed"
 603
 604             if version_cmp(platform.release(), '4.5') < 0:
 605                 logger.info("LDP Test need Linux Kernel 4.5 minimum")
 606                 return "LDP Test need Linux Kernel 4.5 minimum"
 607
 608             # Check if required kernel modules are available with a dryrun modprobe
 609             # Silent accept of modprobe command assumes ok status
 610             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 611                 logger.info("LDP Test needs mpls-router kernel module")
 612                 return "LDP Test needs mpls-router kernel module"
 613             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 614                 logger.info("LDP Test needs mpls-iptunnel kernel module")
 615                 return "LDP Test needs mpls-router kernel module"
 616
 617             self.cmd('/sbin/modprobe mpls-router')
 618             self.cmd('/sbin/modprobe mpls-iptunnel')
 619             self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels')
 620
 621         if self.daemons['eigrpd'] == 1:
 622             eigrpd_path = os.path.join(self.daemondir, 'eigrpd')
 623             if not os.path.isfile(eigrpd_path):
 624                 logger.info("EIGRP Test, but no eigrpd compiled or installed")
 625                 return "EIGRP Test, but no eigrpd compiled or installed"
 626
 627         # Init done - now restarting daemons
 628         self.restartRouter()
 629         return ""
 630     def restartRouter(self):
 631         # Starts actuall daemons without init (ie restart)
 632         # Start Zebra first
 633         if self.daemons['zebra'] == 1:
 634             zebra_path = os.path.join(self.daemondir, 'zebra')
 635             self.cmd('{0} > {1}/{2}-zebra.out 2> {1}/{2}-zebra.err &'.format(
 636                 zebra_path, self.logdir, self.name
 637             ))
 638             self.waitOutput()
 639             logger.debug('{}: {} zebra started'.format(self, self.routertype))
 640             sleep(1, '{}: waiting for zebra to start'.format(self.name))
 641         # Fix Link-Local Addresses
 642         # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
 643         self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done')
 644         # Now start all the other daemons
 645         for daemon in self.daemons:
 646             # Skip disabled daemons and zebra
 647             if self.daemons[daemon] == 0 or daemon == 'zebra':
 648                 continue
 649
 650             daemon_path = os.path.join(self.daemondir, daemon)
 651             self.cmd('{0} > {1}/{2}-{3}.out 2> {1}/{2}-{3}.err &'.format(
 652                 daemon_path, self.logdir, self.name, daemon
 653             ))
 654             self.waitOutput()
 655             logger.debug('{}: {} {} started'.format(self, self.routertype, daemon))
 656     def getStdErr(self, daemon):
 657         return self.getLog('err', daemon)
 658     def getStdOut(self, daemon):
 659         return self.getLog('out', daemon)
 660     def getLog(self, log, daemon):
 661         return self.cmd('cat {}/{}-{}.{}'.format(self.logdir, self.name, daemon, log))
 662     def checkRouterRunning(self):
 663         "Check if router daemons are running and collect crashinfo they don't run"
 664
 665         global fatal_error
 666
 667         daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"')
 668         # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found
 669         if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"):
 670             return "%s: vtysh killed by AddressSanitizer" % (self.name)
 671
 672         for daemon in self.daemons:
 673             if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning):
 674                 sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon))
 675                 # Look for core file
 676                 corefiles = glob.glob('{}/{}_{}_core*.dmp'.format(
 677                     self.logdir, self.name, daemon))
 678                 if (len(corefiles) > 0):
 679                     daemon_path = os.path.join(self.daemondir, daemon)
 680                     backtrace = subprocess.check_output([
 681                         "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0])
 682                     ], shell=True)
 683                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
 684                     sys.stderr.write("%s\n" % backtrace)
 685                 else:
 686                     # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
 687                     if os.path.isfile('{}/{}-{}.log'.format(self.logdir, self.name, daemon)):
 688                         log_tail = subprocess.check_output([
 689                             "tail -n20 {}/{}-{}.log 2> /dev/null".format(
 690                                 self.logdir, self.name, daemon)
 691                             ], shell=True)
 692                         sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon))
 693                         sys.stderr.write("%s\n" % log_tail)
 694
 695                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
 696                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
 697                     return "%s: Daemon %s not running - killed by AddressSanitizer" % (self.name, daemon)
 698
 699                 return "%s: Daemon %s not running" % (self.name, daemon)
 700         return ""
 701     def get_ipv6_linklocal(self):
 702         "Get LinkLocal Addresses from interfaces"
 703
 704         linklocal = []
 705
 706         ifaces = self.cmd('ip -6 address')
 707         # Fix newlines (make them all the same)
 708         ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines()
 709         interface=""
 710         ll_per_if_count=0
 711         for line in ifaces:
 712             m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line)
 713             if m:
 714                 interface = m.group(1)
 715                 ll_per_if_count = 0
 716             m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line)
 717             if m:
 718                 local = m.group(1)
 719                 ll_per_if_count += 1
 720                 if (ll_per_if_count > 1):
 721                     linklocal += [["%s-%s" % (interface, ll_per_if_count), local]]
 722                 else:
 723                     linklocal += [[interface, local]]
 724         return linklocal
 725     def daemon_available(self, daemon):
 726         "Check if specified daemon is installed (and for ldp if kernel supports MPLS)"
 727
 728         daemon_path = os.path.join(self.daemondir, daemon)
 729         if not os.path.isfile(daemon_path):
 730             return False
 731         if (daemon == 'ldpd'):
 732             if version_cmp(platform.release(), '4.5') < 0:
 733                 return False
 734             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 735                 return False
 736             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 737                 return False
 738
 739         return True
 740     def get_routertype(self):
 741         "Return the type of Router (frr or quagga)"
 742
 743         return self.routertype
 744     def report_memory_leaks(self, filename_prefix, testscript):
 745         "Report Memory Leaks to file prefixed with given string"
 746
 747         leakfound = False
 748         filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt"
 749         for daemon in self.daemons:
 750             if (self.daemons[daemon] == 1):
 751                 log = self.getStdErr(daemon)
 752                 if "memstats" in log:
 753                     # Found memory leak
 754                     logger.info('\nRouter {} {} StdErr Log:\n{}'.format(
 755                         self.name, daemon, log))
 756                     if not leakfound:
 757                         leakfound = True
 758                         # Check if file already exists
 759                         fileexists = os.path.isfile(filename)
 760                         leakfile = open(filename, "a")
 761                         if not fileexists:
 762                             # New file - add header
 763                             leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript)
 764                         leakfile.write("## Router %s\n" % self.name)
 765                     leakfile.write("### Process %s\n" % daemon)
 766                     log = re.sub("core_handler: ", "", log)
 767                     log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log)
 768                     log = re.sub("memstats:  ", "    ", log)
 769                     leakfile.write(log)
 770                     leakfile.write("\n")
 771         if leakfound:
 772             leakfile.close()
 773
 774
 775 class LegacySwitch(OVSSwitch):
 776     "A Legacy Switch without OpenFlow"
 777
 778     def __init__(self, name, **params):
 779         OVSSwitch.__init__(self, name, failMode='standalone', **params)
 780         self.switchIP = None