tests/topotests/lib/topotest.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # topotest.py
   5 # Library of helper functions for NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2016 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 import json
  26 import os
  27 import errno
  28 import re
  29 import sys
  30 import glob
  31 import StringIO
  32 import subprocess
  33 import tempfile
  34 import platform
  35 import difflib
  36 import time
  37
  38 from lib.topolog import logger
  39
  40 from mininet.topo import Topo
  41 from mininet.net import Mininet
  42 from mininet.node import Node, OVSSwitch, Host
  43 from mininet.log import setLogLevel, info
  44 from mininet.cli import CLI
  45 from mininet.link import Intf
  46
  47 class json_cmp_result(object):
  48     "json_cmp result class for better assertion messages"
  49
  50     def __init__(self):
  51         self.errors = []
  52
  53     def add_error(self, error):
  54         "Append error message to the result"
  55         for line in error.splitlines():
  56             self.errors.append(line)
  57
  58     def has_errors(self):
  59         "Returns True if there were errors, otherwise False."
  60         return len(self.errors) > 0
  61
  62 def get_test_logdir(node=None, init=False):
  63     """
  64     Return the current test log directory based on PYTEST_CURRENT_TEST
  65     environment variable.
  66     Optional paramters:
  67     node:  when set, adds the node specific log directory to the init dir
  68     init:  when set, initializes the log directory and fixes path permissions
  69     """
  70     cur_test = os.environ['PYTEST_CURRENT_TEST']
  71
  72     ret = '/tmp/topotests/' + cur_test[0:cur_test.find(".py")].replace('/','.')
  73     if node != None:
  74         dir = ret + "/" + node
  75     if init:
  76         os.system('mkdir -p ' + dir)
  77         os.system('chmod -R go+rw /tmp/topotests')
  78     return ret
  79
  80 def json_diff(d1, d2):
  81     """
  82     Returns a string with the difference between JSON data.
  83     """
  84     json_format_opts = {
  85         'indent': 4,
  86         'sort_keys': True,
  87     }
  88     dstr1 = json.dumps(d1, **json_format_opts)
  89     dstr2 = json.dumps(d2, **json_format_opts)
  90     return difflines(dstr2, dstr1, title1='Expected value', title2='Current value', n=0)
  91
  92 def json_cmp(d1, d2):
  93     """
  94     JSON compare function. Receives two parameters:
  95     * `d1`: json value
  96     * `d2`: json subset which we expect
  97
  98     Returns `None` when all keys that `d1` has matches `d2`,
  99     otherwise a string containing what failed.
 100
 101     Note: key absence can be tested by adding a key with value `None`.
 102     """
 103     squeue = [(d1, d2, 'json')]
 104     result = json_cmp_result()
 105     for s in squeue:
 106         nd1, nd2, parent = s
 107         s1, s2 = set(nd1), set(nd2)
 108
 109         # Expect all required fields to exist.
 110         s2_req = set([key for key in nd2 if nd2[key] is not None])
 111         diff = s2_req - s1
 112         if diff != set({}):
 113             result.add_error('expected key(s) {} in {} (have {}):\n{}'.format(
 114                 str(list(diff)), parent, str(list(s1)), json_diff(nd1, nd2)))
 115
 116         for key in s2.intersection(s1):
 117             # Test for non existence of key in d2
 118             if nd2[key] is None:
 119                 result.add_error('"{}" should not exist in {} (have {}):\n{}'.format(
 120                     key, parent, str(s1), json_diff(nd1[key], nd2[key])))
 121                 continue
 122             # If nd1 key is a dict, we have to recurse in it later.
 123             if isinstance(nd2[key], type({})):
 124                 if not isinstance(nd1[key], type({})):
 125                     result.add_error(
 126                         '{}["{}"] has different type than expected '.format(parent, key) +
 127                         '(have {}, expected {}):\n{}'.format(
 128                             type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key])))
 129                     continue
 130                 nparent = '{}["{}"]'.format(parent, key)
 131                 squeue.append((nd1[key], nd2[key], nparent))
 132                 continue
 133             # Check list items
 134             if isinstance(nd2[key], type([])):
 135                 if not isinstance(nd1[key], type([])):
 136                     result.add_error(
 137                         '{}["{}"] has different type than expected '.format(parent, key) +
 138                         '(have {}, expected {}):\n{}'.format(
 139                             type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key])))
 140                     continue
 141                 # Check list size
 142                 if len(nd2[key]) > len(nd1[key]):
 143                     result.add_error(
 144                         '{}["{}"] too few items '.format(parent, key) +
 145                         '(have {}, expected {}:\n {})'.format(
 146                             len(nd1[key]), len(nd2[key]),
 147                             json_diff(nd1[key], nd2[key])))
 148                     continue
 149
 150                 # List all unmatched items errors
 151                 unmatched = []
 152                 for expected in nd2[key]:
 153                     matched = False
 154                     for value in nd1[key]:
 155                         if json_cmp({'json': value}, {'json': expected}) is None:
 156                             matched = True
 157                             break
 158
 159                     if matched:
 160                         break
 161                     if not matched:
 162                         unmatched.append(expected)
 163
 164                 # If there are unmatched items, error out.
 165                 if unmatched:
 166                     result.add_error(
 167                         '{}["{}"] value is different (\n{})'.format(
 168                             parent, key, json_diff(nd1[key], nd2[key])))
 169                 continue
 170
 171             # Compare JSON values
 172             if nd1[key] != nd2[key]:
 173                 result.add_error(
 174                     '{}["{}"] value is different (\n{})'.format(
 175                         parent, key, json_diff(nd1[key], nd2[key])))
 176                 continue
 177
 178     if result.has_errors():
 179         return result
 180
 181     return None
 182
 183 def run_and_expect(func, what, count=20, wait=3):
 184     """
 185     Run `func` and compare the result with `what`. Do it for `count` times
 186     waiting `wait` seconds between tries. By default it tries 20 times with
 187     3 seconds delay between tries.
 188
 189     Returns (True, func-return) on success or
 190     (False, func-return) on failure.
 191     """
 192     while count > 0:
 193         result = func()
 194         if result != what:
 195             time.sleep(wait)
 196             count -= 1
 197             continue
 198         return (True, result)
 199     return (False, result)
 200
 201
 202 def int2dpid(dpid):
 203     "Converting Integer to DPID"
 204
 205     try:
 206         dpid = hex(dpid)[2:]
 207         dpid = '0'*(16-len(dpid))+dpid
 208         return dpid
 209     except IndexError:
 210         raise Exception('Unable to derive default datapath ID - '
 211                         'please either specify a dpid or use a '
 212                         'canonical switch name such as s23.')
 213
 214 def pid_exists(pid):
 215     "Check whether pid exists in the current process table."
 216
 217     if pid <= 0:
 218         return False
 219     try:
 220         os.kill(pid, 0)
 221     except OSError as err:
 222         if err.errno == errno.ESRCH:
 223             # ESRCH == No such process
 224             return False
 225         elif err.errno == errno.EPERM:
 226             # EPERM clearly means there's a process to deny access to
 227             return True
 228         else:
 229             # According to "man 2 kill" possible error values are
 230             # (EINVAL, EPERM, ESRCH)
 231             raise
 232     else:
 233         return True
 234
 235 def get_textdiff(text1, text2, title1="", title2="", **opts):
 236     "Returns empty string if same or formatted diff"
 237
 238     diff = '\n'.join(difflib.unified_diff(text1, text2,
 239            fromfile=title1, tofile=title2, **opts))
 240     # Clean up line endings
 241     diff = os.linesep.join([s for s in diff.splitlines() if s])
 242     return diff
 243
 244 def difflines(text1, text2, title1='', title2='', **opts):
 245     "Wrapper for get_textdiff to avoid string transformations."
 246     text1 = ('\n'.join(text1.rstrip().splitlines()) + '\n').splitlines(1)
 247     text2 = ('\n'.join(text2.rstrip().splitlines()) + '\n').splitlines(1)
 248     return get_textdiff(text1, text2, title1, title2, **opts)
 249
 250 def get_file(content):
 251     """
 252     Generates a temporary file in '/tmp' with `content` and returns the file name.
 253     """
 254     fde = tempfile.NamedTemporaryFile(mode='w', delete=False)
 255     fname = fde.name
 256     fde.write(content)
 257     fde.close()
 258     return fname
 259
 260 def normalize_text(text):
 261     """
 262     Strips formating spaces/tabs and carriage returns.
 263     """
 264     text = re.sub(r'[ \t]+', ' ', text)
 265     text = re.sub(r'\r', '', text)
 266     return text
 267
 268 def version_cmp(v1, v2):
 269     """
 270     Compare two version strings and returns:
 271
 272     * `-1`: if `v1` is less than `v2`
 273     * `0`: if `v1` is equal to `v2`
 274     * `1`: if `v1` is greater than `v2`
 275
 276     Raises `ValueError` if versions are not well formated.
 277     """
 278     vregex = r'(?P<whole>\d+(\.(\d+))*)'
 279     v1m = re.match(vregex, v1)
 280     v2m = re.match(vregex, v2)
 281     if v1m is None or v2m is None:
 282         raise ValueError("got a invalid version string")
 283
 284     # Split values
 285     v1g = v1m.group('whole').split('.')
 286     v2g = v2m.group('whole').split('.')
 287
 288     # Get the longest version string
 289     vnum = len(v1g)
 290     if len(v2g) > vnum:
 291         vnum = len(v2g)
 292
 293     # Reverse list because we are going to pop the tail
 294     v1g.reverse()
 295     v2g.reverse()
 296     for _ in range(vnum):
 297         try:
 298             v1n = int(v1g.pop())
 299         except IndexError:
 300             while v2g:
 301                 v2n = int(v2g.pop())
 302                 if v2n > 0:
 303                     return -1
 304             break
 305
 306         try:
 307             v2n = int(v2g.pop())
 308         except IndexError:
 309             if v1n > 0:
 310                 return 1
 311             while v1g:
 312                 v1n = int(v1g.pop())
 313                 if v1n > 0:
 314                     return 1
 315             break
 316
 317         if v1n > v2n:
 318             return 1
 319         if v1n < v2n:
 320             return -1
 321     return 0
 322
 323 def ip4_route(node):
 324     """
 325     Gets a structured return of the command 'ip route'. It can be used in
 326     conjuction with json_cmp() to provide accurate assert explanations.
 327
 328     Return example:
 329     {
 330         '10.0.1.0/24': {
 331             'dev': 'eth0',
 332             'via': '172.16.0.1',
 333             'proto': '188',
 334         },
 335         '10.0.2.0/24': {
 336             'dev': 'eth1',
 337             'proto': 'kernel',
 338         }
 339     }
 340     """
 341     output = normalize_text(node.run('ip route')).splitlines()
 342     result = {}
 343     for line in output:
 344         columns = line.split(' ')
 345         route = result[columns[0]] = {}
 346         prev = None
 347         for column in columns:
 348             if prev == 'dev':
 349                 route['dev'] = column
 350             if prev == 'via':
 351                 route['via'] = column
 352             if prev == 'proto':
 353                 route['proto'] = column
 354             if prev == 'metric':
 355                 route['metric'] = column
 356             if prev == 'scope':
 357                 route['scope'] = column
 358             prev = column
 359
 360     return result
 361
 362 def ip6_route(node):
 363     """
 364     Gets a structured return of the command 'ip -6 route'. It can be used in
 365     conjuction with json_cmp() to provide accurate assert explanations.
 366
 367     Return example:
 368     {
 369         '2001:db8:1::/64': {
 370             'dev': 'eth0',
 371             'proto': '188',
 372         },
 373         '2001:db8:2::/64': {
 374             'dev': 'eth1',
 375             'proto': 'kernel',
 376         }
 377     }
 378     """
 379     output = normalize_text(node.run('ip -6 route')).splitlines()
 380     result = {}
 381     for line in output:
 382         columns = line.split(' ')
 383         route = result[columns[0]] = {}
 384         prev = None
 385         for column in columns:
 386             if prev == 'dev':
 387                 route['dev'] = column
 388             if prev == 'via':
 389                 route['via'] = column
 390             if prev == 'proto':
 391                 route['proto'] = column
 392             if prev == 'metric':
 393                 route['metric'] = column
 394             if prev == 'pref':
 395                 route['pref'] = column
 396             prev = column
 397
 398     return result
 399
 400 def sleep(amount, reason=None):
 401     """
 402     Sleep wrapper that registers in the log the amount of sleep
 403     """
 404     if reason is None:
 405         logger.info('Sleeping for {} seconds'.format(amount))
 406     else:
 407         logger.info(reason + ' ({} seconds)'.format(amount))
 408
 409     time.sleep(amount)
 410
 411 def checkAddressSanitizerError(output, router, component):
 412     "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise"
 413
 414     addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', output)
 415     if addressSantizerError:
 416         sys.stderr.write("%s: %s triggered an exception by AddressSanitizer\n" % (router, component))
 417         # Sanitizer Error found in log
 418         pidMark = addressSantizerError.group(1)
 419         addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), output, re.DOTALL)
 420         if addressSantizerLog:
 421             callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_back.f_globals['__file__'])
 422             callingProc = sys._getframe(2).f_code.co_name
 423             with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile:
 424                 sys.stderr.write('\n'.join(addressSantizerLog.group(1).splitlines()) + '\n')
 425                 addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2))
 426                 addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router))
 427                 addrSanFile.write('    '+ '\n    '.join(addressSantizerLog.group(1).splitlines()) + '\n')
 428                 addrSanFile.write("\n---------------\n")
 429         return True
 430     return False
 431
 432 def addRouter(topo, name):
 433     "Adding a FRRouter (or Quagga) to Topology"
 434
 435     MyPrivateDirs = ['/etc/frr',
 436                          '/etc/quagga',
 437                          '/var/run/frr',
 438                          '/var/run/quagga',
 439                          '/var/log']
 440     return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs)
 441
 442 def set_sysctl(node, sysctl, value):
 443     "Set a sysctl value and return None on success or an error string"
 444     valuestr = '{}'.format(value)
 445     command = "sysctl {0}={1}".format(sysctl, valuestr)
 446     cmdret = node.cmd(command)
 447
 448     matches = re.search(r'([^ ]+) = ([^\s]+)', cmdret)
 449     if matches is None:
 450         return cmdret
 451     if matches.group(1) != sysctl:
 452         return cmdret
 453     if matches.group(2) != valuestr:
 454         return cmdret
 455
 456     return None
 457
 458 def assert_sysctl(node, sysctl, value):
 459     "Set and assert that the sysctl is set with the specified value."
 460     assert set_sysctl(node, sysctl, value) is None
 461
 462 class LinuxRouter(Node):
 463     "A Node with IPv4/IPv6 forwarding enabled."
 464
 465     def config(self, **params):
 466         super(LinuxRouter, self).config(**params)
 467         # Enable forwarding on the router
 468         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 469         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 470     def terminate(self):
 471         """
 472         Terminate generic LinuxRouter Mininet instance
 473         """
 474         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 475         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 476         super(LinuxRouter, self).terminate()
 477
 478 class Router(Node):
 479     "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine"
 480
 481     def __init__(self, name, **params):
 482         super(Router, self).__init__(name, **params)
 483         self.logdir = params.get('logdir', get_test_logdir(name, True))
 484         self.daemondir = None
 485         self.hasmpls = False
 486         self.routertype = 'frr'
 487         self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0,
 488                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0,
 489                         'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0}
 490         self.daemons_options = {'zebra': ''}
 491         self.reportCores = True
 492
 493     def _config_frr(self, **params):
 494         "Configure FRR binaries"
 495         self.daemondir = params.get('frrdir')
 496         if self.daemondir is None:
 497             self.daemondir = '/usr/lib/frr'
 498
 499         zebra_path = os.path.join(self.daemondir, 'zebra')
 500         if not os.path.isfile(zebra_path):
 501             raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path))
 502
 503     def _config_quagga(self, **params):
 504         "Configure Quagga binaries"
 505         self.daemondir = params.get('quaggadir')
 506         if self.daemondir is None:
 507             self.daemondir = '/usr/lib/quagga'
 508
 509         zebra_path = os.path.join(self.daemondir, 'zebra')
 510         if not os.path.isfile(zebra_path):
 511             raise Exception("Quagga zebra binary doesn't exist at {}".format(zebra_path))
 512
 513     # pylint: disable=W0221
 514     # Some params are only meaningful for the parent class.
 515     def config(self, **params):
 516         super(Router, self).config(**params)
 517
 518         # User did not specify the daemons directory, try to autodetect it.
 519         self.daemondir = params.get('daemondir')
 520         if self.daemondir is None:
 521             self.routertype = params.get('routertype', 'frr')
 522             if self.routertype == 'quagga':
 523                 self._config_quagga(**params)
 524             else:
 525                 self._config_frr(**params)
 526         else:
 527             # Test the provided path
 528             zpath = os.path.join(self.daemondir, 'zebra')
 529             if not os.path.isfile(zpath):
 530                 raise Exception('No zebra binary found in {}'.format(zpath))
 531             # Allow user to specify routertype when the path was specified.
 532             if params.get('routertype') is not None:
 533                 self.routertype = self.params.get('routertype')
 534
 535         # Enable forwarding on the router
 536         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 537         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 538         # Enable coredumps
 539         assert_sysctl(self, 'kernel.core_uses_pid', 1)
 540         assert_sysctl(self, 'fs.suid_dumpable', 1)
 541         #this applies to the kernel not the namespace...
 542         #original on ubuntu 17.x, but apport won't save as in namespace
 543         # |/usr/share/apport/apport %p %s %c %d %P
 544         corefile = '%e_core-sig_%s-pid_%p.dmp'
 545         assert_sysctl(self, 'kernel.core_pattern', corefile)
 546         self.cmd('ulimit -c unlimited')
 547         # Set ownership of config files
 548         self.cmd('chown {0}:{0}vty /etc/{0}'.format(self.routertype))
 549
 550     def terminate(self):
 551         # Delete Running Quagga or FRR Daemons
 552         self.stopRouter()
 553         # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 554         # for d in StringIO.StringIO(rundaemons):
 555         #     self.cmd('kill -7 `cat %s`' % d.rstrip())
 556         #     self.waitOutput()
 557         # Disable forwarding
 558         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 559         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 560         super(Router, self).terminate()
 561         os.system('chmod -R go+rw /tmp/topotests')
 562
 563     def stopRouter(self, wait=True, assertOnError=True):
 564         # Stop Running Quagga or FRR Daemons
 565         rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 566         errors = ""
 567         if re.search(r"No such file or directory", rundaemons):
 568             return errors
 569         if rundaemons is not None:
 570             numRunning = 0
 571             for d in StringIO.StringIO(rundaemons):
 572                 daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 573                 if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 574                     logger.info('{}: stopping {}'.format(
 575                         self.name,
 576                         os.path.basename(d.rstrip().rsplit(".", 1)[0])
 577                     ))
 578                     self.cmd('kill -TERM %s' % daemonpid)
 579                     self.waitOutput()
 580                     if pid_exists(int(daemonpid)):
 581                         numRunning += 1
 582             if wait and numRunning > 0:
 583                 sleep(2, '{}: waiting for daemons stopping'.format(self.name))
 584                 # 2nd round of kill if daemons didn't exit
 585                 for d in StringIO.StringIO(rundaemons):
 586                     daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 587                     if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 588                         logger.info('{}: killing {}'.format(
 589                             self.name,
 590                             os.path.basename(d.rstrip().rsplit(".", 1)[0])
 591                         ))
 592                         self.cmd('kill -7 %s' % daemonpid)
 593                         self.waitOutput()
 594                     self.cmd('rm -- {}'.format(d.rstrip()))
 595         if wait:
 596                 errors = self.checkRouterCores(reportOnce=True)
 597                 if assertOnError and len(errors) > 0:
 598                     assert "Errors found - details follow:" == 0, errors
 599         return errors
 600
 601     def removeIPs(self):
 602         for interface in self.intfNames():
 603             self.cmd('ip address flush', interface)
 604
 605     def checkCapability(self, daemon, param):
 606         if param is not None:
 607             daemon_path = os.path.join(self.daemondir, daemon)
 608             daemon_search_option = param.replace('-','')
 609             output = self.cmd('{0} -h | grep {1}'.format(
 610                 daemon_path, daemon_search_option))
 611             if daemon_search_option not in output:
 612                 return False
 613         return True
 614
 615     def loadConf(self, daemon, source=None, param=None):
 616         # print "Daemons before:", self.daemons
 617         if daemon in self.daemons.keys():
 618             self.daemons[daemon] = 1
 619             if param is not None:
 620                 self.daemons_options[daemon] = param
 621             if source is None:
 622                 self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon))
 623                 self.waitOutput()
 624             else:
 625                 self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon))
 626                 self.waitOutput()
 627             self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon))
 628             self.waitOutput()
 629             self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon))
 630             self.waitOutput()
 631         else:
 632             logger.info('No daemon {} known'.format(daemon))
 633         # print "Daemons after:", self.daemons
 634
 635     def startRouter(self, tgen=None):
 636         # Disable integrated-vtysh-config
 637         self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype)
 638         self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype))
 639         # TODO remove the following lines after all tests are migrated to Topogen.
 640         # Try to find relevant old logfiles in /tmp and delete them
 641         map(os.remove, glob.glob('{}/{}/*.log'.format(self.logdir, self.name)))
 642         # Remove old core files
 643         map(os.remove, glob.glob('{}/{}/*.dmp'.format(self.logdir, self.name)))
 644         # Remove IP addresses from OS first - we have them in zebra.conf
 645         self.removeIPs()
 646         # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher
 647         # No error - but return message and skip all the tests
 648         if self.daemons['ldpd'] == 1:
 649             ldpd_path = os.path.join(self.daemondir, 'ldpd')
 650             if not os.path.isfile(ldpd_path):
 651                 logger.info("LDP Test, but no ldpd compiled or installed")
 652                 return "LDP Test, but no ldpd compiled or installed"
 653
 654             if version_cmp(platform.release(), '4.5') < 0:
 655                 logger.info("LDP Test need Linux Kernel 4.5 minimum")
 656                 return "LDP Test need Linux Kernel 4.5 minimum"
 657             # Check if have mpls
 658             if tgen != None:
 659                 self.hasmpls = tgen.hasmpls
 660                 if self.hasmpls != True:
 661                     logger.info("LDP/MPLS Tests will be skipped, platform missing module(s)")
 662             else:
 663                 # Test for MPLS Kernel modules available
 664                 self.hasmpls = False
 665                 if os.system('/sbin/modprobe mpls-router') != 0:
 666                     logger.info('MPLS tests will not run (missing mpls-router kernel module)')
 667                 elif os.system('/sbin/modprobe mpls-iptunnel') != 0:
 668                     logger.info('MPLS tests will not run (missing mpls-iptunnel kernel module)')
 669                 else:
 670                     self.hasmpls = True
 671             if self.hasmpls != True:
 672                 return "LDP/MPLS Tests need mpls kernel modules"
 673             self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels')
 674
 675         if self.daemons['eigrpd'] == 1:
 676             eigrpd_path = os.path.join(self.daemondir, 'eigrpd')
 677             if not os.path.isfile(eigrpd_path):
 678                 logger.info("EIGRP Test, but no eigrpd compiled or installed")
 679                 return "EIGRP Test, but no eigrpd compiled or installed"
 680
 681         self.restartRouter()
 682         return ""
 683
 684     def restartRouter(self):
 685         # Starts actual daemons without init (ie restart)
 686         # cd to per node directory
 687         self.cmd('cd {}/{}'.format(self.logdir, self.name))
 688         self.cmd('umask 000')
 689         #Re-enable to allow for report per run
 690         self.reportCores = True
 691         # Start Zebra first
 692         if self.daemons['zebra'] == 1:
 693             zebra_path = os.path.join(self.daemondir, 'zebra')
 694             zebra_option = self.daemons_options['zebra']
 695             self.cmd('{0} {1} > zebra.out 2> zebra.err &'.format(
 696                  zebra_path, zebra_option, self.logdir, self.name
 697             ))
 698             self.waitOutput()
 699             logger.debug('{}: {} zebra started'.format(self, self.routertype))
 700             sleep(1, '{}: waiting for zebra to start'.format(self.name))
 701         # Fix Link-Local Addresses
 702         # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
 703         self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done')
 704         # Now start all the other daemons
 705         for daemon in self.daemons:
 706             # Skip disabled daemons and zebra
 707             if self.daemons[daemon] == 0 or daemon == 'zebra':
 708                 continue
 709
 710             daemon_path = os.path.join(self.daemondir, daemon)
 711             self.cmd('{0} > {3}.out 2> {3}.err &'.format(
 712                 daemon_path, self.logdir, self.name, daemon
 713             ))
 714             self.waitOutput()
 715             logger.debug('{}: {} {} started'.format(self, self.routertype, daemon))
 716     def getStdErr(self, daemon):
 717         return self.getLog('err', daemon)
 718     def getStdOut(self, daemon):
 719         return self.getLog('out', daemon)
 720     def getLog(self, log, daemon):
 721         return self.cmd('cat {}/{}/{}.{}'.format(self.logdir, self.name, daemon, log))
 722
 723     def checkRouterCores(self, reportLeaks=True, reportOnce=False):
 724         if reportOnce and not self.reportCores:
 725             return
 726         reportMade = False
 727         traces = ""
 728         for daemon in self.daemons:
 729             if (self.daemons[daemon] == 1):
 730                 # Look for core file
 731                 corefiles = glob.glob('{}/{}/{}_core*.dmp'.format(
 732                     self.logdir, self.name, daemon))
 733                 if (len(corefiles) > 0):
 734                     daemon_path = os.path.join(self.daemondir, daemon)
 735                     backtrace = subprocess.check_output([
 736                         "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0])
 737                     ], shell=True)
 738                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
 739                     sys.stderr.write("%s" % backtrace)
 740                     traces = traces + "\n%s: %s crashed. Core file found - Backtrace follows:\n%s" % (self.name, daemon, backtrace)
 741                     reportMade = True
 742                 elif reportLeaks:
 743                     log = self.getStdErr(daemon)
 744                     if "memstats" in log:
 745                         sys.stderr.write("%s: %s has memory leaks:\n" % (self.name, daemon))
 746                         traces = traces + "\n%s: %s has memory leaks:\n" % (self.name, daemon)
 747                         log = re.sub("core_handler: ", "", log)
 748                         log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n  ## \1", log)
 749                         log = re.sub("memstats:  ", "    ", log)
 750                         sys.stderr.write(log)
 751                         reportMade = True
 752                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
 753                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
 754                     sys.stderr.write("%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon))
 755                     traces = traces + "\n%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon)
 756                     reportMade = True
 757         if reportMade:
 758             self.reportCores = False
 759         return traces
 760
 761     def checkRouterRunning(self):
 762         "Check if router daemons are running and collect crashinfo they don't run"
 763
 764         global fatal_error
 765
 766         daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"')
 767         # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found
 768         if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"):
 769             return "%s: vtysh killed by AddressSanitizer" % (self.name)
 770
 771         for daemon in self.daemons:
 772             if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning):
 773                 sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon))
 774                 # Look for core file
 775                 corefiles = glob.glob('{}/{}/{}_core*.dmp'.format(
 776                     self.logdir, self.name, daemon))
 777                 if (len(corefiles) > 0):
 778                     daemon_path = os.path.join(self.daemondir, daemon)
 779                     backtrace = subprocess.check_output([
 780                         "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0])
 781                     ], shell=True)
 782                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
 783                     sys.stderr.write("%s\n" % backtrace)
 784                 else:
 785                     # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
 786                     if os.path.isfile('{}/{}/{}.log'.format(self.logdir, self.name, daemon)):
 787                         log_tail = subprocess.check_output([
 788                             "tail -n20 {}/{}/{}.log 2> /dev/null".format(
 789                                 self.logdir, self.name, daemon)
 790                             ], shell=True)
 791                         sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon))
 792                         sys.stderr.write("%s\n" % log_tail)
 793
 794                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
 795                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
 796                     return "%s: Daemon %s not running - killed by AddressSanitizer" % (self.name, daemon)
 797
 798                 return "%s: Daemon %s not running" % (self.name, daemon)
 799         return ""
 800     def get_ipv6_linklocal(self):
 801         "Get LinkLocal Addresses from interfaces"
 802
 803         linklocal = []
 804
 805         ifaces = self.cmd('ip -6 address')
 806         # Fix newlines (make them all the same)
 807         ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines()
 808         interface=""
 809         ll_per_if_count=0
 810         for line in ifaces:
 811             m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line)
 812             if m:
 813                 interface = m.group(1)
 814                 ll_per_if_count = 0
 815             m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line)
 816             if m:
 817                 local = m.group(1)
 818                 ll_per_if_count += 1
 819                 if (ll_per_if_count > 1):
 820                     linklocal += [["%s-%s" % (interface, ll_per_if_count), local]]
 821                 else:
 822                     linklocal += [[interface, local]]
 823         return linklocal
 824     def daemon_available(self, daemon):
 825         "Check if specified daemon is installed (and for ldp if kernel supports MPLS)"
 826
 827         daemon_path = os.path.join(self.daemondir, daemon)
 828         if not os.path.isfile(daemon_path):
 829             return False
 830         if (daemon == 'ldpd'):
 831             if version_cmp(platform.release(), '4.5') < 0:
 832                 return False
 833             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 834                 return False
 835             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 836                 return False
 837
 838         return True
 839     def get_routertype(self):
 840         "Return the type of Router (frr or quagga)"
 841
 842         return self.routertype
 843     def report_memory_leaks(self, filename_prefix, testscript):
 844         "Report Memory Leaks to file prefixed with given string"
 845
 846         leakfound = False
 847         filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt"
 848         for daemon in self.daemons:
 849             if (self.daemons[daemon] == 1):
 850                 log = self.getStdErr(daemon)
 851                 if "memstats" in log:
 852                     # Found memory leak
 853                     logger.info('\nRouter {} {} StdErr Log:\n{}'.format(
 854                         self.name, daemon, log))
 855                     if not leakfound:
 856                         leakfound = True
 857                         # Check if file already exists
 858                         fileexists = os.path.isfile(filename)
 859                         leakfile = open(filename, "a")
 860                         if not fileexists:
 861                             # New file - add header
 862                             leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript)
 863                         leakfile.write("## Router %s\n" % self.name)
 864                     leakfile.write("### Process %s\n" % daemon)
 865                     log = re.sub("core_handler: ", "", log)
 866                     log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log)
 867                     log = re.sub("memstats:  ", "    ", log)
 868                     leakfile.write(log)
 869                     leakfile.write("\n")
 870         if leakfound:
 871             leakfile.close()
 872
 873
 874 class LegacySwitch(OVSSwitch):
 875     "A Legacy Switch without OpenFlow"
 876
 877     def __init__(self, name, **params):
 878         OVSSwitch.__init__(self, name, failMode='standalone', **params)
 879         self.switchIP = None