tests/topotests/lib/topotest.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # topotest.py
   5 # Library of helper functions for NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2016 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 import os
  26 import errno
  27 import re
  28 import sys
  29 import glob
  30 import StringIO
  31 import subprocess
  32 import tempfile
  33 import platform
  34 import difflib
  35 import time
  36
  37 from lib.topolog import logger
  38
  39 from mininet.topo import Topo
  40 from mininet.net import Mininet
  41 from mininet.node import Node, OVSSwitch, Host
  42 from mininet.log import setLogLevel, info
  43 from mininet.cli import CLI
  44 from mininet.link import Intf
  45
  46 class json_cmp_result(object):
  47     "json_cmp result class for better assertion messages"
  48
  49     def __init__(self):
  50         self.errors = []
  51
  52     def add_error(self, error):
  53         "Append error message to the result"
  54         self.errors.append(error)
  55
  56     def has_errors(self):
  57         "Returns True if there were errors, otherwise False."
  58         return len(self.errors) > 0
  59
  60
  61 def json_cmp(d1, d2, reason=False):
  62     """
  63     JSON compare function. Receives two parameters:
  64     * `d1`: json value
  65     * `d2`: json subset which we expect
  66
  67     Returns `None` when all keys that `d1` has matches `d2`,
  68     otherwise a string containing what failed.
  69
  70     Note: key absence can be tested by adding a key with value `None`.
  71     """
  72     squeue = [(d1, d2, 'json')]
  73     result = json_cmp_result()
  74     for s in squeue:
  75         nd1, nd2, parent = s
  76         s1, s2 = set(nd1), set(nd2)
  77
  78         # Expect all required fields to exist.
  79         s2_req = set([key for key in nd2 if nd2[key] is not None])
  80         diff = s2_req - s1
  81         if diff != set({}):
  82             result.add_error('expected key(s) {} in {} (have {})'.format(
  83                 str(list(diff)), parent, str(list(s1))))
  84
  85         for key in s2.intersection(s1):
  86             # Test for non existence of key in d2
  87             if nd2[key] is None:
  88                 result.add_error('"{}" should not exist in {} (have {})'.format(
  89                     key, parent, str(s1)))
  90                 continue
  91             # If nd1 key is a dict, we have to recurse in it later.
  92             if isinstance(nd2[key], type({})):
  93                 if not isinstance(nd1[key], type({})):
  94                     result.add_error(
  95                         '{}["{}"] has different type than expected '.format(parent, key) +
  96                         '(have {}, expected {})'.format(type(nd1[key]), type(nd2[key])))
  97                     continue
  98                 nparent = '{}["{}"]'.format(parent, key)
  99                 squeue.append((nd1[key], nd2[key], nparent))
 100                 continue
 101             # Check list items
 102             if isinstance(nd2[key], type([])):
 103                 if not isinstance(nd1[key], type([])):
 104                     result.add_error(
 105                         '{}["{}"] has different type than expected '.format(parent, key) +
 106                         '(have {}, expected {})'.format(type(nd1[key]), type(nd2[key])))
 107                     continue
 108                 # Check list size
 109                 if len(nd2[key]) > len(nd1[key]):
 110                     result.add_error(
 111                         '{}["{}"] too few items '.format(parent, key) +
 112                         '(have ({}) "{}", expected ({}) "{}")'.format(
 113                             len(nd1[key]), str(nd1[key]), len(nd2[key]), str(nd2[key])))
 114                     continue
 115
 116                 # List all unmatched items errors
 117                 unmatched = []
 118                 for expected in nd2[key]:
 119                     matched = False
 120                     for value in nd1[key]:
 121                         if json_cmp({'json': value}, {'json': expected}) is None:
 122                             matched = True
 123                             break
 124
 125                     if matched:
 126                         break
 127                     if not matched:
 128                         unmatched.append(expected)
 129
 130                 # If there are unmatched items, error out.
 131                 if unmatched:
 132                     result.add_error(
 133                         '{}["{}"] value is different (have "{}", expected "{}")'.format(
 134                             parent, key, str(nd1[key]), str(nd2[key])))
 135                 continue
 136
 137             # Compare JSON values
 138             if nd1[key] != nd2[key]:
 139                 result.add_error(
 140                     '{}["{}"] value is different (have "{}", expected "{}")'.format(
 141                         parent, key, str(nd1[key]), str(nd2[key])))
 142                 continue
 143
 144     if result.has_errors():
 145         return result
 146
 147     return None
 148
 149 def run_and_expect(func, what, count=20, wait=3):
 150     """
 151     Run `func` and compare the result with `what`. Do it for `count` times
 152     waiting `wait` seconds between tries. By default it tries 20 times with
 153     3 seconds delay between tries.
 154
 155     Returns (True, func-return) on success or
 156     (False, func-return) on failure.
 157     """
 158     while count > 0:
 159         result = func()
 160         if result != what:
 161             time.sleep(wait)
 162             count -= 1
 163             continue
 164         return (True, result)
 165     return (False, result)
 166
 167
 168 def int2dpid(dpid):
 169     "Converting Integer to DPID"
 170
 171     try:
 172         dpid = hex(dpid)[2:]
 173         dpid = '0'*(16-len(dpid))+dpid
 174         return dpid
 175     except IndexError:
 176         raise Exception('Unable to derive default datapath ID - '
 177                         'please either specify a dpid or use a '
 178                         'canonical switch name such as s23.')
 179
 180 def pid_exists(pid):
 181     "Check whether pid exists in the current process table."
 182
 183     if pid <= 0:
 184         return False
 185     try:
 186         os.kill(pid, 0)
 187     except OSError as err:
 188         if err.errno == errno.ESRCH:
 189             # ESRCH == No such process
 190             return False
 191         elif err.errno == errno.EPERM:
 192             # EPERM clearly means there's a process to deny access to
 193             return True
 194         else:
 195             # According to "man 2 kill" possible error values are
 196             # (EINVAL, EPERM, ESRCH)
 197             raise
 198     else:
 199         return True
 200
 201 def get_textdiff(text1, text2, title1="", title2=""):
 202     "Returns empty string if same or formatted diff"
 203
 204     diff = '\n'.join(difflib.unified_diff(text1, text2,
 205            fromfile=title1, tofile=title2))
 206     # Clean up line endings
 207     diff = os.linesep.join([s for s in diff.splitlines() if s])
 208     return diff
 209
 210 def difflines(text1, text2, title1='', title2=''):
 211     "Wrapper for get_textdiff to avoid string transformations."
 212     text1 = ('\n'.join(text1.rstrip().splitlines()) + '\n').splitlines(1)
 213     text2 = ('\n'.join(text2.rstrip().splitlines()) + '\n').splitlines(1)
 214     return get_textdiff(text1, text2, title1, title2)
 215
 216 def get_file(content):
 217     """
 218     Generates a temporary file in '/tmp' with `content` and returns the file name.
 219     """
 220     fde = tempfile.NamedTemporaryFile(mode='w', delete=False)
 221     fname = fde.name
 222     fde.write(content)
 223     fde.close()
 224     return fname
 225
 226 def normalize_text(text):
 227     """
 228     Strips formating spaces/tabs and carriage returns.
 229     """
 230     text = re.sub(r'[ \t]+', ' ', text)
 231     text = re.sub(r'\r', '', text)
 232     return text
 233
 234 def version_cmp(v1, v2):
 235     """
 236     Compare two version strings and returns:
 237
 238     * `-1`: if `v1` is less than `v2`
 239     * `0`: if `v1` is equal to `v2`
 240     * `1`: if `v1` is greater than `v2`
 241
 242     Raises `ValueError` if versions are not well formated.
 243     """
 244     vregex = r'(?P<whole>\d+(\.(\d+))*)'
 245     v1m = re.match(vregex, v1)
 246     v2m = re.match(vregex, v2)
 247     if v1m is None or v2m is None:
 248         raise ValueError("got a invalid version string")
 249
 250     # Split values
 251     v1g = v1m.group('whole').split('.')
 252     v2g = v2m.group('whole').split('.')
 253
 254     # Get the longest version string
 255     vnum = len(v1g)
 256     if len(v2g) > vnum:
 257         vnum = len(v2g)
 258
 259     # Reverse list because we are going to pop the tail
 260     v1g.reverse()
 261     v2g.reverse()
 262     for _ in range(vnum):
 263         try:
 264             v1n = int(v1g.pop())
 265         except IndexError:
 266             while v2g:
 267                 v2n = int(v2g.pop())
 268                 if v2n > 0:
 269                     return -1
 270             break
 271
 272         try:
 273             v2n = int(v2g.pop())
 274         except IndexError:
 275             if v1n > 0:
 276                 return 1
 277             while v1g:
 278                 v1n = int(v1g.pop())
 279                 if v1n > 0:
 280                     return -1
 281             break
 282
 283         if v1n > v2n:
 284             return 1
 285         if v1n < v2n:
 286             return -1
 287     return 0
 288
 289 def ip4_route(node):
 290     """
 291     Gets a structured return of the command 'ip route'. It can be used in
 292     conjuction with json_cmp() to provide accurate assert explanations.
 293
 294     Return example:
 295     {
 296         '10.0.1.0/24': {
 297             'dev': 'eth0',
 298             'via': '172.16.0.1',
 299             'proto': '188',
 300         },
 301         '10.0.2.0/24': {
 302             'dev': 'eth1',
 303             'proto': 'kernel',
 304         }
 305     }
 306     """
 307     output = normalize_text(node.run('ip route')).splitlines()
 308     result = {}
 309     for line in output:
 310         columns = line.split(' ')
 311         route = result[columns[0]] = {}
 312         prev = None
 313         for column in columns:
 314             if prev == 'dev':
 315                 route['dev'] = column
 316             if prev == 'via':
 317                 route['via'] = column
 318             if prev == 'proto':
 319                 route['proto'] = column
 320             if prev == 'metric':
 321                 route['metric'] = column
 322             if prev == 'scope':
 323                 route['scope'] = column
 324             prev = column
 325
 326     return result
 327
 328 def ip6_route(node):
 329     """
 330     Gets a structured return of the command 'ip -6 route'. It can be used in
 331     conjuction with json_cmp() to provide accurate assert explanations.
 332
 333     Return example:
 334     {
 335         '2001:db8:1::/64': {
 336             'dev': 'eth0',
 337             'proto': '188',
 338         },
 339         '2001:db8:2::/64': {
 340             'dev': 'eth1',
 341             'proto': 'kernel',
 342         }
 343     }
 344     """
 345     output = normalize_text(node.run('ip -6 route')).splitlines()
 346     result = {}
 347     for line in output:
 348         columns = line.split(' ')
 349         route = result[columns[0]] = {}
 350         prev = None
 351         for column in columns:
 352             if prev == 'dev':
 353                 route['dev'] = column
 354             if prev == 'via':
 355                 route['via'] = column
 356             if prev == 'proto':
 357                 route['proto'] = column
 358             if prev == 'metric':
 359                 route['metric'] = column
 360             if prev == 'pref':
 361                 route['pref'] = column
 362             prev = column
 363
 364     return result
 365
 366 def sleep(amount, reason=None):
 367     """
 368     Sleep wrapper that registers in the log the amount of sleep
 369     """
 370     if reason is None:
 371         logger.info('Sleeping for {} seconds'.format(amount))
 372     else:
 373         logger.info(reason + ' ({} seconds)'.format(amount))
 374
 375     time.sleep(amount)
 376
 377 def checkAddressSanitizerError(output, router, component):
 378     "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise"
 379
 380     addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', output)
 381     if addressSantizerError:
 382         sys.stderr.write("%s: %s triggered an exception by AddressSanitizer\n" % (router, component))
 383         # Sanitizer Error found in log
 384         pidMark = addressSantizerError.group(1)
 385         addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), output, re.DOTALL)
 386         if addressSantizerLog:
 387             callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_back.f_globals['__file__'])
 388             callingProc = sys._getframe(2).f_code.co_name
 389             with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile:
 390                 sys.stderr.write('\n'.join(addressSantizerLog.group(1).splitlines()) + '\n')
 391                 addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2))
 392                 addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router))
 393                 addrSanFile.write('    '+ '\n    '.join(addressSantizerLog.group(1).splitlines()) + '\n')
 394                 addrSanFile.write("\n---------------\n")
 395         return True
 396     return False
 397
 398 def addRouter(topo, name):
 399     "Adding a FRRouter (or Quagga) to Topology"
 400
 401     MyPrivateDirs = ['/etc/frr',
 402                          '/etc/quagga',
 403                          '/var/run/frr',
 404                          '/var/run/quagga',
 405                          '/var/log']
 406     return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs)
 407
 408 def set_sysctl(node, sysctl, value):
 409     "Set a sysctl value and return None on success or an error string"
 410     valuestr = '{}'.format(value)
 411     command = "sysctl {0}={1}".format(sysctl, valuestr)
 412     cmdret = node.cmd(command)
 413
 414     matches = re.search(r'([^ ]+) = ([^\s]+)', cmdret)
 415     if matches is None:
 416         return cmdret
 417     if matches.group(1) != sysctl:
 418         return cmdret
 419     if matches.group(2) != valuestr:
 420         return cmdret
 421
 422     return None
 423
 424 def assert_sysctl(node, sysctl, value):
 425     "Set and assert that the sysctl is set with the specified value."
 426     assert set_sysctl(node, sysctl, value) is None
 427
 428 class LinuxRouter(Node):
 429     "A Node with IPv4/IPv6 forwarding enabled."
 430
 431     def config(self, **params):
 432         super(LinuxRouter, self).config(**params)
 433         # Enable forwarding on the router
 434         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 435         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 436     def terminate(self):
 437         """
 438         Terminate generic LinuxRouter Mininet instance
 439         """
 440         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 441         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 442         super(LinuxRouter, self).terminate()
 443
 444 class Router(Node):
 445     "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine"
 446
 447     def __init__(self, name, **params):
 448         super(Router, self).__init__(name, **params)
 449         self.logdir = params.get('logdir', '/tmp')
 450         self.daemondir = None
 451         self.routertype = 'frr'
 452         self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0,
 453                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0,
 454                         'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0}
 455
 456     def _config_frr(self, **params):
 457         "Configure FRR binaries"
 458         self.daemondir = params.get('frrdir')
 459         if self.daemondir is None:
 460             self.daemondir = '/usr/lib/frr'
 461
 462         zebra_path = os.path.join(self.daemondir, 'zebra')
 463         if not os.path.isfile(zebra_path):
 464             raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path))
 465
 466     def _config_quagga(self, **params):
 467         "Configure Quagga binaries"
 468         self.daemondir = params.get('quaggadir')
 469         if self.daemondir is None:
 470             self.daemondir = '/usr/lib/quagga'
 471
 472         zebra_path = os.path.join(self.daemondir, 'zebra')
 473         if not os.path.isfile(zebra_path):
 474             raise Exception("Quagga zebra binary doesn't exist at {}".format(zebra_path))
 475
 476     # pylint: disable=W0221
 477     # Some params are only meaningful for the parent class.
 478     def config(self, **params):
 479         super(Router, self).config(**params)
 480
 481         # User did not specify the daemons directory, try to autodetect it.
 482         self.daemondir = params.get('daemondir')
 483         if self.daemondir is None:
 484             self.routertype = params.get('routertype', 'frr')
 485             if self.routertype == 'quagga':
 486                 self._config_quagga(**params)
 487             else:
 488                 self._config_frr(**params)
 489         else:
 490             # Test the provided path
 491             zpath = os.path.join(self.daemondir, 'zebra')
 492             if not os.path.isfile(zpath):
 493                 raise Exception('No zebra binary found in {}'.format(zpath))
 494             # Allow user to specify routertype when the path was specified.
 495             if params.get('routertype') is not None:
 496                 self.routertype = self.params.get('routertype')
 497
 498         # Enable forwarding on the router
 499         assert_sysctl(self, 'net.ipv4.ip_forward', 1)
 500         assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1)
 501         # Enable coredumps
 502         assert_sysctl(self, 'kernel.core_uses_pid', 1)
 503         assert_sysctl(self, 'fs.suid_dumpable', 2)
 504         corefile = '{}/{}_%e_core-sig_%s-pid_%p.dmp'.format(self.logdir, self.name)
 505         assert_sysctl(self, 'kernel.core_pattern', corefile)
 506         self.cmd('ulimit -c unlimited')
 507         # Set ownership of config files
 508         self.cmd('chown {0}:{0}vty /etc/{0}'.format(self.routertype))
 509
 510     def terminate(self):
 511         # Delete Running Quagga or FRR Daemons
 512         self.stopRouter()
 513         # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 514         # for d in StringIO.StringIO(rundaemons):
 515         #     self.cmd('kill -7 `cat %s`' % d.rstrip())
 516         #     self.waitOutput()
 517         # Disable forwarding
 518         set_sysctl(self, 'net.ipv4.ip_forward', 0)
 519         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
 520         super(Router, self).terminate()
 521     def stopRouter(self):
 522         # Stop Running Quagga or FRR Daemons
 523         rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
 524         if rundaemons is not None:
 525             for d in StringIO.StringIO(rundaemons):
 526                 daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
 527                 if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
 528                     self.cmd('kill -7 %s' % daemonpid)
 529                     self.waitOutput()
 530     def removeIPs(self):
 531         for interface in self.intfNames():
 532             self.cmd('ip address flush', interface)
 533     def loadConf(self, daemon, source=None):
 534         # print "Daemons before:", self.daemons
 535         if daemon in self.daemons.keys():
 536             self.daemons[daemon] = 1
 537             if source is None:
 538                 self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon))
 539                 self.waitOutput()
 540             else:
 541                 self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon))
 542                 self.waitOutput()
 543             self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon))
 544             self.waitOutput()
 545             self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon))
 546             self.waitOutput()
 547         else:
 548             logger.warning('No daemon {} known'.format(daemon))
 549         # print "Daemons after:", self.daemons
 550     def startRouter(self):
 551         # Disable integrated-vtysh-config
 552         self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype)
 553         self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype))
 554         # TODO remove the following lines after all tests are migrated to Topogen.
 555         # Try to find relevant old logfiles in /tmp and delete them
 556         map(os.remove, glob.glob("/tmp/*%s*.log" % self.name))
 557         # Remove old core files
 558         map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name))
 559         # Remove IP addresses from OS first - we have them in zebra.conf
 560         self.removeIPs()
 561         # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher
 562         # No error - but return message and skip all the tests
 563         if self.daemons['ldpd'] == 1:
 564             ldpd_path = os.path.join(self.daemondir, 'ldpd')
 565             if not os.path.isfile(ldpd_path):
 566                 logger.warning("LDP Test, but no ldpd compiled or installed")
 567                 return "LDP Test, but no ldpd compiled or installed"
 568
 569             if version_cmp(platform.release(), '4.5') < 0:
 570                 logger.warning("LDP Test need Linux Kernel 4.5 minimum")
 571                 return "LDP Test need Linux Kernel 4.5 minimum"
 572
 573             # Check if required kernel modules are available with a dryrun modprobe
 574             # Silent accept of modprobe command assumes ok status
 575             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 576                 logger.warning("LDP Test needs mpls-router kernel module")
 577                 return "LDP Test needs mpls-router kernel module"
 578             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 579                 logger.warning("LDP Test needs mpls-iptunnel kernel module")
 580                 return "LDP Test needs mpls-router kernel module"
 581
 582             self.cmd('/sbin/modprobe mpls-router')
 583             self.cmd('/sbin/modprobe mpls-iptunnel')
 584             self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels')
 585         # Init done - now restarting daemons
 586         self.restartRouter()
 587         return ""
 588     def restartRouter(self):
 589         # Starts actuall daemons without init (ie restart)
 590         # Start Zebra first
 591         if self.daemons['zebra'] == 1:
 592             zebra_path = os.path.join(self.daemondir, 'zebra')
 593             self.cmd('{0} > {1}/{2}-zebra.out 2> {1}/{2}-zebra.err &'.format(
 594                 zebra_path, self.logdir, self.name
 595             ))
 596             self.waitOutput()
 597             logger.debug('{}: {} zebra started'.format(self, self.routertype))
 598             time.sleep(1)
 599         # Fix Link-Local Addresses
 600         # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
 601         self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done')
 602         # Now start all the other daemons
 603         for daemon in self.daemons:
 604             # Skip disabled daemons and zebra
 605             if self.daemons[daemon] == 0 or daemon == 'zebra':
 606                 continue
 607
 608             daemon_path = os.path.join(self.daemondir, daemon)
 609             self.cmd('{0} > {1}/{2}-{3}.out 2> {1}/{2}-{3}.err &'.format(
 610                 daemon_path, self.logdir, self.name, daemon
 611             ))
 612             self.waitOutput()
 613             logger.debug('{}: {} {} started'.format(self, self.routertype, daemon))
 614     def getStdErr(self, daemon):
 615         return self.getLog('err', daemon)
 616     def getStdOut(self, daemon):
 617         return self.getLog('out', daemon)
 618     def getLog(self, log, daemon):
 619         return self.cmd('cat {}/{}-{}.{}'.format(self.logdir, self.name, daemon, log))
 620     def checkRouterRunning(self):
 621         "Check if router daemons are running and collect crashinfo they don't run"
 622
 623         global fatal_error
 624
 625         daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"')
 626         # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found
 627         if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"):
 628             return "%s: vtysh killed by AddressSanitizer" % (self.name)
 629
 630         for daemon in self.daemons:
 631             if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning):
 632                 sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon))
 633                 # Look for core file
 634                 corefiles = glob.glob('{}/{}_{}_core*.dmp'.format(
 635                     self.logdir, self.name, daemon))
 636                 if (len(corefiles) > 0):
 637                     daemon_path = os.path.join(self.daemondir, daemon)
 638                     backtrace = subprocess.check_output([
 639                         "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0])
 640                     ], shell=True)
 641                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
 642                     sys.stderr.write("%s\n" % backtrace)
 643                 else:
 644                     # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
 645                     if os.path.isfile('{}/{}-{}.log'.format(self.logdir, self.name, daemon)):
 646                         log_tail = subprocess.check_output([
 647                             "tail -n20 {}/{}-{}.log 2> /dev/null".format(
 648                                 self.logdir, self.name, daemon)
 649                             ], shell=True)
 650                         sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon))
 651                         sys.stderr.write("%s\n" % log_tail)
 652
 653                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
 654                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
 655                     return "%s: Daemon %s not running - killed by AddressSanitizer" % (self.name, daemon)
 656
 657                 return "%s: Daemon %s not running" % (self.name, daemon)
 658         return ""
 659     def get_ipv6_linklocal(self):
 660         "Get LinkLocal Addresses from interfaces"
 661
 662         linklocal = []
 663
 664         ifaces = self.cmd('ip -6 address')
 665         # Fix newlines (make them all the same)
 666         ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines()
 667         interface=""
 668         ll_per_if_count=0
 669         for line in ifaces:
 670             m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line)
 671             if m:
 672                 interface = m.group(1)
 673                 ll_per_if_count = 0
 674             m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line)
 675             if m:
 676                 local = m.group(1)
 677                 ll_per_if_count += 1
 678                 if (ll_per_if_count > 1):
 679                     linklocal += [["%s-%s" % (interface, ll_per_if_count), local]]
 680                 else:
 681                     linklocal += [[interface, local]]
 682         return linklocal
 683     def daemon_available(self, daemon):
 684         "Check if specified daemon is installed (and for ldp if kernel supports MPLS)"
 685
 686         daemon_path = os.path.join(self.daemondir, daemon)
 687         if not os.path.isfile(daemon_path):
 688             return False
 689         if (daemon == 'ldpd'):
 690             if version_cmp(platform.release(), '4.5') < 0:
 691                 return False
 692             if self.cmd('/sbin/modprobe -n mpls-router' ) != "":
 693                 return False
 694             if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "":
 695                 return False
 696
 697         return True
 698     def get_routertype(self):
 699         "Return the type of Router (frr or quagga)"
 700
 701         return self.routertype
 702     def report_memory_leaks(self, filename_prefix, testscript):
 703         "Report Memory Leaks to file prefixed with given string"
 704
 705         leakfound = False
 706         filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt"
 707         for daemon in self.daemons:
 708             if (self.daemons[daemon] == 1):
 709                 log = self.getStdErr(daemon)
 710                 if "memstats" in log:
 711                     # Found memory leak
 712                     logger.info('\nRouter {} {} StdErr Log:\n{}'.format(
 713                         self.name, daemon, log))
 714                     if not leakfound:
 715                         leakfound = True
 716                         # Check if file already exists
 717                         fileexists = os.path.isfile(filename)
 718                         leakfile = open(filename, "a")
 719                         if not fileexists:
 720                             # New file - add header
 721                             leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript)
 722                         leakfile.write("## Router %s\n" % self.name)
 723                     leakfile.write("### Process %s\n" % daemon)
 724                     log = re.sub("core_handler: ", "", log)
 725                     log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log)
 726                     log = re.sub("memstats:  ", "    ", log)
 727                     leakfile.write(log)
 728                     leakfile.write("\n")
 729         if leakfound:
 730             leakfile.close()
 731
 732
 733 class LegacySwitch(OVSSwitch):
 734     "A Legacy Switch without OpenFlow"
 735
 736     def __init__(self, name, **params):
 737         OVSSwitch.__init__(self, name, failMode='standalone', **params)
 738         self.switchIP = None