tests/topotests/ldp_topo1/test_ldp_topo1.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # test_bgp_multiview_topo1.py
   5 # Part of NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2016 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 r"""
  26 test_ldp_topo1.py: Simple FRR LDP Test
  27
  28              +---------+
  29              |    r1   |
  30              | 1.1.1.1 |
  31              +----+----+
  32                   | .1  r1-eth0
  33                   |
  34             ~~~~~~~~~~~~~
  35           ~~     sw0     ~~
  36           ~~ 10.0.1.0/24 ~~
  37             ~~~~~~~~~~~~~
  38                   |10.0.1.0/24
  39                   |
  40                   | .2  r2-eth0
  41              +----+----+
  42              |    r2   |
  43              | 2.2.2.2 |
  44              +--+---+--+
  45     r2-eth2  .2 |   | .2  r2-eth1
  46          ______/     \______
  47         /                   \
  48   ~~~~~~~~~~~~~        ~~~~~~~~~~~~~
  49 ~~     sw2     ~~    ~~     sw1     ~~
  50 ~~ 10.0.3.0/24 ~~    ~~ 10.0.2.0/24 ~~
  51   ~~~~~~~~~~~~~        ~~~~~~~~~~~~~
  52         |                 /    |
  53          \      _________/     |
  54           \    /                \
  55 r3-eth1 .3 |  | .3  r3-eth0      | .4 r4-eth0
  56       +----+--+---+         +----+----+
  57       |     r3    |         |    r4   |
  58       |  3.3.3.3  |         | 4.4.4.4 |
  59       +-----------+         +---------+
  60 """
  61
  62 import os
  63 import re
  64 import sys
  65 import pytest
  66 import json
  67 from functools import partial
  68 from time import sleep
  69 from lib.topolog import logger
  70
  71 # Save the Current Working Directory to find configuration files.
  72 CWD = os.path.dirname(os.path.realpath(__file__))
  73 sys.path.append(os.path.join(CWD, "../"))
  74
  75 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  76 from lib import topotest
  77 from lib.topogen import Topogen, get_topogen
  78
  79 fatal_error = ""
  80
  81 pytestmark = [pytest.mark.ldpd, pytest.mark.ospfd]
  82
  83 #####################################################
  84 ##
  85 ##   Network Topology Definition
  86 ##
  87 #####################################################
  88
  89
  90 def build_topo(tgen):
  91
  92     # Setup Routers
  93     for i in range(1, 5):
  94         tgen.add_router("r%s" % i)
  95
  96     # First switch
  97     switch = tgen.add_switch("sw0")
  98     switch.add_link(tgen.gears["r1"])
  99     switch.add_link(tgen.gears["r2"])
 100     # Second switch
 101     switch = tgen.add_switch("sw1")
 102     switch.add_link(tgen.gears["r2"])
 103     switch.add_link(tgen.gears["r3"])
 104     switch.add_link(tgen.gears["r4"])
 105     # Third switch
 106     switch = tgen.add_switch("sw2")
 107     switch.add_link(tgen.gears["r2"])
 108     switch.add_link(tgen.gears["r3"])
 109
 110
 111 #####################################################
 112 ##
 113 ##   Helper functions
 114 ##
 115 #####################################################
 116
 117
 118 def router_compare_json_output(rname, command, reference, count=60, wait=1):
 119     "Compare router JSON output"
 120
 121     logger.info('Comparing router "%s" "%s" output', rname, command)
 122
 123     tgen = get_topogen()
 124     filename = "{}/{}/{}".format(CWD, rname, reference)
 125     expected = json.loads(open(filename).read())
 126
 127     # Run test function until we get an result.
 128     test_func = partial(topotest.router_json_cmp, tgen.gears[rname], command, expected)
 129     _, diff = topotest.run_and_expect(test_func, None, count, wait)
 130     assertmsg = '"{}" JSON output mismatches the expected result'.format(rname)
 131     assert diff is None, assertmsg
 132
 133
 134 #####################################################
 135 ##
 136 ##   Tests starting
 137 ##
 138 #####################################################
 139
 140
 141 def setup_module(module):
 142     print("\n\n** %s: Setup Topology" % module.__name__)
 143     print("******************************************\n")
 144
 145     thisDir = os.path.dirname(os.path.realpath(__file__))
 146     tgen = Topogen(build_topo, module.__name__)
 147     tgen.start_topology()
 148
 149     net = tgen.net
 150
 151     # Starting Routers
 152     for i in range(1, 5):
 153         net["r%s" % i].loadConf("zebra", "%s/r%s/zebra.conf" % (thisDir, i))
 154         net["r%s" % i].loadConf("ospfd", "%s/r%s/ospfd.conf" % (thisDir, i))
 155         net["r%s" % i].loadConf("ldpd", "%s/r%s/ldpd.conf" % (thisDir, i))
 156         tgen.gears["r%s" % i].start()
 157
 158     # For debugging after starting FRR daemons, uncomment the next line
 159     # tgen.mininet_cli()
 160
 161
 162 def teardown_module(module):
 163     print("\n\n** %s: Shutdown Topology" % module.__name__)
 164     print("******************************************\n")
 165     tgen = get_topogen()
 166     tgen.stop_topology()
 167
 168
 169 def test_router_running():
 170     global fatal_error
 171     net = get_topogen().net
 172
 173     # Skip if previous fatal error condition is raised
 174     if fatal_error != "":
 175         pytest.skip(fatal_error)
 176
 177     print("\n\n** Check if FRR is running on each Router node")
 178     print("******************************************\n")
 179     sleep(5)
 180
 181     # Starting Routers
 182     for i in range(1, 5):
 183         fatal_error = net["r%s" % i].checkRouterRunning()
 184         assert fatal_error == "", fatal_error
 185
 186
 187 def test_mpls_interfaces():
 188     global fatal_error
 189     net = get_topogen().net
 190
 191     # Skip if previous fatal error condition is raised
 192     if fatal_error != "":
 193         pytest.skip(fatal_error)
 194
 195     thisDir = os.path.dirname(os.path.realpath(__file__))
 196
 197     # Verify MPLS Interfaces
 198     print("\n\n** Verifying MPLS Interfaces")
 199     print("******************************************\n")
 200     failures = 0
 201     for i in range(1, 5):
 202         refTableFile = "%s/r%s/show_mpls_ldp_interface.ref"
 203         if os.path.isfile(refTableFile):
 204             # Read expected result from file
 205             expected = open(refTableFile).read().rstrip()
 206             # Fix newlines (make them all the same)
 207             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 208
 209             # Actual output from router
 210             actual = (
 211                 net["r%s" % i]
 212                 .cmd('vtysh -c "show mpls ldp interface" 2> /dev/null')
 213                 .rstrip()
 214             )
 215             # Mask out Timer in Uptime
 216             actual = re.sub(r" [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ", " xx:xx:xx ", actual)
 217             # Fix newlines (make them all the same)
 218             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 219
 220             # Generate Diff
 221             diff = topotest.get_textdiff(
 222                 actual,
 223                 expected,
 224                 title1="actual MPLS LDP interface status",
 225                 title2="expected MPLS LDP interface status",
 226             )
 227
 228             # Empty string if it matches, otherwise diff contains unified diff
 229             if diff:
 230                 sys.stderr.write(
 231                     "r%s failed MPLS LDP Interface status Check:\n%s\n" % (i, diff)
 232                 )
 233                 failures += 1
 234             else:
 235                 print("r%s ok" % i)
 236
 237             if failures > 0:
 238                 fatal_error = "MPLS LDP Interface status failed"
 239
 240             assert (
 241                 failures == 0
 242             ), "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
 243
 244     # Make sure that all daemons are running
 245     for i in range(1, 5):
 246         fatal_error = net["r%s" % i].checkRouterRunning()
 247         assert fatal_error == "", fatal_error
 248
 249
 250 def test_ospf_convergence():
 251     logger.info("Test: check OSPF adjacencies")
 252
 253     # Skip if previous fatal error condition is raised
 254     if fatal_error != "":
 255         pytest.skip(fatal_error)
 256
 257     for rname in ["r1", "r2", "r3", "r4"]:
 258         router_compare_json_output(
 259             rname, "show ip ospf neighbor json", "show_ip_ospf_neighbor.json"
 260         )
 261
 262
 263 def test_mpls_ldp_neighbor_establish():
 264     global fatal_error
 265     net = get_topogen().net
 266
 267     # Skip if previous fatal error condition is raised
 268     if fatal_error != "":
 269         pytest.skip(fatal_error)
 270
 271     neighbors_operational = {
 272         1: 1,
 273         2: 3,
 274         3: 2,
 275         4: 2,
 276     }
 277
 278     # Wait for MPLS LDP neighbors to establish.
 279     print("\n\n** Verify MPLS LDP neighbors to establish")
 280     print("******************************************\n")
 281     timeout = 90
 282     while timeout > 0:
 283         print("Timeout in %s: " % timeout),
 284         sys.stdout.flush()
 285         # Look for any node not yet converged
 286         for i in range(1, 5):
 287             established = (
 288                 net["r%s" % i]
 289                 .cmd('vtysh -c "show mpls ldp neighbor" 2> /dev/null')
 290                 .rstrip()
 291             )
 292
 293             # On current version, we need to make sure they all turn to OPERATIONAL on all lines
 294             #
 295             lines = ("\n".join(established.splitlines()) + "\n").splitlines(1)
 296             # Check all lines to be either table header (starting with ^AF or show OPERATIONAL)
 297             header = r"^AF.*"
 298             operational = r"^ip.*OPERATIONAL.*"
 299             found_operational = 0
 300             for j in range(1, len(lines)):
 301                 if (not re.search(header, lines[j])) and (
 302                     not re.search(operational, lines[j])
 303                 ):
 304                     established = ""  # Empty string shows NOT established
 305                 if re.search(operational, lines[j]):
 306                     found_operational += 1
 307
 308             logger.info("Found operational %d" % found_operational)
 309             if found_operational < 1:
 310                 # Need at least one operational neighbor
 311                 established = ""  # Empty string shows NOT established
 312             else:
 313                 if found_operational != neighbors_operational[i]:
 314                     established = ""
 315             if not established:
 316                 print("Waiting for r%s" % i)
 317                 sys.stdout.flush()
 318                 break
 319         if not established:
 320             sleep(5)
 321             timeout -= 5
 322         else:
 323             print("Done")
 324             break
 325     else:
 326         # Bail out with error if a router fails to converge
 327         fatal_error = "MPLS LDP neighbors did not establish"
 328         assert False, "MPLS LDP neighbors did not establish"
 329
 330     print("MPLS LDP neighbors established.")
 331
 332     if timeout < 60:
 333         # Only wait if we actually went through a convergence
 334         print("\nwaiting 15s for LDP sessions to establish")
 335         sleep(15)
 336
 337     # Make sure that all daemons are running
 338     for i in range(1, 5):
 339         fatal_error = net["r%s" % i].checkRouterRunning()
 340         assert fatal_error == "", fatal_error
 341
 342
 343 def test_mpls_ldp_discovery():
 344     global fatal_error
 345     net = get_topogen().net
 346
 347     # Skip if previous fatal error condition is raised
 348     if fatal_error != "":
 349         pytest.skip(fatal_error)
 350
 351     thisDir = os.path.dirname(os.path.realpath(__file__))
 352
 353     # Verify MPLS LDP discovery
 354     print("\n\n** Verifying MPLS LDP discovery")
 355     print("******************************************\n")
 356     failures = 0
 357     for i in range(1, 5):
 358         refTableFile = "%s/r%s/show_mpls_ldp_discovery.ref" % (thisDir, i)
 359         if os.path.isfile(refTableFile):
 360             # Actual output from router
 361             actual = (
 362                 net["r%s" % i]
 363                 .cmd('vtysh -c "show mpls ldp discovery" 2> /dev/null')
 364                 .rstrip()
 365             )
 366
 367             # Read expected result from file
 368             expected = open(refTableFile).read().rstrip()
 369             # Fix newlines (make them all the same)
 370             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 371
 372             # Actual output from router
 373             actual = (
 374                 net["r%s" % i]
 375                 .cmd('vtysh -c "show mpls ldp discovery" 2> /dev/null')
 376                 .rstrip()
 377             )
 378
 379             # Fix newlines (make them all the same)
 380             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 381
 382             # Generate Diff
 383             diff = topotest.get_textdiff(
 384                 actual,
 385                 expected,
 386                 title1="actual MPLS LDP discovery output",
 387                 title2="expected MPLS LDP discovery output",
 388             )
 389
 390             # Empty string if it matches, otherwise diff contains unified diff
 391             if diff:
 392                 sys.stderr.write(
 393                     "r%s failed MPLS LDP discovery output Check:\n%s\n" % (i, diff)
 394                 )
 395                 failures += 1
 396             else:
 397                 print("r%s ok" % i)
 398
 399             assert (
 400                 failures == 0
 401             ), "MPLS LDP Interface discovery output for router r%s:\n%s" % (i, diff)
 402
 403     # Make sure that all daemons are running
 404     for i in range(1, 5):
 405         fatal_error = net["r%s" % i].checkRouterRunning()
 406         assert fatal_error == "", fatal_error
 407
 408
 409 def test_mpls_ldp_neighbor():
 410     global fatal_error
 411     net = get_topogen().net
 412
 413     # Skip if previous fatal error condition is raised
 414     if fatal_error != "":
 415         pytest.skip(fatal_error)
 416
 417     thisDir = os.path.dirname(os.path.realpath(__file__))
 418
 419     # Verify MPLS LDP neighbor
 420     print("\n\n** Verifying MPLS LDP neighbor")
 421     print("******************************************\n")
 422     failures = 0
 423     for i in range(1, 5):
 424         refTableFile = "%s/r%s/show_mpls_ldp_neighbor.ref" % (thisDir, i)
 425         if os.path.isfile(refTableFile):
 426             # Read expected result from file
 427             expected = open(refTableFile).read().rstrip()
 428             # Fix newlines (make them all the same)
 429             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 430
 431             # Actual output from router
 432             actual = (
 433                 net["r%s" % i]
 434                 .cmd('vtysh -c "show mpls ldp neighbor" 2> /dev/null')
 435                 .rstrip()
 436             )
 437
 438             # Mask out changing parts in output
 439             # Mask out Timer in Uptime
 440             actual = re.sub(
 441                 r"(ipv4 [0-9\.]+ +OPERATIONAL [0-9\.]+ +)[0-9][0-9]:[0-9][0-9]:[0-9][0-9]",
 442                 r"\1xx:xx:xx",
 443                 actual,
 444             )
 445
 446             # Fix newlines (make them all the same)
 447             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 448
 449             # Generate Diff
 450             diff = topotest.get_textdiff(
 451                 actual,
 452                 expected,
 453                 title1="actual MPLS LDP neighbor output",
 454                 title2="expected MPLS LDP neighbor output",
 455             )
 456
 457             # Empty string if it matches, otherwise diff contains unified diff
 458             if diff:
 459                 sys.stderr.write(
 460                     "r%s failed MPLS LDP neighbor output Check:\n%s\n" % (i, diff)
 461                 )
 462                 failures += 1
 463             else:
 464                 print("r%s ok" % i)
 465
 466             assert (
 467                 failures == 0
 468             ), "MPLS LDP Interface neighbor output for router r%s:\n%s" % (i, diff)
 469
 470     # Make sure that all daemons are running
 471     for i in range(1, 5):
 472         fatal_error = net["r%s" % i].checkRouterRunning()
 473         assert fatal_error == "", fatal_error
 474
 475
 476 def test_mpls_ldp_binding():
 477     global fatal_error
 478     net = get_topogen().net
 479
 480     # Skip this test for now until proper sorting of the output
 481     # is implemented
 482     # pytest.skip("Skipping test_mpls_ldp_binding")
 483
 484     # Skip if previous fatal error condition is raised
 485     if fatal_error != "":
 486         pytest.skip(fatal_error)
 487
 488     thisDir = os.path.dirname(os.path.realpath(__file__))
 489
 490     # Verify MPLS LDP binding
 491     print("\n\n** Verifying MPLS LDP binding")
 492     print("******************************************\n")
 493     failures = 0
 494     for i in range(1, 5):
 495         refTableFile = "%s/r%s/show_mpls_ldp_binding.ref" % (thisDir, i)
 496         if os.path.isfile(refTableFile):
 497             # Read expected result from file
 498             expected = open(refTableFile).read().rstrip()
 499             # Fix newlines (make them all the same)
 500             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 501
 502             # Actual output from router
 503             actual = (
 504                 net["r%s" % i]
 505                 .cmd('vtysh -c "show mpls ldp binding" 2> /dev/null')
 506                 .rstrip()
 507             )
 508
 509             # Mask out changing parts in output
 510             # Mask out label
 511             actual = re.sub(
 512                 r"(ipv4 [0-9\./]+ +[0-9\.]+ +)[0-9][0-9] (.*)", r"\1xxx\2", actual
 513             )
 514             actual = re.sub(
 515                 r"(ipv4 [0-9\./]+ +[0-9\.]+ +[a-z\-]+ +)[0-9][0-9] (.*)",
 516                 r"\1xxx\2",
 517                 actual,
 518             )
 519
 520             # Fix newlines (make them all the same)
 521             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 522
 523             # Sort lines which start with "xx via inet "
 524             pattern = r"^\s+[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+"
 525             swapped = True
 526             while swapped:
 527                 swapped = False
 528                 for j in range(1, len(actual)):
 529                     if re.search(pattern, actual[j]) and re.search(
 530                         pattern, actual[j - 1]
 531                     ):
 532                         if actual[j - 1] > actual[j]:
 533                             temp = actual[j - 1]
 534                             actual[j - 1] = actual[j]
 535                             actual[j] = temp
 536                             swapped = True
 537
 538             # Generate Diff
 539             diff = topotest.get_textdiff(
 540                 actual,
 541                 expected,
 542                 title1="actual MPLS LDP binding output",
 543                 title2="expected MPLS LDP binding output",
 544             )
 545
 546             # Empty string if it matches, otherwise diff contains unified diff
 547             if diff:
 548                 sys.stderr.write(
 549                     "r%s failed MPLS LDP binding output Check:\n%s\n" % (i, diff)
 550                 )
 551                 failures += 1
 552             else:
 553                 print("r%s ok" % i)
 554
 555             assert failures == 0, "MPLS LDP binding output for router r%s:\n%s" % (
 556                 i,
 557                 diff,
 558             )
 559
 560     # Make sure that all daemons are running
 561     for i in range(1, 5):
 562         fatal_error = net["r%s" % i].checkRouterRunning()
 563         assert fatal_error == "", fatal_error
 564
 565
 566 def test_zebra_ipv4_routingTable():
 567     global fatal_error
 568     net = get_topogen().net
 569
 570     # Skip if previous fatal error condition is raised
 571     if fatal_error != "":
 572         pytest.skip(fatal_error)
 573
 574     thisDir = os.path.dirname(os.path.realpath(__file__))
 575
 576     # Verify Zebra IPv4 Routing Table
 577     print("\n\n** Verifying Zebra IPv4 Routing Table")
 578     print("******************************************\n")
 579     failures = 0
 580     for i in range(1, 5):
 581         refTableFile = "%s/r%s/show_ipv4_route.ref" % (thisDir, i)
 582         if os.path.isfile(refTableFile):
 583             # Read expected result from file
 584             expected = open(refTableFile).read().rstrip()
 585
 586             # Actual output from router
 587             actual = (
 588                 net["r%s" % i]
 589                 .cmd('vtysh -c "show ip route" 2> /dev/null | grep "^O"')
 590                 .rstrip()
 591             )
 592             # Drop timers on end of line
 593             actual = re.sub(r", [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", "", actual)
 594
 595             # Mask out label - all LDP labels should be >= 10 (2-digit)
 596             #   leaving the implicit labels unmasked
 597             actual = re.sub(r" label [0-9][0-9]+", " label xxx", actual)
 598             #   and translating remaining implicit (single-digit) labels to label implicit-null
 599             actual = re.sub(r" label [0-9]+", " label implicit-null", actual)
 600             # Check if we have implicit labels - if not, then remove them from reference
 601             if not re.search(r" label implicit-null", actual):
 602                 expected = re.sub(r", label implicit-null", "", expected)
 603
 604             # now fix newlines of expected (make them all the same)
 605             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 606
 607             # Fix newlines (make them all the same)
 608             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 609
 610             # Generate Diff
 611             diff = topotest.get_textdiff(
 612                 actual,
 613                 expected,
 614                 title1="actual IPv4 zebra routing table",
 615                 title2="expected IPv4 zebra routing table",
 616             )
 617
 618             # Empty string if it matches, otherwise diff contains unified diff
 619             if diff:
 620                 sys.stderr.write(
 621                     "r%s failed IPv4 Zebra Routing Table Check:\n%s\n" % (i, diff)
 622                 )
 623                 failures += 1
 624             else:
 625                 print("r%s ok" % i)
 626
 627             assert (
 628                 failures == 0
 629             ), "IPv4 Zebra Routing Table verification failed for router r%s:\n%s" % (
 630                 i,
 631                 diff,
 632             )
 633
 634     # Make sure that all daemons are running
 635     for i in range(1, 5):
 636         fatal_error = net["r%s" % i].checkRouterRunning()
 637         assert fatal_error == "", fatal_error
 638
 639
 640 def test_mpls_table():
 641     global fatal_error
 642     net = get_topogen().net
 643
 644     # Skip if previous fatal error condition is raised
 645     if fatal_error != "":
 646         pytest.skip(fatal_error)
 647
 648     thisDir = os.path.dirname(os.path.realpath(__file__))
 649
 650     # Verify MPLS table
 651     print("\n\n** Verifying MPLS table")
 652     print("******************************************\n")
 653     failures = 0
 654
 655     for i in range(1, 5):
 656         refTableFile = "%s/r%s/show_mpls_table.ref" % (thisDir, i)
 657         if os.path.isfile(refTableFile):
 658             # Read expected result from file
 659             expected = open(refTableFile).read()
 660             # Fix newlines (make them all the same)
 661             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 662
 663             # Actual output from router
 664             actual = net["r%s" % i].cmd('vtysh -c "show mpls table" 2> /dev/null')
 665
 666             # Fix inconsistent Label numbers at beginning of line
 667             actual = re.sub(r"(\s+)[0-9]+(\s+LDP)", r"\1XX\2", actual)
 668             # Fix inconsistent Label numbers at end of line
 669             actual = re.sub(
 670                 r"(\s+[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\s+)[0-9][0-9]", r"\1XX", actual
 671             )
 672
 673             # Fix newlines (make them all the same)
 674             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 675
 676             # Sort lines which start with "      XX      LDP"
 677             pattern = r"^\s+[0-9X]+\s+LDP"
 678             swapped = True
 679             while swapped:
 680                 swapped = False
 681                 for j in range(1, len(actual)):
 682                     if re.search(pattern, actual[j]) and re.search(
 683                         pattern, actual[j - 1]
 684                     ):
 685                         if actual[j - 1] > actual[j]:
 686                             temp = actual[j - 1]
 687                             actual[j - 1] = actual[j]
 688                             actual[j] = temp
 689                             swapped = True
 690
 691             # Generate Diff
 692             diff = topotest.get_textdiff(
 693                 actual,
 694                 expected,
 695                 title1="actual MPLS table output",
 696                 title2="expected MPLS table output",
 697             )
 698
 699             # Empty string if it matches, otherwise diff contains unified diff
 700             if diff:
 701                 sys.stderr.write(
 702                     "r%s failed MPLS table output Check:\n%s\n" % (i, diff)
 703                 )
 704                 failures += 1
 705             else:
 706                 print("r%s ok" % i)
 707
 708             assert failures == 0, "MPLS table output for router r%s:\n%s" % (i, diff)
 709
 710     # Make sure that all daemons are running
 711     for i in range(1, 5):
 712         fatal_error = net["r%s" % i].checkRouterRunning()
 713         assert fatal_error == "", fatal_error
 714
 715
 716 def test_linux_mpls_routes():
 717     global fatal_error
 718     net = get_topogen().net
 719
 720     # Skip if previous fatal error condition is raised
 721     if fatal_error != "":
 722         pytest.skip(fatal_error)
 723
 724     thisDir = os.path.dirname(os.path.realpath(__file__))
 725
 726     # Verify Linux Kernel MPLS routes
 727     print("\n\n** Verifying Linux Kernel MPLS routes")
 728     print("******************************************\n")
 729     failures = 0
 730     for i in range(1, 5):
 731         refTableFile = "%s/r%s/ip_mpls_route.ref" % (thisDir, i)
 732         if os.path.isfile(refTableFile):
 733             # Read expected result from file
 734             expected = open(refTableFile).read().rstrip()
 735             # Fix newlines (make them all the same)
 736             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 737
 738             # Actual output from router
 739             actual = (
 740                 net["r%s" % i].cmd("ip -o -family mpls route 2> /dev/null").rstrip()
 741             )
 742
 743             # Mask out label and protocol
 744             actual = re.sub(r"[0-9][0-9] via inet ", "xx via inet ", actual)
 745             actual = re.sub(r"[0-9][0-9] +proto", "xx  proto", actual)
 746             actual = re.sub(r"[0-9][0-9] as to ", "xx as to ", actual)
 747             actual = re.sub(r"[ ]+proto \w+", "  proto xx", actual)
 748
 749             # Sort nexthops
 750             nexthop_sorted = []
 751             for line in actual.splitlines():
 752                 tokens = re.split(r"\\\t", line.strip())
 753                 nexthop_sorted.append(
 754                     "{} {}".format(
 755                         tokens[0].strip(),
 756                         " ".join([token.strip() for token in sorted(tokens[1:])]),
 757                     ).strip()
 758                 )
 759
 760             # Sort lines and fixup differences between old and new iproute
 761             actual = "\n".join(sorted(nexthop_sorted))
 762             actual = re.sub(r"nexthop via", "nexthopvia", actual)
 763             actual = re.sub(r" nexthop as to xx via inet ", " nexthopvia inet ", actual)
 764             actual = re.sub(r" weight 1", "", actual)
 765             actual = re.sub(r" [ ]+", " ", actual)
 766
 767             # put \n back at line ends
 768             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 769
 770             # Generate Diff
 771             diff = topotest.get_textdiff(
 772                 actual,
 773                 expected,
 774                 title1="actual Linux Kernel MPLS route",
 775                 title2="expected Linux Kernel MPLS route",
 776             )
 777
 778             # Empty string if it matches, otherwise diff contains unified diff
 779             if diff:
 780                 sys.stderr.write(
 781                     "r%s failed Linux Kernel MPLS route output Check:\n%s\n" % (i, diff)
 782                 )
 783                 failures += 1
 784             else:
 785                 print("r%s ok" % i)
 786
 787             assert (
 788                 failures == 0
 789             ), "Linux Kernel MPLS route output for router r%s:\n%s" % (i, diff)
 790
 791     # Make sure that all daemons are running
 792     for i in range(1, 5):
 793         fatal_error = net["r%s" % i].checkRouterRunning()
 794         assert fatal_error == "", fatal_error
 795
 796
 797 def test_shutdown_check_stderr():
 798     global fatal_error
 799     net = get_topogen().net
 800
 801     # Skip if previous fatal error condition is raised
 802     if fatal_error != "":
 803         pytest.skip(fatal_error)
 804
 805     if os.environ.get("TOPOTESTS_CHECK_STDERR") is None:
 806         print(
 807             "SKIPPED final check on StdErr output: Disabled (TOPOTESTS_CHECK_STDERR undefined)\n"
 808         )
 809         pytest.skip("Skipping test for Stderr output")
 810
 811     thisDir = os.path.dirname(os.path.realpath(__file__))
 812
 813     print("\n\n** Verifying unexpected STDERR output from daemons")
 814     print("******************************************\n")
 815
 816     for i in range(1, 5):
 817         net["r%s" % i].stopRouter()
 818         log = net["r%s" % i].getStdErr("ldpd")
 819         if log:
 820             print("\nRouter r%s LDPd StdErr Log:\n%s" % (i, log))
 821         log = net["r%s" % i].getStdErr("ospfd")
 822         if log:
 823             print("\nRouter r%s OSPFd StdErr Log:\n%s" % (i, log))
 824         log = net["r%s" % i].getStdErr("zebra")
 825         if log:
 826             print("\nRouter r%s Zebra StdErr Log:\n%s" % (i, log))
 827
 828
 829 def test_shutdown_check_memleak():
 830     global fatal_error
 831     net = get_topogen().net
 832
 833     # Skip if previous fatal error condition is raised
 834     if fatal_error != "":
 835         pytest.skip(fatal_error)
 836
 837     if os.environ.get("TOPOTESTS_CHECK_MEMLEAK") is None:
 838         print(
 839             "SKIPPED final check on Memory leaks: Disabled (TOPOTESTS_CHECK_MEMLEAK undefined)\n"
 840         )
 841         pytest.skip("Skipping test for memory leaks")
 842
 843     thisDir = os.path.dirname(os.path.realpath(__file__))
 844
 845     for i in range(1, 5):
 846         net["r%s" % i].stopRouter()
 847         net["r%s" % i].report_memory_leaks(
 848             os.environ.get("TOPOTESTS_CHECK_MEMLEAK"), os.path.basename(__file__)
 849         )
 850
 851
 852 if __name__ == "__main__":
 853
 854     # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
 855     # retval = pytest.main(["-s", "--tb=no"])
 856     retval = pytest.main(["-s"])
 857     sys.exit(retval)