tests/topotests/all_protocol_startup/test_all_protocol_startup.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # test_all_protocol_startup.py
   5 # Part of NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2017 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 """
  26 test_all_protocol_startup.py: Test of all protocols at same time
  27
  28 """
  29
  30 import os
  31 import re
  32 import sys
  33 import pytest
  34 import glob
  35 from time import sleep
  36
  37
  38 pytestmark = [
  39     pytest.mark.babeld,
  40     pytest.mark.bgpd,
  41     pytest.mark.isisd,
  42     pytest.mark.nhrpd,
  43     pytest.mark.ospfd,
  44     pytest.mark.pbrd,
  45     pytest.mark.ripd,
  46 ]
  47
  48 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  49 from lib import topotest
  50 from lib.topogen import Topogen, get_topogen
  51
  52 fatal_error = ""
  53
  54
  55 #####################################################
  56 ##
  57 ##   Network Topology Definition
  58 ##
  59 #####################################################
  60
  61
  62 def build_topo(tgen):
  63     router = tgen.add_router("r1")
  64     for i in range(0, 10):
  65         tgen.add_switch("sw%d" % i).add_link(router)
  66
  67
  68 #####################################################
  69 ##
  70 ##   Tests starting
  71 ##
  72 #####################################################
  73
  74
  75 def setup_module(module):
  76     global fatal_error
  77
  78     print("\n\n** %s: Setup Topology" % module.__name__)
  79     print("******************************************\n")
  80
  81     thisDir = os.path.dirname(os.path.realpath(__file__))
  82     tgen = Topogen(build_topo, module.__name__)
  83     tgen.start_topology()
  84
  85     net = tgen.net
  86
  87     if net["r1"].get_routertype() != "frr":
  88         fatal_error = "Test is only implemented for FRR"
  89         sys.stderr.write("\n\nTest is only implemented for FRR - Skipping\n\n")
  90         pytest.skip(fatal_error)
  91
  92     # Starting Routers
  93     #
  94     # Main router
  95     for i in range(1, 2):
  96         net["r%s" % i].loadConf("zebra", "%s/r%s/zebra.conf" % (thisDir, i))
  97         net["r%s" % i].loadConf("ripd", "%s/r%s/ripd.conf" % (thisDir, i))
  98         net["r%s" % i].loadConf("ripngd", "%s/r%s/ripngd.conf" % (thisDir, i))
  99         net["r%s" % i].loadConf("ospfd", "%s/r%s/ospfd.conf" % (thisDir, i))
 100         if net["r1"].checkRouterVersion("<", "4.0"):
 101             net["r%s" % i].loadConf(
 102                 "ospf6d", "%s/r%s/ospf6d.conf-pre-v4" % (thisDir, i)
 103             )
 104         else:
 105             net["r%s" % i].loadConf("ospf6d", "%s/r%s/ospf6d.conf" % (thisDir, i))
 106         net["r%s" % i].loadConf("isisd", "%s/r%s/isisd.conf" % (thisDir, i))
 107         net["r%s" % i].loadConf("bgpd", "%s/r%s/bgpd.conf" % (thisDir, i))
 108         if net["r%s" % i].daemon_available("ldpd"):
 109             # Only test LDPd if it's installed and Kernel >= 4.5
 110             net["r%s" % i].loadConf("ldpd", "%s/r%s/ldpd.conf" % (thisDir, i))
 111         net["r%s" % i].loadConf("sharpd")
 112         net["r%s" % i].loadConf("nhrpd", "%s/r%s/nhrpd.conf" % (thisDir, i))
 113         net["r%s" % i].loadConf("babeld", "%s/r%s/babeld.conf" % (thisDir, i))
 114         net["r%s" % i].loadConf("pbrd", "%s/r%s/pbrd.conf" % (thisDir, i))
 115         tgen.gears["r%s" % i].start()
 116
 117     # For debugging after starting FRR daemons, uncomment the next line
 118     # tgen.mininet_cli()
 119
 120
 121 def teardown_module(module):
 122     print("\n\n** %s: Shutdown Topology" % module.__name__)
 123     print("******************************************\n")
 124     tgen = get_topogen()
 125     tgen.stop_topology()
 126
 127
 128 def test_router_running():
 129     global fatal_error
 130     tgen = get_topogen()
 131     net = tgen.net
 132
 133     # Skip if previous fatal error condition is raised
 134     if fatal_error != "":
 135         pytest.skip(fatal_error)
 136
 137     print("\n\n** Check if FRR is running on each Router node")
 138     print("******************************************\n")
 139     sleep(5)
 140
 141     # Starting Routers
 142     for i in range(1, 2):
 143         fatal_error = net["r%s" % i].checkRouterRunning()
 144         assert fatal_error == "", fatal_error
 145
 146     # For debugging after starting FRR daemons, uncomment the next line
 147     # tgen.mininet_cli()
 148
 149
 150 def test_error_messages_vtysh():
 151     global fatal_error
 152     net = get_topogen().net
 153
 154     # Skip if previous fatal error condition is raised
 155     if fatal_error != "":
 156         pytest.skip(fatal_error)
 157
 158     print("\n\n** Check for error messages on VTYSH")
 159     print("******************************************\n")
 160
 161     failures = 0
 162     for i in range(1, 2):
 163         #
 164         # First checking Standard Output
 165         #
 166
 167         # VTYSH output from router
 168         vtystdout = net["r%s" % i].cmd('vtysh -c "show version" 2> /dev/null').rstrip()
 169
 170         # Fix newlines (make them all the same)
 171         vtystdout = ("\n".join(vtystdout.splitlines()) + "\n").rstrip()
 172         # Drop everything starting with "FRRouting X.xx" message
 173         vtystdout = re.sub(r"FRRouting [0-9]+.*", "", vtystdout, flags=re.DOTALL)
 174
 175         if vtystdout == "":
 176             print("r%s StdOut ok" % i)
 177
 178         assert vtystdout == "", "Vtysh StdOut Output check failed for router r%s" % i
 179
 180         #
 181         # Second checking Standard Error
 182         #
 183
 184         # VTYSH StdErr output from router
 185         vtystderr = net["r%s" % i].cmd('vtysh -c "show version" > /dev/null').rstrip()
 186
 187         # Fix newlines (make them all the same)
 188         vtystderr = ("\n".join(vtystderr.splitlines()) + "\n").rstrip()
 189         # # Drop everything starting with "FRRouting X.xx" message
 190         # vtystderr = re.sub(r"FRRouting [0-9]+.*", "", vtystderr, flags=re.DOTALL)
 191
 192         if vtystderr == "":
 193             print("r%s StdErr ok" % i)
 194
 195         assert vtystderr == "", "Vtysh StdErr Output check failed for router r%s" % i
 196
 197     # Make sure that all daemons are running
 198     for i in range(1, 2):
 199         fatal_error = net["r%s" % i].checkRouterRunning()
 200         assert fatal_error == "", fatal_error
 201
 202
 203 def test_error_messages_daemons():
 204     global fatal_error
 205     net = get_topogen().net
 206
 207     # Skip if previous fatal error condition is raised
 208     if fatal_error != "":
 209         pytest.skip(fatal_error)
 210
 211     print("\n\n** Check for error messages in daemons")
 212     print("******************************************\n")
 213
 214     error_logs = ""
 215
 216     for i in range(1, 2):
 217         log = net["r%s" % i].getStdErr("ripd")
 218         if log:
 219             error_logs += "r%s RIPd StdErr Output:\n" % i
 220             error_logs += log
 221         log = net["r%s" % i].getStdErr("ripngd")
 222         if log:
 223             error_logs += "r%s RIPngd StdErr Output:\n" % i
 224             error_logs += log
 225         log = net["r%s" % i].getStdErr("ospfd")
 226         if log:
 227             error_logs += "r%s OSPFd StdErr Output:\n" % i
 228             error_logs += log
 229         log = net["r%s" % i].getStdErr("ospf6d")
 230         if log:
 231             error_logs += "r%s OSPF6d StdErr Output:\n" % i
 232             error_logs += log
 233         log = net["r%s" % i].getStdErr("isisd")
 234         # ISIS shows debugging enabled status on StdErr
 235         # Remove these messages
 236         log = re.sub(r"^IS-IS .* debugging is on.*", "", log).rstrip()
 237         if log:
 238             error_logs += "r%s ISISd StdErr Output:\n" % i
 239             error_logs += log
 240         log = net["r%s" % i].getStdErr("bgpd")
 241         if log:
 242             error_logs += "r%s BGPd StdErr Output:\n" % i
 243             error_logs += log
 244         if net["r%s" % i].daemon_available("ldpd"):
 245             log = net["r%s" % i].getStdErr("ldpd")
 246             if log:
 247                 error_logs += "r%s LDPd StdErr Output:\n" % i
 248                 error_logs += log
 249
 250         log = net["r1"].getStdErr("nhrpd")
 251         # NHRPD shows YANG model not embedded messages
 252         # Ignore these
 253         log = re.sub(r".*YANG model.*not embedded.*", "", log).rstrip()
 254         if log:
 255             error_logs += "r%s NHRPd StdErr Output:\n" % i
 256             error_logs += log
 257
 258         log = net["r1"].getStdErr("babeld")
 259         if log:
 260             error_logs += "r%s BABELd StdErr Output:\n" % i
 261             error_logs += log
 262
 263         log = net["r1"].getStdErr("pbrd")
 264         if log:
 265             error_logs += "r%s PBRd StdErr Output:\n" % i
 266             error_logs += log
 267
 268         log = net["r%s" % i].getStdErr("zebra")
 269         if log:
 270             error_logs += "r%s Zebra StdErr Output:\n" % i
 271             error_logs += log
 272
 273     if error_logs:
 274         sys.stderr.write(
 275             "Failed check for StdErr Output on daemons:\n%s\n" % error_logs
 276         )
 277
 278     # Ignoring the issue if told to ignore (ie not yet fixed)
 279     if error_logs != "":
 280         if os.environ.get("bamboo_TOPOTESTS_ISSUE_349") == "IGNORE":
 281             sys.stderr.write(
 282                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349\n"
 283             )
 284             pytest.skip(
 285                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349"
 286             )
 287
 288     assert error_logs == "", "Daemons report errors to StdErr"
 289
 290
 291 def test_converge_protocols():
 292     global fatal_error
 293     net = get_topogen().net
 294
 295     # Skip if previous fatal error condition is raised
 296     if fatal_error != "":
 297         pytest.skip(fatal_error)
 298
 299     thisDir = os.path.dirname(os.path.realpath(__file__))
 300
 301     print("\n\n** Waiting for protocols convergence")
 302     print("******************************************\n")
 303
 304     # Not really implemented yet - just sleep 60 secs for now
 305     sleep(60)
 306
 307     # Make sure that all daemons are running
 308     failures = 0
 309     for i in range(1, 2):
 310         fatal_error = net["r%s" % i].checkRouterRunning()
 311         assert fatal_error == "", fatal_error
 312
 313         print("Show that v4 routes are right\n")
 314         v4_routesFile = "%s/r%s/ipv4_routes.ref" % (thisDir, i)
 315         expected = (
 316             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v4_routesFile)).rstrip()
 317         )
 318         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 319
 320         actual = (
 321             net["r%s" % i]
 322             .cmd(
 323                 "vtysh -c \"show ip route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 324             )
 325             .rstrip()
 326         )
 327         # Drop time in last update
 328         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 329         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 330         diff = topotest.get_textdiff(
 331             actual,
 332             expected,
 333             title1="Actual IP Routing Table",
 334             title2="Expected IP RoutingTable",
 335         )
 336         if diff:
 337             sys.stderr.write("r%s failed IP Routing table check:\n%s\n" % (i, diff))
 338             failures += 1
 339         else:
 340             print("r%s ok" % i)
 341
 342         assert failures == 0, "IP Routing table failed for r%s\n%s" % (i, diff)
 343
 344         failures = 0
 345
 346         print("Show that v6 routes are right\n")
 347         v6_routesFile = "%s/r%s/ipv6_routes.ref" % (thisDir, i)
 348         expected = (
 349             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v6_routesFile)).rstrip()
 350         )
 351         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 352
 353         actual = (
 354             net["r%s" % i]
 355             .cmd(
 356                 "vtysh -c \"show ipv6 route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 357             )
 358             .rstrip()
 359         )
 360         # Drop time in last update
 361         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 362         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 363         diff = topotest.get_textdiff(
 364             actual,
 365             expected,
 366             title1="Actual IPv6 Routing Table",
 367             title2="Expected IPv6 RoutingTable",
 368         )
 369         if diff:
 370             sys.stderr.write("r%s failed IPv6 Routing table check:\n%s\n" % (i, diff))
 371             failures += 1
 372         else:
 373             print("r%s ok" % i)
 374
 375         assert failures == 0, "IPv6 Routing table failed for r%s\n%s" % (i, diff)
 376
 377
 378 def route_get_nhg_id(route_str):
 379     net = get_topogen().net
 380     output = net["r1"].cmd('vtysh -c "show ip route %s nexthop-group"' % route_str)
 381     match = re.search(r"Nexthop Group ID: (\d+)", output)
 382     assert match is not None, (
 383         "Nexthop Group ID not found for sharpd route %s" % route_str
 384     )
 385
 386     nhg_id = int(match.group(1))
 387     return nhg_id
 388
 389
 390 def verify_nexthop_group(nhg_id, recursive=False, ecmp=0):
 391     net = get_topogen().net
 392     # Verify NHG is valid/installed
 393     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 394
 395     match = re.search(r"Valid", output)
 396     assert match is not None, "Nexthop Group ID=%d not marked Valid" % nhg_id
 397
 398     if ecmp or recursive:
 399         match = re.search(r"Depends:.*\n", output)
 400         assert match is not None, "Nexthop Group ID=%d has no depends" % nhg_id
 401
 402         # list of IDs in group
 403         depends = re.findall(r"\((\d+)\)", match.group(0))
 404
 405         if ecmp:
 406             assert len(depends) == ecmp, (
 407                 "Nexthop Group ID=%d doesn't match ecmp size" % nhg_id
 408             )
 409         else:
 410             # If recursive, we need to look at its resolved group
 411             assert len(depends) == 1, (
 412                 "Nexthop Group ID=%d should only have one recursive depend" % nhg_id
 413             )
 414             resolved_id = int(depends[0])
 415             verify_nexthop_group(resolved_id, False)
 416
 417     else:
 418         match = re.search(r"Installed", output)
 419         assert match is not None, "Nexthop Group ID=%d not marked Installed" % nhg_id
 420
 421
 422 def verify_route_nexthop_group(route_str, recursive=False, ecmp=0):
 423     # Verify route and that zebra created NHGs for and they are valid/installed
 424     nhg_id = route_get_nhg_id(route_str)
 425     verify_nexthop_group(nhg_id, recursive, ecmp)
 426
 427
 428 def test_nexthop_groups():
 429     global fatal_error
 430     net = get_topogen().net
 431
 432     # Skip if previous fatal error condition is raised
 433     if fatal_error != "":
 434         pytest.skip(fatal_error)
 435
 436     print("\n\n** Verifying Nexthop Groups")
 437     print("******************************************\n")
 438
 439     ### Nexthop Group Tests
 440
 441     ## Basic test
 442
 443     # Create a lib nexthop-group
 444     net["r1"].cmd(
 445         'vtysh -c "c t" -c "nexthop-group basic" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
 446     )
 447
 448     # Create with sharpd using nexthop-group
 449     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.1 nexthop-group basic 1"')
 450
 451     verify_route_nexthop_group("2.2.2.1/32")
 452
 453     ## Connected
 454
 455     net["r1"].cmd(
 456         'vtysh -c "c t" -c "nexthop-group connected" -c "nexthop r1-eth1" -c "nexthop r1-eth2"'
 457     )
 458
 459     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.2 nexthop-group connected 1"')
 460
 461     verify_route_nexthop_group("2.2.2.2/32")
 462
 463     ## Recursive
 464
 465     net["r1"].cmd(
 466         'vtysh -c "c t" -c "nexthop-group basic-recursive" -c "nexthop 2.2.2.1"'
 467     )
 468
 469     net["r1"].cmd(
 470         'vtysh -c "sharp install routes 3.3.3.1 nexthop-group basic-recursive 1"'
 471     )
 472
 473     verify_route_nexthop_group("3.3.3.1/32", True)
 474
 475     ## Duplicate
 476
 477     net["r1"].cmd(
 478         'vtysh -c "c t" -c "nexthop-group duplicate" -c "nexthop 2.2.2.1" -c "nexthop 1.1.1.1"'
 479     )
 480
 481     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.2 nexthop-group duplicate 1"')
 482
 483     verify_route_nexthop_group("3.3.3.2/32")
 484
 485     ## Two 4-Way ECMP
 486
 487     net["r1"].cmd(
 488         'vtysh -c "c t" -c "nexthop-group fourA" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2" \
 489             -c "nexthop 1.1.1.3" -c "nexthop 1.1.1.4"'
 490     )
 491
 492     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.1 nexthop-group fourA 1"')
 493
 494     verify_route_nexthop_group("4.4.4.1/32")
 495
 496     net["r1"].cmd(
 497         'vtysh -c "c t" -c "nexthop-group fourB" -c "nexthop 1.1.1.5" -c "nexthop 1.1.1.6" \
 498             -c "nexthop 1.1.1.7" -c "nexthop 1.1.1.8"'
 499     )
 500
 501     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.2 nexthop-group fourB 1"')
 502
 503     verify_route_nexthop_group("4.4.4.2/32")
 504
 505     ## Recursive to 8-Way ECMP
 506
 507     net["r1"].cmd(
 508         'vtysh -c "c t" -c "nexthop-group eight-recursive" -c "nexthop 4.4.4.1" -c "nexthop 4.4.4.2"'
 509     )
 510
 511     net["r1"].cmd(
 512         'vtysh -c "sharp install routes 5.5.5.1 nexthop-group eight-recursive 1"'
 513     )
 514
 515     verify_route_nexthop_group("5.5.5.1/32")
 516
 517     ## 4-way ECMP Routes Pointing to Each Other
 518
 519     # This is to check for a bug with NH resolution where
 520     # routes would infintely resolve to each other blowing
 521     # up the resolved-> nexthop pointer.
 522
 523     net["r1"].cmd(
 524         'vtysh -c "c t" -c "nexthop-group infinite-recursive" -c "nexthop 6.6.6.1" -c "nexthop 6.6.6.2" \
 525             -c "nexthop 6.6.6.3" -c "nexthop 6.6.6.4"'
 526     )
 527
 528     # static route nexthops can recurse to
 529
 530     net["r1"].cmd('vtysh -c "c t" -c "ip route 6.6.6.0/24 1.1.1.1"')
 531
 532     # Make routes that point to themselves in ecmp
 533
 534     net["r1"].cmd(
 535         'vtysh -c "sharp install routes 6.6.6.4 nexthop-group infinite-recursive 1"'
 536     )
 537
 538     net["r1"].cmd(
 539         'vtysh -c "sharp install routes 6.6.6.3 nexthop-group infinite-recursive 1"'
 540     )
 541
 542     net["r1"].cmd(
 543         'vtysh -c "sharp install routes 6.6.6.2 nexthop-group infinite-recursive 1"'
 544     )
 545
 546     net["r1"].cmd(
 547         'vtysh -c "sharp install routes 6.6.6.1 nexthop-group infinite-recursive 1"'
 548     )
 549
 550     # Get routes and test if has too many (duplicate) nexthops
 551     nhg_id = route_get_nhg_id("6.6.6.1/32")
 552     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 553
 554     dups = re.findall(r"(via 1\.1\.1\.1)", output)
 555
 556     # Should find 3, itself is inactive
 557     assert len(dups) == 3, (
 558         "Route 6.6.6.1/32 with Nexthop Group ID=%d has wrong number of resolved nexthops"
 559         % nhg_id
 560     )
 561
 562     ## Remove all NHG routes
 563
 564     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.1 1"')
 565     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.2 1"')
 566     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.1 1"')
 567     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.2 1"')
 568     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.1 1"')
 569     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.2 1"')
 570     net["r1"].cmd('vtysh -c "sharp remove routes 5.5.5.1 1"')
 571     net["r1"].cmd('vtysh -c "sharp remove routes 6.6.6.1 4"')
 572     net["r1"].cmd('vtysh -c "c t" -c "no ip route 6.6.6.0/24 1.1.1.1"')
 573
 574
 575 def test_rip_status():
 576     global fatal_error
 577     net = get_topogen().net
 578
 579     # Skip if previous fatal error condition is raised
 580     if fatal_error != "":
 581         pytest.skip(fatal_error)
 582
 583     thisDir = os.path.dirname(os.path.realpath(__file__))
 584
 585     print("\n\n** Verifying RIP status")
 586     print("******************************************\n")
 587     failures = 0
 588     for i in range(1, 2):
 589         refTableFile = "%s/r%s/rip_status.ref" % (thisDir, i)
 590         if os.path.isfile(refTableFile):
 591             # Read expected result from file
 592             expected = open(refTableFile).read().rstrip()
 593             # Fix newlines (make them all the same)
 594             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 595
 596             # Actual output from router
 597             actual = (
 598                 net["r%s" % i]
 599                 .cmd('vtysh -c "show ip rip status" 2> /dev/null')
 600                 .rstrip()
 601             )
 602             # Drop time in next due
 603             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 604             # Drop time in last update
 605             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 606             # Fix newlines (make them all the same)
 607             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 608
 609             # Generate Diff
 610             diff = topotest.get_textdiff(
 611                 actual,
 612                 expected,
 613                 title1="actual IP RIP status",
 614                 title2="expected IP RIP status",
 615             )
 616
 617             # Empty string if it matches, otherwise diff contains unified diff
 618             if diff:
 619                 sys.stderr.write("r%s failed IP RIP status check:\n%s\n" % (i, diff))
 620                 failures += 1
 621             else:
 622                 print("r%s ok" % i)
 623
 624             assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff)
 625
 626     # Make sure that all daemons are running
 627     for i in range(1, 2):
 628         fatal_error = net["r%s" % i].checkRouterRunning()
 629         assert fatal_error == "", fatal_error
 630
 631
 632 def test_ripng_status():
 633     global fatal_error
 634     net = get_topogen().net
 635
 636     # Skip if previous fatal error condition is raised
 637     if fatal_error != "":
 638         pytest.skip(fatal_error)
 639
 640     thisDir = os.path.dirname(os.path.realpath(__file__))
 641
 642     print("\n\n** Verifying RIPng status")
 643     print("******************************************\n")
 644     failures = 0
 645     for i in range(1, 2):
 646         refTableFile = "%s/r%s/ripng_status.ref" % (thisDir, i)
 647         if os.path.isfile(refTableFile):
 648             # Read expected result from file
 649             expected = open(refTableFile).read().rstrip()
 650             # Fix newlines (make them all the same)
 651             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 652
 653             # Actual output from router
 654             actual = (
 655                 net["r%s" % i]
 656                 .cmd('vtysh -c "show ipv6 ripng status" 2> /dev/null')
 657                 .rstrip()
 658             )
 659             # Mask out Link-Local mac address portion. They are random...
 660             actual = re.sub(r" fe80::[0-9a-f:]+", " fe80::XXXX:XXXX:XXXX:XXXX", actual)
 661             # Drop time in next due
 662             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 663             # Drop time in last update
 664             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 665             # Fix newlines (make them all the same)
 666             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 667
 668             # Generate Diff
 669             diff = topotest.get_textdiff(
 670                 actual,
 671                 expected,
 672                 title1="actual IPv6 RIPng status",
 673                 title2="expected IPv6 RIPng status",
 674             )
 675
 676             # Empty string if it matches, otherwise diff contains unified diff
 677             if diff:
 678                 sys.stderr.write(
 679                     "r%s failed IPv6 RIPng status check:\n%s\n" % (i, diff)
 680                 )
 681                 failures += 1
 682             else:
 683                 print("r%s ok" % i)
 684
 685             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (
 686                 i,
 687                 diff,
 688             )
 689
 690     # Make sure that all daemons are running
 691     for i in range(1, 2):
 692         fatal_error = net["r%s" % i].checkRouterRunning()
 693         assert fatal_error == "", fatal_error
 694
 695
 696 def test_ospfv2_interfaces():
 697     global fatal_error
 698     net = get_topogen().net
 699
 700     # Skip if previous fatal error condition is raised
 701     if fatal_error != "":
 702         pytest.skip(fatal_error)
 703
 704     thisDir = os.path.dirname(os.path.realpath(__file__))
 705
 706     print("\n\n** Verifying OSPFv2 interfaces")
 707     print("******************************************\n")
 708     failures = 0
 709     for i in range(1, 2):
 710         refTableFile = "%s/r%s/show_ip_ospf_interface.ref" % (thisDir, i)
 711         if os.path.isfile(refTableFile):
 712             # Read expected result from file
 713             expected = open(refTableFile).read().rstrip()
 714             # Fix newlines (make them all the same)
 715             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 716
 717             # Actual output from router
 718             actual = (
 719                 net["r%s" % i]
 720                 .cmd('vtysh -c "show ip ospf interface" 2> /dev/null')
 721                 .rstrip()
 722             )
 723             # Mask out Bandwidth portion. They may change..
 724             actual = re.sub(r"BW [0-9]+ Mbit", "BW XX Mbit", actual)
 725             actual = re.sub(r"ifindex [0-9]+", "ifindex X", actual)
 726
 727             # Drop time in next due
 728             actual = re.sub(r"Hello due in [0-9\.]+s", "Hello due in XX.XXXs", actual)
 729             actual = re.sub(
 730                 r"Hello due in [0-9\.]+ usecs", "Hello due in XX.XXXs", actual
 731             )
 732             # Fix 'MTU mismatch detection: enabled' vs 'MTU mismatch detection:enabled' - accept both
 733             actual = re.sub(
 734                 r"MTU mismatch detection:([a-z]+.*)",
 735                 r"MTU mismatch detection: \1",
 736                 actual,
 737             )
 738             # Fix newlines (make them all the same)
 739             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 740
 741             # Generate Diff
 742             diff = topotest.get_textdiff(
 743                 actual,
 744                 expected,
 745                 title1="actual SHOW IP OSPF INTERFACE",
 746                 title2="expected SHOW IP OSPF INTERFACE",
 747             )
 748
 749             # Empty string if it matches, otherwise diff contains unified diff
 750             if diff:
 751                 sys.stderr.write(
 752                     "r%s failed SHOW IP OSPF INTERFACE check:\n%s\n" % (i, diff)
 753                 )
 754                 failures += 1
 755             else:
 756                 print("r%s ok" % i)
 757
 758             # Ignoring the issue if told to ignore (ie not yet fixed)
 759             if failures != 0:
 760                 if os.environ.get("bamboo_TOPOTESTS_ISSUE_348") == "IGNORE":
 761                     sys.stderr.write(
 762                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348\n"
 763                     )
 764                     pytest.skip(
 765                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348"
 766                     )
 767
 768             assert (
 769                 failures == 0
 770             ), "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff)
 771
 772     # Make sure that all daemons are running
 773     for i in range(1, 2):
 774         fatal_error = net["r%s" % i].checkRouterRunning()
 775         assert fatal_error == "", fatal_error
 776
 777
 778 def test_isis_interfaces():
 779     global fatal_error
 780     net = get_topogen().net
 781
 782     # Skip if previous fatal error condition is raised
 783     if fatal_error != "":
 784         pytest.skip(fatal_error)
 785
 786     thisDir = os.path.dirname(os.path.realpath(__file__))
 787
 788     print("\n\n** Verifying ISIS interfaces")
 789     print("******************************************\n")
 790     failures = 0
 791     for i in range(1, 2):
 792         refTableFile = "%s/r%s/show_isis_interface_detail.ref" % (thisDir, i)
 793         if os.path.isfile(refTableFile):
 794             # Read expected result from file
 795             expected = open(refTableFile).read().rstrip()
 796             # Fix newlines (make them all the same)
 797             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 798
 799             # Actual output from router
 800             actual = (
 801                 net["r%s" % i]
 802                 .cmd('vtysh -c "show isis interface detail" 2> /dev/null')
 803                 .rstrip()
 804             )
 805             # Mask out Link-Local mac address portion. They are random...
 806             actual = re.sub(r"fe80::[0-9a-f:]+", "fe80::XXXX:XXXX:XXXX:XXXX", actual)
 807             # Mask out SNPA mac address portion. They are random...
 808             actual = re.sub(r"SNPA: [0-9a-f\.]+", "SNPA: XXXX.XXXX.XXXX", actual)
 809             # Mask out Circuit ID number
 810             actual = re.sub(r"Circuit Id: 0x[0-9a-f]+", "Circuit Id: 0xXX", actual)
 811             # Fix newlines (make them all the same)
 812             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 813
 814             # Generate Diff
 815             diff = topotest.get_textdiff(
 816                 actual,
 817                 expected,
 818                 title1="actual SHOW ISIS INTERFACE DETAIL",
 819                 title2="expected SHOW ISIS OSPF6 INTERFACE DETAIL",
 820             )
 821
 822             # Empty string if it matches, otherwise diff contains unified diff
 823             if diff:
 824                 sys.stderr.write(
 825                     "r%s failed SHOW ISIS INTERFACE DETAIL check:\n%s\n" % (i, diff)
 826                 )
 827                 failures += 1
 828             else:
 829                 print("r%s ok" % i)
 830
 831             assert (
 832                 failures == 0
 833             ), "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff)
 834
 835     # Make sure that all daemons are running
 836     for i in range(1, 2):
 837         fatal_error = net["r%s" % i].checkRouterRunning()
 838         assert fatal_error == "", fatal_error
 839
 840
 841 def test_bgp_summary():
 842     global fatal_error
 843     net = get_topogen().net
 844
 845     # Skip if previous fatal error condition is raised
 846     if fatal_error != "":
 847         pytest.skip(fatal_error)
 848
 849     thisDir = os.path.dirname(os.path.realpath(__file__))
 850
 851     print("\n\n** Verifying BGP Summary")
 852     print("******************************************\n")
 853     failures = 0
 854     for i in range(1, 2):
 855         refTableFile = "%s/r%s/show_ip_bgp_summary.ref" % (thisDir, i)
 856         if os.path.isfile(refTableFile):
 857             # Read expected result from file
 858             expected_original = open(refTableFile).read().rstrip()
 859
 860             for arguments in [
 861                 "",
 862                 "remote-as internal",
 863                 "remote-as external",
 864                 "remote-as 100",
 865                 "remote-as 123",
 866                 "neighbor 192.168.7.10",
 867                 "neighbor 192.168.7.10",
 868                 "neighbor fc00:0:0:8::1000",
 869                 "neighbor 10.0.0.1",
 870                 "terse",
 871                 "remote-as internal terse",
 872                 "remote-as external terse",
 873                 "remote-as 100 terse",
 874                 "remote-as 123 terse",
 875                 "neighbor 192.168.7.10 terse",
 876                 "neighbor 192.168.7.10 terse",
 877                 "neighbor fc00:0:0:8::1000 terse",
 878                 "neighbor 10.0.0.1 terse",
 879             ]:
 880                 # Actual output from router
 881                 actual = (
 882                     net["r%s" % i]
 883                     .cmd(
 884                         'vtysh -c "show ip bgp summary ' + arguments + '" 2> /dev/null'
 885                     )
 886                     .rstrip()
 887                 )
 888
 889                 # Mask out "using XXiXX bytes" portion. They are random...
 890                 actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
 891                 # Mask out "using XiXXX KiB" portion. They are random...
 892                 actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
 893
 894                 # Remove extra summaries which exist with newer versions
 895
 896                 # Remove summary lines (changed recently)
 897                 actual = re.sub(r"Total number.*", "", actual)
 898                 actual = re.sub(r"Displayed.*", "", actual)
 899                 # Remove IPv4 Unicast Summary (Title only)
 900                 actual = re.sub(r"IPv4 Unicast Summary \(VRF default\):", "", actual)
 901                 # Remove IPv4 Multicast Summary (all of it)
 902                 actual = re.sub(r"IPv4 Multicast Summary \(VRF default\):", "", actual)
 903                 actual = re.sub(r"No IPv4 Multicast neighbor is configured", "", actual)
 904                 # Remove IPv4 VPN Summary (all of it)
 905                 actual = re.sub(r"IPv4 VPN Summary \(VRF default\):", "", actual)
 906                 actual = re.sub(r"No IPv4 VPN neighbor is configured", "", actual)
 907                 # Remove IPv4 Encap Summary (all of it)
 908                 actual = re.sub(r"IPv4 Encap Summary \(VRF default\):", "", actual)
 909                 actual = re.sub(r"No IPv4 Encap neighbor is configured", "", actual)
 910                 # Remove Unknown Summary (all of it)
 911                 actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
 912                 actual = re.sub(r"No Unknown neighbor is configured", "", actual)
 913                 # Make Connect/Active/Idle the same (change them all to Active)
 914                 actual = re.sub(r" Connect ", "  Active ", actual)
 915                 actual = re.sub(r"    Idle ", "  Active ", actual)
 916
 917                 actual = re.sub(
 918                     r"IPv4 labeled-unicast Summary \(VRF default\):", "", actual
 919                 )
 920                 actual = re.sub(
 921                     r"No IPv4 labeled-unicast neighbor is configured", "", actual
 922                 )
 923
 924                 expected = expected_original
 925                 # apply argumentss on expected output
 926                 if "internal" in arguments or "remote-as 100" in arguments:
 927                     expected = re.sub(r".+\s+200\s+.+", "", expected)
 928                 elif "external" in arguments:
 929                     expected = re.sub(r".+\s+100\s+.+Active.+", "", expected)
 930                 elif "remote-as 123" in arguments:
 931                     expected = re.sub(
 932                         r"(192.168.7.(1|2)0|fc00:0:0:8::(1|2)000).+Active.+",
 933                         "",
 934                         expected,
 935                     )
 936                     expected = re.sub(r"\nNeighbor.+Desc", "", expected)
 937                     expected = expected + "% No matching neighbor\n"
 938                 elif "192.168.7.10" in arguments:
 939                     expected = re.sub(
 940                         r"(192.168.7.20|fc00:0:0:8::(1|2)000).+Active.+", "", expected
 941                     )
 942                 elif "fc00:0:0:8::1000" in arguments:
 943                     expected = re.sub(
 944                         r"(192.168.7.(1|2)0|fc00:0:0:8::2000).+Active.+", "", expected
 945                     )
 946                 elif "10.0.0.1" in arguments:
 947                     expected = "No such neighbor in this view/vrf"
 948
 949                 if "terse" in arguments:
 950                     expected = re.sub(r"BGP table version .+", "", expected)
 951                     expected = re.sub(r"RIB entries .+", "", expected)
 952                     expected = re.sub(r"Peers [0-9]+, using .+", "", expected)
 953
 954                 # Strip empty lines
 955                 actual = actual.lstrip().rstrip()
 956                 expected = expected.lstrip().rstrip()
 957                 actual = re.sub(r"\n+", "\n", actual)
 958                 expected = re.sub(r"\n+", "\n", expected)
 959
 960                 # reapply initial formatting
 961                 if "terse" in arguments:
 962                     actual = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", actual)
 963                     expected = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", expected)
 964                 else:
 965                     actual = re.sub(r"KiB of memory\n", "KiB of memory\n\n", actual)
 966                     expected = re.sub(r"KiB of memory\n", "KiB of memory\n\n", expected)
 967
 968                 # realign expected neighbor columns if needed
 969                 try:
 970                     idx_actual = (
 971                         re.search(r"(Neighbor\s+V\s+)", actual).group(1).find("V")
 972                     )
 973                     idx_expected = (
 974                         re.search(r"(Neighbor\s+V\s+)", expected).group(1).find("V")
 975                     )
 976                     idx_diff = idx_expected - idx_actual
 977                     if idx_diff > 0:
 978                         # Neighbor        V         AS MsgRcvd MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd
 979                         expected = re.sub(" " * idx_diff + "V ", "V ", expected)
 980                         # 192.168.7.10    4        100       0       0        0    0    0    never       Active
 981                         expected = re.sub(" " * idx_diff + "4 ", "4 ", expected)
 982                 except AttributeError:
 983                     pass
 984
 985                 # Fix newlines (make them all the same)
 986                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 987                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 988
 989                 # Generate Diff
 990                 diff = topotest.get_textdiff(
 991                     actual,
 992                     expected,
 993                     title1="actual SHOW IP BGP SUMMARY " + arguments.upper(),
 994                     title2="expected SHOW IP BGP SUMMARY " + arguments.upper(),
 995                 )
 996
 997                 # Empty string if it matches, otherwise diff contains unified diff
 998                 if diff:
 999                     sys.stderr.write(
1000                         "r%s failed SHOW IP BGP SUMMARY check:\n%s\n" % (i, diff)
1001                     )
1002                     failures += 1
1003                 else:
1004                     print("r%s ok" % i)
1005
1006                 assert (
1007                     failures == 0
1008                 ), "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (
1009                     i,
1010                     diff,
1011                 )
1012
1013     # Make sure that all daemons are running
1014     for i in range(1, 2):
1015         fatal_error = net["r%s" % i].checkRouterRunning()
1016         assert fatal_error == "", fatal_error
1017
1018
1019 def test_bgp_ipv6_summary():
1020     global fatal_error
1021     net = get_topogen().net
1022
1023     # Skip if previous fatal error condition is raised
1024     if fatal_error != "":
1025         pytest.skip(fatal_error)
1026
1027     thisDir = os.path.dirname(os.path.realpath(__file__))
1028
1029     print("\n\n** Verifying BGP IPv6 Summary")
1030     print("******************************************\n")
1031     failures = 0
1032     for i in range(1, 2):
1033         refTableFile = "%s/r%s/show_bgp_ipv6_summary.ref" % (thisDir, i)
1034         if os.path.isfile(refTableFile):
1035             # Read expected result from file
1036             expected = open(refTableFile).read().rstrip()
1037             # Fix newlines (make them all the same)
1038             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1039
1040             # Actual output from router
1041             actual = (
1042                 net["r%s" % i]
1043                 .cmd('vtysh -c "show bgp ipv6 summary" 2> /dev/null')
1044                 .rstrip()
1045             )
1046             # Mask out "using XXiXX bytes" portion. They are random...
1047             actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
1048             # Mask out "using XiXXX KiB" portion. They are random...
1049             actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
1050             #
1051             # Remove extra summaries which exist with newer versions
1052             #
1053             # Remove summary lines (changed recently)
1054             actual = re.sub(r"Total number.*", "", actual)
1055             actual = re.sub(r"Displayed.*", "", actual)
1056             # Remove IPv4 Unicast Summary (Title only)
1057             actual = re.sub(r"IPv6 Unicast Summary \(VRF default\):", "", actual)
1058             # Remove IPv4 Multicast Summary (all of it)
1059             actual = re.sub(r"IPv6 Multicast Summary \(VRF default\):", "", actual)
1060             actual = re.sub(r"No IPv6 Multicast neighbor is configured", "", actual)
1061             # Remove IPv4 VPN Summary (all of it)
1062             actual = re.sub(r"IPv6 VPN Summary \(VRF default\):", "", actual)
1063             actual = re.sub(r"No IPv6 VPN neighbor is configured", "", actual)
1064             # Remove IPv4 Encap Summary (all of it)
1065             actual = re.sub(r"IPv6 Encap Summary \(VRF default\):", "", actual)
1066             actual = re.sub(r"No IPv6 Encap neighbor is configured", "", actual)
1067             # Remove Unknown Summary (all of it)
1068             actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
1069             actual = re.sub(r"No Unknown neighbor is configured", "", actual)
1070             # Make Connect/Active/Idle the same (change them all to Active)
1071             actual = re.sub(r" Connect ", "  Active ", actual)
1072             actual = re.sub(r"    Idle ", "  Active ", actual)
1073
1074             # Remove Labeled Unicast Summary (all of it)
1075             actual = re.sub(
1076                 r"IPv6 labeled-unicast Summary \(VRF default\):", "", actual
1077             )
1078             actual = re.sub(
1079                 r"No IPv6 labeled-unicast neighbor is configured", "", actual
1080             )
1081
1082             # Strip empty lines
1083             actual = actual.lstrip()
1084             actual = actual.rstrip()
1085             #
1086             # Fix newlines (make them all the same)
1087             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1088
1089             # Generate Diff
1090             diff = topotest.get_textdiff(
1091                 actual,
1092                 expected,
1093                 title1="actual SHOW BGP IPv6 SUMMARY",
1094                 title2="expected SHOW BGP IPv6 SUMMARY",
1095             )
1096
1097             # Empty string if it matches, otherwise diff contains unified diff
1098             if diff:
1099                 sys.stderr.write(
1100                     "r%s failed SHOW BGP IPv6 SUMMARY check:\n%s\n" % (i, diff)
1101                 )
1102                 failures += 1
1103             else:
1104                 print("r%s ok" % i)
1105
1106             assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (
1107                 i,
1108                 diff,
1109             )
1110
1111     # Make sure that all daemons are running
1112     for i in range(1, 2):
1113         fatal_error = net["r%s" % i].checkRouterRunning()
1114         assert fatal_error == "", fatal_error
1115
1116
1117 def test_nht():
1118     global fatal_error
1119     net = get_topogen().net
1120
1121     # Skip if previous fatal error condition is raised
1122     if fatal_error != "":
1123         pytest.skip(fatal_error)
1124
1125     print("\n\n**** Test that nexthop tracking is at least nominally working ****\n")
1126
1127     thisDir = os.path.dirname(os.path.realpath(__file__))
1128
1129     for i in range(1, 2):
1130         nhtFile = "%s/r%s/ip_nht.ref" % (thisDir, i)
1131         expected = open(nhtFile).read().rstrip()
1132         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1133
1134         actual = net["r%s" % i].cmd('vtysh -c "show ip nht" 2> /dev/null').rstrip()
1135         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1136         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1137
1138         diff = topotest.get_textdiff(
1139             actual,
1140             expected,
1141             title1="Actual `show ip nht`",
1142             title2="Expected `show ip nht`",
1143         )
1144
1145         if diff:
1146             assert 0, "r%s failed ip nht check:\n%s\n" % (i, diff)
1147         else:
1148             print("show ip nht is ok\n")
1149
1150         nhtFile = "%s/r%s/ipv6_nht.ref" % (thisDir, i)
1151         expected = open(nhtFile).read().rstrip()
1152         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1153
1154         actual = net["r%s" % i].cmd('vtysh -c "show ipv6 nht" 2> /dev/null').rstrip()
1155         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1156         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1157
1158         diff = topotest.get_textdiff(
1159             actual,
1160             expected,
1161             title1="Actual `show ip nht`",
1162             title2="Expected `show ip nht`",
1163         )
1164
1165         if diff:
1166             assert 0, "r%s failed ipv6 nht check:\n%s\n" % (i, diff)
1167         else:
1168             print("show ipv6 nht is ok\n")
1169
1170
1171 def test_bgp_ipv4():
1172     global fatal_error
1173     net = get_topogen().net
1174
1175     # Skip if previous fatal error condition is raised
1176     if fatal_error != "":
1177         pytest.skip(fatal_error)
1178
1179     thisDir = os.path.dirname(os.path.realpath(__file__))
1180
1181     print("\n\n** Verifying BGP IPv4")
1182     print("******************************************\n")
1183     diffresult = {}
1184     for i in range(1, 2):
1185         success = 0
1186         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv4*.ref" % (thisDir, i)):
1187             if os.path.isfile(refTableFile):
1188                 # Read expected result from file
1189                 expected = open(refTableFile).read().rstrip()
1190                 # Fix newlines (make them all the same)
1191                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1192
1193                 # Actual output from router
1194                 actual = (
1195                     net["r%s" % i].cmd('vtysh -c "show bgp ipv4" 2> /dev/null').rstrip()
1196                 )
1197                 # Remove summary line (changed recently)
1198                 actual = re.sub(r"Total number.*", "", actual)
1199                 actual = re.sub(r"Displayed.*", "", actual)
1200                 actual = actual.rstrip()
1201                 # Fix newlines (make them all the same)
1202                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1203
1204                 # Generate Diff
1205                 diff = topotest.get_textdiff(
1206                     actual,
1207                     expected,
1208                     title1="actual SHOW BGP IPv4",
1209                     title2="expected SHOW BGP IPv4",
1210                 )
1211
1212                 # Empty string if it matches, otherwise diff contains unified diff
1213                 if diff:
1214                     diffresult[refTableFile] = diff
1215                 else:
1216                     success = 1
1217                     print("template %s matched: r%s ok" % (refTableFile, i))
1218                     break
1219
1220         if not success:
1221             resultstr = "No template matched.\n"
1222             for f in diffresult.keys():
1223                 resultstr += "template %s: r%s failed SHOW BGP IPv4 check:\n%s\n" % (
1224                     f,
1225                     i,
1226                     diffresult[f],
1227                 )
1228             raise AssertionError(
1229                 "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, resultstr)
1230             )
1231
1232     # Make sure that all daemons are running
1233     for i in range(1, 2):
1234         fatal_error = net["r%s" % i].checkRouterRunning()
1235         assert fatal_error == "", fatal_error
1236
1237
1238 def test_bgp_ipv6():
1239     global fatal_error
1240     net = get_topogen().net
1241
1242     # Skip if previous fatal error condition is raised
1243     if fatal_error != "":
1244         pytest.skip(fatal_error)
1245
1246     thisDir = os.path.dirname(os.path.realpath(__file__))
1247
1248     print("\n\n** Verifying BGP IPv6")
1249     print("******************************************\n")
1250     diffresult = {}
1251     for i in range(1, 2):
1252         success = 0
1253         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv6*.ref" % (thisDir, i)):
1254             if os.path.isfile(refTableFile):
1255                 # Read expected result from file
1256                 expected = open(refTableFile).read().rstrip()
1257                 # Fix newlines (make them all the same)
1258                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1259
1260                 # Actual output from router
1261                 actual = (
1262                     net["r%s" % i].cmd('vtysh -c "show bgp ipv6" 2> /dev/null').rstrip()
1263                 )
1264                 # Remove summary line (changed recently)
1265                 actual = re.sub(r"Total number.*", "", actual)
1266                 actual = re.sub(r"Displayed.*", "", actual)
1267                 actual = actual.rstrip()
1268                 # Fix newlines (make them all the same)
1269                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1270
1271                 # Generate Diff
1272                 diff = topotest.get_textdiff(
1273                     actual,
1274                     expected,
1275                     title1="actual SHOW BGP IPv6",
1276                     title2="expected SHOW BGP IPv6",
1277                 )
1278
1279                 # Empty string if it matches, otherwise diff contains unified diff
1280                 if diff:
1281                     diffresult[refTableFile] = diff
1282                 else:
1283                     success = 1
1284                     print("template %s matched: r%s ok" % (refTableFile, i))
1285
1286         if not success:
1287             resultstr = "No template matched.\n"
1288             for f in diffresult.keys():
1289                 resultstr += "template %s: r%s failed SHOW BGP IPv6 check:\n%s\n" % (
1290                     f,
1291                     i,
1292                     diffresult[f],
1293                 )
1294             raise AssertionError(
1295                 "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, resultstr)
1296             )
1297
1298     # Make sure that all daemons are running
1299     for i in range(1, 2):
1300         fatal_error = net["r%s" % i].checkRouterRunning()
1301         assert fatal_error == "", fatal_error
1302
1303
1304 def test_route_map():
1305     global fatal_error
1306     net = get_topogen().net
1307
1308     if fatal_error != "":
1309         pytest.skip(fatal_error)
1310
1311     thisDir = os.path.dirname(os.path.realpath(__file__))
1312
1313     print("\n\n** Verifying some basic routemap forward references\n")
1314     print("*******************************************************\n")
1315     failures = 0
1316     for i in range(1, 2):
1317         refroutemap = "%s/r%s/show_route_map.ref" % (thisDir, i)
1318         if os.path.isfile(refroutemap):
1319             expected = open(refroutemap).read().rstrip()
1320             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1321
1322             actual = (
1323                 net["r%s" % i].cmd('vtysh -c "show route-map" 2> /dev/null').rstrip()
1324             )
1325             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1326
1327             diff = topotest.get_textdiff(
1328                 actual,
1329                 expected,
1330                 title1="actual show route-map",
1331                 title2="expected show route-map",
1332             )
1333
1334             if diff:
1335                 sys.stderr.write(
1336                     "r%s failed show route-map command Check:\n%s\n" % (i, diff)
1337                 )
1338                 failures += 1
1339             else:
1340                 print("r%s ok" % i)
1341
1342             assert (
1343                 failures == 0
1344             ), "Show route-map command failed for router r%s:\n%s" % (i, diff)
1345
1346
1347 def test_nexthop_groups_with_route_maps():
1348     global fatal_error
1349     net = get_topogen().net
1350
1351     # Skip if previous fatal error condition is raised
1352     if fatal_error != "":
1353         pytest.skip(fatal_error)
1354
1355     print("\n\n** Verifying Nexthop Groups With Route-Maps")
1356     print("******************************************\n")
1357
1358     ### Nexthop Group With Route-Map Tests
1359
1360     # Create a lib nexthop-group
1361     net["r1"].cmd(
1362         'vtysh -c "c t" -c "nexthop-group test" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
1363     )
1364
1365     ## Route-Map Proto Source
1366
1367     route_str = "2.2.2.1"
1368     src_str = "192.168.0.1"
1369
1370     net["r1"].cmd(
1371         'vtysh -c "c t" -c "route-map NH-SRC permit 111" -c "set src %s"' % src_str
1372     )
1373     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NH-SRC"')
1374
1375     net["r1"].cmd('vtysh -c "sharp install routes %s nexthop-group test 1"' % route_str)
1376
1377     verify_route_nexthop_group("%s/32" % route_str)
1378
1379     # Only a valid test on linux using nexthop objects
1380     if sys.platform.startswith("linux"):
1381         output = net["r1"].cmd("ip route show %s/32" % route_str)
1382         match = re.search(r"src %s" % src_str, output)
1383         assert match is not None, "Route %s/32 not installed with src %s" % (
1384             route_str,
1385             src_str,
1386         )
1387
1388     # Remove NHG routes and route-map
1389     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % route_str)
1390     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NH-SRC"')
1391     net["r1"].cmd(
1392         'vtysh -c "c t" -c "no route-map NH-SRC permit 111" # -c "set src %s"' % src_str
1393     )
1394     net["r1"].cmd('vtysh -c "c t" -c "no route-map NH-SRC"')
1395
1396     ## Route-Map Deny/Permit with same nexthop group
1397
1398     permit_route_str = "3.3.3.1"
1399     deny_route_str = "3.3.3.2"
1400
1401     net["r1"].cmd(
1402         'vtysh -c "c t" -c "ip prefix-list NOPE seq 5 permit %s/32"' % permit_route_str
1403     )
1404     net["r1"].cmd(
1405         'vtysh -c "c t" -c "route-map NOPE permit 111" -c "match ip address prefix-list NOPE"'
1406     )
1407     net["r1"].cmd('vtysh -c "c t" -c "route-map NOPE deny 222"')
1408     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NOPE"')
1409
1410     # This route should be permitted
1411     net["r1"].cmd(
1412         'vtysh -c "sharp install routes %s nexthop-group test 1"' % permit_route_str
1413     )
1414
1415     verify_route_nexthop_group("%s/32" % permit_route_str)
1416
1417     # This route should be denied
1418     net["r1"].cmd(
1419         'vtysh -c "sharp install routes %s nexthop-group test 1"' % deny_route_str
1420     )
1421
1422     nhg_id = route_get_nhg_id(deny_route_str)
1423     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
1424
1425     match = re.search(r"Valid", output)
1426     assert match is None, "Nexthop Group ID=%d should not be marked Valid" % nhg_id
1427
1428     match = re.search(r"Installed", output)
1429     assert match is None, "Nexthop Group ID=%d should not be marked Installed" % nhg_id
1430
1431     # Remove NHG routes and route-map
1432     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % permit_route_str)
1433     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % deny_route_str)
1434     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NOPE"')
1435     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE permit 111"')
1436     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE deny 222"')
1437     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE"')
1438     net["r1"].cmd(
1439         'vtysh -c "c t" -c "no ip prefix-list NOPE seq 5 permit %s/32"'
1440         % permit_route_str
1441     )
1442
1443
1444 def test_nexthop_group_replace():
1445     global fatal_error
1446     net = get_topogen().net
1447
1448     # Skip if previous fatal error condition is raised
1449     if fatal_error != "":
1450         pytest.skip(fatal_error)
1451
1452     print("\n\n** Verifying Nexthop Groups")
1453     print("******************************************\n")
1454
1455     ### Nexthop Group Tests
1456
1457     ## 2-Way ECMP Directly Connected
1458
1459     net["r1"].cmd(
1460         'vtysh -c "c t" -c "nexthop-group replace" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1461     )
1462
1463     # Create with sharpd using nexthop-group
1464     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.1 nexthop-group replace 1"')
1465
1466     verify_route_nexthop_group("3.3.3.1/32")
1467
1468     # Change the nexthop group
1469     net["r1"].cmd(
1470         'vtysh -c "c t" -c "nexthop-group replace" -c "no nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.3 r1-eth1 onlink" -c "nexthop 1.1.1.4 r1-eth4 onlink"'
1471     )
1472
1473     # Verify it updated. We can just check install and ecmp count here.
1474     verify_route_nexthop_group("3.3.3.1/32", False, 3)
1475
1476
1477 def test_mpls_interfaces():
1478     global fatal_error
1479     net = get_topogen().net
1480
1481     # Skip if previous fatal error condition is raised
1482     if fatal_error != "":
1483         pytest.skip(fatal_error)
1484
1485     # Skip if no LDP installed or old kernel
1486     if net["r1"].daemon_available("ldpd") == False:
1487         pytest.skip("No MPLS or kernel < 4.5")
1488
1489     thisDir = os.path.dirname(os.path.realpath(__file__))
1490
1491     print("\n\n** Verifying MPLS Interfaces")
1492     print("******************************************\n")
1493     failures = 0
1494     for i in range(1, 2):
1495         refTableFile = "%s/r%s/show_mpls_ldp_interface.ref" % (thisDir, i)
1496         if os.path.isfile(refTableFile):
1497             # Read expected result from file
1498             expected = open(refTableFile).read().rstrip()
1499             # Fix newlines (make them all the same)
1500             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1501
1502             # Actual output from router
1503             actual = (
1504                 net["r%s" % i]
1505                 .cmd('vtysh -c "show mpls ldp interface" 2> /dev/null')
1506                 .rstrip()
1507             )
1508             # Mask out Timer in Uptime
1509             actual = re.sub(r" [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ", " xx:xx:xx ", actual)
1510             # Fix newlines (make them all the same)
1511             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1512
1513             # Generate Diff
1514             diff = topotest.get_textdiff(
1515                 actual,
1516                 expected,
1517                 title1="actual MPLS LDP interface status",
1518                 title2="expected MPLS LDP interface status",
1519             )
1520
1521             # Empty string if it matches, otherwise diff contains unified diff
1522             if diff:
1523                 sys.stderr.write(
1524                     "r%s failed MPLS LDP Interface status Check:\n%s\n" % (i, diff)
1525                 )
1526                 failures += 1
1527             else:
1528                 print("r%s ok" % i)
1529
1530             if failures > 0:
1531                 fatal_error = "MPLS LDP Interface status failed"
1532
1533             assert (
1534                 failures == 0
1535             ), "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
1536
1537     # Make sure that all daemons are running
1538     for i in range(1, 2):
1539         fatal_error = net["r%s" % i].checkRouterRunning()
1540         assert fatal_error == "", fatal_error
1541
1542
1543 def test_shutdown_check_stderr():
1544     global fatal_error
1545     net = get_topogen().net
1546
1547     # Skip if previous fatal error condition is raised
1548     if fatal_error != "":
1549         pytest.skip(fatal_error)
1550
1551     print("\n\n** Verifying unexpected STDERR output from daemons")
1552     print("******************************************\n")
1553
1554     if os.environ.get("TOPOTESTS_CHECK_STDERR") is None:
1555         print(
1556             "SKIPPED final check on StdErr output: Disabled (TOPOTESTS_CHECK_STDERR undefined)\n"
1557         )
1558         pytest.skip("Skipping test for Stderr output")
1559
1560     thisDir = os.path.dirname(os.path.realpath(__file__))
1561
1562     print("thisDir=" + thisDir)
1563
1564     net["r1"].stopRouter()
1565
1566     log = net["r1"].getStdErr("ripd")
1567     if log:
1568         print("\nRIPd StdErr Log:\n" + log)
1569     log = net["r1"].getStdErr("ripngd")
1570     if log:
1571         print("\nRIPngd StdErr Log:\n" + log)
1572     log = net["r1"].getStdErr("ospfd")
1573     if log:
1574         print("\nOSPFd StdErr Log:\n" + log)
1575     log = net["r1"].getStdErr("ospf6d")
1576     if log:
1577         print("\nOSPF6d StdErr Log:\n" + log)
1578     log = net["r1"].getStdErr("isisd")
1579     if log:
1580         print("\nISISd StdErr Log:\n" + log)
1581     log = net["r1"].getStdErr("bgpd")
1582     if log:
1583         print("\nBGPd StdErr Log:\n" + log)
1584
1585     log = net["r1"].getStdErr("nhrpd")
1586     if log:
1587         print("\nNHRPd StdErr Log:\n" + log)
1588
1589     log = net["r1"].getStdErr("pbrd")
1590     if log:
1591         print("\nPBRd StdErr Log:\n" + log)
1592
1593     log = net["r1"].getStdErr("babeld")
1594     if log:
1595         print("\nBABELd StdErr Log:\n" + log)
1596
1597     if net["r1"].daemon_available("ldpd"):
1598         log = net["r1"].getStdErr("ldpd")
1599         if log:
1600             print("\nLDPd StdErr Log:\n" + log)
1601     log = net["r1"].getStdErr("zebra")
1602     if log:
1603         print("\nZebra StdErr Log:\n" + log)
1604
1605
1606 def test_shutdown_check_memleak():
1607     global fatal_error
1608     net = get_topogen().net
1609
1610     # Skip if previous fatal error condition is raised
1611     if fatal_error != "":
1612         pytest.skip(fatal_error)
1613
1614     if os.environ.get("TOPOTESTS_CHECK_MEMLEAK") is None:
1615         print(
1616             "SKIPPED final check on Memory leaks: Disabled (TOPOTESTS_CHECK_MEMLEAK undefined)\n"
1617         )
1618         pytest.skip("Skipping test for memory leaks")
1619
1620     thisDir = os.path.dirname(os.path.realpath(__file__))
1621
1622     for i in range(1, 2):
1623         net["r%s" % i].stopRouter()
1624         net["r%s" % i].report_memory_leaks(
1625             os.environ.get("TOPOTESTS_CHECK_MEMLEAK"), os.path.basename(__file__)
1626         )
1627
1628
1629 if __name__ == "__main__":
1630     # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
1631     # retval = pytest.main(["-s", "--tb=no"])
1632     retval = pytest.main(["-s"])
1633     sys.exit(retval)