tests/topotests/all_protocol_startup/test_all_protocol_startup.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: ISC
   3
   4 #
   5 # test_all_protocol_startup.py
   6 # Part of NetDEF Topology Tests
   7 #
   8 # Copyright (c) 2017 by
   9 # Network Device Education Foundation, Inc. ("NetDEF")
  10 #
  11
  12 """
  13 test_all_protocol_startup.py: Test of all protocols at same time
  14
  15 """
  16
  17 import os
  18 import re
  19 import sys
  20 import pytest
  21 import glob
  22 from time import sleep
  23
  24 pytestmark = [
  25     pytest.mark.babeld,
  26     pytest.mark.bgpd,
  27     pytest.mark.isisd,
  28     pytest.mark.nhrpd,
  29     pytest.mark.ospfd,
  30     pytest.mark.pbrd,
  31     pytest.mark.ripd,
  32 ]
  33
  34 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  35 from lib import topotest
  36 from lib.topogen import Topogen, get_topogen
  37 from lib.common_config import (
  38     required_linux_kernel_version,
  39 )
  40
  41 fatal_error = ""
  42
  43
  44 #####################################################
  45 ##
  46 ##   Network Topology Definition
  47 ##
  48 #####################################################
  49
  50
  51 def build_topo(tgen):
  52     router = tgen.add_router("r1")
  53     for i in range(0, 10):
  54         tgen.add_switch("sw%d" % i).add_link(router)
  55
  56
  57 #####################################################
  58 ##
  59 ##   Tests starting
  60 ##
  61 #####################################################
  62
  63
  64 def setup_module(module):
  65     global fatal_error
  66
  67     print("\n\n** %s: Setup Topology" % module.__name__)
  68     print("******************************************\n")
  69
  70     thisDir = os.path.dirname(os.path.realpath(__file__))
  71     tgen = Topogen(build_topo, module.__name__)
  72     tgen.start_topology()
  73
  74     net = tgen.net
  75
  76     if net["r1"].get_routertype() != "frr":
  77         fatal_error = "Test is only implemented for FRR"
  78         sys.stderr.write("\n\nTest is only implemented for FRR - Skipping\n\n")
  79         pytest.skip(fatal_error)
  80
  81     # Starting Routers
  82     #
  83     # Main router
  84     for i in range(1, 2):
  85         net["r%s" % i].loadConf("mgmtd", "%s/r%s/zebra.conf" % (thisDir, i))
  86         net["r%s" % i].loadConf("zebra", "%s/r%s/zebra.conf" % (thisDir, i))
  87         net["r%s" % i].loadConf("ripd", "%s/r%s/ripd.conf" % (thisDir, i))
  88         net["r%s" % i].loadConf("ripngd", "%s/r%s/ripngd.conf" % (thisDir, i))
  89         net["r%s" % i].loadConf("ospfd", "%s/r%s/ospfd.conf" % (thisDir, i))
  90         if net["r1"].checkRouterVersion("<", "4.0"):
  91             net["r%s" % i].loadConf(
  92                 "ospf6d", "%s/r%s/ospf6d.conf-pre-v4" % (thisDir, i)
  93             )
  94         else:
  95             net["r%s" % i].loadConf("ospf6d", "%s/r%s/ospf6d.conf" % (thisDir, i))
  96         net["r%s" % i].loadConf("isisd", "%s/r%s/isisd.conf" % (thisDir, i))
  97         net["r%s" % i].loadConf("bgpd", "%s/r%s/bgpd.conf" % (thisDir, i))
  98         if net["r%s" % i].daemon_available("ldpd"):
  99             # Only test LDPd if it's installed and Kernel >= 4.5
 100             net["r%s" % i].loadConf("ldpd", "%s/r%s/ldpd.conf" % (thisDir, i))
 101         net["r%s" % i].loadConf("sharpd")
 102         net["r%s" % i].loadConf("nhrpd", "%s/r%s/nhrpd.conf" % (thisDir, i))
 103         net["r%s" % i].loadConf("babeld", "%s/r%s/babeld.conf" % (thisDir, i))
 104         net["r%s" % i].loadConf("pbrd", "%s/r%s/pbrd.conf" % (thisDir, i))
 105         tgen.gears["r%s" % i].start()
 106
 107     # For debugging after starting FRR daemons, uncomment the next line
 108     # tgen.mininet_cli()
 109
 110
 111 def teardown_module(module):
 112     print("\n\n** %s: Shutdown Topology" % module.__name__)
 113     print("******************************************\n")
 114     tgen = get_topogen()
 115     tgen.stop_topology()
 116
 117
 118 def test_router_running():
 119     global fatal_error
 120     tgen = get_topogen()
 121     net = tgen.net
 122
 123     # Skip if previous fatal error condition is raised
 124     if fatal_error != "":
 125         pytest.skip(fatal_error)
 126
 127     print("\n\n** Check if FRR is running on each Router node")
 128     print("******************************************\n")
 129     sleep(5)
 130
 131     # Starting Routers
 132     for i in range(1, 2):
 133         fatal_error = net["r%s" % i].checkRouterRunning()
 134         assert fatal_error == "", fatal_error
 135
 136     # For debugging after starting FRR daemons, uncomment the next line
 137     # tgen.mininet_cli()
 138
 139
 140 def test_error_messages_vtysh():
 141     global fatal_error
 142     net = get_topogen().net
 143
 144     # Skip if previous fatal error condition is raised
 145     if fatal_error != "":
 146         pytest.skip(fatal_error)
 147
 148     print("\n\n** Check for error messages on VTYSH")
 149     print("******************************************\n")
 150
 151     failures = 0
 152     for i in range(1, 2):
 153         #
 154         # First checking Standard Output
 155         #
 156
 157         # VTYSH output from router
 158         vtystdout = net["r%s" % i].cmd('vtysh -c "show version" 2> /dev/null').rstrip()
 159
 160         # Fix newlines (make them all the same)
 161         vtystdout = ("\n".join(vtystdout.splitlines()) + "\n").rstrip()
 162         # Drop everything starting with "FRRouting X.xx" message
 163         vtystdout = re.sub(r"FRRouting [0-9]+.*", "", vtystdout, flags=re.DOTALL)
 164
 165         if vtystdout == "":
 166             print("r%s StdOut ok" % i)
 167
 168         assert vtystdout == "", "Vtysh StdOut Output check failed for router r%s" % i
 169
 170         #
 171         # Second checking Standard Error
 172         #
 173
 174         # VTYSH StdErr output from router
 175         vtystderr = net["r%s" % i].cmd('vtysh -c "show version" > /dev/null').rstrip()
 176
 177         # Fix newlines (make them all the same)
 178         vtystderr = ("\n".join(vtystderr.splitlines()) + "\n").rstrip()
 179         # # Drop everything starting with "FRRouting X.xx" message
 180         # vtystderr = re.sub(r"FRRouting [0-9]+.*", "", vtystderr, flags=re.DOTALL)
 181
 182         if vtystderr == "":
 183             print("r%s StdErr ok" % i)
 184
 185         assert vtystderr == "", "Vtysh StdErr Output check failed for router r%s" % i
 186
 187     # Make sure that all daemons are running
 188     for i in range(1, 2):
 189         fatal_error = net["r%s" % i].checkRouterRunning()
 190         assert fatal_error == "", fatal_error
 191
 192
 193 def test_error_messages_daemons():
 194     global fatal_error
 195     net = get_topogen().net
 196
 197     # Skip if previous fatal error condition is raised
 198     if fatal_error != "":
 199         pytest.skip(fatal_error)
 200
 201     print("\n\n** Check for error messages in daemons")
 202     print("******************************************\n")
 203
 204     error_logs = ""
 205
 206     for i in range(1, 2):
 207         log = net["r%s" % i].getStdErr("ripd")
 208         if log:
 209             error_logs += "r%s RIPd StdErr Output:\n" % i
 210             error_logs += log
 211         log = net["r%s" % i].getStdErr("ripngd")
 212         if log:
 213             error_logs += "r%s RIPngd StdErr Output:\n" % i
 214             error_logs += log
 215         log = net["r%s" % i].getStdErr("ospfd")
 216         if log:
 217             error_logs += "r%s OSPFd StdErr Output:\n" % i
 218             error_logs += log
 219         log = net["r%s" % i].getStdErr("ospf6d")
 220         if log:
 221             error_logs += "r%s OSPF6d StdErr Output:\n" % i
 222             error_logs += log
 223         log = net["r%s" % i].getStdErr("isisd")
 224         # ISIS shows debugging enabled status on StdErr
 225         # Remove these messages
 226         log = re.sub(r"^IS-IS .* debugging is on.*", "", log).rstrip()
 227         if log:
 228             error_logs += "r%s ISISd StdErr Output:\n" % i
 229             error_logs += log
 230         log = net["r%s" % i].getStdErr("bgpd")
 231         if log:
 232             error_logs += "r%s BGPd StdErr Output:\n" % i
 233             error_logs += log
 234         if net["r%s" % i].daemon_available("ldpd"):
 235             log = net["r%s" % i].getStdErr("ldpd")
 236             if log:
 237                 error_logs += "r%s LDPd StdErr Output:\n" % i
 238                 error_logs += log
 239
 240         log = net["r1"].getStdErr("nhrpd")
 241         # NHRPD shows YANG model not embedded messages
 242         # Ignore these
 243         log = re.sub(r".*YANG model.*not embedded.*", "", log).rstrip()
 244         if log:
 245             error_logs += "r%s NHRPd StdErr Output:\n" % i
 246             error_logs += log
 247
 248         log = net["r1"].getStdErr("babeld")
 249         if log:
 250             error_logs += "r%s BABELd StdErr Output:\n" % i
 251             error_logs += log
 252
 253         log = net["r1"].getStdErr("pbrd")
 254         if log:
 255             error_logs += "r%s PBRd StdErr Output:\n" % i
 256             error_logs += log
 257
 258         log = net["r%s" % i].getStdErr("zebra")
 259         if log:
 260             error_logs += "r%s Zebra StdErr Output:\n" % i
 261             error_logs += log
 262
 263     if error_logs:
 264         sys.stderr.write(
 265             "Failed check for StdErr Output on daemons:\n%s\n" % error_logs
 266         )
 267
 268     # Ignoring the issue if told to ignore (ie not yet fixed)
 269     if error_logs != "":
 270         if os.environ.get("bamboo_TOPOTESTS_ISSUE_349") == "IGNORE":
 271             sys.stderr.write(
 272                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349\n"
 273             )
 274             pytest.skip(
 275                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349"
 276             )
 277
 278     assert error_logs == "", "Daemons report errors to StdErr"
 279
 280
 281 def test_converge_protocols():
 282     global fatal_error
 283     net = get_topogen().net
 284
 285     # Skip if previous fatal error condition is raised
 286     if fatal_error != "":
 287         pytest.skip(fatal_error)
 288
 289     thisDir = os.path.dirname(os.path.realpath(__file__))
 290
 291     # We need loopback to have a link local so it always is the
 292     # "selected" router for fe80::/64 when we static compare below.
 293     print("Adding link-local to loopback for stable results")
 294     cmd = (
 295         "mac=`cat /sys/class/net/lo/address`; echo lo: $mac;"
 296         " [ -z \"$mac\" ] && continue; IFS=':'; set $mac; unset IFS;"
 297         " ip address add dev lo scope link"
 298         " fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64"
 299     )
 300     net["r1"].cmd_raises(cmd)
 301
 302     print("\n\n** Waiting for protocols convergence")
 303     print("******************************************\n")
 304
 305     # Not really implemented yet - just sleep 60 secs for now
 306     sleep(5)
 307
 308     # Make sure that all daemons are running
 309     failures = 0
 310     for i in range(1, 2):
 311         fatal_error = net["r%s" % i].checkRouterRunning()
 312         assert fatal_error == "", fatal_error
 313
 314         print("Show that v4 routes are right\n")
 315         v4_routesFile = "%s/r%s/ipv4_routes.ref" % (thisDir, i)
 316         expected = (
 317             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v4_routesFile)).rstrip()
 318         )
 319         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 320
 321         actual = (
 322             net["r%s" % i]
 323             .cmd(
 324                 "vtysh -c \"show ip route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 325             )
 326             .rstrip()
 327         )
 328         # Drop time in last update
 329         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 330         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 331         diff = topotest.get_textdiff(
 332             actual,
 333             expected,
 334             title1="Actual IP Routing Table",
 335             title2="Expected IP RoutingTable",
 336         )
 337         if diff:
 338             sys.stderr.write("r%s failed IP Routing table check:\n%s\n" % (i, diff))
 339             failures += 1
 340         else:
 341             print("r%s ok" % i)
 342
 343         assert failures == 0, "IP Routing table failed for r%s\n%s" % (i, diff)
 344
 345         failures = 0
 346
 347         print("Show that v6 routes are right\n")
 348         v6_routesFile = "%s/r%s/ipv6_routes.ref" % (thisDir, i)
 349         expected = (
 350             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v6_routesFile)).rstrip()
 351         )
 352         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 353
 354         actual = (
 355             net["r%s" % i]
 356             .cmd(
 357                 "vtysh -c \"show ipv6 route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 358             )
 359             .rstrip()
 360         )
 361         # Drop time in last update
 362         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 363         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 364         diff = topotest.get_textdiff(
 365             actual,
 366             expected,
 367             title1="Actual IPv6 Routing Table",
 368             title2="Expected IPv6 RoutingTable",
 369         )
 370         if diff:
 371             sys.stderr.write("r%s failed IPv6 Routing table check:\n%s\n" % (i, diff))
 372             failures += 1
 373         else:
 374             print("r%s ok" % i)
 375
 376         assert failures == 0, "IPv6 Routing table failed for r%s\n%s" % (i, diff)
 377
 378
 379 def route_get_nhg_id(route_str):
 380     net = get_topogen().net
 381     output = net["r1"].cmd('vtysh -c "show ip route %s nexthop-group"' % route_str)
 382     match = re.search(r"Nexthop Group ID: (\d+)", output)
 383     assert match is not None, (
 384         "Nexthop Group ID not found for sharpd route %s" % route_str
 385     )
 386
 387     nhg_id = int(match.group(1))
 388     return nhg_id
 389
 390
 391 def verify_nexthop_group(nhg_id, recursive=False, ecmp=0):
 392     net = get_topogen().net
 393     count = 0
 394     valid = None
 395     ecmpcount = None
 396     depends = None
 397     resolved_id = None
 398     installed = None
 399     found = False
 400
 401     while not found and count < 10:
 402         count += 1
 403         # Verify NHG is valid/installed
 404         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 405         valid = re.search(r"Valid", output)
 406         if valid is None:
 407             found = False
 408             sleep(1)
 409             continue
 410
 411         if ecmp or recursive:
 412             ecmpcount = re.search(r"Depends:.*\n", output)
 413             if ecmpcount is None:
 414                 found = False
 415                 sleep(1)
 416                 continue
 417
 418             # list of IDs in group
 419             depends = re.findall(r"\((\d+)\)", ecmpcount.group(0))
 420
 421             if ecmp:
 422                 if len(depends) != ecmp:
 423                     found = False
 424                     sleep(1)
 425                     continue
 426             else:
 427                 # If recursive, we need to look at its resolved group
 428                 if len(depends) != 1:
 429                     found = False
 430                     sleep(1)
 431                     continue
 432
 433                 resolved_id = int(depends[0])
 434                 verify_nexthop_group(resolved_id, False)
 435         else:
 436             installed = re.search(r"Installed", output)
 437             if installed is None:
 438                 found = False
 439                 sleep(1)
 440                 continue
 441         found = True
 442
 443     assert valid is not None, "Nexthop Group ID=%d not marked Valid" % nhg_id
 444     if ecmp or recursive:
 445         assert ecmpcount is not None, "Nexthop Group ID=%d has no depends" % nhg_id
 446         if ecmp:
 447             assert len(depends) == ecmp, (
 448                 "Nexthop Group ID=%d doesn't match ecmp size" % nhg_id
 449             )
 450         else:
 451             assert len(depends) == 1, (
 452                 "Nexthop Group ID=%d should only have one recursive depend" % nhg_id
 453             )
 454     else:
 455         assert installed is not None, (
 456             "Nexthop Group ID=%d not marked Installed" % nhg_id
 457         )
 458
 459
 460 def verify_route_nexthop_group(route_str, recursive=False, ecmp=0):
 461     # Verify route and that zebra created NHGs for and they are valid/installed
 462     nhg_id = route_get_nhg_id(route_str)
 463     verify_nexthop_group(nhg_id, recursive, ecmp)
 464
 465
 466 def test_nexthop_groups():
 467     global fatal_error
 468     net = get_topogen().net
 469
 470     # Skip if previous fatal error condition is raised
 471     if fatal_error != "":
 472         pytest.skip(fatal_error)
 473
 474     print("\n\n** Verifying Nexthop Groups")
 475     print("******************************************\n")
 476
 477     ### Nexthop Group Tests
 478
 479     ## Basic test
 480
 481     # Create a lib nexthop-group
 482     net["r1"].cmd(
 483         'vtysh -c "c t" -c "nexthop-group basic" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
 484     )
 485
 486     # Create with sharpd using nexthop-group
 487     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.1 nexthop-group basic 1"')
 488     verify_route_nexthop_group("2.2.2.1/32")
 489
 490     ## Connected
 491
 492     net["r1"].cmd(
 493         'vtysh -c "c t" -c "nexthop-group connected" -c "nexthop r1-eth1" -c "nexthop r1-eth2"'
 494     )
 495
 496     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.2 nexthop-group connected 1"')
 497     verify_route_nexthop_group("2.2.2.2/32")
 498
 499     ## Recursive
 500
 501     net["r1"].cmd(
 502         'vtysh -c "c t" -c "nexthop-group basic-recursive" -c "nexthop 2.2.2.1"'
 503     )
 504
 505     net["r1"].cmd(
 506         'vtysh -c "sharp install routes 3.3.3.1 nexthop-group basic-recursive 1"'
 507     )
 508
 509     verify_route_nexthop_group("3.3.3.1/32", True)
 510
 511     ## Duplicate
 512
 513     net["r1"].cmd(
 514         'vtysh -c "c t" -c "nexthop-group duplicate" -c "nexthop 2.2.2.1" -c "nexthop 1.1.1.1"'
 515     )
 516
 517     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.2 nexthop-group duplicate 1"')
 518
 519     verify_route_nexthop_group("3.3.3.2/32")
 520
 521     ## Two 4-Way ECMP
 522
 523     net["r1"].cmd(
 524         'vtysh -c "c t" -c "nexthop-group fourA" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2" \
 525             -c "nexthop 1.1.1.3" -c "nexthop 1.1.1.4"'
 526     )
 527
 528     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.1 nexthop-group fourA 1"')
 529
 530     verify_route_nexthop_group("4.4.4.1/32")
 531
 532     net["r1"].cmd(
 533         'vtysh -c "c t" -c "nexthop-group fourB" -c "nexthop 1.1.1.5" -c "nexthop 1.1.1.6" \
 534             -c "nexthop 1.1.1.7" -c "nexthop 1.1.1.8"'
 535     )
 536
 537     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.2 nexthop-group fourB 1"')
 538
 539     verify_route_nexthop_group("4.4.4.2/32")
 540
 541     ## Recursive to 8-Way ECMP
 542
 543     net["r1"].cmd(
 544         'vtysh -c "c t" -c "nexthop-group eight-recursive" -c "nexthop 4.4.4.1" -c "nexthop 4.4.4.2"'
 545     )
 546
 547     net["r1"].cmd(
 548         'vtysh -c "sharp install routes 5.5.5.1 nexthop-group eight-recursive 1"'
 549     )
 550
 551     verify_route_nexthop_group("5.5.5.1/32")
 552
 553     ## 4-way ECMP Routes Pointing to Each Other
 554
 555     # This is to check for a bug with NH resolution where
 556     # routes would infintely resolve to each other blowing
 557     # up the resolved-> nexthop pointer.
 558
 559     net["r1"].cmd(
 560         'vtysh -c "c t" -c "nexthop-group infinite-recursive" -c "nexthop 6.6.6.1" -c "nexthop 6.6.6.2" \
 561             -c "nexthop 6.6.6.3" -c "nexthop 6.6.6.4"'
 562     )
 563
 564     # static route nexthops can recurse to
 565
 566     net["r1"].cmd('vtysh -c "c t" -c "ip route 6.6.6.0/24 1.1.1.1"')
 567
 568     # Make routes that point to themselves in ecmp
 569
 570     net["r1"].cmd(
 571         'vtysh -c "sharp install routes 6.6.6.4 nexthop-group infinite-recursive 1"'
 572     )
 573     sleep(5)
 574
 575     net["r1"].cmd(
 576         'vtysh -c "sharp install routes 6.6.6.3 nexthop-group infinite-recursive 1"'
 577     )
 578     sleep(5)
 579
 580     net["r1"].cmd(
 581         'vtysh -c "sharp install routes 6.6.6.2 nexthop-group infinite-recursive 1"'
 582     )
 583     sleep(5)
 584
 585     net["r1"].cmd(
 586         'vtysh -c "sharp install routes 6.6.6.1 nexthop-group infinite-recursive 1"'
 587     )
 588
 589     # Get routes and test if has too many (duplicate) nexthops
 590     count = 0
 591     dups = []
 592     nhg_id = route_get_nhg_id("6.6.6.1/32")
 593     while (len(dups) != 3) and count < 10:
 594         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 595
 596         dups = re.findall(r"(via 1\.1\.1\.1)", output)
 597         if len(dups) != 3:
 598             count += 1
 599             sleep(1)
 600
 601     # Should find 3, itself is inactive
 602     assert len(dups) == 3, (
 603         "Route 6.6.6.1/32 with Nexthop Group ID=%d has wrong number of resolved nexthops"
 604         % nhg_id
 605     )
 606
 607     ## Remove all NHG routes
 608
 609     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.1 1"')
 610     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.2 1"')
 611     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.1 1"')
 612     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.2 1"')
 613     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.1 1"')
 614     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.2 1"')
 615     net["r1"].cmd('vtysh -c "sharp remove routes 5.5.5.1 1"')
 616     net["r1"].cmd('vtysh -c "sharp remove routes 6.6.6.1 4"')
 617     net["r1"].cmd('vtysh -c "c t" -c "no ip route 6.6.6.0/24 1.1.1.1"')
 618
 619
 620 def test_rip_status():
 621     global fatal_error
 622     net = get_topogen().net
 623
 624     # Skip if previous fatal error condition is raised
 625     if fatal_error != "":
 626         pytest.skip(fatal_error)
 627
 628     thisDir = os.path.dirname(os.path.realpath(__file__))
 629
 630     print("\n\n** Verifying RIP status")
 631     print("******************************************\n")
 632     failures = 0
 633     for i in range(1, 2):
 634         refTableFile = "%s/r%s/rip_status.ref" % (thisDir, i)
 635         if os.path.isfile(refTableFile):
 636             # Read expected result from file
 637             expected = open(refTableFile).read().rstrip()
 638             # Fix newlines (make them all the same)
 639             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 640
 641             # Actual output from router
 642             actual = (
 643                 net["r%s" % i]
 644                 .cmd('vtysh -c "show ip rip status" 2> /dev/null')
 645                 .rstrip()
 646             )
 647             # Drop time in next due
 648             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 649             # Drop time in last update
 650             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 651             # Fix newlines (make them all the same)
 652             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 653
 654             # Generate Diff
 655             diff = topotest.get_textdiff(
 656                 actual,
 657                 expected,
 658                 title1="actual IP RIP status",
 659                 title2="expected IP RIP status",
 660             )
 661
 662             # Empty string if it matches, otherwise diff contains unified diff
 663             if diff:
 664                 sys.stderr.write("r%s failed IP RIP status check:\n%s\n" % (i, diff))
 665                 failures += 1
 666             else:
 667                 print("r%s ok" % i)
 668
 669             assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff)
 670
 671     # Make sure that all daemons are running
 672     for i in range(1, 2):
 673         fatal_error = net["r%s" % i].checkRouterRunning()
 674         assert fatal_error == "", fatal_error
 675
 676
 677 def test_ripng_status():
 678     global fatal_error
 679     net = get_topogen().net
 680
 681     # Skip if previous fatal error condition is raised
 682     if fatal_error != "":
 683         pytest.skip(fatal_error)
 684
 685     thisDir = os.path.dirname(os.path.realpath(__file__))
 686
 687     print("\n\n** Verifying RIPng status")
 688     print("******************************************\n")
 689     failures = 0
 690     for i in range(1, 2):
 691         refTableFile = "%s/r%s/ripng_status.ref" % (thisDir, i)
 692         if os.path.isfile(refTableFile):
 693             # Read expected result from file
 694             expected = open(refTableFile).read().rstrip()
 695             # Fix newlines (make them all the same)
 696             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 697
 698             # Actual output from router
 699             actual = (
 700                 net["r%s" % i]
 701                 .cmd('vtysh -c "show ipv6 ripng status" 2> /dev/null')
 702                 .rstrip()
 703             )
 704             # Mask out Link-Local mac address portion. They are random...
 705             actual = re.sub(r" fe80::[0-9a-f:]+", " fe80::XXXX:XXXX:XXXX:XXXX", actual)
 706             # Drop time in next due
 707             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 708             # Drop time in last update
 709             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 710             # Fix newlines (make them all the same)
 711             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 712
 713             # Generate Diff
 714             diff = topotest.get_textdiff(
 715                 actual,
 716                 expected,
 717                 title1="actual IPv6 RIPng status",
 718                 title2="expected IPv6 RIPng status",
 719             )
 720
 721             # Empty string if it matches, otherwise diff contains unified diff
 722             if diff:
 723                 sys.stderr.write(
 724                     "r%s failed IPv6 RIPng status check:\n%s\n" % (i, diff)
 725                 )
 726                 failures += 1
 727             else:
 728                 print("r%s ok" % i)
 729
 730             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (
 731                 i,
 732                 diff,
 733             )
 734
 735     # Make sure that all daemons are running
 736     for i in range(1, 2):
 737         fatal_error = net["r%s" % i].checkRouterRunning()
 738         assert fatal_error == "", fatal_error
 739
 740
 741 def test_ospfv2_interfaces():
 742     global fatal_error
 743     net = get_topogen().net
 744
 745     # Skip if previous fatal error condition is raised
 746     if fatal_error != "":
 747         pytest.skip(fatal_error)
 748
 749     thisDir = os.path.dirname(os.path.realpath(__file__))
 750
 751     print("\n\n** Verifying OSPFv2 interfaces")
 752     print("******************************************\n")
 753     failures = 0
 754     for i in range(1, 2):
 755         refTableFile = "%s/r%s/show_ip_ospf_interface.ref" % (thisDir, i)
 756         if os.path.isfile(refTableFile):
 757             # Read expected result from file
 758             expected = open(refTableFile).read().rstrip()
 759             # Fix newlines (make them all the same)
 760             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 761
 762             # Actual output from router
 763             actual = (
 764                 net["r%s" % i]
 765                 .cmd('vtysh -c "show ip ospf interface" 2> /dev/null')
 766                 .rstrip()
 767             )
 768             # Mask out Bandwidth portion. They may change..
 769             actual = re.sub(r"BW [0-9]+ Mbit", "BW XX Mbit", actual)
 770             actual = re.sub(r"ifindex [0-9]+", "ifindex X", actual)
 771
 772             # Drop time in next due
 773             actual = re.sub(r"Hello due in [0-9\.]+s", "Hello due in XX.XXXs", actual)
 774             actual = re.sub(
 775                 r"Hello due in [0-9\.]+ usecs", "Hello due in XX.XXXs", actual
 776             )
 777             # Fix 'MTU mismatch detection: enabled' vs 'MTU mismatch detection:enabled' - accept both
 778             actual = re.sub(
 779                 r"MTU mismatch detection:([a-z]+.*)",
 780                 r"MTU mismatch detection: \1",
 781                 actual,
 782             )
 783             # Fix newlines (make them all the same)
 784             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 785
 786             # Generate Diff
 787             diff = topotest.get_textdiff(
 788                 actual,
 789                 expected,
 790                 title1="actual SHOW IP OSPF INTERFACE",
 791                 title2="expected SHOW IP OSPF INTERFACE",
 792             )
 793
 794             # Empty string if it matches, otherwise diff contains unified diff
 795             if diff:
 796                 sys.stderr.write(
 797                     "r%s failed SHOW IP OSPF INTERFACE check:\n%s\n" % (i, diff)
 798                 )
 799                 failures += 1
 800             else:
 801                 print("r%s ok" % i)
 802
 803             # Ignoring the issue if told to ignore (ie not yet fixed)
 804             if failures != 0:
 805                 if os.environ.get("bamboo_TOPOTESTS_ISSUE_348") == "IGNORE":
 806                     sys.stderr.write(
 807                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348\n"
 808                     )
 809                     pytest.skip(
 810                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348"
 811                     )
 812
 813             assert (
 814                 failures == 0
 815             ), "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff)
 816
 817     # Make sure that all daemons are running
 818     for i in range(1, 2):
 819         fatal_error = net["r%s" % i].checkRouterRunning()
 820         assert fatal_error == "", fatal_error
 821
 822
 823 def test_isis_interfaces():
 824     global fatal_error
 825     net = get_topogen().net
 826
 827     # Skip if previous fatal error condition is raised
 828     if fatal_error != "":
 829         pytest.skip(fatal_error)
 830
 831     thisDir = os.path.dirname(os.path.realpath(__file__))
 832
 833     print("\n\n** Verifying ISIS interfaces")
 834     print("******************************************\n")
 835     failures = 0
 836     for i in range(1, 2):
 837         refTableFile = "%s/r%s/show_isis_interface_detail.ref" % (thisDir, i)
 838         if os.path.isfile(refTableFile):
 839             # Read expected result from file
 840             expected = open(refTableFile).read().rstrip()
 841             # Fix newlines (make them all the same)
 842             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 843
 844             # Actual output from router
 845             actual = (
 846                 net["r%s" % i]
 847                 .cmd('vtysh -c "show isis interface detail" 2> /dev/null')
 848                 .rstrip()
 849             )
 850             # Mask out Link-Local mac address portion. They are random...
 851             actual = re.sub(r"fe80::[0-9a-f:]+", "fe80::XXXX:XXXX:XXXX:XXXX", actual)
 852             # Mask out SNPA mac address portion. They are random...
 853             actual = re.sub(r"SNPA: [0-9a-f\.]+", "SNPA: XXXX.XXXX.XXXX", actual)
 854             # Mask out Circuit ID number
 855             actual = re.sub(r"Circuit Id: 0x[0-9a-f]+", "Circuit Id: 0xXX", actual)
 856             # Fix newlines (make them all the same)
 857             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 858
 859             # Generate Diff
 860             diff = topotest.get_textdiff(
 861                 actual,
 862                 expected,
 863                 title1="actual SHOW ISIS INTERFACE DETAIL",
 864                 title2="expected SHOW ISIS OSPF6 INTERFACE DETAIL",
 865             )
 866
 867             # Empty string if it matches, otherwise diff contains unified diff
 868             if diff:
 869                 sys.stderr.write(
 870                     "r%s failed SHOW ISIS INTERFACE DETAIL check:\n%s\n" % (i, diff)
 871                 )
 872                 failures += 1
 873             else:
 874                 print("r%s ok" % i)
 875
 876             assert (
 877                 failures == 0
 878             ), "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff)
 879
 880     # Make sure that all daemons are running
 881     for i in range(1, 2):
 882         fatal_error = net["r%s" % i].checkRouterRunning()
 883         assert fatal_error == "", fatal_error
 884
 885
 886 def test_bgp_summary():
 887     global fatal_error
 888     net = get_topogen().net
 889
 890     # Skip if previous fatal error condition is raised
 891     if fatal_error != "":
 892         pytest.skip(fatal_error)
 893
 894     thisDir = os.path.dirname(os.path.realpath(__file__))
 895
 896     print("\n\n** Verifying BGP Summary")
 897     print("******************************************\n")
 898     failures = 0
 899     for i in range(1, 2):
 900         refTableFile = "%s/r%s/show_ip_bgp_summary.ref" % (thisDir, i)
 901         if os.path.isfile(refTableFile):
 902             # Read expected result from file
 903             expected_original = open(refTableFile).read().rstrip()
 904
 905             for arguments in [
 906                 "",
 907                 "remote-as internal",
 908                 "remote-as external",
 909                 "remote-as 100",
 910                 "remote-as 123",
 911                 "neighbor 192.168.7.10",
 912                 "neighbor 192.168.7.10",
 913                 "neighbor fc00:0:0:8::1000",
 914                 "neighbor 10.0.0.1",
 915                 "terse",
 916                 "remote-as internal terse",
 917                 "remote-as external terse",
 918                 "remote-as 100 terse",
 919                 "remote-as 123 terse",
 920                 "neighbor 192.168.7.10 terse",
 921                 "neighbor 192.168.7.10 terse",
 922                 "neighbor fc00:0:0:8::1000 terse",
 923                 "neighbor 10.0.0.1 terse",
 924             ]:
 925                 # Actual output from router
 926                 actual = (
 927                     net["r%s" % i]
 928                     .cmd(
 929                         'vtysh -c "show ip bgp summary ' + arguments + '" 2> /dev/null'
 930                     )
 931                     .rstrip()
 932                 )
 933
 934                 # Mask out "using XXiXX bytes" portion. They are random...
 935                 actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
 936                 # Mask out "using XiXXX KiB" portion. They are random...
 937                 actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
 938
 939                 # Remove extra summaries which exist with newer versions
 940
 941                 # Remove summary lines (changed recently)
 942                 actual = re.sub(r"Total number.*", "", actual)
 943                 actual = re.sub(r"Displayed.*", "", actual)
 944                 # Remove IPv4 Unicast Summary (Title only)
 945                 actual = re.sub(r"IPv4 Unicast Summary \(VRF default\):", "", actual)
 946                 # Remove IPv4 Multicast Summary (all of it)
 947                 actual = re.sub(r"IPv4 Multicast Summary \(VRF default\):", "", actual)
 948                 actual = re.sub(r"No IPv4 Multicast neighbor is configured", "", actual)
 949                 # Remove IPv4 VPN Summary (all of it)
 950                 actual = re.sub(r"IPv4 VPN Summary \(VRF default\):", "", actual)
 951                 actual = re.sub(r"No IPv4 VPN neighbor is configured", "", actual)
 952                 # Remove IPv4 Encap Summary (all of it)
 953                 actual = re.sub(r"IPv4 Encap Summary \(VRF default\):", "", actual)
 954                 actual = re.sub(r"No IPv4 Encap neighbor is configured", "", actual)
 955                 # Remove Unknown Summary (all of it)
 956                 actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
 957                 actual = re.sub(r"No Unknown neighbor is configured", "", actual)
 958                 # Make Connect/Active/Idle the same (change them all to Active)
 959                 actual = re.sub(r" Connect ", "  Active ", actual)
 960                 actual = re.sub(r"    Idle ", "  Active ", actual)
 961
 962                 actual = re.sub(
 963                     r"IPv4 labeled-unicast Summary \(VRF default\):", "", actual
 964                 )
 965                 actual = re.sub(
 966                     r"No IPv4 labeled-unicast neighbor is configured", "", actual
 967                 )
 968
 969                 expected = expected_original
 970                 # apply argumentss on expected output
 971                 if "internal" in arguments or "remote-as 100" in arguments:
 972                     expected = re.sub(r".+\s+200\s+.+", "", expected)
 973                 elif "external" in arguments:
 974                     expected = re.sub(r".+\s+100\s+.+Active.+", "", expected)
 975                 elif "remote-as 123" in arguments:
 976                     expected = re.sub(
 977                         r"(192.168.7.(1|2)0|fc00:0:0:8::(1|2)000).+Active.+",
 978                         "",
 979                         expected,
 980                     )
 981                     expected = re.sub(r"\nNeighbor.+Desc", "", expected)
 982                     expected = expected + "% No matching neighbor\n"
 983                 elif "192.168.7.10" in arguments:
 984                     expected = re.sub(
 985                         r"(192.168.7.20|fc00:0:0:8::(1|2)000).+Active.+", "", expected
 986                     )
 987                 elif "fc00:0:0:8::1000" in arguments:
 988                     expected = re.sub(
 989                         r"(192.168.7.(1|2)0|fc00:0:0:8::2000).+Active.+", "", expected
 990                     )
 991                 elif "10.0.0.1" in arguments:
 992                     expected = "No such neighbor in this view/vrf"
 993
 994                 if "terse" in arguments:
 995                     expected = re.sub(r"BGP table version .+", "", expected)
 996                     expected = re.sub(r"RIB entries .+", "", expected)
 997                     expected = re.sub(r"Peers [0-9]+, using .+", "", expected)
 998
 999                 # Strip empty lines
1000                 actual = actual.lstrip().rstrip()
1001                 expected = expected.lstrip().rstrip()
1002                 actual = re.sub(r"\n+", "\n", actual)
1003                 expected = re.sub(r"\n+", "\n", expected)
1004
1005                 # reapply initial formatting
1006                 if "terse" in arguments:
1007                     actual = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", actual)
1008                     expected = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", expected)
1009                 else:
1010                     actual = re.sub(r"KiB of memory\n", "KiB of memory\n\n", actual)
1011                     expected = re.sub(r"KiB of memory\n", "KiB of memory\n\n", expected)
1012
1013                 # realign expected neighbor columns if needed
1014                 try:
1015                     idx_actual = (
1016                         re.search(r"(Neighbor\s+V\s+)", actual).group(1).find("V")
1017                     )
1018                     idx_expected = (
1019                         re.search(r"(Neighbor\s+V\s+)", expected).group(1).find("V")
1020                     )
1021                     idx_diff = idx_expected - idx_actual
1022                     if idx_diff > 0:
1023                         # Neighbor        V         AS MsgRcvd MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd
1024                         expected = re.sub(" " * idx_diff + "V ", "V ", expected)
1025                         # 192.168.7.10    4        100       0       0        0    0    0    never       Active
1026                         expected = re.sub(" " * idx_diff + "4 ", "4 ", expected)
1027                 except AttributeError:
1028                     pass
1029
1030                 # Fix newlines (make them all the same)
1031                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1032                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1033
1034                 # Generate Diff
1035                 diff = topotest.get_textdiff(
1036                     actual,
1037                     expected,
1038                     title1="actual SHOW IP BGP SUMMARY " + arguments.upper(),
1039                     title2="expected SHOW IP BGP SUMMARY " + arguments.upper(),
1040                 )
1041
1042                 # Empty string if it matches, otherwise diff contains unified diff
1043                 if diff:
1044                     sys.stderr.write(
1045                         "r%s failed SHOW IP BGP SUMMARY check:\n%s\n" % (i, diff)
1046                     )
1047                     failures += 1
1048                 else:
1049                     print("r%s ok" % i)
1050
1051                 assert (
1052                     failures == 0
1053                 ), "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (
1054                     i,
1055                     diff,
1056                 )
1057
1058     # Make sure that all daemons are running
1059     for i in range(1, 2):
1060         fatal_error = net["r%s" % i].checkRouterRunning()
1061         assert fatal_error == "", fatal_error
1062
1063
1064 def test_bgp_ipv6_summary():
1065     global fatal_error
1066     net = get_topogen().net
1067
1068     # Skip if previous fatal error condition is raised
1069     if fatal_error != "":
1070         pytest.skip(fatal_error)
1071
1072     thisDir = os.path.dirname(os.path.realpath(__file__))
1073
1074     print("\n\n** Verifying BGP IPv6 Summary")
1075     print("******************************************\n")
1076     failures = 0
1077     for i in range(1, 2):
1078         refTableFile = "%s/r%s/show_bgp_ipv6_summary.ref" % (thisDir, i)
1079         if os.path.isfile(refTableFile):
1080             # Read expected result from file
1081             expected = open(refTableFile).read().rstrip()
1082             # Fix newlines (make them all the same)
1083             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1084
1085             # Actual output from router
1086             actual = (
1087                 net["r%s" % i]
1088                 .cmd('vtysh -c "show bgp ipv6 summary" 2> /dev/null')
1089                 .rstrip()
1090             )
1091             # Mask out "using XXiXX bytes" portion. They are random...
1092             actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
1093             # Mask out "using XiXXX KiB" portion. They are random...
1094             actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
1095             #
1096             # Remove extra summaries which exist with newer versions
1097             #
1098             # Remove summary lines (changed recently)
1099             actual = re.sub(r"Total number.*", "", actual)
1100             actual = re.sub(r"Displayed.*", "", actual)
1101             # Remove IPv4 Unicast Summary (Title only)
1102             actual = re.sub(r"IPv6 Unicast Summary \(VRF default\):", "", actual)
1103             # Remove IPv4 Multicast Summary (all of it)
1104             actual = re.sub(r"IPv6 Multicast Summary \(VRF default\):", "", actual)
1105             actual = re.sub(r"No IPv6 Multicast neighbor is configured", "", actual)
1106             # Remove IPv4 VPN Summary (all of it)
1107             actual = re.sub(r"IPv6 VPN Summary \(VRF default\):", "", actual)
1108             actual = re.sub(r"No IPv6 VPN neighbor is configured", "", actual)
1109             # Remove IPv4 Encap Summary (all of it)
1110             actual = re.sub(r"IPv6 Encap Summary \(VRF default\):", "", actual)
1111             actual = re.sub(r"No IPv6 Encap neighbor is configured", "", actual)
1112             # Remove Unknown Summary (all of it)
1113             actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
1114             actual = re.sub(r"No Unknown neighbor is configured", "", actual)
1115             # Make Connect/Active/Idle the same (change them all to Active)
1116             actual = re.sub(r" Connect ", "  Active ", actual)
1117             actual = re.sub(r"    Idle ", "  Active ", actual)
1118
1119             # Remove Labeled Unicast Summary (all of it)
1120             actual = re.sub(
1121                 r"IPv6 labeled-unicast Summary \(VRF default\):", "", actual
1122             )
1123             actual = re.sub(
1124                 r"No IPv6 labeled-unicast neighbor is configured", "", actual
1125             )
1126
1127             # Strip empty lines
1128             actual = actual.lstrip()
1129             actual = actual.rstrip()
1130             #
1131             # Fix newlines (make them all the same)
1132             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1133
1134             # Generate Diff
1135             diff = topotest.get_textdiff(
1136                 actual,
1137                 expected,
1138                 title1="actual SHOW BGP IPv6 SUMMARY",
1139                 title2="expected SHOW BGP IPv6 SUMMARY",
1140             )
1141
1142             # Empty string if it matches, otherwise diff contains unified diff
1143             if diff:
1144                 sys.stderr.write(
1145                     "r%s failed SHOW BGP IPv6 SUMMARY check:\n%s\n" % (i, diff)
1146                 )
1147                 failures += 1
1148             else:
1149                 print("r%s ok" % i)
1150
1151             assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (
1152                 i,
1153                 diff,
1154             )
1155
1156     # Make sure that all daemons are running
1157     for i in range(1, 2):
1158         fatal_error = net["r%s" % i].checkRouterRunning()
1159         assert fatal_error == "", fatal_error
1160
1161
1162 def test_nht():
1163     global fatal_error
1164     net = get_topogen().net
1165
1166     # Skip if previous fatal error condition is raised
1167     if fatal_error != "":
1168         pytest.skip(fatal_error)
1169
1170     print("\n\n**** Test that nexthop tracking is at least nominally working ****\n")
1171
1172     thisDir = os.path.dirname(os.path.realpath(__file__))
1173
1174     for i in range(1, 2):
1175         nhtFile = "%s/r%s/ip_nht.ref" % (thisDir, i)
1176         expected = open(nhtFile).read().rstrip()
1177         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1178
1179         actual = net["r%s" % i].cmd('vtysh -c "show ip nht" 2> /dev/null').rstrip()
1180         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1181         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1182
1183         diff = topotest.get_textdiff(
1184             actual,
1185             expected,
1186             title1="Actual `show ip nht`",
1187             title2="Expected `show ip nht`",
1188         )
1189
1190         if diff:
1191             assert 0, "r%s failed ip nht check:\n%s\n" % (i, diff)
1192         else:
1193             print("show ip nht is ok\n")
1194
1195         nhtFile = "%s/r%s/ipv6_nht.ref" % (thisDir, i)
1196         expected = open(nhtFile).read().rstrip()
1197         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1198
1199         actual = net["r%s" % i].cmd('vtysh -c "show ipv6 nht" 2> /dev/null').rstrip()
1200         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1201         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1202
1203         diff = topotest.get_textdiff(
1204             actual,
1205             expected,
1206             title1="Actual `show ip nht`",
1207             title2="Expected `show ip nht`",
1208         )
1209
1210         if diff:
1211             assert 0, "r%s failed ipv6 nht check:\n%s\n" % (i, diff)
1212         else:
1213             print("show ipv6 nht is ok\n")
1214
1215
1216 def test_bgp_ipv4():
1217     global fatal_error
1218     net = get_topogen().net
1219
1220     # Skip if previous fatal error condition is raised
1221     if fatal_error != "":
1222         pytest.skip(fatal_error)
1223
1224     thisDir = os.path.dirname(os.path.realpath(__file__))
1225
1226     print("\n\n** Verifying BGP IPv4")
1227     print("******************************************\n")
1228     diffresult = {}
1229     for i in range(1, 2):
1230         success = 0
1231         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv4*.ref" % (thisDir, i)):
1232             if os.path.isfile(refTableFile):
1233                 # Read expected result from file
1234                 expected = open(refTableFile).read().rstrip()
1235                 # Fix newlines (make them all the same)
1236                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1237
1238                 # Actual output from router
1239                 actual = (
1240                     net["r%s" % i].cmd('vtysh -c "show bgp ipv4" 2> /dev/null').rstrip()
1241                 )
1242                 # Remove summary line (changed recently)
1243                 actual = re.sub(r"Total number.*", "", actual)
1244                 actual = re.sub(r"Displayed.*", "", actual)
1245                 actual = actual.rstrip()
1246                 # Fix newlines (make them all the same)
1247                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1248
1249                 # Generate Diff
1250                 diff = topotest.get_textdiff(
1251                     actual,
1252                     expected,
1253                     title1="actual SHOW BGP IPv4",
1254                     title2="expected SHOW BGP IPv4",
1255                 )
1256
1257                 # Empty string if it matches, otherwise diff contains unified diff
1258                 if diff:
1259                     diffresult[refTableFile] = diff
1260                 else:
1261                     success = 1
1262                     print("template %s matched: r%s ok" % (refTableFile, i))
1263                     break
1264
1265         if not success:
1266             resultstr = "No template matched.\n"
1267             for f in diffresult.keys():
1268                 resultstr += "template %s: r%s failed SHOW BGP IPv4 check:\n%s\n" % (
1269                     f,
1270                     i,
1271                     diffresult[f],
1272                 )
1273             raise AssertionError(
1274                 "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, resultstr)
1275             )
1276
1277     # Make sure that all daemons are running
1278     for i in range(1, 2):
1279         fatal_error = net["r%s" % i].checkRouterRunning()
1280         assert fatal_error == "", fatal_error
1281
1282
1283 def test_bgp_ipv6():
1284     global fatal_error
1285     net = get_topogen().net
1286
1287     # Skip if previous fatal error condition is raised
1288     if fatal_error != "":
1289         pytest.skip(fatal_error)
1290
1291     thisDir = os.path.dirname(os.path.realpath(__file__))
1292
1293     print("\n\n** Verifying BGP IPv6")
1294     print("******************************************\n")
1295     diffresult = {}
1296     for i in range(1, 2):
1297         success = 0
1298         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv6*.ref" % (thisDir, i)):
1299             if os.path.isfile(refTableFile):
1300                 # Read expected result from file
1301                 expected = open(refTableFile).read().rstrip()
1302                 # Fix newlines (make them all the same)
1303                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1304
1305                 # Actual output from router
1306                 actual = (
1307                     net["r%s" % i].cmd('vtysh -c "show bgp ipv6" 2> /dev/null').rstrip()
1308                 )
1309                 # Remove summary line (changed recently)
1310                 actual = re.sub(r"Total number.*", "", actual)
1311                 actual = re.sub(r"Displayed.*", "", actual)
1312                 actual = actual.rstrip()
1313                 # Fix newlines (make them all the same)
1314                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1315
1316                 # Generate Diff
1317                 diff = topotest.get_textdiff(
1318                     actual,
1319                     expected,
1320                     title1="actual SHOW BGP IPv6",
1321                     title2="expected SHOW BGP IPv6",
1322                 )
1323
1324                 # Empty string if it matches, otherwise diff contains unified diff
1325                 if diff:
1326                     diffresult[refTableFile] = diff
1327                 else:
1328                     success = 1
1329                     print("template %s matched: r%s ok" % (refTableFile, i))
1330
1331         if not success:
1332             resultstr = "No template matched.\n"
1333             for f in diffresult.keys():
1334                 resultstr += "template %s: r%s failed SHOW BGP IPv6 check:\n%s\n" % (
1335                     f,
1336                     i,
1337                     diffresult[f],
1338                 )
1339             raise AssertionError(
1340                 "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, resultstr)
1341             )
1342
1343     # Make sure that all daemons are running
1344     for i in range(1, 2):
1345         fatal_error = net["r%s" % i].checkRouterRunning()
1346         assert fatal_error == "", fatal_error
1347
1348
1349 def test_route_map():
1350     global fatal_error
1351     net = get_topogen().net
1352
1353     if fatal_error != "":
1354         pytest.skip(fatal_error)
1355
1356     thisDir = os.path.dirname(os.path.realpath(__file__))
1357
1358     print("\n\n** Verifying some basic routemap forward references\n")
1359     print("*******************************************************\n")
1360     failures = 0
1361     for i in range(1, 2):
1362         refroutemap = "%s/r%s/show_route_map.ref" % (thisDir, i)
1363         if os.path.isfile(refroutemap):
1364             expected = open(refroutemap).read().rstrip()
1365             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1366
1367             actual = (
1368                 net["r%s" % i].cmd('vtysh -c "show route-map" 2> /dev/null').rstrip()
1369             )
1370             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1371
1372             diff = topotest.get_textdiff(
1373                 actual,
1374                 expected,
1375                 title1="actual show route-map",
1376                 title2="expected show route-map",
1377             )
1378
1379             if diff:
1380                 sys.stderr.write(
1381                     "r%s failed show route-map command Check:\n%s\n" % (i, diff)
1382                 )
1383                 failures += 1
1384             else:
1385                 print("r%s ok" % i)
1386
1387             assert (
1388                 failures == 0
1389             ), "Show route-map command failed for router r%s:\n%s" % (i, diff)
1390
1391
1392 def test_nexthop_groups_with_route_maps():
1393     global fatal_error
1394     net = get_topogen().net
1395
1396     # Skip if previous fatal error condition is raised
1397     if fatal_error != "":
1398         pytest.skip(fatal_error)
1399
1400     print("\n\n** Verifying Nexthop Groups With Route-Maps")
1401     print("******************************************\n")
1402
1403     ### Nexthop Group With Route-Map Tests
1404
1405     # Create a lib nexthop-group
1406     net["r1"].cmd(
1407         'vtysh -c "c t" -c "nexthop-group test" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
1408     )
1409
1410     ## Route-Map Proto Source
1411
1412     route_str = "2.2.2.1"
1413     src_str = "192.168.0.1"
1414
1415     net["r1"].cmd(
1416         'vtysh -c "c t" -c "route-map NH-SRC permit 111" -c "set src %s"' % src_str
1417     )
1418     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NH-SRC"')
1419
1420     net["r1"].cmd('vtysh -c "sharp install routes %s nexthop-group test 1"' % route_str)
1421
1422     verify_route_nexthop_group("%s/32" % route_str)
1423
1424     # Only a valid test on linux using nexthop objects
1425     if sys.platform.startswith("linux"):
1426         output = net["r1"].cmd("ip route show %s/32" % route_str)
1427         match = re.search(r"src %s" % src_str, output)
1428         assert match is not None, "Route %s/32 not installed with src %s" % (
1429             route_str,
1430             src_str,
1431         )
1432
1433     # Remove NHG routes and route-map
1434     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % route_str)
1435     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NH-SRC"')
1436     net["r1"].cmd(
1437         'vtysh -c "c t" -c "no route-map NH-SRC permit 111" # -c "set src %s"' % src_str
1438     )
1439     net["r1"].cmd('vtysh -c "c t" -c "no route-map NH-SRC"')
1440
1441     ## Route-Map Deny/Permit with same nexthop group
1442
1443     permit_route_str = "3.3.3.1"
1444     deny_route_str = "3.3.3.2"
1445
1446     net["r1"].cmd(
1447         'vtysh -c "c t" -c "ip prefix-list NOPE seq 5 permit %s/32"' % permit_route_str
1448     )
1449     net["r1"].cmd(
1450         'vtysh -c "c t" -c "route-map NOPE permit 111" -c "match ip address prefix-list NOPE"'
1451     )
1452     net["r1"].cmd('vtysh -c "c t" -c "route-map NOPE deny 222"')
1453     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NOPE"')
1454
1455     # This route should be permitted
1456     net["r1"].cmd(
1457         'vtysh -c "sharp install routes %s nexthop-group test 1"' % permit_route_str
1458     )
1459
1460     verify_route_nexthop_group("%s/32" % permit_route_str)
1461
1462     # This route should be denied
1463     net["r1"].cmd(
1464         'vtysh -c "sharp install routes %s nexthop-group test 1"' % deny_route_str
1465     )
1466
1467     nhg_id = route_get_nhg_id(deny_route_str)
1468     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
1469
1470     match = re.search(r"Valid", output)
1471     assert match is None, "Nexthop Group ID=%d should not be marked Valid" % nhg_id
1472
1473     match = re.search(r"Installed", output)
1474     assert match is None, "Nexthop Group ID=%d should not be marked Installed" % nhg_id
1475
1476     # Remove NHG routes and route-map
1477     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % permit_route_str)
1478     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % deny_route_str)
1479     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NOPE"')
1480     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE permit 111"')
1481     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE deny 222"')
1482     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE"')
1483     net["r1"].cmd(
1484         'vtysh -c "c t" -c "no ip prefix-list NOPE seq 5 permit %s/32"'
1485         % permit_route_str
1486     )
1487
1488
1489 def test_nexthop_group_replace():
1490     global fatal_error
1491     net = get_topogen().net
1492
1493     # Skip if previous fatal error condition is raised
1494     if fatal_error != "":
1495         pytest.skip(fatal_error)
1496
1497     print("\n\n** Verifying Nexthop Groups")
1498     print("******************************************\n")
1499
1500     ### Nexthop Group Tests
1501
1502     ## 2-Way ECMP Directly Connected
1503
1504     net["r1"].cmd(
1505         'vtysh -c "c t" -c "nexthop-group replace" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1506     )
1507
1508     # At the moment there is absolutely no real easy way to query sharpd
1509     # for the nexthop group actually installed.  If it is not installed
1510     # sharpd will just transmit the nexthops down instead of the nexthop
1511     # group id.  Leading to a situation where the replace is not actually
1512     # being tested.  So let's just wait some time here because this
1513     # is hard and this test fails all the time
1514     sleep(5)
1515
1516     # Create with sharpd using nexthop-group
1517     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.1 nexthop-group replace 1"')
1518
1519     verify_route_nexthop_group("3.3.3.1/32")
1520
1521     # Change the nexthop group
1522     net["r1"].cmd(
1523         'vtysh -c "c t" -c "nexthop-group replace" -c "no nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.3 r1-eth1 onlink" -c "nexthop 1.1.1.4 r1-eth4 onlink"'
1524     )
1525
1526     # Verify it updated. We can just check install and ecmp count here.
1527     verify_route_nexthop_group("3.3.3.1/32", False, 3)
1528
1529
1530 def test_mpls_interfaces():
1531     global fatal_error
1532     net = get_topogen().net
1533
1534     # Skip if previous fatal error condition is raised
1535     if fatal_error != "":
1536         pytest.skip(fatal_error)
1537
1538     # Skip if no LDP installed or old kernel
1539     if net["r1"].daemon_available("ldpd") == False:
1540         pytest.skip("No MPLS or kernel < 4.5")
1541
1542     thisDir = os.path.dirname(os.path.realpath(__file__))
1543
1544     print("\n\n** Verifying MPLS Interfaces")
1545     print("******************************************\n")
1546     failures = 0
1547     for i in range(1, 2):
1548         refTableFile = "%s/r%s/show_mpls_ldp_interface.ref" % (thisDir, i)
1549         if os.path.isfile(refTableFile):
1550             # Read expected result from file
1551             expected = open(refTableFile).read().rstrip()
1552             # Fix newlines (make them all the same)
1553             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1554
1555             # Actual output from router
1556             actual = (
1557                 net["r%s" % i]
1558                 .cmd('vtysh -c "show mpls ldp interface" 2> /dev/null')
1559                 .rstrip()
1560             )
1561             # Mask out Timer in Uptime
1562             actual = re.sub(r" [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ", " xx:xx:xx ", actual)
1563             # Fix newlines (make them all the same)
1564             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1565
1566             # Generate Diff
1567             diff = topotest.get_textdiff(
1568                 actual,
1569                 expected,
1570                 title1="actual MPLS LDP interface status",
1571                 title2="expected MPLS LDP interface status",
1572             )
1573
1574             # Empty string if it matches, otherwise diff contains unified diff
1575             if diff:
1576                 sys.stderr.write(
1577                     "r%s failed MPLS LDP Interface status Check:\n%s\n" % (i, diff)
1578                 )
1579                 failures += 1
1580             else:
1581                 print("r%s ok" % i)
1582
1583             if failures > 0:
1584                 fatal_error = "MPLS LDP Interface status failed"
1585
1586             assert (
1587                 failures == 0
1588             ), "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
1589
1590     # Make sure that all daemons are running
1591     for i in range(1, 2):
1592         fatal_error = net["r%s" % i].checkRouterRunning()
1593         assert fatal_error == "", fatal_error
1594
1595
1596 def test_resilient_nexthop_group():
1597     net = get_topogen().net
1598
1599     result = required_linux_kernel_version("5.19")
1600     if result is not True:
1601         pytest.skip("Kernel requirements are not met, kernel version should be >= 5.19")
1602
1603     net["r1"].cmd(
1604         'vtysh -c "conf" -c "nexthop-group resilience" -c "resilient buckets 64 idle-timer 128 unbalanced-timer 256" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1605     )
1606
1607     output = net["r1"].cmd('vtysh -c "show nexthop-group rib sharp"')
1608     output = re.findall(r"Buckets", output)
1609
1610     verify_nexthop_group(185483878)
1611     assert len(output) == 1, "Resilient NHG not created in zebra"
1612
1613
1614 def test_shutdown_check_stderr():
1615     global fatal_error
1616     net = get_topogen().net
1617
1618     # Skip if previous fatal error condition is raised
1619     if fatal_error != "":
1620         pytest.skip(fatal_error)
1621
1622     print("\n\n** Verifying unexpected STDERR output from daemons")
1623     print("******************************************\n")
1624
1625     if os.environ.get("TOPOTESTS_CHECK_STDERR") is None:
1626         print(
1627             "SKIPPED final check on StdErr output: Disabled (TOPOTESTS_CHECK_STDERR undefined)\n"
1628         )
1629         pytest.skip("Skipping test for Stderr output")
1630
1631     thisDir = os.path.dirname(os.path.realpath(__file__))
1632
1633     print("thisDir=" + thisDir)
1634
1635     net["r1"].stopRouter()
1636
1637     log = net["r1"].getStdErr("ripd")
1638     if log:
1639         print("\nRIPd StdErr Log:\n" + log)
1640     log = net["r1"].getStdErr("ripngd")
1641     if log:
1642         print("\nRIPngd StdErr Log:\n" + log)
1643     log = net["r1"].getStdErr("ospfd")
1644     if log:
1645         print("\nOSPFd StdErr Log:\n" + log)
1646     log = net["r1"].getStdErr("ospf6d")
1647     if log:
1648         print("\nOSPF6d StdErr Log:\n" + log)
1649     log = net["r1"].getStdErr("isisd")
1650     if log:
1651         print("\nISISd StdErr Log:\n" + log)
1652     log = net["r1"].getStdErr("bgpd")
1653     if log:
1654         print("\nBGPd StdErr Log:\n" + log)
1655
1656     log = net["r1"].getStdErr("nhrpd")
1657     if log:
1658         print("\nNHRPd StdErr Log:\n" + log)
1659
1660     log = net["r1"].getStdErr("pbrd")
1661     if log:
1662         print("\nPBRd StdErr Log:\n" + log)
1663
1664     log = net["r1"].getStdErr("babeld")
1665     if log:
1666         print("\nBABELd StdErr Log:\n" + log)
1667
1668     if net["r1"].daemon_available("ldpd"):
1669         log = net["r1"].getStdErr("ldpd")
1670         if log:
1671             print("\nLDPd StdErr Log:\n" + log)
1672     log = net["r1"].getStdErr("zebra")
1673     if log:
1674         print("\nZebra StdErr Log:\n" + log)
1675
1676
1677 def test_shutdown_check_memleak():
1678     global fatal_error
1679     net = get_topogen().net
1680
1681     # Skip if previous fatal error condition is raised
1682     if fatal_error != "":
1683         pytest.skip(fatal_error)
1684
1685     if os.environ.get("TOPOTESTS_CHECK_MEMLEAK") is None:
1686         print(
1687             "SKIPPED final check on Memory leaks: Disabled (TOPOTESTS_CHECK_MEMLEAK undefined)\n"
1688         )
1689         pytest.skip("Skipping test for memory leaks")
1690
1691     thisDir = os.path.dirname(os.path.realpath(__file__))
1692
1693     for i in range(1, 2):
1694         net["r%s" % i].stopRouter()
1695         net["r%s" % i].report_memory_leaks(
1696             os.environ.get("TOPOTESTS_CHECK_MEMLEAK"), os.path.basename(__file__)
1697         )
1698
1699
1700 if __name__ == "__main__":
1701     # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
1702     # retval = pytest.main(["-s", "--tb=no"])
1703     retval = pytest.main(["-s"])
1704     sys.exit(retval)