tests/topotests/all_protocol_startup/test_all_protocol_startup.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: ISC
   3
   4 #
   5 # test_all_protocol_startup.py
   6 # Part of NetDEF Topology Tests
   7 #
   8 # Copyright (c) 2017 by
   9 # Network Device Education Foundation, Inc. ("NetDEF")
  10 #
  11
  12 """
  13 test_all_protocol_startup.py: Test of all protocols at same time
  14
  15 """
  16
  17 import os
  18 import re
  19 import sys
  20 import pytest
  21 import glob
  22 from time import sleep
  23
  24 pytestmark = [
  25     pytest.mark.babeld,
  26     pytest.mark.bgpd,
  27     pytest.mark.isisd,
  28     pytest.mark.nhrpd,
  29     pytest.mark.ospfd,
  30     pytest.mark.pbrd,
  31     pytest.mark.ripd,
  32 ]
  33
  34 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  35 from lib import topotest
  36 from lib.topogen import Topogen, get_topogen
  37 from lib.common_config import (
  38     required_linux_kernel_version,
  39 )
  40
  41 fatal_error = ""
  42
  43
  44 #####################################################
  45 ##
  46 ##   Network Topology Definition
  47 ##
  48 #####################################################
  49
  50
  51 def build_topo(tgen):
  52     router = tgen.add_router("r1")
  53     for i in range(0, 10):
  54         tgen.add_switch("sw%d" % i).add_link(router)
  55
  56
  57 #####################################################
  58 ##
  59 ##   Tests starting
  60 ##
  61 #####################################################
  62
  63
  64 def setup_module(module):
  65     global fatal_error
  66
  67     print("\n\n** %s: Setup Topology" % module.__name__)
  68     print("******************************************\n")
  69
  70     thisDir = os.path.dirname(os.path.realpath(__file__))
  71     tgen = Topogen(build_topo, module.__name__)
  72     tgen.start_topology()
  73
  74     net = tgen.net
  75
  76     if net["r1"].get_routertype() != "frr":
  77         fatal_error = "Test is only implemented for FRR"
  78         sys.stderr.write("\n\nTest is only implemented for FRR - Skipping\n\n")
  79         pytest.skip(fatal_error)
  80
  81     # Starting Routers
  82     #
  83     # Main router
  84     for i in range(1, 2):
  85         net["r%s" % i].loadConf("zebra", "%s/r%s/zebra.conf" % (thisDir, i))
  86         net["r%s" % i].loadConf("ripd", "%s/r%s/ripd.conf" % (thisDir, i))
  87         net["r%s" % i].loadConf("ripngd", "%s/r%s/ripngd.conf" % (thisDir, i))
  88         net["r%s" % i].loadConf("ospfd", "%s/r%s/ospfd.conf" % (thisDir, i))
  89         if net["r1"].checkRouterVersion("<", "4.0"):
  90             net["r%s" % i].loadConf(
  91                 "ospf6d", "%s/r%s/ospf6d.conf-pre-v4" % (thisDir, i)
  92             )
  93         else:
  94             net["r%s" % i].loadConf("ospf6d", "%s/r%s/ospf6d.conf" % (thisDir, i))
  95         net["r%s" % i].loadConf("isisd", "%s/r%s/isisd.conf" % (thisDir, i))
  96         net["r%s" % i].loadConf("bgpd", "%s/r%s/bgpd.conf" % (thisDir, i))
  97         if net["r%s" % i].daemon_available("ldpd"):
  98             # Only test LDPd if it's installed and Kernel >= 4.5
  99             net["r%s" % i].loadConf("ldpd", "%s/r%s/ldpd.conf" % (thisDir, i))
 100         net["r%s" % i].loadConf("sharpd")
 101         net["r%s" % i].loadConf("nhrpd", "%s/r%s/nhrpd.conf" % (thisDir, i))
 102         net["r%s" % i].loadConf("babeld", "%s/r%s/babeld.conf" % (thisDir, i))
 103         net["r%s" % i].loadConf("pbrd", "%s/r%s/pbrd.conf" % (thisDir, i))
 104         tgen.gears["r%s" % i].start()
 105
 106     # For debugging after starting FRR daemons, uncomment the next line
 107     # tgen.mininet_cli()
 108
 109
 110 def teardown_module(module):
 111     print("\n\n** %s: Shutdown Topology" % module.__name__)
 112     print("******************************************\n")
 113     tgen = get_topogen()
 114     tgen.stop_topology()
 115
 116
 117 def test_router_running():
 118     global fatal_error
 119     tgen = get_topogen()
 120     net = tgen.net
 121
 122     # Skip if previous fatal error condition is raised
 123     if fatal_error != "":
 124         pytest.skip(fatal_error)
 125
 126     print("\n\n** Check if FRR is running on each Router node")
 127     print("******************************************\n")
 128     sleep(5)
 129
 130     # Starting Routers
 131     for i in range(1, 2):
 132         fatal_error = net["r%s" % i].checkRouterRunning()
 133         assert fatal_error == "", fatal_error
 134
 135     # For debugging after starting FRR daemons, uncomment the next line
 136     # tgen.mininet_cli()
 137
 138
 139 def test_error_messages_vtysh():
 140     global fatal_error
 141     net = get_topogen().net
 142
 143     # Skip if previous fatal error condition is raised
 144     if fatal_error != "":
 145         pytest.skip(fatal_error)
 146
 147     print("\n\n** Check for error messages on VTYSH")
 148     print("******************************************\n")
 149
 150     failures = 0
 151     for i in range(1, 2):
 152         #
 153         # First checking Standard Output
 154         #
 155
 156         # VTYSH output from router
 157         vtystdout = net["r%s" % i].cmd('vtysh -c "show version" 2> /dev/null').rstrip()
 158
 159         # Fix newlines (make them all the same)
 160         vtystdout = ("\n".join(vtystdout.splitlines()) + "\n").rstrip()
 161         # Drop everything starting with "FRRouting X.xx" message
 162         vtystdout = re.sub(r"FRRouting [0-9]+.*", "", vtystdout, flags=re.DOTALL)
 163
 164         if vtystdout == "":
 165             print("r%s StdOut ok" % i)
 166
 167         assert vtystdout == "", "Vtysh StdOut Output check failed for router r%s" % i
 168
 169         #
 170         # Second checking Standard Error
 171         #
 172
 173         # VTYSH StdErr output from router
 174         vtystderr = net["r%s" % i].cmd('vtysh -c "show version" > /dev/null').rstrip()
 175
 176         # Fix newlines (make them all the same)
 177         vtystderr = ("\n".join(vtystderr.splitlines()) + "\n").rstrip()
 178         # # Drop everything starting with "FRRouting X.xx" message
 179         # vtystderr = re.sub(r"FRRouting [0-9]+.*", "", vtystderr, flags=re.DOTALL)
 180
 181         if vtystderr == "":
 182             print("r%s StdErr ok" % i)
 183
 184         assert vtystderr == "", "Vtysh StdErr Output check failed for router r%s" % i
 185
 186     # Make sure that all daemons are running
 187     for i in range(1, 2):
 188         fatal_error = net["r%s" % i].checkRouterRunning()
 189         assert fatal_error == "", fatal_error
 190
 191
 192 def test_error_messages_daemons():
 193     global fatal_error
 194     net = get_topogen().net
 195
 196     # Skip if previous fatal error condition is raised
 197     if fatal_error != "":
 198         pytest.skip(fatal_error)
 199
 200     print("\n\n** Check for error messages in daemons")
 201     print("******************************************\n")
 202
 203     error_logs = ""
 204
 205     for i in range(1, 2):
 206         log = net["r%s" % i].getStdErr("ripd")
 207         if log:
 208             error_logs += "r%s RIPd StdErr Output:\n" % i
 209             error_logs += log
 210         log = net["r%s" % i].getStdErr("ripngd")
 211         if log:
 212             error_logs += "r%s RIPngd StdErr Output:\n" % i
 213             error_logs += log
 214         log = net["r%s" % i].getStdErr("ospfd")
 215         if log:
 216             error_logs += "r%s OSPFd StdErr Output:\n" % i
 217             error_logs += log
 218         log = net["r%s" % i].getStdErr("ospf6d")
 219         if log:
 220             error_logs += "r%s OSPF6d StdErr Output:\n" % i
 221             error_logs += log
 222         log = net["r%s" % i].getStdErr("isisd")
 223         # ISIS shows debugging enabled status on StdErr
 224         # Remove these messages
 225         log = re.sub(r"^IS-IS .* debugging is on.*", "", log).rstrip()
 226         if log:
 227             error_logs += "r%s ISISd StdErr Output:\n" % i
 228             error_logs += log
 229         log = net["r%s" % i].getStdErr("bgpd")
 230         if log:
 231             error_logs += "r%s BGPd StdErr Output:\n" % i
 232             error_logs += log
 233         if net["r%s" % i].daemon_available("ldpd"):
 234             log = net["r%s" % i].getStdErr("ldpd")
 235             if log:
 236                 error_logs += "r%s LDPd StdErr Output:\n" % i
 237                 error_logs += log
 238
 239         log = net["r1"].getStdErr("nhrpd")
 240         # NHRPD shows YANG model not embedded messages
 241         # Ignore these
 242         log = re.sub(r".*YANG model.*not embedded.*", "", log).rstrip()
 243         if log:
 244             error_logs += "r%s NHRPd StdErr Output:\n" % i
 245             error_logs += log
 246
 247         log = net["r1"].getStdErr("babeld")
 248         if log:
 249             error_logs += "r%s BABELd StdErr Output:\n" % i
 250             error_logs += log
 251
 252         log = net["r1"].getStdErr("pbrd")
 253         if log:
 254             error_logs += "r%s PBRd StdErr Output:\n" % i
 255             error_logs += log
 256
 257         log = net["r%s" % i].getStdErr("zebra")
 258         if log:
 259             error_logs += "r%s Zebra StdErr Output:\n" % i
 260             error_logs += log
 261
 262     if error_logs:
 263         sys.stderr.write(
 264             "Failed check for StdErr Output on daemons:\n%s\n" % error_logs
 265         )
 266
 267     # Ignoring the issue if told to ignore (ie not yet fixed)
 268     if error_logs != "":
 269         if os.environ.get("bamboo_TOPOTESTS_ISSUE_349") == "IGNORE":
 270             sys.stderr.write(
 271                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349\n"
 272             )
 273             pytest.skip(
 274                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349"
 275             )
 276
 277     assert error_logs == "", "Daemons report errors to StdErr"
 278
 279
 280 def test_converge_protocols():
 281     global fatal_error
 282     net = get_topogen().net
 283
 284     # Skip if previous fatal error condition is raised
 285     if fatal_error != "":
 286         pytest.skip(fatal_error)
 287
 288     thisDir = os.path.dirname(os.path.realpath(__file__))
 289
 290     print("\n\n** Waiting for protocols convergence")
 291     print("******************************************\n")
 292
 293     # Not really implemented yet - just sleep 60 secs for now
 294     sleep(5)
 295
 296     # Make sure that all daemons are running
 297     failures = 0
 298     for i in range(1, 2):
 299         fatal_error = net["r%s" % i].checkRouterRunning()
 300         assert fatal_error == "", fatal_error
 301
 302         print("Show that v4 routes are right\n")
 303         v4_routesFile = "%s/r%s/ipv4_routes.ref" % (thisDir, i)
 304         expected = (
 305             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v4_routesFile)).rstrip()
 306         )
 307         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 308
 309         actual = (
 310             net["r%s" % i]
 311             .cmd(
 312                 "vtysh -c \"show ip route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 313             )
 314             .rstrip()
 315         )
 316         # Drop time in last update
 317         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 318         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 319         diff = topotest.get_textdiff(
 320             actual,
 321             expected,
 322             title1="Actual IP Routing Table",
 323             title2="Expected IP RoutingTable",
 324         )
 325         if diff:
 326             sys.stderr.write("r%s failed IP Routing table check:\n%s\n" % (i, diff))
 327             failures += 1
 328         else:
 329             print("r%s ok" % i)
 330
 331         assert failures == 0, "IP Routing table failed for r%s\n%s" % (i, diff)
 332
 333         failures = 0
 334
 335         print("Show that v6 routes are right\n")
 336         v6_routesFile = "%s/r%s/ipv6_routes.ref" % (thisDir, i)
 337         expected = (
 338             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v6_routesFile)).rstrip()
 339         )
 340         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 341
 342         actual = (
 343             net["r%s" % i]
 344             .cmd(
 345                 "vtysh -c \"show ipv6 route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 346             )
 347             .rstrip()
 348         )
 349         # Drop time in last update
 350         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 351         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 352         diff = topotest.get_textdiff(
 353             actual,
 354             expected,
 355             title1="Actual IPv6 Routing Table",
 356             title2="Expected IPv6 RoutingTable",
 357         )
 358         if diff:
 359             sys.stderr.write("r%s failed IPv6 Routing table check:\n%s\n" % (i, diff))
 360             failures += 1
 361         else:
 362             print("r%s ok" % i)
 363
 364         assert failures == 0, "IPv6 Routing table failed for r%s\n%s" % (i, diff)
 365
 366
 367 def route_get_nhg_id(route_str):
 368     net = get_topogen().net
 369     output = net["r1"].cmd('vtysh -c "show ip route %s nexthop-group"' % route_str)
 370     match = re.search(r"Nexthop Group ID: (\d+)", output)
 371     assert match is not None, (
 372         "Nexthop Group ID not found for sharpd route %s" % route_str
 373     )
 374
 375     nhg_id = int(match.group(1))
 376     return nhg_id
 377
 378
 379 def verify_nexthop_group(nhg_id, recursive=False, ecmp=0):
 380     net = get_topogen().net
 381     count = 0
 382     valid = None
 383     ecmpcount = None
 384     depends = None
 385     resolved_id = None
 386     installed = None
 387     found = False
 388
 389     while not found and count < 10:
 390         count += 1
 391         # Verify NHG is valid/installed
 392         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 393         valid = re.search(r"Valid", output)
 394         if valid is None:
 395             found = False
 396             sleep(1)
 397             continue
 398
 399         if ecmp or recursive:
 400             ecmpcount = re.search(r"Depends:.*\n", output)
 401             if ecmpcount is None:
 402                 found = False
 403                 sleep(1)
 404                 continue
 405
 406             # list of IDs in group
 407             depends = re.findall(r"\((\d+)\)", ecmpcount.group(0))
 408
 409             if ecmp:
 410                 if len(depends) != ecmp:
 411                     found = False
 412                     sleep(1)
 413                     continue
 414             else:
 415                 # If recursive, we need to look at its resolved group
 416                 if len(depends) != 1:
 417                     found = False
 418                     sleep(1)
 419                     continue
 420
 421                 resolved_id = int(depends[0])
 422                 verify_nexthop_group(resolved_id, False)
 423         else:
 424             installed = re.search(r"Installed", output)
 425             if installed is None:
 426                 found = False
 427                 sleep(1)
 428                 continue
 429         found = True
 430
 431     assert valid is not None, "Nexthop Group ID=%d not marked Valid" % nhg_id
 432     if ecmp or recursive:
 433         assert ecmpcount is not None, "Nexthop Group ID=%d has no depends" % nhg_id
 434         if ecmp:
 435             assert len(depends) == ecmp, (
 436                 "Nexthop Group ID=%d doesn't match ecmp size" % nhg_id
 437             )
 438         else:
 439             assert len(depends) == 1, (
 440                 "Nexthop Group ID=%d should only have one recursive depend" % nhg_id
 441             )
 442     else:
 443         assert installed is not None, (
 444             "Nexthop Group ID=%d not marked Installed" % nhg_id
 445         )
 446
 447
 448 def verify_route_nexthop_group(route_str, recursive=False, ecmp=0):
 449     # Verify route and that zebra created NHGs for and they are valid/installed
 450     nhg_id = route_get_nhg_id(route_str)
 451     verify_nexthop_group(nhg_id, recursive, ecmp)
 452
 453
 454 def test_nexthop_groups():
 455     global fatal_error
 456     net = get_topogen().net
 457
 458     # Skip if previous fatal error condition is raised
 459     if fatal_error != "":
 460         pytest.skip(fatal_error)
 461
 462     print("\n\n** Verifying Nexthop Groups")
 463     print("******************************************\n")
 464
 465     ### Nexthop Group Tests
 466
 467     ## Basic test
 468
 469     # Create a lib nexthop-group
 470     net["r1"].cmd(
 471         'vtysh -c "c t" -c "nexthop-group basic" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
 472     )
 473
 474     # Create with sharpd using nexthop-group
 475     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.1 nexthop-group basic 1"')
 476     verify_route_nexthop_group("2.2.2.1/32")
 477
 478     ## Connected
 479
 480     net["r1"].cmd(
 481         'vtysh -c "c t" -c "nexthop-group connected" -c "nexthop r1-eth1" -c "nexthop r1-eth2"'
 482     )
 483
 484     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.2 nexthop-group connected 1"')
 485     verify_route_nexthop_group("2.2.2.2/32")
 486
 487     ## Recursive
 488
 489     net["r1"].cmd(
 490         'vtysh -c "c t" -c "nexthop-group basic-recursive" -c "nexthop 2.2.2.1"'
 491     )
 492
 493     net["r1"].cmd(
 494         'vtysh -c "sharp install routes 3.3.3.1 nexthop-group basic-recursive 1"'
 495     )
 496
 497     verify_route_nexthop_group("3.3.3.1/32", True)
 498
 499     ## Duplicate
 500
 501     net["r1"].cmd(
 502         'vtysh -c "c t" -c "nexthop-group duplicate" -c "nexthop 2.2.2.1" -c "nexthop 1.1.1.1"'
 503     )
 504
 505     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.2 nexthop-group duplicate 1"')
 506
 507     verify_route_nexthop_group("3.3.3.2/32")
 508
 509     ## Two 4-Way ECMP
 510
 511     net["r1"].cmd(
 512         'vtysh -c "c t" -c "nexthop-group fourA" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2" \
 513             -c "nexthop 1.1.1.3" -c "nexthop 1.1.1.4"'
 514     )
 515
 516     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.1 nexthop-group fourA 1"')
 517
 518     verify_route_nexthop_group("4.4.4.1/32")
 519
 520     net["r1"].cmd(
 521         'vtysh -c "c t" -c "nexthop-group fourB" -c "nexthop 1.1.1.5" -c "nexthop 1.1.1.6" \
 522             -c "nexthop 1.1.1.7" -c "nexthop 1.1.1.8"'
 523     )
 524
 525     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.2 nexthop-group fourB 1"')
 526
 527     verify_route_nexthop_group("4.4.4.2/32")
 528
 529     ## Recursive to 8-Way ECMP
 530
 531     net["r1"].cmd(
 532         'vtysh -c "c t" -c "nexthop-group eight-recursive" -c "nexthop 4.4.4.1" -c "nexthop 4.4.4.2"'
 533     )
 534
 535     net["r1"].cmd(
 536         'vtysh -c "sharp install routes 5.5.5.1 nexthop-group eight-recursive 1"'
 537     )
 538
 539     verify_route_nexthop_group("5.5.5.1/32")
 540
 541     ## 4-way ECMP Routes Pointing to Each Other
 542
 543     # This is to check for a bug with NH resolution where
 544     # routes would infintely resolve to each other blowing
 545     # up the resolved-> nexthop pointer.
 546
 547     net["r1"].cmd(
 548         'vtysh -c "c t" -c "nexthop-group infinite-recursive" -c "nexthop 6.6.6.1" -c "nexthop 6.6.6.2" \
 549             -c "nexthop 6.6.6.3" -c "nexthop 6.6.6.4"'
 550     )
 551
 552     # static route nexthops can recurse to
 553
 554     net["r1"].cmd('vtysh -c "c t" -c "ip route 6.6.6.0/24 1.1.1.1"')
 555
 556     # Make routes that point to themselves in ecmp
 557
 558     net["r1"].cmd(
 559         'vtysh -c "sharp install routes 6.6.6.4 nexthop-group infinite-recursive 1"'
 560     )
 561     sleep(5)
 562
 563     net["r1"].cmd(
 564         'vtysh -c "sharp install routes 6.6.6.3 nexthop-group infinite-recursive 1"'
 565     )
 566     sleep(5)
 567
 568     net["r1"].cmd(
 569         'vtysh -c "sharp install routes 6.6.6.2 nexthop-group infinite-recursive 1"'
 570     )
 571     sleep(5)
 572
 573     net["r1"].cmd(
 574         'vtysh -c "sharp install routes 6.6.6.1 nexthop-group infinite-recursive 1"'
 575     )
 576
 577     # Get routes and test if has too many (duplicate) nexthops
 578     count = 0
 579     dups = []
 580     nhg_id = route_get_nhg_id("6.6.6.1/32")
 581     while (len(dups) != 3) and count < 10:
 582         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 583
 584         dups = re.findall(r"(via 1\.1\.1\.1)", output)
 585         if len(dups) != 3:
 586             count += 1
 587             sleep(1)
 588
 589     # Should find 3, itself is inactive
 590     assert len(dups) == 3, (
 591         "Route 6.6.6.1/32 with Nexthop Group ID=%d has wrong number of resolved nexthops"
 592         % nhg_id
 593     )
 594
 595     ## Remove all NHG routes
 596
 597     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.1 1"')
 598     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.2 1"')
 599     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.1 1"')
 600     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.2 1"')
 601     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.1 1"')
 602     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.2 1"')
 603     net["r1"].cmd('vtysh -c "sharp remove routes 5.5.5.1 1"')
 604     net["r1"].cmd('vtysh -c "sharp remove routes 6.6.6.1 4"')
 605     net["r1"].cmd('vtysh -c "c t" -c "no ip route 6.6.6.0/24 1.1.1.1"')
 606
 607
 608 def test_rip_status():
 609     global fatal_error
 610     net = get_topogen().net
 611
 612     # Skip if previous fatal error condition is raised
 613     if fatal_error != "":
 614         pytest.skip(fatal_error)
 615
 616     thisDir = os.path.dirname(os.path.realpath(__file__))
 617
 618     print("\n\n** Verifying RIP status")
 619     print("******************************************\n")
 620     failures = 0
 621     for i in range(1, 2):
 622         refTableFile = "%s/r%s/rip_status.ref" % (thisDir, i)
 623         if os.path.isfile(refTableFile):
 624             # Read expected result from file
 625             expected = open(refTableFile).read().rstrip()
 626             # Fix newlines (make them all the same)
 627             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 628
 629             # Actual output from router
 630             actual = (
 631                 net["r%s" % i]
 632                 .cmd('vtysh -c "show ip rip status" 2> /dev/null')
 633                 .rstrip()
 634             )
 635             # Drop time in next due
 636             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 637             # Drop time in last update
 638             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 639             # Fix newlines (make them all the same)
 640             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 641
 642             # Generate Diff
 643             diff = topotest.get_textdiff(
 644                 actual,
 645                 expected,
 646                 title1="actual IP RIP status",
 647                 title2="expected IP RIP status",
 648             )
 649
 650             # Empty string if it matches, otherwise diff contains unified diff
 651             if diff:
 652                 sys.stderr.write("r%s failed IP RIP status check:\n%s\n" % (i, diff))
 653                 failures += 1
 654             else:
 655                 print("r%s ok" % i)
 656
 657             assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff)
 658
 659     # Make sure that all daemons are running
 660     for i in range(1, 2):
 661         fatal_error = net["r%s" % i].checkRouterRunning()
 662         assert fatal_error == "", fatal_error
 663
 664
 665 def test_ripng_status():
 666     global fatal_error
 667     net = get_topogen().net
 668
 669     # Skip if previous fatal error condition is raised
 670     if fatal_error != "":
 671         pytest.skip(fatal_error)
 672
 673     thisDir = os.path.dirname(os.path.realpath(__file__))
 674
 675     print("\n\n** Verifying RIPng status")
 676     print("******************************************\n")
 677     failures = 0
 678     for i in range(1, 2):
 679         refTableFile = "%s/r%s/ripng_status.ref" % (thisDir, i)
 680         if os.path.isfile(refTableFile):
 681             # Read expected result from file
 682             expected = open(refTableFile).read().rstrip()
 683             # Fix newlines (make them all the same)
 684             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 685
 686             # Actual output from router
 687             actual = (
 688                 net["r%s" % i]
 689                 .cmd('vtysh -c "show ipv6 ripng status" 2> /dev/null')
 690                 .rstrip()
 691             )
 692             # Mask out Link-Local mac address portion. They are random...
 693             actual = re.sub(r" fe80::[0-9a-f:]+", " fe80::XXXX:XXXX:XXXX:XXXX", actual)
 694             # Drop time in next due
 695             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 696             # Drop time in last update
 697             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 698             # Fix newlines (make them all the same)
 699             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 700
 701             # Generate Diff
 702             diff = topotest.get_textdiff(
 703                 actual,
 704                 expected,
 705                 title1="actual IPv6 RIPng status",
 706                 title2="expected IPv6 RIPng status",
 707             )
 708
 709             # Empty string if it matches, otherwise diff contains unified diff
 710             if diff:
 711                 sys.stderr.write(
 712                     "r%s failed IPv6 RIPng status check:\n%s\n" % (i, diff)
 713                 )
 714                 failures += 1
 715             else:
 716                 print("r%s ok" % i)
 717
 718             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (
 719                 i,
 720                 diff,
 721             )
 722
 723     # Make sure that all daemons are running
 724     for i in range(1, 2):
 725         fatal_error = net["r%s" % i].checkRouterRunning()
 726         assert fatal_error == "", fatal_error
 727
 728
 729 def test_ospfv2_interfaces():
 730     global fatal_error
 731     net = get_topogen().net
 732
 733     # Skip if previous fatal error condition is raised
 734     if fatal_error != "":
 735         pytest.skip(fatal_error)
 736
 737     thisDir = os.path.dirname(os.path.realpath(__file__))
 738
 739     print("\n\n** Verifying OSPFv2 interfaces")
 740     print("******************************************\n")
 741     failures = 0
 742     for i in range(1, 2):
 743         refTableFile = "%s/r%s/show_ip_ospf_interface.ref" % (thisDir, i)
 744         if os.path.isfile(refTableFile):
 745             # Read expected result from file
 746             expected = open(refTableFile).read().rstrip()
 747             # Fix newlines (make them all the same)
 748             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 749
 750             # Actual output from router
 751             actual = (
 752                 net["r%s" % i]
 753                 .cmd('vtysh -c "show ip ospf interface" 2> /dev/null')
 754                 .rstrip()
 755             )
 756             # Mask out Bandwidth portion. They may change..
 757             actual = re.sub(r"BW [0-9]+ Mbit", "BW XX Mbit", actual)
 758             actual = re.sub(r"ifindex [0-9]+", "ifindex X", actual)
 759
 760             # Drop time in next due
 761             actual = re.sub(r"Hello due in [0-9\.]+s", "Hello due in XX.XXXs", actual)
 762             actual = re.sub(
 763                 r"Hello due in [0-9\.]+ usecs", "Hello due in XX.XXXs", actual
 764             )
 765             # Fix 'MTU mismatch detection: enabled' vs 'MTU mismatch detection:enabled' - accept both
 766             actual = re.sub(
 767                 r"MTU mismatch detection:([a-z]+.*)",
 768                 r"MTU mismatch detection: \1",
 769                 actual,
 770             )
 771             # Fix newlines (make them all the same)
 772             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 773
 774             # Generate Diff
 775             diff = topotest.get_textdiff(
 776                 actual,
 777                 expected,
 778                 title1="actual SHOW IP OSPF INTERFACE",
 779                 title2="expected SHOW IP OSPF INTERFACE",
 780             )
 781
 782             # Empty string if it matches, otherwise diff contains unified diff
 783             if diff:
 784                 sys.stderr.write(
 785                     "r%s failed SHOW IP OSPF INTERFACE check:\n%s\n" % (i, diff)
 786                 )
 787                 failures += 1
 788             else:
 789                 print("r%s ok" % i)
 790
 791             # Ignoring the issue if told to ignore (ie not yet fixed)
 792             if failures != 0:
 793                 if os.environ.get("bamboo_TOPOTESTS_ISSUE_348") == "IGNORE":
 794                     sys.stderr.write(
 795                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348\n"
 796                     )
 797                     pytest.skip(
 798                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348"
 799                     )
 800
 801             assert (
 802                 failures == 0
 803             ), "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff)
 804
 805     # Make sure that all daemons are running
 806     for i in range(1, 2):
 807         fatal_error = net["r%s" % i].checkRouterRunning()
 808         assert fatal_error == "", fatal_error
 809
 810
 811 def test_isis_interfaces():
 812     global fatal_error
 813     net = get_topogen().net
 814
 815     # Skip if previous fatal error condition is raised
 816     if fatal_error != "":
 817         pytest.skip(fatal_error)
 818
 819     thisDir = os.path.dirname(os.path.realpath(__file__))
 820
 821     print("\n\n** Verifying ISIS interfaces")
 822     print("******************************************\n")
 823     failures = 0
 824     for i in range(1, 2):
 825         refTableFile = "%s/r%s/show_isis_interface_detail.ref" % (thisDir, i)
 826         if os.path.isfile(refTableFile):
 827             # Read expected result from file
 828             expected = open(refTableFile).read().rstrip()
 829             # Fix newlines (make them all the same)
 830             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 831
 832             # Actual output from router
 833             actual = (
 834                 net["r%s" % i]
 835                 .cmd('vtysh -c "show isis interface detail" 2> /dev/null')
 836                 .rstrip()
 837             )
 838             # Mask out Link-Local mac address portion. They are random...
 839             actual = re.sub(r"fe80::[0-9a-f:]+", "fe80::XXXX:XXXX:XXXX:XXXX", actual)
 840             # Mask out SNPA mac address portion. They are random...
 841             actual = re.sub(r"SNPA: [0-9a-f\.]+", "SNPA: XXXX.XXXX.XXXX", actual)
 842             # Mask out Circuit ID number
 843             actual = re.sub(r"Circuit Id: 0x[0-9a-f]+", "Circuit Id: 0xXX", actual)
 844             # Fix newlines (make them all the same)
 845             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 846
 847             # Generate Diff
 848             diff = topotest.get_textdiff(
 849                 actual,
 850                 expected,
 851                 title1="actual SHOW ISIS INTERFACE DETAIL",
 852                 title2="expected SHOW ISIS OSPF6 INTERFACE DETAIL",
 853             )
 854
 855             # Empty string if it matches, otherwise diff contains unified diff
 856             if diff:
 857                 sys.stderr.write(
 858                     "r%s failed SHOW ISIS INTERFACE DETAIL check:\n%s\n" % (i, diff)
 859                 )
 860                 failures += 1
 861             else:
 862                 print("r%s ok" % i)
 863
 864             assert (
 865                 failures == 0
 866             ), "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff)
 867
 868     # Make sure that all daemons are running
 869     for i in range(1, 2):
 870         fatal_error = net["r%s" % i].checkRouterRunning()
 871         assert fatal_error == "", fatal_error
 872
 873
 874 def test_bgp_summary():
 875     global fatal_error
 876     net = get_topogen().net
 877
 878     # Skip if previous fatal error condition is raised
 879     if fatal_error != "":
 880         pytest.skip(fatal_error)
 881
 882     thisDir = os.path.dirname(os.path.realpath(__file__))
 883
 884     print("\n\n** Verifying BGP Summary")
 885     print("******************************************\n")
 886     failures = 0
 887     for i in range(1, 2):
 888         refTableFile = "%s/r%s/show_ip_bgp_summary.ref" % (thisDir, i)
 889         if os.path.isfile(refTableFile):
 890             # Read expected result from file
 891             expected_original = open(refTableFile).read().rstrip()
 892
 893             for arguments in [
 894                 "",
 895                 "remote-as internal",
 896                 "remote-as external",
 897                 "remote-as 100",
 898                 "remote-as 123",
 899                 "neighbor 192.168.7.10",
 900                 "neighbor 192.168.7.10",
 901                 "neighbor fc00:0:0:8::1000",
 902                 "neighbor 10.0.0.1",
 903                 "terse",
 904                 "remote-as internal terse",
 905                 "remote-as external terse",
 906                 "remote-as 100 terse",
 907                 "remote-as 123 terse",
 908                 "neighbor 192.168.7.10 terse",
 909                 "neighbor 192.168.7.10 terse",
 910                 "neighbor fc00:0:0:8::1000 terse",
 911                 "neighbor 10.0.0.1 terse",
 912             ]:
 913                 # Actual output from router
 914                 actual = (
 915                     net["r%s" % i]
 916                     .cmd(
 917                         'vtysh -c "show ip bgp summary ' + arguments + '" 2> /dev/null'
 918                     )
 919                     .rstrip()
 920                 )
 921
 922                 # Mask out "using XXiXX bytes" portion. They are random...
 923                 actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
 924                 # Mask out "using XiXXX KiB" portion. They are random...
 925                 actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
 926
 927                 # Remove extra summaries which exist with newer versions
 928
 929                 # Remove summary lines (changed recently)
 930                 actual = re.sub(r"Total number.*", "", actual)
 931                 actual = re.sub(r"Displayed.*", "", actual)
 932                 # Remove IPv4 Unicast Summary (Title only)
 933                 actual = re.sub(r"IPv4 Unicast Summary \(VRF default\):", "", actual)
 934                 # Remove IPv4 Multicast Summary (all of it)
 935                 actual = re.sub(r"IPv4 Multicast Summary \(VRF default\):", "", actual)
 936                 actual = re.sub(r"No IPv4 Multicast neighbor is configured", "", actual)
 937                 # Remove IPv4 VPN Summary (all of it)
 938                 actual = re.sub(r"IPv4 VPN Summary \(VRF default\):", "", actual)
 939                 actual = re.sub(r"No IPv4 VPN neighbor is configured", "", actual)
 940                 # Remove IPv4 Encap Summary (all of it)
 941                 actual = re.sub(r"IPv4 Encap Summary \(VRF default\):", "", actual)
 942                 actual = re.sub(r"No IPv4 Encap neighbor is configured", "", actual)
 943                 # Remove Unknown Summary (all of it)
 944                 actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
 945                 actual = re.sub(r"No Unknown neighbor is configured", "", actual)
 946                 # Make Connect/Active/Idle the same (change them all to Active)
 947                 actual = re.sub(r" Connect ", "  Active ", actual)
 948                 actual = re.sub(r"    Idle ", "  Active ", actual)
 949
 950                 actual = re.sub(
 951                     r"IPv4 labeled-unicast Summary \(VRF default\):", "", actual
 952                 )
 953                 actual = re.sub(
 954                     r"No IPv4 labeled-unicast neighbor is configured", "", actual
 955                 )
 956
 957                 expected = expected_original
 958                 # apply argumentss on expected output
 959                 if "internal" in arguments or "remote-as 100" in arguments:
 960                     expected = re.sub(r".+\s+200\s+.+", "", expected)
 961                 elif "external" in arguments:
 962                     expected = re.sub(r".+\s+100\s+.+Active.+", "", expected)
 963                 elif "remote-as 123" in arguments:
 964                     expected = re.sub(
 965                         r"(192.168.7.(1|2)0|fc00:0:0:8::(1|2)000).+Active.+",
 966                         "",
 967                         expected,
 968                     )
 969                     expected = re.sub(r"\nNeighbor.+Desc", "", expected)
 970                     expected = expected + "% No matching neighbor\n"
 971                 elif "192.168.7.10" in arguments:
 972                     expected = re.sub(
 973                         r"(192.168.7.20|fc00:0:0:8::(1|2)000).+Active.+", "", expected
 974                     )
 975                 elif "fc00:0:0:8::1000" in arguments:
 976                     expected = re.sub(
 977                         r"(192.168.7.(1|2)0|fc00:0:0:8::2000).+Active.+", "", expected
 978                     )
 979                 elif "10.0.0.1" in arguments:
 980                     expected = "No such neighbor in this view/vrf"
 981
 982                 if "terse" in arguments:
 983                     expected = re.sub(r"BGP table version .+", "", expected)
 984                     expected = re.sub(r"RIB entries .+", "", expected)
 985                     expected = re.sub(r"Peers [0-9]+, using .+", "", expected)
 986
 987                 # Strip empty lines
 988                 actual = actual.lstrip().rstrip()
 989                 expected = expected.lstrip().rstrip()
 990                 actual = re.sub(r"\n+", "\n", actual)
 991                 expected = re.sub(r"\n+", "\n", expected)
 992
 993                 # reapply initial formatting
 994                 if "terse" in arguments:
 995                     actual = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", actual)
 996                     expected = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", expected)
 997                 else:
 998                     actual = re.sub(r"KiB of memory\n", "KiB of memory\n\n", actual)
 999                     expected = re.sub(r"KiB of memory\n", "KiB of memory\n\n", expected)
1000
1001                 # realign expected neighbor columns if needed
1002                 try:
1003                     idx_actual = (
1004                         re.search(r"(Neighbor\s+V\s+)", actual).group(1).find("V")
1005                     )
1006                     idx_expected = (
1007                         re.search(r"(Neighbor\s+V\s+)", expected).group(1).find("V")
1008                     )
1009                     idx_diff = idx_expected - idx_actual
1010                     if idx_diff > 0:
1011                         # Neighbor        V         AS MsgRcvd MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd
1012                         expected = re.sub(" " * idx_diff + "V ", "V ", expected)
1013                         # 192.168.7.10    4        100       0       0        0    0    0    never       Active
1014                         expected = re.sub(" " * idx_diff + "4 ", "4 ", expected)
1015                 except AttributeError:
1016                     pass
1017
1018                 # Fix newlines (make them all the same)
1019                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1020                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1021
1022                 # Generate Diff
1023                 diff = topotest.get_textdiff(
1024                     actual,
1025                     expected,
1026                     title1="actual SHOW IP BGP SUMMARY " + arguments.upper(),
1027                     title2="expected SHOW IP BGP SUMMARY " + arguments.upper(),
1028                 )
1029
1030                 # Empty string if it matches, otherwise diff contains unified diff
1031                 if diff:
1032                     sys.stderr.write(
1033                         "r%s failed SHOW IP BGP SUMMARY check:\n%s\n" % (i, diff)
1034                     )
1035                     failures += 1
1036                 else:
1037                     print("r%s ok" % i)
1038
1039                 assert (
1040                     failures == 0
1041                 ), "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (
1042                     i,
1043                     diff,
1044                 )
1045
1046     # Make sure that all daemons are running
1047     for i in range(1, 2):
1048         fatal_error = net["r%s" % i].checkRouterRunning()
1049         assert fatal_error == "", fatal_error
1050
1051
1052 def test_bgp_ipv6_summary():
1053     global fatal_error
1054     net = get_topogen().net
1055
1056     # Skip if previous fatal error condition is raised
1057     if fatal_error != "":
1058         pytest.skip(fatal_error)
1059
1060     thisDir = os.path.dirname(os.path.realpath(__file__))
1061
1062     print("\n\n** Verifying BGP IPv6 Summary")
1063     print("******************************************\n")
1064     failures = 0
1065     for i in range(1, 2):
1066         refTableFile = "%s/r%s/show_bgp_ipv6_summary.ref" % (thisDir, i)
1067         if os.path.isfile(refTableFile):
1068             # Read expected result from file
1069             expected = open(refTableFile).read().rstrip()
1070             # Fix newlines (make them all the same)
1071             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1072
1073             # Actual output from router
1074             actual = (
1075                 net["r%s" % i]
1076                 .cmd('vtysh -c "show bgp ipv6 summary" 2> /dev/null')
1077                 .rstrip()
1078             )
1079             # Mask out "using XXiXX bytes" portion. They are random...
1080             actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
1081             # Mask out "using XiXXX KiB" portion. They are random...
1082             actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
1083             #
1084             # Remove extra summaries which exist with newer versions
1085             #
1086             # Remove summary lines (changed recently)
1087             actual = re.sub(r"Total number.*", "", actual)
1088             actual = re.sub(r"Displayed.*", "", actual)
1089             # Remove IPv4 Unicast Summary (Title only)
1090             actual = re.sub(r"IPv6 Unicast Summary \(VRF default\):", "", actual)
1091             # Remove IPv4 Multicast Summary (all of it)
1092             actual = re.sub(r"IPv6 Multicast Summary \(VRF default\):", "", actual)
1093             actual = re.sub(r"No IPv6 Multicast neighbor is configured", "", actual)
1094             # Remove IPv4 VPN Summary (all of it)
1095             actual = re.sub(r"IPv6 VPN Summary \(VRF default\):", "", actual)
1096             actual = re.sub(r"No IPv6 VPN neighbor is configured", "", actual)
1097             # Remove IPv4 Encap Summary (all of it)
1098             actual = re.sub(r"IPv6 Encap Summary \(VRF default\):", "", actual)
1099             actual = re.sub(r"No IPv6 Encap neighbor is configured", "", actual)
1100             # Remove Unknown Summary (all of it)
1101             actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
1102             actual = re.sub(r"No Unknown neighbor is configured", "", actual)
1103             # Make Connect/Active/Idle the same (change them all to Active)
1104             actual = re.sub(r" Connect ", "  Active ", actual)
1105             actual = re.sub(r"    Idle ", "  Active ", actual)
1106
1107             # Remove Labeled Unicast Summary (all of it)
1108             actual = re.sub(
1109                 r"IPv6 labeled-unicast Summary \(VRF default\):", "", actual
1110             )
1111             actual = re.sub(
1112                 r"No IPv6 labeled-unicast neighbor is configured", "", actual
1113             )
1114
1115             # Strip empty lines
1116             actual = actual.lstrip()
1117             actual = actual.rstrip()
1118             #
1119             # Fix newlines (make them all the same)
1120             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1121
1122             # Generate Diff
1123             diff = topotest.get_textdiff(
1124                 actual,
1125                 expected,
1126                 title1="actual SHOW BGP IPv6 SUMMARY",
1127                 title2="expected SHOW BGP IPv6 SUMMARY",
1128             )
1129
1130             # Empty string if it matches, otherwise diff contains unified diff
1131             if diff:
1132                 sys.stderr.write(
1133                     "r%s failed SHOW BGP IPv6 SUMMARY check:\n%s\n" % (i, diff)
1134                 )
1135                 failures += 1
1136             else:
1137                 print("r%s ok" % i)
1138
1139             assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (
1140                 i,
1141                 diff,
1142             )
1143
1144     # Make sure that all daemons are running
1145     for i in range(1, 2):
1146         fatal_error = net["r%s" % i].checkRouterRunning()
1147         assert fatal_error == "", fatal_error
1148
1149
1150 def test_nht():
1151     global fatal_error
1152     net = get_topogen().net
1153
1154     # Skip if previous fatal error condition is raised
1155     if fatal_error != "":
1156         pytest.skip(fatal_error)
1157
1158     print("\n\n**** Test that nexthop tracking is at least nominally working ****\n")
1159
1160     thisDir = os.path.dirname(os.path.realpath(__file__))
1161
1162     for i in range(1, 2):
1163         nhtFile = "%s/r%s/ip_nht.ref" % (thisDir, i)
1164         expected = open(nhtFile).read().rstrip()
1165         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1166
1167         actual = net["r%s" % i].cmd('vtysh -c "show ip nht" 2> /dev/null').rstrip()
1168         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1169         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1170
1171         diff = topotest.get_textdiff(
1172             actual,
1173             expected,
1174             title1="Actual `show ip nht`",
1175             title2="Expected `show ip nht`",
1176         )
1177
1178         if diff:
1179             assert 0, "r%s failed ip nht check:\n%s\n" % (i, diff)
1180         else:
1181             print("show ip nht is ok\n")
1182
1183         nhtFile = "%s/r%s/ipv6_nht.ref" % (thisDir, i)
1184         expected = open(nhtFile).read().rstrip()
1185         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1186
1187         actual = net["r%s" % i].cmd('vtysh -c "show ipv6 nht" 2> /dev/null').rstrip()
1188         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1189         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1190
1191         diff = topotest.get_textdiff(
1192             actual,
1193             expected,
1194             title1="Actual `show ip nht`",
1195             title2="Expected `show ip nht`",
1196         )
1197
1198         if diff:
1199             assert 0, "r%s failed ipv6 nht check:\n%s\n" % (i, diff)
1200         else:
1201             print("show ipv6 nht is ok\n")
1202
1203
1204 def test_bgp_ipv4():
1205     global fatal_error
1206     net = get_topogen().net
1207
1208     # Skip if previous fatal error condition is raised
1209     if fatal_error != "":
1210         pytest.skip(fatal_error)
1211
1212     thisDir = os.path.dirname(os.path.realpath(__file__))
1213
1214     print("\n\n** Verifying BGP IPv4")
1215     print("******************************************\n")
1216     diffresult = {}
1217     for i in range(1, 2):
1218         success = 0
1219         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv4*.ref" % (thisDir, i)):
1220             if os.path.isfile(refTableFile):
1221                 # Read expected result from file
1222                 expected = open(refTableFile).read().rstrip()
1223                 # Fix newlines (make them all the same)
1224                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1225
1226                 # Actual output from router
1227                 actual = (
1228                     net["r%s" % i].cmd('vtysh -c "show bgp ipv4" 2> /dev/null').rstrip()
1229                 )
1230                 # Remove summary line (changed recently)
1231                 actual = re.sub(r"Total number.*", "", actual)
1232                 actual = re.sub(r"Displayed.*", "", actual)
1233                 actual = actual.rstrip()
1234                 # Fix newlines (make them all the same)
1235                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1236
1237                 # Generate Diff
1238                 diff = topotest.get_textdiff(
1239                     actual,
1240                     expected,
1241                     title1="actual SHOW BGP IPv4",
1242                     title2="expected SHOW BGP IPv4",
1243                 )
1244
1245                 # Empty string if it matches, otherwise diff contains unified diff
1246                 if diff:
1247                     diffresult[refTableFile] = diff
1248                 else:
1249                     success = 1
1250                     print("template %s matched: r%s ok" % (refTableFile, i))
1251                     break
1252
1253         if not success:
1254             resultstr = "No template matched.\n"
1255             for f in diffresult.keys():
1256                 resultstr += "template %s: r%s failed SHOW BGP IPv4 check:\n%s\n" % (
1257                     f,
1258                     i,
1259                     diffresult[f],
1260                 )
1261             raise AssertionError(
1262                 "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, resultstr)
1263             )
1264
1265     # Make sure that all daemons are running
1266     for i in range(1, 2):
1267         fatal_error = net["r%s" % i].checkRouterRunning()
1268         assert fatal_error == "", fatal_error
1269
1270
1271 def test_bgp_ipv6():
1272     global fatal_error
1273     net = get_topogen().net
1274
1275     # Skip if previous fatal error condition is raised
1276     if fatal_error != "":
1277         pytest.skip(fatal_error)
1278
1279     thisDir = os.path.dirname(os.path.realpath(__file__))
1280
1281     print("\n\n** Verifying BGP IPv6")
1282     print("******************************************\n")
1283     diffresult = {}
1284     for i in range(1, 2):
1285         success = 0
1286         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv6*.ref" % (thisDir, i)):
1287             if os.path.isfile(refTableFile):
1288                 # Read expected result from file
1289                 expected = open(refTableFile).read().rstrip()
1290                 # Fix newlines (make them all the same)
1291                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1292
1293                 # Actual output from router
1294                 actual = (
1295                     net["r%s" % i].cmd('vtysh -c "show bgp ipv6" 2> /dev/null').rstrip()
1296                 )
1297                 # Remove summary line (changed recently)
1298                 actual = re.sub(r"Total number.*", "", actual)
1299                 actual = re.sub(r"Displayed.*", "", actual)
1300                 actual = actual.rstrip()
1301                 # Fix newlines (make them all the same)
1302                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1303
1304                 # Generate Diff
1305                 diff = topotest.get_textdiff(
1306                     actual,
1307                     expected,
1308                     title1="actual SHOW BGP IPv6",
1309                     title2="expected SHOW BGP IPv6",
1310                 )
1311
1312                 # Empty string if it matches, otherwise diff contains unified diff
1313                 if diff:
1314                     diffresult[refTableFile] = diff
1315                 else:
1316                     success = 1
1317                     print("template %s matched: r%s ok" % (refTableFile, i))
1318
1319         if not success:
1320             resultstr = "No template matched.\n"
1321             for f in diffresult.keys():
1322                 resultstr += "template %s: r%s failed SHOW BGP IPv6 check:\n%s\n" % (
1323                     f,
1324                     i,
1325                     diffresult[f],
1326                 )
1327             raise AssertionError(
1328                 "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, resultstr)
1329             )
1330
1331     # Make sure that all daemons are running
1332     for i in range(1, 2):
1333         fatal_error = net["r%s" % i].checkRouterRunning()
1334         assert fatal_error == "", fatal_error
1335
1336
1337 def test_route_map():
1338     global fatal_error
1339     net = get_topogen().net
1340
1341     if fatal_error != "":
1342         pytest.skip(fatal_error)
1343
1344     thisDir = os.path.dirname(os.path.realpath(__file__))
1345
1346     print("\n\n** Verifying some basic routemap forward references\n")
1347     print("*******************************************************\n")
1348     failures = 0
1349     for i in range(1, 2):
1350         refroutemap = "%s/r%s/show_route_map.ref" % (thisDir, i)
1351         if os.path.isfile(refroutemap):
1352             expected = open(refroutemap).read().rstrip()
1353             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1354
1355             actual = (
1356                 net["r%s" % i].cmd('vtysh -c "show route-map" 2> /dev/null').rstrip()
1357             )
1358             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1359
1360             diff = topotest.get_textdiff(
1361                 actual,
1362                 expected,
1363                 title1="actual show route-map",
1364                 title2="expected show route-map",
1365             )
1366
1367             if diff:
1368                 sys.stderr.write(
1369                     "r%s failed show route-map command Check:\n%s\n" % (i, diff)
1370                 )
1371                 failures += 1
1372             else:
1373                 print("r%s ok" % i)
1374
1375             assert (
1376                 failures == 0
1377             ), "Show route-map command failed for router r%s:\n%s" % (i, diff)
1378
1379
1380 def test_nexthop_groups_with_route_maps():
1381     global fatal_error
1382     net = get_topogen().net
1383
1384     # Skip if previous fatal error condition is raised
1385     if fatal_error != "":
1386         pytest.skip(fatal_error)
1387
1388     print("\n\n** Verifying Nexthop Groups With Route-Maps")
1389     print("******************************************\n")
1390
1391     ### Nexthop Group With Route-Map Tests
1392
1393     # Create a lib nexthop-group
1394     net["r1"].cmd(
1395         'vtysh -c "c t" -c "nexthop-group test" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
1396     )
1397
1398     ## Route-Map Proto Source
1399
1400     route_str = "2.2.2.1"
1401     src_str = "192.168.0.1"
1402
1403     net["r1"].cmd(
1404         'vtysh -c "c t" -c "route-map NH-SRC permit 111" -c "set src %s"' % src_str
1405     )
1406     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NH-SRC"')
1407
1408     net["r1"].cmd('vtysh -c "sharp install routes %s nexthop-group test 1"' % route_str)
1409
1410     verify_route_nexthop_group("%s/32" % route_str)
1411
1412     # Only a valid test on linux using nexthop objects
1413     if sys.platform.startswith("linux"):
1414         output = net["r1"].cmd("ip route show %s/32" % route_str)
1415         match = re.search(r"src %s" % src_str, output)
1416         assert match is not None, "Route %s/32 not installed with src %s" % (
1417             route_str,
1418             src_str,
1419         )
1420
1421     # Remove NHG routes and route-map
1422     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % route_str)
1423     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NH-SRC"')
1424     net["r1"].cmd(
1425         'vtysh -c "c t" -c "no route-map NH-SRC permit 111" # -c "set src %s"' % src_str
1426     )
1427     net["r1"].cmd('vtysh -c "c t" -c "no route-map NH-SRC"')
1428
1429     ## Route-Map Deny/Permit with same nexthop group
1430
1431     permit_route_str = "3.3.3.1"
1432     deny_route_str = "3.3.3.2"
1433
1434     net["r1"].cmd(
1435         'vtysh -c "c t" -c "ip prefix-list NOPE seq 5 permit %s/32"' % permit_route_str
1436     )
1437     net["r1"].cmd(
1438         'vtysh -c "c t" -c "route-map NOPE permit 111" -c "match ip address prefix-list NOPE"'
1439     )
1440     net["r1"].cmd('vtysh -c "c t" -c "route-map NOPE deny 222"')
1441     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NOPE"')
1442
1443     # This route should be permitted
1444     net["r1"].cmd(
1445         'vtysh -c "sharp install routes %s nexthop-group test 1"' % permit_route_str
1446     )
1447
1448     verify_route_nexthop_group("%s/32" % permit_route_str)
1449
1450     # This route should be denied
1451     net["r1"].cmd(
1452         'vtysh -c "sharp install routes %s nexthop-group test 1"' % deny_route_str
1453     )
1454
1455     nhg_id = route_get_nhg_id(deny_route_str)
1456     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
1457
1458     match = re.search(r"Valid", output)
1459     assert match is None, "Nexthop Group ID=%d should not be marked Valid" % nhg_id
1460
1461     match = re.search(r"Installed", output)
1462     assert match is None, "Nexthop Group ID=%d should not be marked Installed" % nhg_id
1463
1464     # Remove NHG routes and route-map
1465     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % permit_route_str)
1466     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % deny_route_str)
1467     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NOPE"')
1468     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE permit 111"')
1469     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE deny 222"')
1470     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE"')
1471     net["r1"].cmd(
1472         'vtysh -c "c t" -c "no ip prefix-list NOPE seq 5 permit %s/32"'
1473         % permit_route_str
1474     )
1475
1476
1477 def test_nexthop_group_replace():
1478     global fatal_error
1479     net = get_topogen().net
1480
1481     # Skip if previous fatal error condition is raised
1482     if fatal_error != "":
1483         pytest.skip(fatal_error)
1484
1485     print("\n\n** Verifying Nexthop Groups")
1486     print("******************************************\n")
1487
1488     ### Nexthop Group Tests
1489
1490     ## 2-Way ECMP Directly Connected
1491
1492     net["r1"].cmd(
1493         'vtysh -c "c t" -c "nexthop-group replace" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1494     )
1495
1496     # At the moment there is absolutely no real easy way to query sharpd
1497     # for the nexthop group actually installed.  If it is not installed
1498     # sharpd will just transmit the nexthops down instead of the nexthop
1499     # group id.  Leading to a situation where the replace is not actually
1500     # being tested.  So let's just wait some time here because this
1501     # is hard and this test fails all the time
1502     sleep(5)
1503
1504     # Create with sharpd using nexthop-group
1505     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.1 nexthop-group replace 1"')
1506
1507     verify_route_nexthop_group("3.3.3.1/32")
1508
1509     # Change the nexthop group
1510     net["r1"].cmd(
1511         'vtysh -c "c t" -c "nexthop-group replace" -c "no nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.3 r1-eth1 onlink" -c "nexthop 1.1.1.4 r1-eth4 onlink"'
1512     )
1513
1514     # Verify it updated. We can just check install and ecmp count here.
1515     verify_route_nexthop_group("3.3.3.1/32", False, 3)
1516
1517
1518 def test_mpls_interfaces():
1519     global fatal_error
1520     net = get_topogen().net
1521
1522     # Skip if previous fatal error condition is raised
1523     if fatal_error != "":
1524         pytest.skip(fatal_error)
1525
1526     # Skip if no LDP installed or old kernel
1527     if net["r1"].daemon_available("ldpd") == False:
1528         pytest.skip("No MPLS or kernel < 4.5")
1529
1530     thisDir = os.path.dirname(os.path.realpath(__file__))
1531
1532     print("\n\n** Verifying MPLS Interfaces")
1533     print("******************************************\n")
1534     failures = 0
1535     for i in range(1, 2):
1536         refTableFile = "%s/r%s/show_mpls_ldp_interface.ref" % (thisDir, i)
1537         if os.path.isfile(refTableFile):
1538             # Read expected result from file
1539             expected = open(refTableFile).read().rstrip()
1540             # Fix newlines (make them all the same)
1541             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1542
1543             # Actual output from router
1544             actual = (
1545                 net["r%s" % i]
1546                 .cmd('vtysh -c "show mpls ldp interface" 2> /dev/null')
1547                 .rstrip()
1548             )
1549             # Mask out Timer in Uptime
1550             actual = re.sub(r" [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ", " xx:xx:xx ", actual)
1551             # Fix newlines (make them all the same)
1552             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1553
1554             # Generate Diff
1555             diff = topotest.get_textdiff(
1556                 actual,
1557                 expected,
1558                 title1="actual MPLS LDP interface status",
1559                 title2="expected MPLS LDP interface status",
1560             )
1561
1562             # Empty string if it matches, otherwise diff contains unified diff
1563             if diff:
1564                 sys.stderr.write(
1565                     "r%s failed MPLS LDP Interface status Check:\n%s\n" % (i, diff)
1566                 )
1567                 failures += 1
1568             else:
1569                 print("r%s ok" % i)
1570
1571             if failures > 0:
1572                 fatal_error = "MPLS LDP Interface status failed"
1573
1574             assert (
1575                 failures == 0
1576             ), "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
1577
1578     # Make sure that all daemons are running
1579     for i in range(1, 2):
1580         fatal_error = net["r%s" % i].checkRouterRunning()
1581         assert fatal_error == "", fatal_error
1582
1583
1584 def test_resilient_nexthop_group():
1585     net = get_topogen().net
1586
1587     result = required_linux_kernel_version("5.19")
1588     if result is not True:
1589         pytest.skip("Kernel requirements are not met, kernel version should be >= 5.19")
1590
1591     net["r1"].cmd(
1592         'vtysh -c "conf" -c "nexthop-group resilience" -c "resilient buckets 64 idle-timer 128 unbalanced-timer 256" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1593     )
1594
1595     output = net["r1"].cmd('vtysh -c "show nexthop-group rib sharp"')
1596     output = re.findall(r"Buckets", output)
1597
1598     verify_nexthop_group(185483878)
1599     assert len(output) == 1, "Resilient NHG not created in zebra"
1600
1601
1602 def test_shutdown_check_stderr():
1603     global fatal_error
1604     net = get_topogen().net
1605
1606     # Skip if previous fatal error condition is raised
1607     if fatal_error != "":
1608         pytest.skip(fatal_error)
1609
1610     print("\n\n** Verifying unexpected STDERR output from daemons")
1611     print("******************************************\n")
1612
1613     if os.environ.get("TOPOTESTS_CHECK_STDERR") is None:
1614         print(
1615             "SKIPPED final check on StdErr output: Disabled (TOPOTESTS_CHECK_STDERR undefined)\n"
1616         )
1617         pytest.skip("Skipping test for Stderr output")
1618
1619     thisDir = os.path.dirname(os.path.realpath(__file__))
1620
1621     print("thisDir=" + thisDir)
1622
1623     net["r1"].stopRouter()
1624
1625     log = net["r1"].getStdErr("ripd")
1626     if log:
1627         print("\nRIPd StdErr Log:\n" + log)
1628     log = net["r1"].getStdErr("ripngd")
1629     if log:
1630         print("\nRIPngd StdErr Log:\n" + log)
1631     log = net["r1"].getStdErr("ospfd")
1632     if log:
1633         print("\nOSPFd StdErr Log:\n" + log)
1634     log = net["r1"].getStdErr("ospf6d")
1635     if log:
1636         print("\nOSPF6d StdErr Log:\n" + log)
1637     log = net["r1"].getStdErr("isisd")
1638     if log:
1639         print("\nISISd StdErr Log:\n" + log)
1640     log = net["r1"].getStdErr("bgpd")
1641     if log:
1642         print("\nBGPd StdErr Log:\n" + log)
1643
1644     log = net["r1"].getStdErr("nhrpd")
1645     if log:
1646         print("\nNHRPd StdErr Log:\n" + log)
1647
1648     log = net["r1"].getStdErr("pbrd")
1649     if log:
1650         print("\nPBRd StdErr Log:\n" + log)
1651
1652     log = net["r1"].getStdErr("babeld")
1653     if log:
1654         print("\nBABELd StdErr Log:\n" + log)
1655
1656     if net["r1"].daemon_available("ldpd"):
1657         log = net["r1"].getStdErr("ldpd")
1658         if log:
1659             print("\nLDPd StdErr Log:\n" + log)
1660     log = net["r1"].getStdErr("zebra")
1661     if log:
1662         print("\nZebra StdErr Log:\n" + log)
1663
1664
1665 def test_shutdown_check_memleak():
1666     global fatal_error
1667     net = get_topogen().net
1668
1669     # Skip if previous fatal error condition is raised
1670     if fatal_error != "":
1671         pytest.skip(fatal_error)
1672
1673     if os.environ.get("TOPOTESTS_CHECK_MEMLEAK") is None:
1674         print(
1675             "SKIPPED final check on Memory leaks: Disabled (TOPOTESTS_CHECK_MEMLEAK undefined)\n"
1676         )
1677         pytest.skip("Skipping test for memory leaks")
1678
1679     thisDir = os.path.dirname(os.path.realpath(__file__))
1680
1681     for i in range(1, 2):
1682         net["r%s" % i].stopRouter()
1683         net["r%s" % i].report_memory_leaks(
1684             os.environ.get("TOPOTESTS_CHECK_MEMLEAK"), os.path.basename(__file__)
1685         )
1686
1687
1688 if __name__ == "__main__":
1689     # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
1690     # retval = pytest.main(["-s", "--tb=no"])
1691     retval = pytest.main(["-s"])
1692     sys.exit(retval)