tests/topotests/all_protocol_startup/test_all_protocol_startup.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # test_all_protocol_startup.py
   5 # Part of NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2017 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 """
  26 test_all_protocol_startup.py: Test of all protocols at same time
  27
  28 """
  29
  30 import os
  31 import re
  32 import sys
  33 import pytest
  34 import glob
  35 from time import sleep
  36
  37 pytestmark = [
  38     pytest.mark.babeld,
  39     pytest.mark.bgpd,
  40     pytest.mark.isisd,
  41     pytest.mark.nhrpd,
  42     pytest.mark.ospfd,
  43     pytest.mark.pbrd,
  44     pytest.mark.ripd,
  45 ]
  46
  47 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
  48 from lib import topotest
  49 from lib.topogen import Topogen, get_topogen
  50 from lib.common_config import (
  51     required_linux_kernel_version,
  52 )
  53
  54 fatal_error = ""
  55
  56
  57 #####################################################
  58 ##
  59 ##   Network Topology Definition
  60 ##
  61 #####################################################
  62
  63
  64 def build_topo(tgen):
  65     router = tgen.add_router("r1")
  66     for i in range(0, 10):
  67         tgen.add_switch("sw%d" % i).add_link(router)
  68
  69
  70 #####################################################
  71 ##
  72 ##   Tests starting
  73 ##
  74 #####################################################
  75
  76
  77 def setup_module(module):
  78     global fatal_error
  79
  80     print("\n\n** %s: Setup Topology" % module.__name__)
  81     print("******************************************\n")
  82
  83     thisDir = os.path.dirname(os.path.realpath(__file__))
  84     tgen = Topogen(build_topo, module.__name__)
  85     tgen.start_topology()
  86
  87     net = tgen.net
  88
  89     if net["r1"].get_routertype() != "frr":
  90         fatal_error = "Test is only implemented for FRR"
  91         sys.stderr.write("\n\nTest is only implemented for FRR - Skipping\n\n")
  92         pytest.skip(fatal_error)
  93
  94     # Starting Routers
  95     #
  96     # Main router
  97     for i in range(1, 2):
  98         net["r%s" % i].loadConf("zebra", "%s/r%s/zebra.conf" % (thisDir, i))
  99         net["r%s" % i].loadConf("ripd", "%s/r%s/ripd.conf" % (thisDir, i))
 100         net["r%s" % i].loadConf("ripngd", "%s/r%s/ripngd.conf" % (thisDir, i))
 101         net["r%s" % i].loadConf("ospfd", "%s/r%s/ospfd.conf" % (thisDir, i))
 102         if net["r1"].checkRouterVersion("<", "4.0"):
 103             net["r%s" % i].loadConf(
 104                 "ospf6d", "%s/r%s/ospf6d.conf-pre-v4" % (thisDir, i)
 105             )
 106         else:
 107             net["r%s" % i].loadConf("ospf6d", "%s/r%s/ospf6d.conf" % (thisDir, i))
 108         net["r%s" % i].loadConf("isisd", "%s/r%s/isisd.conf" % (thisDir, i))
 109         net["r%s" % i].loadConf("bgpd", "%s/r%s/bgpd.conf" % (thisDir, i))
 110         if net["r%s" % i].daemon_available("ldpd"):
 111             # Only test LDPd if it's installed and Kernel >= 4.5
 112             net["r%s" % i].loadConf("ldpd", "%s/r%s/ldpd.conf" % (thisDir, i))
 113         net["r%s" % i].loadConf("sharpd")
 114         net["r%s" % i].loadConf("nhrpd", "%s/r%s/nhrpd.conf" % (thisDir, i))
 115         net["r%s" % i].loadConf("babeld", "%s/r%s/babeld.conf" % (thisDir, i))
 116         net["r%s" % i].loadConf("pbrd", "%s/r%s/pbrd.conf" % (thisDir, i))
 117         tgen.gears["r%s" % i].start()
 118
 119     # For debugging after starting FRR daemons, uncomment the next line
 120     # tgen.mininet_cli()
 121
 122
 123 def teardown_module(module):
 124     print("\n\n** %s: Shutdown Topology" % module.__name__)
 125     print("******************************************\n")
 126     tgen = get_topogen()
 127     tgen.stop_topology()
 128
 129
 130 def test_router_running():
 131     global fatal_error
 132     tgen = get_topogen()
 133     net = tgen.net
 134
 135     # Skip if previous fatal error condition is raised
 136     if fatal_error != "":
 137         pytest.skip(fatal_error)
 138
 139     print("\n\n** Check if FRR is running on each Router node")
 140     print("******************************************\n")
 141     sleep(5)
 142
 143     # Starting Routers
 144     for i in range(1, 2):
 145         fatal_error = net["r%s" % i].checkRouterRunning()
 146         assert fatal_error == "", fatal_error
 147
 148     # For debugging after starting FRR daemons, uncomment the next line
 149     # tgen.mininet_cli()
 150
 151
 152 def test_error_messages_vtysh():
 153     global fatal_error
 154     net = get_topogen().net
 155
 156     # Skip if previous fatal error condition is raised
 157     if fatal_error != "":
 158         pytest.skip(fatal_error)
 159
 160     print("\n\n** Check for error messages on VTYSH")
 161     print("******************************************\n")
 162
 163     failures = 0
 164     for i in range(1, 2):
 165         #
 166         # First checking Standard Output
 167         #
 168
 169         # VTYSH output from router
 170         vtystdout = net["r%s" % i].cmd('vtysh -c "show version" 2> /dev/null').rstrip()
 171
 172         # Fix newlines (make them all the same)
 173         vtystdout = ("\n".join(vtystdout.splitlines()) + "\n").rstrip()
 174         # Drop everything starting with "FRRouting X.xx" message
 175         vtystdout = re.sub(r"FRRouting [0-9]+.*", "", vtystdout, flags=re.DOTALL)
 176
 177         if vtystdout == "":
 178             print("r%s StdOut ok" % i)
 179
 180         assert vtystdout == "", "Vtysh StdOut Output check failed for router r%s" % i
 181
 182         #
 183         # Second checking Standard Error
 184         #
 185
 186         # VTYSH StdErr output from router
 187         vtystderr = net["r%s" % i].cmd('vtysh -c "show version" > /dev/null').rstrip()
 188
 189         # Fix newlines (make them all the same)
 190         vtystderr = ("\n".join(vtystderr.splitlines()) + "\n").rstrip()
 191         # # Drop everything starting with "FRRouting X.xx" message
 192         # vtystderr = re.sub(r"FRRouting [0-9]+.*", "", vtystderr, flags=re.DOTALL)
 193
 194         if vtystderr == "":
 195             print("r%s StdErr ok" % i)
 196
 197         assert vtystderr == "", "Vtysh StdErr Output check failed for router r%s" % i
 198
 199     # Make sure that all daemons are running
 200     for i in range(1, 2):
 201         fatal_error = net["r%s" % i].checkRouterRunning()
 202         assert fatal_error == "", fatal_error
 203
 204
 205 def test_error_messages_daemons():
 206     global fatal_error
 207     net = get_topogen().net
 208
 209     # Skip if previous fatal error condition is raised
 210     if fatal_error != "":
 211         pytest.skip(fatal_error)
 212
 213     print("\n\n** Check for error messages in daemons")
 214     print("******************************************\n")
 215
 216     error_logs = ""
 217
 218     for i in range(1, 2):
 219         log = net["r%s" % i].getStdErr("ripd")
 220         if log:
 221             error_logs += "r%s RIPd StdErr Output:\n" % i
 222             error_logs += log
 223         log = net["r%s" % i].getStdErr("ripngd")
 224         if log:
 225             error_logs += "r%s RIPngd StdErr Output:\n" % i
 226             error_logs += log
 227         log = net["r%s" % i].getStdErr("ospfd")
 228         if log:
 229             error_logs += "r%s OSPFd StdErr Output:\n" % i
 230             error_logs += log
 231         log = net["r%s" % i].getStdErr("ospf6d")
 232         if log:
 233             error_logs += "r%s OSPF6d StdErr Output:\n" % i
 234             error_logs += log
 235         log = net["r%s" % i].getStdErr("isisd")
 236         # ISIS shows debugging enabled status on StdErr
 237         # Remove these messages
 238         log = re.sub(r"^IS-IS .* debugging is on.*", "", log).rstrip()
 239         if log:
 240             error_logs += "r%s ISISd StdErr Output:\n" % i
 241             error_logs += log
 242         log = net["r%s" % i].getStdErr("bgpd")
 243         if log:
 244             error_logs += "r%s BGPd StdErr Output:\n" % i
 245             error_logs += log
 246         if net["r%s" % i].daemon_available("ldpd"):
 247             log = net["r%s" % i].getStdErr("ldpd")
 248             if log:
 249                 error_logs += "r%s LDPd StdErr Output:\n" % i
 250                 error_logs += log
 251
 252         log = net["r1"].getStdErr("nhrpd")
 253         # NHRPD shows YANG model not embedded messages
 254         # Ignore these
 255         log = re.sub(r".*YANG model.*not embedded.*", "", log).rstrip()
 256         if log:
 257             error_logs += "r%s NHRPd StdErr Output:\n" % i
 258             error_logs += log
 259
 260         log = net["r1"].getStdErr("babeld")
 261         if log:
 262             error_logs += "r%s BABELd StdErr Output:\n" % i
 263             error_logs += log
 264
 265         log = net["r1"].getStdErr("pbrd")
 266         if log:
 267             error_logs += "r%s PBRd StdErr Output:\n" % i
 268             error_logs += log
 269
 270         log = net["r%s" % i].getStdErr("zebra")
 271         if log:
 272             error_logs += "r%s Zebra StdErr Output:\n" % i
 273             error_logs += log
 274
 275     if error_logs:
 276         sys.stderr.write(
 277             "Failed check for StdErr Output on daemons:\n%s\n" % error_logs
 278         )
 279
 280     # Ignoring the issue if told to ignore (ie not yet fixed)
 281     if error_logs != "":
 282         if os.environ.get("bamboo_TOPOTESTS_ISSUE_349") == "IGNORE":
 283             sys.stderr.write(
 284                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349\n"
 285             )
 286             pytest.skip(
 287                 "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/349"
 288             )
 289
 290     assert error_logs == "", "Daemons report errors to StdErr"
 291
 292
 293 def test_converge_protocols():
 294     global fatal_error
 295     net = get_topogen().net
 296
 297     # Skip if previous fatal error condition is raised
 298     if fatal_error != "":
 299         pytest.skip(fatal_error)
 300
 301     thisDir = os.path.dirname(os.path.realpath(__file__))
 302
 303     print("\n\n** Waiting for protocols convergence")
 304     print("******************************************\n")
 305
 306     # Not really implemented yet - just sleep 60 secs for now
 307     sleep(5)
 308
 309     # Make sure that all daemons are running
 310     failures = 0
 311     for i in range(1, 2):
 312         fatal_error = net["r%s" % i].checkRouterRunning()
 313         assert fatal_error == "", fatal_error
 314
 315         print("Show that v4 routes are right\n")
 316         v4_routesFile = "%s/r%s/ipv4_routes.ref" % (thisDir, i)
 317         expected = (
 318             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v4_routesFile)).rstrip()
 319         )
 320         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 321
 322         actual = (
 323             net["r%s" % i]
 324             .cmd(
 325                 "vtysh -c \"show ip route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 326             )
 327             .rstrip()
 328         )
 329         # Drop time in last update
 330         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 331         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 332         diff = topotest.get_textdiff(
 333             actual,
 334             expected,
 335             title1="Actual IP Routing Table",
 336             title2="Expected IP RoutingTable",
 337         )
 338         if diff:
 339             sys.stderr.write("r%s failed IP Routing table check:\n%s\n" % (i, diff))
 340             failures += 1
 341         else:
 342             print("r%s ok" % i)
 343
 344         assert failures == 0, "IP Routing table failed for r%s\n%s" % (i, diff)
 345
 346         failures = 0
 347
 348         print("Show that v6 routes are right\n")
 349         v6_routesFile = "%s/r%s/ipv6_routes.ref" % (thisDir, i)
 350         expected = (
 351             net["r%s" % i].cmd("sort {} 2> /dev/null".format(v6_routesFile)).rstrip()
 352         )
 353         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 354
 355         actual = (
 356             net["r%s" % i]
 357             .cmd(
 358                 "vtysh -c \"show ipv6 route\" | sed -e '/^Codes: /,/^\\s*$/d' | sort 2> /dev/null"
 359             )
 360             .rstrip()
 361         )
 362         # Drop time in last update
 363         actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 364         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 365         diff = topotest.get_textdiff(
 366             actual,
 367             expected,
 368             title1="Actual IPv6 Routing Table",
 369             title2="Expected IPv6 RoutingTable",
 370         )
 371         if diff:
 372             sys.stderr.write("r%s failed IPv6 Routing table check:\n%s\n" % (i, diff))
 373             failures += 1
 374         else:
 375             print("r%s ok" % i)
 376
 377         assert failures == 0, "IPv6 Routing table failed for r%s\n%s" % (i, diff)
 378
 379
 380 def route_get_nhg_id(route_str):
 381     net = get_topogen().net
 382     output = net["r1"].cmd('vtysh -c "show ip route %s nexthop-group"' % route_str)
 383     match = re.search(r"Nexthop Group ID: (\d+)", output)
 384     assert match is not None, (
 385         "Nexthop Group ID not found for sharpd route %s" % route_str
 386     )
 387
 388     nhg_id = int(match.group(1))
 389     return nhg_id
 390
 391
 392 def verify_nexthop_group(nhg_id, recursive=False, ecmp=0):
 393     net = get_topogen().net
 394     count = 0
 395     valid = None
 396     ecmpcount = None
 397     depends = None
 398     resolved_id = None
 399     installed = None
 400     found = False
 401
 402     while not found and count < 10:
 403         count += 1
 404         # Verify NHG is valid/installed
 405         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 406         valid = re.search(r"Valid", output)
 407         if valid is None:
 408             found = False
 409             sleep(1)
 410             continue
 411
 412         if ecmp or recursive:
 413             ecmpcount = re.search(r"Depends:.*\n", output)
 414             if ecmpcount is None:
 415                 found = False
 416                 sleep(1)
 417                 continue
 418
 419             # list of IDs in group
 420             depends = re.findall(r"\((\d+)\)", ecmpcount.group(0))
 421
 422             if ecmp:
 423                 if len(depends) != ecmp:
 424                     found = False
 425                     sleep(1)
 426                     continue
 427             else:
 428                 # If recursive, we need to look at its resolved group
 429                 if len(depends) != 1:
 430                     found = False
 431                     sleep(1)
 432                     continue
 433
 434                 resolved_id = int(depends[0])
 435                 verify_nexthop_group(resolved_id, False)
 436         else:
 437             installed = re.search(r"Installed", output)
 438             if installed is None:
 439                 found = False
 440                 sleep(1)
 441                 continue
 442         found = True
 443
 444     assert valid is not None, "Nexthop Group ID=%d not marked Valid" % nhg_id
 445     if ecmp or recursive:
 446         assert ecmpcount is not None, "Nexthop Group ID=%d has no depends" % nhg_id
 447         if ecmp:
 448             assert len(depends) == ecmp, (
 449                 "Nexthop Group ID=%d doesn't match ecmp size" % nhg_id
 450             )
 451         else:
 452             assert len(depends) == 1, (
 453                 "Nexthop Group ID=%d should only have one recursive depend" % nhg_id
 454             )
 455     else:
 456         assert installed is not None, (
 457             "Nexthop Group ID=%d not marked Installed" % nhg_id
 458         )
 459
 460
 461 def verify_route_nexthop_group(route_str, recursive=False, ecmp=0):
 462     # Verify route and that zebra created NHGs for and they are valid/installed
 463     nhg_id = route_get_nhg_id(route_str)
 464     verify_nexthop_group(nhg_id, recursive, ecmp)
 465
 466
 467 def test_nexthop_groups():
 468     global fatal_error
 469     net = get_topogen().net
 470
 471     # Skip if previous fatal error condition is raised
 472     if fatal_error != "":
 473         pytest.skip(fatal_error)
 474
 475     print("\n\n** Verifying Nexthop Groups")
 476     print("******************************************\n")
 477
 478     ### Nexthop Group Tests
 479
 480     ## Basic test
 481
 482     # Create a lib nexthop-group
 483     net["r1"].cmd(
 484         'vtysh -c "c t" -c "nexthop-group basic" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
 485     )
 486
 487     # Create with sharpd using nexthop-group
 488     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.1 nexthop-group basic 1"')
 489     verify_route_nexthop_group("2.2.2.1/32")
 490
 491     ## Connected
 492
 493     net["r1"].cmd(
 494         'vtysh -c "c t" -c "nexthop-group connected" -c "nexthop r1-eth1" -c "nexthop r1-eth2"'
 495     )
 496
 497     net["r1"].cmd('vtysh -c "sharp install routes 2.2.2.2 nexthop-group connected 1"')
 498     verify_route_nexthop_group("2.2.2.2/32")
 499
 500     ## Recursive
 501
 502     net["r1"].cmd(
 503         'vtysh -c "c t" -c "nexthop-group basic-recursive" -c "nexthop 2.2.2.1"'
 504     )
 505
 506     net["r1"].cmd(
 507         'vtysh -c "sharp install routes 3.3.3.1 nexthop-group basic-recursive 1"'
 508     )
 509
 510     verify_route_nexthop_group("3.3.3.1/32", True)
 511
 512     ## Duplicate
 513
 514     net["r1"].cmd(
 515         'vtysh -c "c t" -c "nexthop-group duplicate" -c "nexthop 2.2.2.1" -c "nexthop 1.1.1.1"'
 516     )
 517
 518     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.2 nexthop-group duplicate 1"')
 519
 520     verify_route_nexthop_group("3.3.3.2/32")
 521
 522     ## Two 4-Way ECMP
 523
 524     net["r1"].cmd(
 525         'vtysh -c "c t" -c "nexthop-group fourA" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2" \
 526             -c "nexthop 1.1.1.3" -c "nexthop 1.1.1.4"'
 527     )
 528
 529     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.1 nexthop-group fourA 1"')
 530
 531     verify_route_nexthop_group("4.4.4.1/32")
 532
 533     net["r1"].cmd(
 534         'vtysh -c "c t" -c "nexthop-group fourB" -c "nexthop 1.1.1.5" -c "nexthop 1.1.1.6" \
 535             -c "nexthop 1.1.1.7" -c "nexthop 1.1.1.8"'
 536     )
 537
 538     net["r1"].cmd('vtysh -c "sharp install routes 4.4.4.2 nexthop-group fourB 1"')
 539
 540     verify_route_nexthop_group("4.4.4.2/32")
 541
 542     ## Recursive to 8-Way ECMP
 543
 544     net["r1"].cmd(
 545         'vtysh -c "c t" -c "nexthop-group eight-recursive" -c "nexthop 4.4.4.1" -c "nexthop 4.4.4.2"'
 546     )
 547
 548     net["r1"].cmd(
 549         'vtysh -c "sharp install routes 5.5.5.1 nexthop-group eight-recursive 1"'
 550     )
 551
 552     verify_route_nexthop_group("5.5.5.1/32")
 553
 554     ## 4-way ECMP Routes Pointing to Each Other
 555
 556     # This is to check for a bug with NH resolution where
 557     # routes would infintely resolve to each other blowing
 558     # up the resolved-> nexthop pointer.
 559
 560     net["r1"].cmd(
 561         'vtysh -c "c t" -c "nexthop-group infinite-recursive" -c "nexthop 6.6.6.1" -c "nexthop 6.6.6.2" \
 562             -c "nexthop 6.6.6.3" -c "nexthop 6.6.6.4"'
 563     )
 564
 565     # static route nexthops can recurse to
 566
 567     net["r1"].cmd('vtysh -c "c t" -c "ip route 6.6.6.0/24 1.1.1.1"')
 568
 569     # Make routes that point to themselves in ecmp
 570
 571     net["r1"].cmd(
 572         'vtysh -c "sharp install routes 6.6.6.4 nexthop-group infinite-recursive 1"'
 573     )
 574     sleep(5)
 575
 576     net["r1"].cmd(
 577         'vtysh -c "sharp install routes 6.6.6.3 nexthop-group infinite-recursive 1"'
 578     )
 579     sleep(5)
 580
 581     net["r1"].cmd(
 582         'vtysh -c "sharp install routes 6.6.6.2 nexthop-group infinite-recursive 1"'
 583     )
 584     sleep(5)
 585
 586     net["r1"].cmd(
 587         'vtysh -c "sharp install routes 6.6.6.1 nexthop-group infinite-recursive 1"'
 588     )
 589
 590     # Get routes and test if has too many (duplicate) nexthops
 591     count = 0
 592     dups = []
 593     nhg_id = route_get_nhg_id("6.6.6.1/32")
 594     while (len(dups) != 3) and count < 10:
 595         output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
 596
 597         dups = re.findall(r"(via 1\.1\.1\.1)", output)
 598         if len(dups) != 3:
 599             count += 1
 600             sleep(1)
 601
 602     # Should find 3, itself is inactive
 603     assert len(dups) == 3, (
 604         "Route 6.6.6.1/32 with Nexthop Group ID=%d has wrong number of resolved nexthops"
 605         % nhg_id
 606     )
 607
 608     ## Remove all NHG routes
 609
 610     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.1 1"')
 611     net["r1"].cmd('vtysh -c "sharp remove routes 2.2.2.2 1"')
 612     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.1 1"')
 613     net["r1"].cmd('vtysh -c "sharp remove routes 3.3.3.2 1"')
 614     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.1 1"')
 615     net["r1"].cmd('vtysh -c "sharp remove routes 4.4.4.2 1"')
 616     net["r1"].cmd('vtysh -c "sharp remove routes 5.5.5.1 1"')
 617     net["r1"].cmd('vtysh -c "sharp remove routes 6.6.6.1 4"')
 618     net["r1"].cmd('vtysh -c "c t" -c "no ip route 6.6.6.0/24 1.1.1.1"')
 619
 620
 621 def test_rip_status():
 622     global fatal_error
 623     net = get_topogen().net
 624
 625     # Skip if previous fatal error condition is raised
 626     if fatal_error != "":
 627         pytest.skip(fatal_error)
 628
 629     thisDir = os.path.dirname(os.path.realpath(__file__))
 630
 631     print("\n\n** Verifying RIP status")
 632     print("******************************************\n")
 633     failures = 0
 634     for i in range(1, 2):
 635         refTableFile = "%s/r%s/rip_status.ref" % (thisDir, i)
 636         if os.path.isfile(refTableFile):
 637             # Read expected result from file
 638             expected = open(refTableFile).read().rstrip()
 639             # Fix newlines (make them all the same)
 640             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 641
 642             # Actual output from router
 643             actual = (
 644                 net["r%s" % i]
 645                 .cmd('vtysh -c "show ip rip status" 2> /dev/null')
 646                 .rstrip()
 647             )
 648             # Drop time in next due
 649             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 650             # Drop time in last update
 651             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 652             # Fix newlines (make them all the same)
 653             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 654
 655             # Generate Diff
 656             diff = topotest.get_textdiff(
 657                 actual,
 658                 expected,
 659                 title1="actual IP RIP status",
 660                 title2="expected IP RIP status",
 661             )
 662
 663             # Empty string if it matches, otherwise diff contains unified diff
 664             if diff:
 665                 sys.stderr.write("r%s failed IP RIP status check:\n%s\n" % (i, diff))
 666                 failures += 1
 667             else:
 668                 print("r%s ok" % i)
 669
 670             assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff)
 671
 672     # Make sure that all daemons are running
 673     for i in range(1, 2):
 674         fatal_error = net["r%s" % i].checkRouterRunning()
 675         assert fatal_error == "", fatal_error
 676
 677
 678 def test_ripng_status():
 679     global fatal_error
 680     net = get_topogen().net
 681
 682     # Skip if previous fatal error condition is raised
 683     if fatal_error != "":
 684         pytest.skip(fatal_error)
 685
 686     thisDir = os.path.dirname(os.path.realpath(__file__))
 687
 688     print("\n\n** Verifying RIPng status")
 689     print("******************************************\n")
 690     failures = 0
 691     for i in range(1, 2):
 692         refTableFile = "%s/r%s/ripng_status.ref" % (thisDir, i)
 693         if os.path.isfile(refTableFile):
 694             # Read expected result from file
 695             expected = open(refTableFile).read().rstrip()
 696             # Fix newlines (make them all the same)
 697             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 698
 699             # Actual output from router
 700             actual = (
 701                 net["r%s" % i]
 702                 .cmd('vtysh -c "show ipv6 ripng status" 2> /dev/null')
 703                 .rstrip()
 704             )
 705             # Mask out Link-Local mac address portion. They are random...
 706             actual = re.sub(r" fe80::[0-9a-f:]+", " fe80::XXXX:XXXX:XXXX:XXXX", actual)
 707             # Drop time in next due
 708             actual = re.sub(r"in [0-9]+ seconds", "in XX seconds", actual)
 709             # Drop time in last update
 710             actual = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", actual)
 711             # Fix newlines (make them all the same)
 712             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 713
 714             # Generate Diff
 715             diff = topotest.get_textdiff(
 716                 actual,
 717                 expected,
 718                 title1="actual IPv6 RIPng status",
 719                 title2="expected IPv6 RIPng status",
 720             )
 721
 722             # Empty string if it matches, otherwise diff contains unified diff
 723             if diff:
 724                 sys.stderr.write(
 725                     "r%s failed IPv6 RIPng status check:\n%s\n" % (i, diff)
 726                 )
 727                 failures += 1
 728             else:
 729                 print("r%s ok" % i)
 730
 731             assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (
 732                 i,
 733                 diff,
 734             )
 735
 736     # Make sure that all daemons are running
 737     for i in range(1, 2):
 738         fatal_error = net["r%s" % i].checkRouterRunning()
 739         assert fatal_error == "", fatal_error
 740
 741
 742 def test_ospfv2_interfaces():
 743     global fatal_error
 744     net = get_topogen().net
 745
 746     # Skip if previous fatal error condition is raised
 747     if fatal_error != "":
 748         pytest.skip(fatal_error)
 749
 750     thisDir = os.path.dirname(os.path.realpath(__file__))
 751
 752     print("\n\n** Verifying OSPFv2 interfaces")
 753     print("******************************************\n")
 754     failures = 0
 755     for i in range(1, 2):
 756         refTableFile = "%s/r%s/show_ip_ospf_interface.ref" % (thisDir, i)
 757         if os.path.isfile(refTableFile):
 758             # Read expected result from file
 759             expected = open(refTableFile).read().rstrip()
 760             # Fix newlines (make them all the same)
 761             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 762
 763             # Actual output from router
 764             actual = (
 765                 net["r%s" % i]
 766                 .cmd('vtysh -c "show ip ospf interface" 2> /dev/null')
 767                 .rstrip()
 768             )
 769             # Mask out Bandwidth portion. They may change..
 770             actual = re.sub(r"BW [0-9]+ Mbit", "BW XX Mbit", actual)
 771             actual = re.sub(r"ifindex [0-9]+", "ifindex X", actual)
 772
 773             # Drop time in next due
 774             actual = re.sub(r"Hello due in [0-9\.]+s", "Hello due in XX.XXXs", actual)
 775             actual = re.sub(
 776                 r"Hello due in [0-9\.]+ usecs", "Hello due in XX.XXXs", actual
 777             )
 778             # Fix 'MTU mismatch detection: enabled' vs 'MTU mismatch detection:enabled' - accept both
 779             actual = re.sub(
 780                 r"MTU mismatch detection:([a-z]+.*)",
 781                 r"MTU mismatch detection: \1",
 782                 actual,
 783             )
 784             # Fix newlines (make them all the same)
 785             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 786
 787             # Generate Diff
 788             diff = topotest.get_textdiff(
 789                 actual,
 790                 expected,
 791                 title1="actual SHOW IP OSPF INTERFACE",
 792                 title2="expected SHOW IP OSPF INTERFACE",
 793             )
 794
 795             # Empty string if it matches, otherwise diff contains unified diff
 796             if diff:
 797                 sys.stderr.write(
 798                     "r%s failed SHOW IP OSPF INTERFACE check:\n%s\n" % (i, diff)
 799                 )
 800                 failures += 1
 801             else:
 802                 print("r%s ok" % i)
 803
 804             # Ignoring the issue if told to ignore (ie not yet fixed)
 805             if failures != 0:
 806                 if os.environ.get("bamboo_TOPOTESTS_ISSUE_348") == "IGNORE":
 807                     sys.stderr.write(
 808                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348\n"
 809                     )
 810                     pytest.skip(
 811                         "Known issue - IGNORING. See https://github.com/FRRouting/frr/issues/348"
 812                     )
 813
 814             assert (
 815                 failures == 0
 816             ), "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff)
 817
 818     # Make sure that all daemons are running
 819     for i in range(1, 2):
 820         fatal_error = net["r%s" % i].checkRouterRunning()
 821         assert fatal_error == "", fatal_error
 822
 823
 824 def test_isis_interfaces():
 825     global fatal_error
 826     net = get_topogen().net
 827
 828     # Skip if previous fatal error condition is raised
 829     if fatal_error != "":
 830         pytest.skip(fatal_error)
 831
 832     thisDir = os.path.dirname(os.path.realpath(__file__))
 833
 834     print("\n\n** Verifying ISIS interfaces")
 835     print("******************************************\n")
 836     failures = 0
 837     for i in range(1, 2):
 838         refTableFile = "%s/r%s/show_isis_interface_detail.ref" % (thisDir, i)
 839         if os.path.isfile(refTableFile):
 840             # Read expected result from file
 841             expected = open(refTableFile).read().rstrip()
 842             # Fix newlines (make them all the same)
 843             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
 844
 845             # Actual output from router
 846             actual = (
 847                 net["r%s" % i]
 848                 .cmd('vtysh -c "show isis interface detail" 2> /dev/null')
 849                 .rstrip()
 850             )
 851             # Mask out Link-Local mac address portion. They are random...
 852             actual = re.sub(r"fe80::[0-9a-f:]+", "fe80::XXXX:XXXX:XXXX:XXXX", actual)
 853             # Mask out SNPA mac address portion. They are random...
 854             actual = re.sub(r"SNPA: [0-9a-f\.]+", "SNPA: XXXX.XXXX.XXXX", actual)
 855             # Mask out Circuit ID number
 856             actual = re.sub(r"Circuit Id: 0x[0-9a-f]+", "Circuit Id: 0xXX", actual)
 857             # Fix newlines (make them all the same)
 858             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
 859
 860             # Generate Diff
 861             diff = topotest.get_textdiff(
 862                 actual,
 863                 expected,
 864                 title1="actual SHOW ISIS INTERFACE DETAIL",
 865                 title2="expected SHOW ISIS OSPF6 INTERFACE DETAIL",
 866             )
 867
 868             # Empty string if it matches, otherwise diff contains unified diff
 869             if diff:
 870                 sys.stderr.write(
 871                     "r%s failed SHOW ISIS INTERFACE DETAIL check:\n%s\n" % (i, diff)
 872                 )
 873                 failures += 1
 874             else:
 875                 print("r%s ok" % i)
 876
 877             assert (
 878                 failures == 0
 879             ), "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff)
 880
 881     # Make sure that all daemons are running
 882     for i in range(1, 2):
 883         fatal_error = net["r%s" % i].checkRouterRunning()
 884         assert fatal_error == "", fatal_error
 885
 886
 887 def test_bgp_summary():
 888     global fatal_error
 889     net = get_topogen().net
 890
 891     # Skip if previous fatal error condition is raised
 892     if fatal_error != "":
 893         pytest.skip(fatal_error)
 894
 895     thisDir = os.path.dirname(os.path.realpath(__file__))
 896
 897     print("\n\n** Verifying BGP Summary")
 898     print("******************************************\n")
 899     failures = 0
 900     for i in range(1, 2):
 901         refTableFile = "%s/r%s/show_ip_bgp_summary.ref" % (thisDir, i)
 902         if os.path.isfile(refTableFile):
 903             # Read expected result from file
 904             expected_original = open(refTableFile).read().rstrip()
 905
 906             for arguments in [
 907                 "",
 908                 "remote-as internal",
 909                 "remote-as external",
 910                 "remote-as 100",
 911                 "remote-as 123",
 912                 "neighbor 192.168.7.10",
 913                 "neighbor 192.168.7.10",
 914                 "neighbor fc00:0:0:8::1000",
 915                 "neighbor 10.0.0.1",
 916                 "terse",
 917                 "remote-as internal terse",
 918                 "remote-as external terse",
 919                 "remote-as 100 terse",
 920                 "remote-as 123 terse",
 921                 "neighbor 192.168.7.10 terse",
 922                 "neighbor 192.168.7.10 terse",
 923                 "neighbor fc00:0:0:8::1000 terse",
 924                 "neighbor 10.0.0.1 terse",
 925             ]:
 926                 # Actual output from router
 927                 actual = (
 928                     net["r%s" % i]
 929                     .cmd(
 930                         'vtysh -c "show ip bgp summary ' + arguments + '" 2> /dev/null'
 931                     )
 932                     .rstrip()
 933                 )
 934
 935                 # Mask out "using XXiXX bytes" portion. They are random...
 936                 actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
 937                 # Mask out "using XiXXX KiB" portion. They are random...
 938                 actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
 939
 940                 # Remove extra summaries which exist with newer versions
 941
 942                 # Remove summary lines (changed recently)
 943                 actual = re.sub(r"Total number.*", "", actual)
 944                 actual = re.sub(r"Displayed.*", "", actual)
 945                 # Remove IPv4 Unicast Summary (Title only)
 946                 actual = re.sub(r"IPv4 Unicast Summary \(VRF default\):", "", actual)
 947                 # Remove IPv4 Multicast Summary (all of it)
 948                 actual = re.sub(r"IPv4 Multicast Summary \(VRF default\):", "", actual)
 949                 actual = re.sub(r"No IPv4 Multicast neighbor is configured", "", actual)
 950                 # Remove IPv4 VPN Summary (all of it)
 951                 actual = re.sub(r"IPv4 VPN Summary \(VRF default\):", "", actual)
 952                 actual = re.sub(r"No IPv4 VPN neighbor is configured", "", actual)
 953                 # Remove IPv4 Encap Summary (all of it)
 954                 actual = re.sub(r"IPv4 Encap Summary \(VRF default\):", "", actual)
 955                 actual = re.sub(r"No IPv4 Encap neighbor is configured", "", actual)
 956                 # Remove Unknown Summary (all of it)
 957                 actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
 958                 actual = re.sub(r"No Unknown neighbor is configured", "", actual)
 959                 # Make Connect/Active/Idle the same (change them all to Active)
 960                 actual = re.sub(r" Connect ", "  Active ", actual)
 961                 actual = re.sub(r"    Idle ", "  Active ", actual)
 962
 963                 actual = re.sub(
 964                     r"IPv4 labeled-unicast Summary \(VRF default\):", "", actual
 965                 )
 966                 actual = re.sub(
 967                     r"No IPv4 labeled-unicast neighbor is configured", "", actual
 968                 )
 969
 970                 expected = expected_original
 971                 # apply argumentss on expected output
 972                 if "internal" in arguments or "remote-as 100" in arguments:
 973                     expected = re.sub(r".+\s+200\s+.+", "", expected)
 974                 elif "external" in arguments:
 975                     expected = re.sub(r".+\s+100\s+.+Active.+", "", expected)
 976                 elif "remote-as 123" in arguments:
 977                     expected = re.sub(
 978                         r"(192.168.7.(1|2)0|fc00:0:0:8::(1|2)000).+Active.+",
 979                         "",
 980                         expected,
 981                     )
 982                     expected = re.sub(r"\nNeighbor.+Desc", "", expected)
 983                     expected = expected + "% No matching neighbor\n"
 984                 elif "192.168.7.10" in arguments:
 985                     expected = re.sub(
 986                         r"(192.168.7.20|fc00:0:0:8::(1|2)000).+Active.+", "", expected
 987                     )
 988                 elif "fc00:0:0:8::1000" in arguments:
 989                     expected = re.sub(
 990                         r"(192.168.7.(1|2)0|fc00:0:0:8::2000).+Active.+", "", expected
 991                     )
 992                 elif "10.0.0.1" in arguments:
 993                     expected = "No such neighbor in this view/vrf"
 994
 995                 if "terse" in arguments:
 996                     expected = re.sub(r"BGP table version .+", "", expected)
 997                     expected = re.sub(r"RIB entries .+", "", expected)
 998                     expected = re.sub(r"Peers [0-9]+, using .+", "", expected)
 999
1000                 # Strip empty lines
1001                 actual = actual.lstrip().rstrip()
1002                 expected = expected.lstrip().rstrip()
1003                 actual = re.sub(r"\n+", "\n", actual)
1004                 expected = re.sub(r"\n+", "\n", expected)
1005
1006                 # reapply initial formatting
1007                 if "terse" in arguments:
1008                     actual = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", actual)
1009                     expected = re.sub(r" vrf-id 0\n", " vrf-id 0\n\n", expected)
1010                 else:
1011                     actual = re.sub(r"KiB of memory\n", "KiB of memory\n\n", actual)
1012                     expected = re.sub(r"KiB of memory\n", "KiB of memory\n\n", expected)
1013
1014                 # realign expected neighbor columns if needed
1015                 try:
1016                     idx_actual = (
1017                         re.search(r"(Neighbor\s+V\s+)", actual).group(1).find("V")
1018                     )
1019                     idx_expected = (
1020                         re.search(r"(Neighbor\s+V\s+)", expected).group(1).find("V")
1021                     )
1022                     idx_diff = idx_expected - idx_actual
1023                     if idx_diff > 0:
1024                         # Neighbor        V         AS MsgRcvd MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd
1025                         expected = re.sub(" " * idx_diff + "V ", "V ", expected)
1026                         # 192.168.7.10    4        100       0       0        0    0    0    never       Active
1027                         expected = re.sub(" " * idx_diff + "4 ", "4 ", expected)
1028                 except AttributeError:
1029                     pass
1030
1031                 # Fix newlines (make them all the same)
1032                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1033                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1034
1035                 # Generate Diff
1036                 diff = topotest.get_textdiff(
1037                     actual,
1038                     expected,
1039                     title1="actual SHOW IP BGP SUMMARY " + arguments.upper(),
1040                     title2="expected SHOW IP BGP SUMMARY " + arguments.upper(),
1041                 )
1042
1043                 # Empty string if it matches, otherwise diff contains unified diff
1044                 if diff:
1045                     sys.stderr.write(
1046                         "r%s failed SHOW IP BGP SUMMARY check:\n%s\n" % (i, diff)
1047                     )
1048                     failures += 1
1049                 else:
1050                     print("r%s ok" % i)
1051
1052                 assert (
1053                     failures == 0
1054                 ), "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (
1055                     i,
1056                     diff,
1057                 )
1058
1059     # Make sure that all daemons are running
1060     for i in range(1, 2):
1061         fatal_error = net["r%s" % i].checkRouterRunning()
1062         assert fatal_error == "", fatal_error
1063
1064
1065 def test_bgp_ipv6_summary():
1066     global fatal_error
1067     net = get_topogen().net
1068
1069     # Skip if previous fatal error condition is raised
1070     if fatal_error != "":
1071         pytest.skip(fatal_error)
1072
1073     thisDir = os.path.dirname(os.path.realpath(__file__))
1074
1075     print("\n\n** Verifying BGP IPv6 Summary")
1076     print("******************************************\n")
1077     failures = 0
1078     for i in range(1, 2):
1079         refTableFile = "%s/r%s/show_bgp_ipv6_summary.ref" % (thisDir, i)
1080         if os.path.isfile(refTableFile):
1081             # Read expected result from file
1082             expected = open(refTableFile).read().rstrip()
1083             # Fix newlines (make them all the same)
1084             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1085
1086             # Actual output from router
1087             actual = (
1088                 net["r%s" % i]
1089                 .cmd('vtysh -c "show bgp ipv6 summary" 2> /dev/null')
1090                 .rstrip()
1091             )
1092             # Mask out "using XXiXX bytes" portion. They are random...
1093             actual = re.sub(r"using [0-9]+ bytes", "using XXXX bytes", actual)
1094             # Mask out "using XiXXX KiB" portion. They are random...
1095             actual = re.sub(r"using [0-9]+ KiB", "using XXXX KiB", actual)
1096             #
1097             # Remove extra summaries which exist with newer versions
1098             #
1099             # Remove summary lines (changed recently)
1100             actual = re.sub(r"Total number.*", "", actual)
1101             actual = re.sub(r"Displayed.*", "", actual)
1102             # Remove IPv4 Unicast Summary (Title only)
1103             actual = re.sub(r"IPv6 Unicast Summary \(VRF default\):", "", actual)
1104             # Remove IPv4 Multicast Summary (all of it)
1105             actual = re.sub(r"IPv6 Multicast Summary \(VRF default\):", "", actual)
1106             actual = re.sub(r"No IPv6 Multicast neighbor is configured", "", actual)
1107             # Remove IPv4 VPN Summary (all of it)
1108             actual = re.sub(r"IPv6 VPN Summary \(VRF default\):", "", actual)
1109             actual = re.sub(r"No IPv6 VPN neighbor is configured", "", actual)
1110             # Remove IPv4 Encap Summary (all of it)
1111             actual = re.sub(r"IPv6 Encap Summary \(VRF default\):", "", actual)
1112             actual = re.sub(r"No IPv6 Encap neighbor is configured", "", actual)
1113             # Remove Unknown Summary (all of it)
1114             actual = re.sub(r"Unknown Summary \(VRF default\):", "", actual)
1115             actual = re.sub(r"No Unknown neighbor is configured", "", actual)
1116             # Make Connect/Active/Idle the same (change them all to Active)
1117             actual = re.sub(r" Connect ", "  Active ", actual)
1118             actual = re.sub(r"    Idle ", "  Active ", actual)
1119
1120             # Remove Labeled Unicast Summary (all of it)
1121             actual = re.sub(
1122                 r"IPv6 labeled-unicast Summary \(VRF default\):", "", actual
1123             )
1124             actual = re.sub(
1125                 r"No IPv6 labeled-unicast neighbor is configured", "", actual
1126             )
1127
1128             # Strip empty lines
1129             actual = actual.lstrip()
1130             actual = actual.rstrip()
1131             #
1132             # Fix newlines (make them all the same)
1133             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1134
1135             # Generate Diff
1136             diff = topotest.get_textdiff(
1137                 actual,
1138                 expected,
1139                 title1="actual SHOW BGP IPv6 SUMMARY",
1140                 title2="expected SHOW BGP IPv6 SUMMARY",
1141             )
1142
1143             # Empty string if it matches, otherwise diff contains unified diff
1144             if diff:
1145                 sys.stderr.write(
1146                     "r%s failed SHOW BGP IPv6 SUMMARY check:\n%s\n" % (i, diff)
1147                 )
1148                 failures += 1
1149             else:
1150                 print("r%s ok" % i)
1151
1152             assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (
1153                 i,
1154                 diff,
1155             )
1156
1157     # Make sure that all daemons are running
1158     for i in range(1, 2):
1159         fatal_error = net["r%s" % i].checkRouterRunning()
1160         assert fatal_error == "", fatal_error
1161
1162
1163 def test_nht():
1164     global fatal_error
1165     net = get_topogen().net
1166
1167     # Skip if previous fatal error condition is raised
1168     if fatal_error != "":
1169         pytest.skip(fatal_error)
1170
1171     print("\n\n**** Test that nexthop tracking is at least nominally working ****\n")
1172
1173     thisDir = os.path.dirname(os.path.realpath(__file__))
1174
1175     for i in range(1, 2):
1176         nhtFile = "%s/r%s/ip_nht.ref" % (thisDir, i)
1177         expected = open(nhtFile).read().rstrip()
1178         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1179
1180         actual = net["r%s" % i].cmd('vtysh -c "show ip nht" 2> /dev/null').rstrip()
1181         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1182         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1183
1184         diff = topotest.get_textdiff(
1185             actual,
1186             expected,
1187             title1="Actual `show ip nht`",
1188             title2="Expected `show ip nht`",
1189         )
1190
1191         if diff:
1192             assert 0, "r%s failed ip nht check:\n%s\n" % (i, diff)
1193         else:
1194             print("show ip nht is ok\n")
1195
1196         nhtFile = "%s/r%s/ipv6_nht.ref" % (thisDir, i)
1197         expected = open(nhtFile).read().rstrip()
1198         expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1199
1200         actual = net["r%s" % i].cmd('vtysh -c "show ipv6 nht" 2> /dev/null').rstrip()
1201         actual = re.sub(r"fd [0-9]+", "fd XX", actual)
1202         actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1203
1204         diff = topotest.get_textdiff(
1205             actual,
1206             expected,
1207             title1="Actual `show ip nht`",
1208             title2="Expected `show ip nht`",
1209         )
1210
1211         if diff:
1212             assert 0, "r%s failed ipv6 nht check:\n%s\n" % (i, diff)
1213         else:
1214             print("show ipv6 nht is ok\n")
1215
1216
1217 def test_bgp_ipv4():
1218     global fatal_error
1219     net = get_topogen().net
1220
1221     # Skip if previous fatal error condition is raised
1222     if fatal_error != "":
1223         pytest.skip(fatal_error)
1224
1225     thisDir = os.path.dirname(os.path.realpath(__file__))
1226
1227     print("\n\n** Verifying BGP IPv4")
1228     print("******************************************\n")
1229     diffresult = {}
1230     for i in range(1, 2):
1231         success = 0
1232         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv4*.ref" % (thisDir, i)):
1233             if os.path.isfile(refTableFile):
1234                 # Read expected result from file
1235                 expected = open(refTableFile).read().rstrip()
1236                 # Fix newlines (make them all the same)
1237                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1238
1239                 # Actual output from router
1240                 actual = (
1241                     net["r%s" % i].cmd('vtysh -c "show bgp ipv4" 2> /dev/null').rstrip()
1242                 )
1243                 # Remove summary line (changed recently)
1244                 actual = re.sub(r"Total number.*", "", actual)
1245                 actual = re.sub(r"Displayed.*", "", actual)
1246                 actual = actual.rstrip()
1247                 # Fix newlines (make them all the same)
1248                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1249
1250                 # Generate Diff
1251                 diff = topotest.get_textdiff(
1252                     actual,
1253                     expected,
1254                     title1="actual SHOW BGP IPv4",
1255                     title2="expected SHOW BGP IPv4",
1256                 )
1257
1258                 # Empty string if it matches, otherwise diff contains unified diff
1259                 if diff:
1260                     diffresult[refTableFile] = diff
1261                 else:
1262                     success = 1
1263                     print("template %s matched: r%s ok" % (refTableFile, i))
1264                     break
1265
1266         if not success:
1267             resultstr = "No template matched.\n"
1268             for f in diffresult.keys():
1269                 resultstr += "template %s: r%s failed SHOW BGP IPv4 check:\n%s\n" % (
1270                     f,
1271                     i,
1272                     diffresult[f],
1273                 )
1274             raise AssertionError(
1275                 "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, resultstr)
1276             )
1277
1278     # Make sure that all daemons are running
1279     for i in range(1, 2):
1280         fatal_error = net["r%s" % i].checkRouterRunning()
1281         assert fatal_error == "", fatal_error
1282
1283
1284 def test_bgp_ipv6():
1285     global fatal_error
1286     net = get_topogen().net
1287
1288     # Skip if previous fatal error condition is raised
1289     if fatal_error != "":
1290         pytest.skip(fatal_error)
1291
1292     thisDir = os.path.dirname(os.path.realpath(__file__))
1293
1294     print("\n\n** Verifying BGP IPv6")
1295     print("******************************************\n")
1296     diffresult = {}
1297     for i in range(1, 2):
1298         success = 0
1299         for refTableFile in glob.glob("%s/r%s/show_bgp_ipv6*.ref" % (thisDir, i)):
1300             if os.path.isfile(refTableFile):
1301                 # Read expected result from file
1302                 expected = open(refTableFile).read().rstrip()
1303                 # Fix newlines (make them all the same)
1304                 expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1305
1306                 # Actual output from router
1307                 actual = (
1308                     net["r%s" % i].cmd('vtysh -c "show bgp ipv6" 2> /dev/null').rstrip()
1309                 )
1310                 # Remove summary line (changed recently)
1311                 actual = re.sub(r"Total number.*", "", actual)
1312                 actual = re.sub(r"Displayed.*", "", actual)
1313                 actual = actual.rstrip()
1314                 # Fix newlines (make them all the same)
1315                 actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1316
1317                 # Generate Diff
1318                 diff = topotest.get_textdiff(
1319                     actual,
1320                     expected,
1321                     title1="actual SHOW BGP IPv6",
1322                     title2="expected SHOW BGP IPv6",
1323                 )
1324
1325                 # Empty string if it matches, otherwise diff contains unified diff
1326                 if diff:
1327                     diffresult[refTableFile] = diff
1328                 else:
1329                     success = 1
1330                     print("template %s matched: r%s ok" % (refTableFile, i))
1331
1332         if not success:
1333             resultstr = "No template matched.\n"
1334             for f in diffresult.keys():
1335                 resultstr += "template %s: r%s failed SHOW BGP IPv6 check:\n%s\n" % (
1336                     f,
1337                     i,
1338                     diffresult[f],
1339                 )
1340             raise AssertionError(
1341                 "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, resultstr)
1342             )
1343
1344     # Make sure that all daemons are running
1345     for i in range(1, 2):
1346         fatal_error = net["r%s" % i].checkRouterRunning()
1347         assert fatal_error == "", fatal_error
1348
1349
1350 def test_route_map():
1351     global fatal_error
1352     net = get_topogen().net
1353
1354     if fatal_error != "":
1355         pytest.skip(fatal_error)
1356
1357     thisDir = os.path.dirname(os.path.realpath(__file__))
1358
1359     print("\n\n** Verifying some basic routemap forward references\n")
1360     print("*******************************************************\n")
1361     failures = 0
1362     for i in range(1, 2):
1363         refroutemap = "%s/r%s/show_route_map.ref" % (thisDir, i)
1364         if os.path.isfile(refroutemap):
1365             expected = open(refroutemap).read().rstrip()
1366             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1367
1368             actual = (
1369                 net["r%s" % i].cmd('vtysh -c "show route-map" 2> /dev/null').rstrip()
1370             )
1371             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1372
1373             diff = topotest.get_textdiff(
1374                 actual,
1375                 expected,
1376                 title1="actual show route-map",
1377                 title2="expected show route-map",
1378             )
1379
1380             if diff:
1381                 sys.stderr.write(
1382                     "r%s failed show route-map command Check:\n%s\n" % (i, diff)
1383                 )
1384                 failures += 1
1385             else:
1386                 print("r%s ok" % i)
1387
1388             assert (
1389                 failures == 0
1390             ), "Show route-map command failed for router r%s:\n%s" % (i, diff)
1391
1392
1393 def test_nexthop_groups_with_route_maps():
1394     global fatal_error
1395     net = get_topogen().net
1396
1397     # Skip if previous fatal error condition is raised
1398     if fatal_error != "":
1399         pytest.skip(fatal_error)
1400
1401     print("\n\n** Verifying Nexthop Groups With Route-Maps")
1402     print("******************************************\n")
1403
1404     ### Nexthop Group With Route-Map Tests
1405
1406     # Create a lib nexthop-group
1407     net["r1"].cmd(
1408         'vtysh -c "c t" -c "nexthop-group test" -c "nexthop 1.1.1.1" -c "nexthop 1.1.1.2"'
1409     )
1410
1411     ## Route-Map Proto Source
1412
1413     route_str = "2.2.2.1"
1414     src_str = "192.168.0.1"
1415
1416     net["r1"].cmd(
1417         'vtysh -c "c t" -c "route-map NH-SRC permit 111" -c "set src %s"' % src_str
1418     )
1419     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NH-SRC"')
1420
1421     net["r1"].cmd('vtysh -c "sharp install routes %s nexthop-group test 1"' % route_str)
1422
1423     verify_route_nexthop_group("%s/32" % route_str)
1424
1425     # Only a valid test on linux using nexthop objects
1426     if sys.platform.startswith("linux"):
1427         output = net["r1"].cmd("ip route show %s/32" % route_str)
1428         match = re.search(r"src %s" % src_str, output)
1429         assert match is not None, "Route %s/32 not installed with src %s" % (
1430             route_str,
1431             src_str,
1432         )
1433
1434     # Remove NHG routes and route-map
1435     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % route_str)
1436     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NH-SRC"')
1437     net["r1"].cmd(
1438         'vtysh -c "c t" -c "no route-map NH-SRC permit 111" # -c "set src %s"' % src_str
1439     )
1440     net["r1"].cmd('vtysh -c "c t" -c "no route-map NH-SRC"')
1441
1442     ## Route-Map Deny/Permit with same nexthop group
1443
1444     permit_route_str = "3.3.3.1"
1445     deny_route_str = "3.3.3.2"
1446
1447     net["r1"].cmd(
1448         'vtysh -c "c t" -c "ip prefix-list NOPE seq 5 permit %s/32"' % permit_route_str
1449     )
1450     net["r1"].cmd(
1451         'vtysh -c "c t" -c "route-map NOPE permit 111" -c "match ip address prefix-list NOPE"'
1452     )
1453     net["r1"].cmd('vtysh -c "c t" -c "route-map NOPE deny 222"')
1454     net["r1"].cmd('vtysh -c "c t" -c "ip protocol sharp route-map NOPE"')
1455
1456     # This route should be permitted
1457     net["r1"].cmd(
1458         'vtysh -c "sharp install routes %s nexthop-group test 1"' % permit_route_str
1459     )
1460
1461     verify_route_nexthop_group("%s/32" % permit_route_str)
1462
1463     # This route should be denied
1464     net["r1"].cmd(
1465         'vtysh -c "sharp install routes %s nexthop-group test 1"' % deny_route_str
1466     )
1467
1468     nhg_id = route_get_nhg_id(deny_route_str)
1469     output = net["r1"].cmd('vtysh -c "show nexthop-group rib %d"' % nhg_id)
1470
1471     match = re.search(r"Valid", output)
1472     assert match is None, "Nexthop Group ID=%d should not be marked Valid" % nhg_id
1473
1474     match = re.search(r"Installed", output)
1475     assert match is None, "Nexthop Group ID=%d should not be marked Installed" % nhg_id
1476
1477     # Remove NHG routes and route-map
1478     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % permit_route_str)
1479     net["r1"].cmd('vtysh -c "sharp remove routes %s 1"' % deny_route_str)
1480     net["r1"].cmd('vtysh -c "c t" -c "no ip protocol sharp route-map NOPE"')
1481     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE permit 111"')
1482     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE deny 222"')
1483     net["r1"].cmd('vtysh -c "c t" -c "no route-map NOPE"')
1484     net["r1"].cmd(
1485         'vtysh -c "c t" -c "no ip prefix-list NOPE seq 5 permit %s/32"'
1486         % permit_route_str
1487     )
1488
1489
1490 def test_nexthop_group_replace():
1491     global fatal_error
1492     net = get_topogen().net
1493
1494     # Skip if previous fatal error condition is raised
1495     if fatal_error != "":
1496         pytest.skip(fatal_error)
1497
1498     print("\n\n** Verifying Nexthop Groups")
1499     print("******************************************\n")
1500
1501     ### Nexthop Group Tests
1502
1503     ## 2-Way ECMP Directly Connected
1504
1505     net["r1"].cmd(
1506         'vtysh -c "c t" -c "nexthop-group replace" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1507     )
1508
1509     # At the moment there is absolutely no real easy way to query sharpd
1510     # for the nexthop group actually installed.  If it is not installed
1511     # sharpd will just transmit the nexthops down instead of the nexthop
1512     # group id.  Leading to a situation where the replace is not actually
1513     # being tested.  So let's just wait some time here because this
1514     # is hard and this test fails all the time
1515     sleep(5)
1516
1517     # Create with sharpd using nexthop-group
1518     net["r1"].cmd('vtysh -c "sharp install routes 3.3.3.1 nexthop-group replace 1"')
1519
1520     verify_route_nexthop_group("3.3.3.1/32")
1521
1522     # Change the nexthop group
1523     net["r1"].cmd(
1524         'vtysh -c "c t" -c "nexthop-group replace" -c "no nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.3 r1-eth1 onlink" -c "nexthop 1.1.1.4 r1-eth4 onlink"'
1525     )
1526
1527     # Verify it updated. We can just check install and ecmp count here.
1528     verify_route_nexthop_group("3.3.3.1/32", False, 3)
1529
1530
1531 def test_mpls_interfaces():
1532     global fatal_error
1533     net = get_topogen().net
1534
1535     # Skip if previous fatal error condition is raised
1536     if fatal_error != "":
1537         pytest.skip(fatal_error)
1538
1539     # Skip if no LDP installed or old kernel
1540     if net["r1"].daemon_available("ldpd") == False:
1541         pytest.skip("No MPLS or kernel < 4.5")
1542
1543     thisDir = os.path.dirname(os.path.realpath(__file__))
1544
1545     print("\n\n** Verifying MPLS Interfaces")
1546     print("******************************************\n")
1547     failures = 0
1548     for i in range(1, 2):
1549         refTableFile = "%s/r%s/show_mpls_ldp_interface.ref" % (thisDir, i)
1550         if os.path.isfile(refTableFile):
1551             # Read expected result from file
1552             expected = open(refTableFile).read().rstrip()
1553             # Fix newlines (make them all the same)
1554             expected = ("\n".join(expected.splitlines()) + "\n").splitlines(1)
1555
1556             # Actual output from router
1557             actual = (
1558                 net["r%s" % i]
1559                 .cmd('vtysh -c "show mpls ldp interface" 2> /dev/null')
1560                 .rstrip()
1561             )
1562             # Mask out Timer in Uptime
1563             actual = re.sub(r" [0-9][0-9]:[0-9][0-9]:[0-9][0-9] ", " xx:xx:xx ", actual)
1564             # Fix newlines (make them all the same)
1565             actual = ("\n".join(actual.splitlines()) + "\n").splitlines(1)
1566
1567             # Generate Diff
1568             diff = topotest.get_textdiff(
1569                 actual,
1570                 expected,
1571                 title1="actual MPLS LDP interface status",
1572                 title2="expected MPLS LDP interface status",
1573             )
1574
1575             # Empty string if it matches, otherwise diff contains unified diff
1576             if diff:
1577                 sys.stderr.write(
1578                     "r%s failed MPLS LDP Interface status Check:\n%s\n" % (i, diff)
1579                 )
1580                 failures += 1
1581             else:
1582                 print("r%s ok" % i)
1583
1584             if failures > 0:
1585                 fatal_error = "MPLS LDP Interface status failed"
1586
1587             assert (
1588                 failures == 0
1589             ), "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff)
1590
1591     # Make sure that all daemons are running
1592     for i in range(1, 2):
1593         fatal_error = net["r%s" % i].checkRouterRunning()
1594         assert fatal_error == "", fatal_error
1595
1596
1597 def test_resilient_nexthop_group():
1598     net = get_topogen().net
1599
1600     result = required_linux_kernel_version("5.19")
1601     if result is not True:
1602         pytest.skip("Kernel requirements are not met, kernel version should be >= 5.19")
1603
1604     net["r1"].cmd(
1605         'vtysh -c "conf" -c "nexthop-group resilience" -c "resilient buckets 64 idle-timer 128 unbalanced-timer 256" -c "nexthop 1.1.1.1 r1-eth1 onlink" -c "nexthop 1.1.1.2 r1-eth2 onlink"'
1606     )
1607
1608     output = net["r1"].cmd('vtysh -c "show nexthop-group rib sharp"')
1609     output = re.findall(r"Buckets", output)
1610
1611     verify_nexthop_group(185483878)
1612     assert len(output) == 1, "Resilient NHG not created in zebra"
1613
1614
1615 def test_shutdown_check_stderr():
1616     global fatal_error
1617     net = get_topogen().net
1618
1619     # Skip if previous fatal error condition is raised
1620     if fatal_error != "":
1621         pytest.skip(fatal_error)
1622
1623     print("\n\n** Verifying unexpected STDERR output from daemons")
1624     print("******************************************\n")
1625
1626     if os.environ.get("TOPOTESTS_CHECK_STDERR") is None:
1627         print(
1628             "SKIPPED final check on StdErr output: Disabled (TOPOTESTS_CHECK_STDERR undefined)\n"
1629         )
1630         pytest.skip("Skipping test for Stderr output")
1631
1632     thisDir = os.path.dirname(os.path.realpath(__file__))
1633
1634     print("thisDir=" + thisDir)
1635
1636     net["r1"].stopRouter()
1637
1638     log = net["r1"].getStdErr("ripd")
1639     if log:
1640         print("\nRIPd StdErr Log:\n" + log)
1641     log = net["r1"].getStdErr("ripngd")
1642     if log:
1643         print("\nRIPngd StdErr Log:\n" + log)
1644     log = net["r1"].getStdErr("ospfd")
1645     if log:
1646         print("\nOSPFd StdErr Log:\n" + log)
1647     log = net["r1"].getStdErr("ospf6d")
1648     if log:
1649         print("\nOSPF6d StdErr Log:\n" + log)
1650     log = net["r1"].getStdErr("isisd")
1651     if log:
1652         print("\nISISd StdErr Log:\n" + log)
1653     log = net["r1"].getStdErr("bgpd")
1654     if log:
1655         print("\nBGPd StdErr Log:\n" + log)
1656
1657     log = net["r1"].getStdErr("nhrpd")
1658     if log:
1659         print("\nNHRPd StdErr Log:\n" + log)
1660
1661     log = net["r1"].getStdErr("pbrd")
1662     if log:
1663         print("\nPBRd StdErr Log:\n" + log)
1664
1665     log = net["r1"].getStdErr("babeld")
1666     if log:
1667         print("\nBABELd StdErr Log:\n" + log)
1668
1669     if net["r1"].daemon_available("ldpd"):
1670         log = net["r1"].getStdErr("ldpd")
1671         if log:
1672             print("\nLDPd StdErr Log:\n" + log)
1673     log = net["r1"].getStdErr("zebra")
1674     if log:
1675         print("\nZebra StdErr Log:\n" + log)
1676
1677
1678 def test_shutdown_check_memleak():
1679     global fatal_error
1680     net = get_topogen().net
1681
1682     # Skip if previous fatal error condition is raised
1683     if fatal_error != "":
1684         pytest.skip(fatal_error)
1685
1686     if os.environ.get("TOPOTESTS_CHECK_MEMLEAK") is None:
1687         print(
1688             "SKIPPED final check on Memory leaks: Disabled (TOPOTESTS_CHECK_MEMLEAK undefined)\n"
1689         )
1690         pytest.skip("Skipping test for memory leaks")
1691
1692     thisDir = os.path.dirname(os.path.realpath(__file__))
1693
1694     for i in range(1, 2):
1695         net["r%s" % i].stopRouter()
1696         net["r%s" % i].report_memory_leaks(
1697             os.environ.get("TOPOTESTS_CHECK_MEMLEAK"), os.path.basename(__file__)
1698         )
1699
1700
1701 if __name__ == "__main__":
1702     # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
1703     # retval = pytest.main(["-s", "--tb=no"])
1704     retval = pytest.main(["-s"])
1705     sys.exit(retval)