tests/topotests/bgp_link_bw_ip/test_bgp_linkbw_ip.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: ISC
   3
   4 #
   5 # test_bgp_linkbw_ip.py
   6 #
   7 # Copyright (c) 2020 by
   8 # Cumulus Networks, Inc
   9 # Vivek Venkatraman
  10 #
  11
  12 """
  13 test_bgp_linkbw_ip.py: Test weighted ECMP using BGP link-bandwidth
  14 """
  15
  16 import os
  17 import sys
  18 from functools import partial
  19 import pytest
  20 import json
  21
  22 # Save the Current Working Directory to find configuration files.
  23 CWD = os.path.dirname(os.path.realpath(__file__))
  24 sys.path.append(os.path.join(CWD, "../"))
  25
  26 # pylint: disable=C0413
  27 # Import topogen and topotest helpers
  28 from lib import topotest
  29 from lib.topogen import Topogen, TopoRouter, get_topogen
  30 from lib.topolog import logger
  31
  32 # Required to instantiate the topology builder class.
  33
  34 pytestmark = [pytest.mark.bgpd]
  35
  36
  37 """
  38 This topology is for validating one of the primary use cases for
  39 weighted ECMP (a.k.a. Unequal cost multipath) using BGP link-bandwidth:
  40 https://tools.ietf.org/html/draft-mohanty-bess-ebgp-dmz
  41
  42 The topology consists of two PODs. Pod-1 consists of a spine switch
  43 and two leaf switches, with two servers attached to the first leaf and
  44 one to the second leaf. Pod-2 consists of one spine and one leaf, with
  45 one server connected to the leaf. The PODs are connected by a super-spine
  46 switch.
  47
  48 Note that the use of the term "switch" above is in keeping with common
  49 data-center terminology. These devices are all regular routers; for
  50 this scenario, the servers are also routers as they have to announce
  51 anycast IP (VIP) addresses via BGP.
  52 """
  53
  54
  55 def build_topo(tgen):
  56     "Build function"
  57
  58     # Create 10 routers - 1 super-spine, 2 spines, 3 leafs
  59     # and 4 servers
  60     routers = {}
  61     for i in range(1, 11):
  62         routers[i] = tgen.add_router("r{}".format(i))
  63
  64     # Create 13 "switches" - to interconnect the above routers
  65     switches = {}
  66     for i in range(1, 14):
  67         switches[i] = tgen.add_switch("s{}".format(i))
  68
  69     # Interconnect R1 (super-spine) to R2 and R3 (the two spines)
  70     switches[1].add_link(tgen.gears["r1"])
  71     switches[1].add_link(tgen.gears["r2"])
  72     switches[2].add_link(tgen.gears["r1"])
  73     switches[2].add_link(tgen.gears["r3"])
  74
  75     # Interconnect R2 (spine in pod-1) to R4 and R5 (the associated
  76     # leaf switches)
  77     switches[3].add_link(tgen.gears["r2"])
  78     switches[3].add_link(tgen.gears["r4"])
  79     switches[4].add_link(tgen.gears["r2"])
  80     switches[4].add_link(tgen.gears["r5"])
  81
  82     # Interconnect R3 (spine in pod-2) to R6 (associated leaf)
  83     switches[5].add_link(tgen.gears["r3"])
  84     switches[5].add_link(tgen.gears["r6"])
  85
  86     # Interconnect leaf switches to servers
  87     switches[6].add_link(tgen.gears["r4"])
  88     switches[6].add_link(tgen.gears["r7"])
  89     switches[7].add_link(tgen.gears["r4"])
  90     switches[7].add_link(tgen.gears["r8"])
  91     switches[8].add_link(tgen.gears["r5"])
  92     switches[8].add_link(tgen.gears["r9"])
  93     switches[9].add_link(tgen.gears["r6"])
  94     switches[9].add_link(tgen.gears["r10"])
  95
  96     # Create empty networks for the servers
  97     switches[10].add_link(tgen.gears["r7"])
  98     switches[11].add_link(tgen.gears["r8"])
  99     switches[12].add_link(tgen.gears["r9"])
 100     switches[13].add_link(tgen.gears["r10"])
 101
 102
 103 def setup_module(mod):
 104     "Sets up the pytest environment"
 105     tgen = Topogen(build_topo, mod.__name__)
 106     tgen.start_topology()
 107
 108     router_list = tgen.routers()
 109     for rname, router in router_list.items():
 110         router.load_config(
 111             TopoRouter.RD_ZEBRA, os.path.join(CWD, "{}/zebra.conf".format(rname))
 112         )
 113         router.load_config(
 114             TopoRouter.RD_BGP, os.path.join(CWD, "{}/bgpd.conf".format(rname))
 115         )
 116
 117     # Initialize all routers.
 118     tgen.start_router()
 119
 120     # tgen.mininet_cli()
 121
 122
 123 def teardown_module(mod):
 124     "Teardown the pytest environment"
 125     tgen = get_topogen()
 126     tgen.stop_topology()
 127
 128
 129 def test_bgp_linkbw_adv():
 130     "Test #1: Test BGP link-bandwidth advertisement based on number of multipaths"
 131     logger.info(
 132         "\nTest #1: Test BGP link-bandwidth advertisement based on number of multipaths"
 133     )
 134
 135     tgen = get_topogen()
 136     if tgen.routers_have_failure():
 137         pytest.skip("skipped because of router(s) failure")
 138
 139     r1 = tgen.gears["r1"]
 140     r2 = tgen.gears["r2"]
 141
 142     # Configure anycast IP on server r7
 143     logger.info("Configure anycast IP on server r7")
 144
 145     tgen.net["r7"].cmd("ip addr add 198.10.1.1/32 dev r7-eth1")
 146
 147     # Check on spine router r2 for link-bw advertisement by leaf router r4
 148     logger.info("Check on spine router r2 for link-bw advertisement by leaf router r4")
 149
 150     json_file = "{}/r2/bgp-route-1.json".format(CWD)
 151     expected = json.loads(open(json_file).read())
 152     test_func = partial(
 153         topotest.router_json_cmp, r2, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 154     )
 155     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 156     assertmsg = "JSON output mismatch on spine router r2"
 157     assert result is None, assertmsg
 158
 159     # Check on spine router r2 that default weight is used as there is no multipath
 160     logger.info(
 161         "Check on spine router r2 that default weight is used as there is no multipath"
 162     )
 163
 164     json_file = "{}/r2/ip-route-1.json".format(CWD)
 165     expected = json.loads(open(json_file).read())
 166     test_func = partial(
 167         topotest.router_json_cmp, r2, "show ip route 198.10.1.1/32 json", expected
 168     )
 169     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 170     assertmsg = "JSON output mismatch on spine router r2"
 171     assert result is None, assertmsg
 172
 173     # Check on super-spine router r1 that link-bw has been propagated by spine router r2
 174     logger.info(
 175         "Check on super-spine router r1 that link-bw has been propagated by spine router r2"
 176     )
 177
 178     json_file = "{}/r1/bgp-route-1.json".format(CWD)
 179     expected = json.loads(open(json_file).read())
 180     test_func = partial(
 181         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 182     )
 183     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 184     assertmsg = "JSON output mismatch on super-spine router r1"
 185     assert result is None, assertmsg
 186
 187
 188 def test_bgp_cumul_linkbw():
 189     "Test #2: Test cumulative link-bandwidth propagation"
 190     logger.info("\nTest #2: Test cumulative link-bandwidth propagation")
 191
 192     tgen = get_topogen()
 193     if tgen.routers_have_failure():
 194         pytest.skip("skipped because of router(s) failure")
 195
 196     r1 = tgen.gears["r1"]
 197     r2 = tgen.gears["r2"]
 198     r4 = tgen.gears["r4"]
 199
 200     # Configure anycast IP on additional server r8
 201     logger.info("Configure anycast IP on server r8")
 202
 203     tgen.net["r8"].cmd("ip addr add 198.10.1.1/32 dev r8-eth1")
 204
 205     # Check multipath on leaf router r4
 206     logger.info("Check multipath on leaf router r4")
 207
 208     json_file = "{}/r4/bgp-route-1.json".format(CWD)
 209     expected = json.loads(open(json_file).read())
 210     test_func = partial(
 211         topotest.router_json_cmp, r4, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 212     )
 213     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 214     assertmsg = "JSON output mismatch on leaf router r4"
 215     assert result is None, assertmsg
 216
 217     # Check regular ECMP is in effect on leaf router r4
 218     logger.info("Check regular ECMP is in effect on leaf router r4")
 219
 220     json_file = "{}/r4/ip-route-1.json".format(CWD)
 221     expected = json.loads(open(json_file).read())
 222     test_func = partial(
 223         topotest.router_json_cmp, r4, "show ip route 198.10.1.1/32 json", expected
 224     )
 225     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 226     assertmsg = "JSON output mismatch on leaf router r4"
 227     assert result is None, assertmsg
 228
 229     # Check on spine router r2 that leaf has propagated the cumulative link-bw based on num-multipaths
 230     logger.info(
 231         "Check on spine router r2 that leaf has propagated the cumulative link-bw based on num-multipaths"
 232     )
 233
 234     json_file = "{}/r2/bgp-route-2.json".format(CWD)
 235     expected = json.loads(open(json_file).read())
 236     test_func = partial(
 237         topotest.router_json_cmp, r2, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 238     )
 239     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 240     assertmsg = "JSON output mismatch on spine router r2"
 241     assert result is None, assertmsg
 242
 243
 244 def test_weighted_ecmp():
 245     "Test #3: Test weighted ECMP - multipath with next hop weights"
 246     logger.info("\nTest #3: Test weighted ECMP - multipath with next hop weights")
 247
 248     tgen = get_topogen()
 249     if tgen.routers_have_failure():
 250         pytest.skip("skipped because of router(s) failure")
 251
 252     r1 = tgen.gears["r1"]
 253     r2 = tgen.gears["r2"]
 254     r3 = tgen.gears["r3"]
 255
 256     # Configure anycast IP on additional server r9
 257     logger.info("Configure anycast IP on server r9")
 258
 259     tgen.net["r9"].cmd("ip addr add 198.10.1.1/32 dev r9-eth1")
 260
 261     # Check multipath on spine router r2
 262     logger.info("Check multipath on spine router r2")
 263     json_file = "{}/r2/bgp-route-3.json".format(CWD)
 264     expected = json.loads(open(json_file).read())
 265     test_func = partial(
 266         topotest.router_json_cmp, r2, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 267     )
 268     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 269     assertmsg = "JSON output mismatch on spine router r2"
 270     assert result is None, assertmsg
 271
 272     # Check weighted ECMP is in effect on the spine router r2
 273     logger.info("Check weighted ECMP is in effect on the spine router r2")
 274
 275     json_file = "{}/r2/ip-route-2.json".format(CWD)
 276     expected = json.loads(open(json_file).read())
 277     test_func = partial(
 278         topotest.router_json_cmp, r2, "show ip route 198.10.1.1/32 json", expected
 279     )
 280     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 281     assertmsg = "JSON output mismatch on spine router r2"
 282     assert result is None, assertmsg
 283
 284     # Configure anycast IP on additional server r10
 285     logger.info("Configure anycast IP on server r10")
 286
 287     tgen.net["r10"].cmd("ip addr add 198.10.1.1/32 dev r10-eth1")
 288
 289     # Check if bandwidth is properly encoded with non IEEE floatig-point (uint32) format on r3
 290     logger.info(
 291         "Check if bandwidth is properly encoded with non IEEE floatig-point (uint32) format on r3"
 292     )
 293     json_file = "{}/r3/bgp-route-1.json".format(CWD)
 294     expected = json.loads(open(json_file).read())
 295     test_func = partial(
 296         topotest.router_json_cmp, r3, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 297     )
 298     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 299     assertmsg = "JSON output mismatch on r3"
 300     assert result is None, assertmsg
 301
 302     # Check multipath on super-spine router r1
 303     logger.info("Check multipath on super-spine router r1")
 304     json_file = "{}/r1/bgp-route-2.json".format(CWD)
 305     expected = json.loads(open(json_file).read())
 306     test_func = partial(
 307         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 308     )
 309     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 310     assertmsg = "JSON output mismatch on super-spine router r1"
 311     assert result is None, assertmsg
 312
 313     # Check weighted ECMP is in effect on the super-spine router r1
 314     logger.info("Check weighted ECMP is in effect on the super-spine router r1")
 315     json_file = "{}/r1/ip-route-1.json".format(CWD)
 316     expected = json.loads(open(json_file).read())
 317     test_func = partial(
 318         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 319     )
 320     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 321     assertmsg = "JSON output mismatch on super-spine router r1"
 322     assert result is None, assertmsg
 323
 324
 325 def test_weighted_ecmp_link_flap():
 326     "Test #4: Test weighted ECMP rebalancing upon change (link flap)"
 327     logger.info("\nTest #4: Test weighted ECMP rebalancing upon change (link flap)")
 328
 329     tgen = get_topogen()
 330     if tgen.routers_have_failure():
 331         pytest.skip("skipped because of router(s) failure")
 332
 333     r1 = tgen.gears["r1"]
 334     r2 = tgen.gears["r2"]
 335
 336     # Bring down link on server r9
 337     logger.info("Bring down link on server r9")
 338
 339     tgen.net["r9"].cmd("ip link set dev r9-eth1 down")
 340
 341     # Check spine router r2 has only one path
 342     logger.info("Check spine router r2 has only one path")
 343
 344     json_file = "{}/r2/ip-route-3.json".format(CWD)
 345     expected = json.loads(open(json_file).read())
 346     test_func = partial(
 347         topotest.router_json_cmp, r2, "show ip route 198.10.1.1/32 json", expected
 348     )
 349     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 350     assertmsg = "JSON output mismatch on spine router r2"
 351     assert result is None, assertmsg
 352
 353     # Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1
 354     logger.info(
 355         "Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1"
 356     )
 357
 358     json_file = "{}/r1/bgp-route-3.json".format(CWD)
 359     expected = json.loads(open(json_file).read())
 360     test_func = partial(
 361         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 362     )
 363     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 364     assertmsg = "JSON output mismatch on super-spine router r1"
 365     assert result is None, assertmsg
 366
 367     json_file = "{}/r1/ip-route-2.json".format(CWD)
 368     expected = json.loads(open(json_file).read())
 369     test_func = partial(
 370         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 371     )
 372     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 373     assertmsg = "JSON output mismatch on super-spine router r1"
 374     assert result is None, assertmsg
 375
 376     # Bring up link on server r9
 377     logger.info("Bring up link on server r9")
 378
 379     tgen.net["r9"].cmd("ip link set dev r9-eth1 up")
 380
 381     # Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1
 382     logger.info(
 383         "Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1"
 384     )
 385
 386     json_file = "{}/r1/bgp-route-2.json".format(CWD)
 387     expected = json.loads(open(json_file).read())
 388     test_func = partial(
 389         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 390     )
 391     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 392     assertmsg = "JSON output mismatch on super-spine router r1"
 393     assert result is None, assertmsg
 394
 395     json_file = "{}/r1/ip-route-1.json".format(CWD)
 396     expected = json.loads(open(json_file).read())
 397     test_func = partial(
 398         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 399     )
 400     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 401     assertmsg = "JSON output mismatch on super-spine router r1"
 402     assert result is None, assertmsg
 403
 404
 405 def test_weighted_ecmp_second_anycast_ip():
 406     "Test #5: Test weighted ECMP for a second anycast IP"
 407     logger.info("\nTest #5: Test weighted ECMP for a second anycast IP")
 408
 409     tgen = get_topogen()
 410     if tgen.routers_have_failure():
 411         pytest.skip("skipped because of router(s) failure")
 412
 413     r1 = tgen.gears["r1"]
 414     r2 = tgen.gears["r2"]
 415
 416     # Configure anycast IP on additional server r7, r9 and r10
 417     logger.info("Configure anycast IP on server r7, r9 and r10")
 418
 419     tgen.net["r7"].cmd("ip addr add 198.10.1.11/32 dev r7-eth1")
 420     tgen.net["r9"].cmd("ip addr add 198.10.1.11/32 dev r9-eth1")
 421     tgen.net["r10"].cmd("ip addr add 198.10.1.11/32 dev r10-eth1")
 422
 423     # Check link-bandwidth and weighted ECMP on super-spine router r1
 424     logger.info("Check link-bandwidth and weighted ECMP on super-spine router r1")
 425
 426     json_file = "{}/r1/bgp-route-4.json".format(CWD)
 427     expected = json.loads(open(json_file).read())
 428     test_func = partial(
 429         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.11/32 json", expected
 430     )
 431     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 432     assertmsg = "JSON output mismatch on super-spine router r1"
 433     assert result is None, assertmsg
 434
 435     json_file = "{}/r1/ip-route-3.json".format(CWD)
 436     expected = json.loads(open(json_file).read())
 437     test_func = partial(
 438         topotest.router_json_cmp, r1, "show ip route 198.10.1.11/32 json", expected
 439     )
 440     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 441     assertmsg = "JSON output mismatch on super-spine router r1"
 442     assert result is None, assertmsg
 443
 444
 445 def test_paths_with_and_without_linkbw():
 446     "Test #6: Test paths with and without link-bandwidth - receiver should resort to regular ECMP"
 447     logger.info(
 448         "\nTest #6: Test paths with and without link-bandwidth - receiver should resort to regular ECMP"
 449     )
 450
 451     tgen = get_topogen()
 452     if tgen.routers_have_failure():
 453         pytest.skip("skipped because of router(s) failure")
 454
 455     r1 = tgen.gears["r1"]
 456
 457     # Configure leaf router r6 to not advertise any link-bandwidth
 458     logger.info("Configure leaf router r6 to not advertise any link-bandwidth")
 459
 460     tgen.net["r6"].cmd(
 461         'vtysh -c "conf t" -c "router bgp 65303" -c "address-family ipv4 unicast" -c "no neighbor 11.1.3.1 route-map anycast_ip out"'
 462     )
 463
 464     # Check link-bandwidth change on super-spine router r1
 465     logger.info("Check link-bandwidth change on super-spine router r1")
 466
 467     json_file = "{}/r1/bgp-route-5.json".format(CWD)
 468     expected = json.loads(open(json_file).read())
 469     test_func = partial(
 470         topotest.router_json_cmp, r1, "show bgp ipv4 uni 198.10.1.1/32 json", expected
 471     )
 472     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 473     assertmsg = "JSON output mismatch on super-spine router r1"
 474     assert result is None, assertmsg
 475
 476     # Check super-spine router r1 resorts to regular ECMP
 477     logger.info("Check super-spine router r1 resorts to regular ECMP")
 478
 479     json_file = "{}/r1/ip-route-4.json".format(CWD)
 480     expected = json.loads(open(json_file).read())
 481     test_func = partial(
 482         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 483     )
 484     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 485     assertmsg = "JSON output mismatch on super-spine router r1"
 486     assert result is None, assertmsg
 487
 488     json_file = "{}/r1/ip-route-5.json".format(CWD)
 489     expected = json.loads(open(json_file).read())
 490     test_func = partial(
 491         topotest.router_json_cmp, r1, "show ip route 198.10.1.11/32 json", expected
 492     )
 493     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 494     assertmsg = "JSON output mismatch on super-spine router r1"
 495     assert result is None, assertmsg
 496
 497
 498 def test_linkbw_handling_options():
 499     "Test #7: Test different options for processing link-bandwidth on the receiver"
 500     logger.info(
 501         "\nTest #7: Test different options for processing link-bandwidth on the receiver"
 502     )
 503
 504     tgen = get_topogen()
 505     if tgen.routers_have_failure():
 506         pytest.skip("skipped because of router(s) failure")
 507
 508     r1 = tgen.gears["r1"]
 509
 510     # Configure super-spine r1 to skip multipaths without link-bandwidth
 511     logger.info("Configure super-spine r1 to skip multipaths without link-bandwidth")
 512
 513     tgen.net["r1"].cmd(
 514         'vtysh -c "conf t" -c "router bgp 65101" -c "bgp bestpath bandwidth skip-missing"'
 515     )
 516
 517     # Check super-spine router r1 resorts to only one path as other path is skipped
 518     logger.info(
 519         "Check super-spine router r1 resorts to only one path as other path is skipped"
 520     )
 521
 522     json_file = "{}/r1/ip-route-6.json".format(CWD)
 523     expected = json.loads(open(json_file).read())
 524     test_func = partial(
 525         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 526     )
 527     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 528     assertmsg = "JSON output mismatch on super-spine router r1"
 529     assert result is None, assertmsg
 530
 531     json_file = "{}/r1/ip-route-7.json".format(CWD)
 532     expected = json.loads(open(json_file).read())
 533     test_func = partial(
 534         topotest.router_json_cmp, r1, "show ip route 198.10.1.11/32 json", expected
 535     )
 536     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 537     assertmsg = "JSON output mismatch on super-spine router r1"
 538     assert result is None, assertmsg
 539
 540     # Configure super-spine r1 to use default-weight for multipaths without link-bandwidth
 541     logger.info(
 542         "Configure super-spine r1 to use default-weight for multipaths without link-bandwidth"
 543     )
 544
 545     tgen.net["r1"].cmd(
 546         'vtysh -c "conf t" -c "router bgp 65101" -c "bgp bestpath bandwidth default-weight-for-missing"'
 547     )
 548
 549     # Check super-spine router r1 uses ECMP with weight 1 for path without link-bandwidth
 550     logger.info(
 551         "Check super-spine router r1 uses ECMP with weight 1 for path without link-bandwidth"
 552     )
 553
 554     json_file = "{}/r1/ip-route-8.json".format(CWD)
 555     expected = json.loads(open(json_file).read())
 556     test_func = partial(
 557         topotest.router_json_cmp, r1, "show ip route 198.10.1.1/32 json", expected
 558     )
 559     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 560     assertmsg = "JSON output mismatch on super-spine router r1"
 561     assert result is None, assertmsg
 562
 563     json_file = "{}/r1/ip-route-9.json".format(CWD)
 564     expected = json.loads(open(json_file).read())
 565     test_func = partial(
 566         topotest.router_json_cmp, r1, "show ip route 198.10.1.11/32 json", expected
 567     )
 568     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 569     assertmsg = "JSON output mismatch on super-spine router r1"
 570     assert result is None, assertmsg
 571
 572
 573 if __name__ == "__main__":
 574     args = ["-s"] + sys.argv[1:]
 575     sys.exit(pytest.main(args))