tests/topotests/bgp_link_bw_ip/test_bgp_linkbw_ip.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # test_bgp_linkbw_ip.py
   5 #
   6 # Copyright (c) 2020 by
   7 # Cumulus Networks, Inc
   8 # Vivek Venkatraman
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 """
  26 test_bgp_linkbw_ip.py: Test weighted ECMP using BGP link-bandwidth
  27 """
  28
  29 import os
  30 import re
  31 import sys
  32 from functools import partial
  33 import pytest
  34 import json
  35
  36 # Save the Current Working Directory to find configuration files.
  37 CWD = os.path.dirname(os.path.realpath(__file__))
  38 sys.path.append(os.path.join(CWD, '../'))
  39
  40 # pylint: disable=C0413
  41 # Import topogen and topotest helpers
  42 from lib import topotest
  43 from lib.topogen import Topogen, TopoRouter, get_topogen
  44 from lib.topolog import logger
  45
  46 # Required to instantiate the topology builder class.
  47 from mininet.topo import Topo
  48
  49 """
  50 This topology is for validating one of the primary use cases for
  51 weighted ECMP (a.k.a. Unequal cost multipath) using BGP link-bandwidth:
  52 https://tools.ietf.org/html/draft-mohanty-bess-ebgp-dmz
  53
  54 The topology consists of two PODs. Pod-1 consists of a spine switch
  55 and two leaf switches, with two servers attached to the first leaf and
  56 one to the second leaf. Pod-2 consists of one spine and one leaf, with
  57 one server connected to the leaf. The PODs are connected by a super-spine
  58 switch.
  59
  60 Note that the use of the term "switch" above is in keeping with common
  61 data-center terminology. These devices are all regular routers; for
  62 this scenario, the servers are also routers as they have to announce
  63 anycast IP (VIP) addresses via BGP.
  64 """
  65
  66 class BgpLinkBwTopo(Topo):
  67     "Test topology builder"
  68     def build(self, *_args, **_opts):
  69         "Build function"
  70         tgen = get_topogen(self)
  71
  72         # Create 10 routers - 1 super-spine, 2 spines, 3 leafs
  73         # and 4 servers
  74         routers = {}
  75         for i in range(1, 11):
  76             routers[i] = tgen.add_router('r{}'.format(i))
  77
  78         # Create 13 "switches" - to interconnect the above routers
  79         switches = {}
  80         for i in range(1, 14):
  81             switches[i] = tgen.add_switch('s{}'.format(i))
  82
  83         # Interconnect R1 (super-spine) to R2 and R3 (the two spines)
  84         switches[1].add_link(tgen.gears['r1'])
  85         switches[1].add_link(tgen.gears['r2'])
  86         switches[2].add_link(tgen.gears['r1'])
  87         switches[2].add_link(tgen.gears['r3'])
  88
  89         # Interconnect R2 (spine in pod-1) to R4 and R5 (the associated
  90         # leaf switches)
  91         switches[3].add_link(tgen.gears['r2'])
  92         switches[3].add_link(tgen.gears['r4'])
  93         switches[4].add_link(tgen.gears['r2'])
  94         switches[4].add_link(tgen.gears['r5'])
  95
  96         # Interconnect R3 (spine in pod-2) to R6 (associated leaf)
  97         switches[5].add_link(tgen.gears['r3'])
  98         switches[5].add_link(tgen.gears['r6'])
  99
 100         # Interconnect leaf switches to servers
 101         switches[6].add_link(tgen.gears['r4'])
 102         switches[6].add_link(tgen.gears['r7'])
 103         switches[7].add_link(tgen.gears['r4'])
 104         switches[7].add_link(tgen.gears['r8'])
 105         switches[8].add_link(tgen.gears['r5'])
 106         switches[8].add_link(tgen.gears['r9'])
 107         switches[9].add_link(tgen.gears['r6'])
 108         switches[9].add_link(tgen.gears['r10'])
 109
 110         # Create empty networks for the servers
 111         switches[10].add_link(tgen.gears['r7'])
 112         switches[11].add_link(tgen.gears['r8'])
 113         switches[12].add_link(tgen.gears['r9'])
 114         switches[13].add_link(tgen.gears['r10'])
 115
 116 def setup_module(mod):
 117     "Sets up the pytest environment"
 118     tgen = Topogen(BgpLinkBwTopo, mod.__name__)
 119     tgen.start_topology()
 120
 121     router_list = tgen.routers()
 122     for rname, router in router_list.items():
 123         router.load_config(
 124             TopoRouter.RD_ZEBRA,
 125             os.path.join(CWD, '{}/zebra.conf'.format(rname))
 126         )
 127         router.load_config(
 128             TopoRouter.RD_BGP,
 129             os.path.join(CWD, '{}/bgpd.conf'.format(rname))
 130         )
 131
 132     # Initialize all routers.
 133     tgen.start_router()
 134
 135     #tgen.mininet_cli()
 136
 137 def teardown_module(mod):
 138     "Teardown the pytest environment"
 139     tgen = get_topogen()
 140     tgen.stop_topology()
 141
 142 def test_bgp_linkbw_adv():
 143     "Test #1: Test BGP link-bandwidth advertisement based on number of multipaths"
 144     logger.info('\nTest #1: Test BGP link-bandwidth advertisement based on number of multipaths')
 145
 146     tgen = get_topogen()
 147     if tgen.routers_have_failure():
 148         pytest.skip('skipped because of router(s) failure')
 149
 150     r1 = tgen.gears['r1']
 151     r2 = tgen.gears['r2']
 152
 153     # Configure anycast IP on server r7
 154     logger.info('Configure anycast IP on server r7')
 155
 156     tgen.net['r7'].cmd('ip addr add 198.10.1.1/32 dev r7-eth1')
 157
 158     # Check on spine router r2 for link-bw advertisement by leaf router r4
 159     logger.info('Check on spine router r2 for link-bw advertisement by leaf router r4')
 160
 161     json_file = '{}/r2/bgp-route-1.json'.format(CWD)
 162     expected = json.loads(open(json_file).read())
 163     test_func = partial(topotest.router_json_cmp,
 164                         r2, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 165     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 166     assertmsg = 'JSON output mismatch on spine router r2'
 167     assert result is None, assertmsg
 168
 169     # Check on spine router r2 that default weight is used as there is no multipath
 170     logger.info('Check on spine router r2 that default weight is used as there is no multipath')
 171
 172     json_file = '{}/r2/ip-route-1.json'.format(CWD)
 173     expected = json.loads(open(json_file).read())
 174     test_func = partial(topotest.router_json_cmp,
 175                         r2, 'show ip route 198.10.1.1/32 json', expected)
 176     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 177     assertmsg = 'JSON output mismatch on spine router r2'
 178     assert result is None, assertmsg
 179
 180     # Check on super-spine router r1 that link-bw has been propagated by spine router r2
 181     logger.info('Check on super-spine router r1 that link-bw has been propagated by spine router r2')
 182
 183     json_file = '{}/r1/bgp-route-1.json'.format(CWD)
 184     expected = json.loads(open(json_file).read())
 185     test_func = partial(topotest.router_json_cmp,
 186                         r1, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 187     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 188     assertmsg = 'JSON output mismatch on super-spine router r1'
 189     assert result is None, assertmsg
 190
 191 def test_bgp_cumul_linkbw():
 192     "Test #2: Test cumulative link-bandwidth propagation"
 193     logger.info('\nTest #2: Test cumulative link-bandwidth propagation')
 194
 195     tgen = get_topogen()
 196     if tgen.routers_have_failure():
 197         pytest.skip('skipped because of router(s) failure')
 198
 199     r1 = tgen.gears['r1']
 200     r2 = tgen.gears['r2']
 201     r4 = tgen.gears['r4']
 202
 203     # Configure anycast IP on additional server r8
 204     logger.info('Configure anycast IP on server r8')
 205
 206     tgen.net['r8'].cmd('ip addr add 198.10.1.1/32 dev r8-eth1')
 207
 208     # Check multipath on leaf router r4
 209     logger.info('Check multipath on leaf router r4')
 210
 211     json_file = '{}/r4/bgp-route-1.json'.format(CWD)
 212     expected = json.loads(open(json_file).read())
 213     test_func = partial(topotest.router_json_cmp,
 214                         r4, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 215     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 216     assertmsg = 'JSON output mismatch on leaf router r4'
 217     assert result is None, assertmsg
 218
 219     # Check regular ECMP is in effect on leaf router r4
 220     logger.info('Check regular ECMP is in effect on leaf router r4')
 221
 222     json_file = '{}/r4/ip-route-1.json'.format(CWD)
 223     expected = json.loads(open(json_file).read())
 224     test_func = partial(topotest.router_json_cmp,
 225                         r4, 'show ip route 198.10.1.1/32 json', expected)
 226     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 227     assertmsg = 'JSON output mismatch on leaf router r4'
 228     assert result is None, assertmsg
 229
 230     # Check on spine router r2 that leaf has propagated the cumulative link-bw based on num-multipaths
 231     logger.info('Check on spine router r2 that leaf has propagated the cumulative link-bw based on num-multipaths')
 232
 233     json_file = '{}/r2/bgp-route-2.json'.format(CWD)
 234     expected = json.loads(open(json_file).read())
 235     test_func = partial(topotest.router_json_cmp,
 236                         r2, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 237     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 238     assertmsg = 'JSON output mismatch on spine router r2'
 239     assert result is None, assertmsg
 240
 241 def test_weighted_ecmp():
 242     "Test #3: Test weighted ECMP - multipath with next hop weights"
 243     logger.info('\nTest #3: Test weighted ECMP - multipath with next hop weights')
 244
 245     tgen = get_topogen()
 246     if tgen.routers_have_failure():
 247         pytest.skip('skipped because of router(s) failure')
 248
 249     r1 = tgen.gears['r1']
 250     r2 = tgen.gears['r2']
 251
 252     # Configure anycast IP on additional server r9
 253     logger.info('Configure anycast IP on server r9')
 254
 255     tgen.net['r9'].cmd('ip addr add 198.10.1.1/32 dev r9-eth1')
 256
 257     # Check multipath on spine router r2
 258     logger.info('Check multipath on spine router r2')
 259     json_file = '{}/r2/bgp-route-3.json'.format(CWD)
 260     expected = json.loads(open(json_file).read())
 261     test_func = partial(topotest.router_json_cmp,
 262                         r2, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 263     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 264     assertmsg = 'JSON output mismatch on spine router r2'
 265     assert result is None, assertmsg
 266
 267     # Check weighted ECMP is in effect on the spine router r2
 268     logger.info('Check weighted ECMP is in effect on the spine router r2')
 269
 270     json_file = '{}/r2/ip-route-2.json'.format(CWD)
 271     expected = json.loads(open(json_file).read())
 272     test_func = partial(topotest.router_json_cmp,
 273                         r2, 'show ip route 198.10.1.1/32 json', expected)
 274     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 275     assertmsg = 'JSON output mismatch on spine router r2'
 276     assert result is None, assertmsg
 277
 278     # Configure anycast IP on additional server r10
 279     logger.info('Configure anycast IP on server r10')
 280
 281     tgen.net['r10'].cmd('ip addr add 198.10.1.1/32 dev r10-eth1')
 282
 283     # Check multipath on super-spine router r1
 284     logger.info('Check multipath on super-spine router r1')
 285     json_file = '{}/r1/bgp-route-2.json'.format(CWD)
 286     expected = json.loads(open(json_file).read())
 287     test_func = partial(topotest.router_json_cmp,
 288                         r1, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 289     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 290     assertmsg = 'JSON output mismatch on super-spine router r1'
 291     assert result is None, assertmsg
 292
 293     # Check weighted ECMP is in effect on the super-spine router r1
 294     logger.info('Check weighted ECMP is in effect on the super-spine router r1')
 295     json_file = '{}/r1/ip-route-1.json'.format(CWD)
 296     expected = json.loads(open(json_file).read())
 297     test_func = partial(topotest.router_json_cmp,
 298                         r1, 'show ip route 198.10.1.1/32 json', expected)
 299     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 300     assertmsg = 'JSON output mismatch on super-spine router r1'
 301     assert result is None, assertmsg
 302
 303 def test_weighted_ecmp_link_flap():
 304     "Test #4: Test weighted ECMP rebalancing upon change (link flap)"
 305     logger.info('\nTest #4: Test weighted ECMP rebalancing upon change (link flap)')
 306
 307     tgen = get_topogen()
 308     if tgen.routers_have_failure():
 309         pytest.skip('skipped because of router(s) failure')
 310
 311     r1 = tgen.gears['r1']
 312     r2 = tgen.gears['r2']
 313
 314     # Bring down link on server r9
 315     logger.info('Bring down link on server r9')
 316
 317     tgen.net['r9'].cmd('ip link set dev r9-eth1 down')
 318
 319     # Check spine router r2 has only one path
 320     logger.info('Check spine router r2 has only one path')
 321
 322     json_file = '{}/r2/ip-route-3.json'.format(CWD)
 323     expected = json.loads(open(json_file).read())
 324     test_func = partial(topotest.router_json_cmp,
 325                         r2, 'show ip route 198.10.1.1/32 json', expected)
 326     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 327     assertmsg = 'JSON output mismatch on spine router r2'
 328     assert result is None, assertmsg
 329
 330     # Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1
 331     logger.info('Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1')
 332
 333     json_file = '{}/r1/bgp-route-3.json'.format(CWD)
 334     expected = json.loads(open(json_file).read())
 335     test_func = partial(topotest.router_json_cmp,
 336                         r1, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 337     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 338     assertmsg = 'JSON output mismatch on super-spine router r1'
 339     assert result is None, assertmsg
 340
 341     json_file = '{}/r1/ip-route-2.json'.format(CWD)
 342     expected = json.loads(open(json_file).read())
 343     test_func = partial(topotest.router_json_cmp,
 344                         r1, 'show ip route 198.10.1.1/32 json', expected)
 345     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 346     assertmsg = 'JSON output mismatch on super-spine router r1'
 347     assert result is None, assertmsg
 348
 349     # Bring up link on server r9
 350     logger.info('Bring up link on server r9')
 351
 352     tgen.net['r9'].cmd('ip link set dev r9-eth1 up')
 353
 354     # Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1
 355     logger.info('Check link-bandwidth change and weighted ECMP rebalance on super-spine router r1')
 356
 357     json_file = '{}/r1/bgp-route-2.json'.format(CWD)
 358     expected = json.loads(open(json_file).read())
 359     test_func = partial(topotest.router_json_cmp,
 360                         r1, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 361     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 362     assertmsg = 'JSON output mismatch on super-spine router r1'
 363     assert result is None, assertmsg
 364
 365     json_file = '{}/r1/ip-route-1.json'.format(CWD)
 366     expected = json.loads(open(json_file).read())
 367     test_func = partial(topotest.router_json_cmp,
 368                         r1, 'show ip route 198.10.1.1/32 json', expected)
 369     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 370     assertmsg = 'JSON output mismatch on super-spine router r1'
 371     assert result is None, assertmsg
 372
 373 def test_weighted_ecmp_second_anycast_ip():
 374     "Test #5: Test weighted ECMP for a second anycast IP"
 375     logger.info('\nTest #5: Test weighted ECMP for a second anycast IP')
 376
 377     tgen = get_topogen()
 378     if tgen.routers_have_failure():
 379         pytest.skip('skipped because of router(s) failure')
 380
 381     r1 = tgen.gears['r1']
 382     r2 = tgen.gears['r2']
 383
 384     # Configure anycast IP on additional server r7, r9 and r10
 385     logger.info('Configure anycast IP on server r7, r9 and r10')
 386
 387     tgen.net['r7'].cmd('ip addr add 198.10.1.11/32 dev r7-eth1')
 388     tgen.net['r9'].cmd('ip addr add 198.10.1.11/32 dev r9-eth1')
 389     tgen.net['r10'].cmd('ip addr add 198.10.1.11/32 dev r10-eth1')
 390
 391     # Check link-bandwidth and weighted ECMP on super-spine router r1
 392     logger.info('Check link-bandwidth and weighted ECMP on super-spine router r1')
 393
 394     json_file = '{}/r1/bgp-route-4.json'.format(CWD)
 395     expected = json.loads(open(json_file).read())
 396     test_func = partial(topotest.router_json_cmp,
 397                         r1, 'show bgp ipv4 uni 198.10.1.11/32 json', expected)
 398     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 399     assertmsg = 'JSON output mismatch on super-spine router r1'
 400     assert result is None, assertmsg
 401
 402     json_file = '{}/r1/ip-route-3.json'.format(CWD)
 403     expected = json.loads(open(json_file).read())
 404     test_func = partial(topotest.router_json_cmp,
 405                         r1, 'show ip route 198.10.1.11/32 json', expected)
 406     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 407     assertmsg = 'JSON output mismatch on super-spine router r1'
 408     assert result is None, assertmsg
 409
 410 def test_paths_with_and_without_linkbw():
 411     "Test #6: Test paths with and without link-bandwidth - receiver should resort to regular ECMP"
 412     logger.info('\nTest #6: Test paths with and without link-bandwidth - receiver should resort to regular ECMP')
 413
 414     tgen = get_topogen()
 415     if tgen.routers_have_failure():
 416         pytest.skip('skipped because of router(s) failure')
 417
 418     r1 = tgen.gears['r1']
 419
 420     # Configure leaf router r6 to not advertise any link-bandwidth
 421     logger.info('Configure leaf router r6 to not advertise any link-bandwidth')
 422
 423     tgen.net['r6'].cmd('vtysh -c \"conf t\" -c \"router bgp 65303\" -c \"address-family ipv4 unicast\" -c \"no neighbor 11.1.3.1 route-map anycast_ip out\"')
 424
 425     # Check link-bandwidth change on super-spine router r1
 426     logger.info('Check link-bandwidth change on super-spine router r1')
 427
 428     json_file = '{}/r1/bgp-route-5.json'.format(CWD)
 429     expected = json.loads(open(json_file).read())
 430     test_func = partial(topotest.router_json_cmp,
 431                         r1, 'show bgp ipv4 uni 198.10.1.1/32 json', expected)
 432     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 433     assertmsg = 'JSON output mismatch on super-spine router r1'
 434     assert result is None, assertmsg
 435
 436     # Check super-spine router r1 resorts to regular ECMP
 437     logger.info('Check super-spine router r1 resorts to regular ECMP')
 438
 439     json_file = '{}/r1/ip-route-4.json'.format(CWD)
 440     expected = json.loads(open(json_file).read())
 441     test_func = partial(topotest.router_json_cmp,
 442                         r1, 'show ip route 198.10.1.1/32 json', expected)
 443     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 444     assertmsg = 'JSON output mismatch on super-spine router r1'
 445     assert result is None, assertmsg
 446
 447     json_file = '{}/r1/ip-route-5.json'.format(CWD)
 448     expected = json.loads(open(json_file).read())
 449     test_func = partial(topotest.router_json_cmp,
 450                         r1, 'show ip route 198.10.1.11/32 json', expected)
 451     _, result = topotest.run_and_expect(test_func, None, count=50, wait=0.5)
 452     assertmsg = 'JSON output mismatch on super-spine router r1'
 453     assert result is None, assertmsg
 454
 455 def test_linkbw_handling_options():
 456     "Test #7: Test different options for processing link-bandwidth on the receiver"
 457     logger.info('\nTest #7: Test different options for processing link-bandwidth on the receiver')
 458
 459     tgen = get_topogen()
 460     if tgen.routers_have_failure():
 461         pytest.skip('skipped because of router(s) failure')
 462
 463     r1 = tgen.gears['r1']
 464
 465     # Configure super-spine r1 to skip multipaths without link-bandwidth
 466     logger.info('Configure super-spine r1 to skip multipaths without link-bandwidth')
 467
 468     tgen.net['r1'].cmd('vtysh -c \"conf t\" -c \"router bgp 65101\" -c \"bgp bestpath bandwidth skip-missing\"')
 469
 470     # Check super-spine router r1 resorts to only one path as other path is skipped
 471     logger.info('Check super-spine router r1 resorts to only one path as other path is skipped')
 472
 473     json_file = '{}/r1/ip-route-6.json'.format(CWD)
 474     expected = json.loads(open(json_file).read())
 475     test_func = partial(topotest.router_json_cmp,
 476                         r1, 'show ip route 198.10.1.1/32 json', expected)
 477     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 478     assertmsg = 'JSON output mismatch on super-spine router r1'
 479     assert result is None, assertmsg
 480
 481     json_file = '{}/r1/ip-route-7.json'.format(CWD)
 482     expected = json.loads(open(json_file).read())
 483     test_func = partial(topotest.router_json_cmp,
 484                         r1, 'show ip route 198.10.1.11/32 json', expected)
 485     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 486     assertmsg = 'JSON output mismatch on super-spine router r1'
 487     assert result is None, assertmsg
 488
 489     # Configure super-spine r1 to use default-weight for multipaths without link-bandwidth
 490     logger.info('Configure super-spine r1 to use default-weight for multipaths without link-bandwidth')
 491
 492     tgen.net['r1'].cmd('vtysh -c \"conf t\" -c \"router bgp 65101\" -c \"bgp bestpath bandwidth default-weight-for-missing\"')
 493
 494     # Check super-spine router r1 uses ECMP with weight 1 for path without link-bandwidth
 495     logger.info('Check super-spine router r1 uses ECMP with weight 1 for path without link-bandwidth')
 496
 497     json_file = '{}/r1/ip-route-8.json'.format(CWD)
 498     expected = json.loads(open(json_file).read())
 499     test_func = partial(topotest.router_json_cmp,
 500                         r1, 'show ip route 198.10.1.1/32 json', expected)
 501     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 502     assertmsg = 'JSON output mismatch on super-spine router r1'
 503     assert result is None, assertmsg
 504
 505     json_file = '{}/r1/ip-route-9.json'.format(CWD)
 506     expected = json.loads(open(json_file).read())
 507     test_func = partial(topotest.router_json_cmp,
 508                         r1, 'show ip route 198.10.1.11/32 json', expected)
 509     _, result = topotest.run_and_expect(test_func, None, count=200, wait=0.5)
 510     assertmsg = 'JSON output mismatch on super-spine router r1'
 511     assert result is None, assertmsg
 512
 513 if __name__ == '__main__':
 514     args = ["-s"] + sys.argv[1:]
 515     sys.exit(pytest.main(args))