tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py

   1 #!/usr/bin/env python
   2
   3 #
   4 # test_ospf_gr_topo1.py
   5 # Part of NetDEF Topology Tests
   6 #
   7 # Copyright (c) 2021 by
   8 # Network Device Education Foundation, Inc. ("NetDEF")
   9 #
  10 # Permission to use, copy, modify, and/or distribute this software
  11 # for any purpose with or without fee is hereby granted, provided
  12 # that the above copyright notice and this permission notice appear
  13 # in all copies.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES
  16 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  17 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR
  18 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
  19 # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
  20 # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
  21 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
  22 # OF THIS SOFTWARE.
  23 #
  24
  25 """
  26 test_ospf_gr_topo1.py:
  27
  28              +---------+
  29              |   RT1   |
  30              | 1.1.1.1 |
  31              +---------+
  32                   |eth-rt2
  33                   |
  34                   |10.0.1.0/24
  35                   |
  36                   |eth-rt1
  37              +---------+
  38              |   RT2   |
  39              | 2.2.2.2 |
  40              +---------+
  41                   |eth-rt3
  42                   |
  43                   |10.0.2.0/24
  44                   |
  45                   |eth-rt2
  46              +---------+
  47              |   RT3   |
  48              | 3.3.3.3 |
  49              +---------+
  50           eth-rt4|  |eth-rt6
  51                  |  |
  52      10.0.3.0/24 |  |  10.0.4.0/24
  53        +---------+  +--------+
  54        |                     |
  55        |eth-rt3              |eth-rt3
  56   +---------+           +---------+
  57   |   RT4   |           |   RT6   |
  58   | 4.4.4.4 |           | 6.6.6.6 |
  59   +---------+           +---------+
  60        |eth-rt5              |eth-rt7
  61        |                     |
  62        |10.0.5.0/24          |10.0.6.0/24
  63        |                     |
  64        |eth-rt4              |eth-rt6
  65   +---------+           +---------+
  66   |   RT5   |           |   RT7   |
  67   | 5.5.5.5 |           | 7.7.7.7 |
  68   +---------+           +---------+
  69 """
  70
  71 import os
  72 import sys
  73 import pytest
  74 import json
  75 from time import sleep
  76 from functools import partial
  77
  78 # Save the Current Working Directory to find configuration files.
  79 CWD = os.path.dirname(os.path.realpath(__file__))
  80 sys.path.append(os.path.join(CWD, "../"))
  81
  82 # pylint: disable=C0413
  83 # Import topogen and topotest helpers
  84 from lib import topotest
  85 from lib.topogen import Topogen, TopoRouter, get_topogen
  86 from lib.topolog import logger
  87 from lib.common_config import (
  88     kill_router_daemons,
  89     start_router_daemons,
  90 )
  91
  92 # Required to instantiate the topology builder class.
  93
  94 pytestmark = [pytest.mark.ospfd]
  95
  96 # Global multi-dimensional dictionary containing all expected outputs
  97 outputs = {}
  98
  99
 100 def build_topo(tgen):
 101     #
 102     # Define FRR Routers
 103     #
 104     for router in ["rt1", "rt2", "rt3", "rt4", "rt5", "rt6", "rt7"]:
 105         tgen.add_router(router)
 106
 107     #
 108     # Define connections
 109     #
 110     switch = tgen.add_switch("s1")
 111     switch.add_link(tgen.gears["rt1"], nodeif="eth-rt2")
 112     switch.add_link(tgen.gears["rt2"], nodeif="eth-rt1")
 113
 114     switch = tgen.add_switch("s2")
 115     switch.add_link(tgen.gears["rt1"], nodeif="stub1")
 116
 117     switch = tgen.add_switch("s3")
 118     switch.add_link(tgen.gears["rt2"], nodeif="eth-rt3")
 119     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt2")
 120
 121     switch = tgen.add_switch("s4")
 122     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt4")
 123     switch.add_link(tgen.gears["rt4"], nodeif="eth-rt3")
 124
 125     switch = tgen.add_switch("s5")
 126     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt6")
 127     switch.add_link(tgen.gears["rt6"], nodeif="eth-rt3")
 128
 129     switch = tgen.add_switch("s6")
 130     switch.add_link(tgen.gears["rt4"], nodeif="eth-rt5")
 131     switch.add_link(tgen.gears["rt5"], nodeif="eth-rt4")
 132
 133     switch = tgen.add_switch("s7")
 134     switch.add_link(tgen.gears["rt6"], nodeif="eth-rt7")
 135     switch.add_link(tgen.gears["rt7"], nodeif="eth-rt6")
 136
 137     switch = tgen.add_switch("s8")
 138     switch.add_link(tgen.gears["rt7"], nodeif="stub1")
 139
 140
 141 def setup_module(mod):
 142     return pytest.skip("OSPF GR helper mode is currently broken")
 143
 144     "Sets up the pytest environment"
 145     tgen = Topogen(build_topo, mod.__name__)
 146     tgen.start_topology()
 147
 148     router_list = tgen.routers()
 149
 150     # For all registered routers, load the zebra configuration file
 151     for rname, router in router_list.items():
 152         router.load_config(
 153             TopoRouter.RD_ZEBRA, os.path.join(CWD, "{}/zebra.conf".format(rname))
 154         )
 155         router.load_config(
 156             TopoRouter.RD_OSPF, os.path.join(CWD, "{}/ospfd.conf".format(rname))
 157         )
 158
 159     tgen.start_router()
 160
 161
 162 def teardown_module(mod):
 163     "Teardown the pytest environment"
 164     tgen = get_topogen()
 165
 166     # This function tears down the whole topology.
 167     tgen.stop_topology()
 168
 169
 170 def router_compare_json_output(rname, command, reference, tries):
 171     "Compare router JSON output"
 172
 173     logger.info('Comparing router "%s" "%s" output', rname, command)
 174
 175     tgen = get_topogen()
 176     filename = "{}/{}/{}".format(CWD, rname, reference)
 177     expected = json.loads(open(filename).read())
 178
 179     test_func = partial(topotest.router_json_cmp, tgen.gears[rname], command, expected)
 180     _, diff = topotest.run_and_expect(test_func, None, count=tries, wait=0.5)
 181     assertmsg = '"{}" JSON output mismatches the expected result'.format(rname)
 182     assert diff is None, assertmsg
 183
 184
 185 def expect_grace_lsa(restarting, area, helper):
 186     """
 187     Check if the given helper neighbor has already received a Grace-LSA from
 188     the router performing a graceful restart.
 189     """
 190     tgen = get_topogen()
 191
 192     logger.info(
 193         "'{}': checking if a Grace-LSA was received from '{}'".format(
 194             helper, restarting
 195         )
 196     )
 197     test_func = partial(
 198         topotest.router_json_cmp,
 199         tgen.gears[helper],
 200         "show ip ospf database opaque-link json",
 201         {
 202             "linkLocalOpaqueLsa": {
 203                 "areas": {
 204                     area: [
 205                         {
 206                             "advertisingRouter": restarting,
 207                             "opaqueType": "Grace-LSA",
 208                         }
 209                     ]
 210                 }
 211             }
 212         },
 213     )
 214     _, result = topotest.run_and_expect(test_func, None, count=60, wait=1)
 215     assertmsg = '"{}" didn\'t receive a Grace-LSA from "{}"'.format(helper, restarting)
 216
 217     assert result is None, assertmsg
 218
 219
 220 def check_routers(initial_convergence=False, exiting=None, restarting=None):
 221     for rname in ["rt1", "rt2", "rt3", "rt4", "rt5", "rt6", "rt7"]:
 222         # Check the RIB first, which should be preserved across restarts in
 223         # all routers of the routing domain.
 224         # If we are not on initial convergence *but* we are checking
 225         # after a restart.  Looking in the zebra rib for installed
 226         # is a recipe for test failure.  Why?  because if we are restarting
 227         # then ospf is in the process of establishing neighbors and passing
 228         # new routes to zebra.  Zebra will not mark the route as installed
 229         # when it receives a replacement from ospf until it has finished
 230         # processing it.  Let's give it a few seconds to allow this to happen
 231         # under load.
 232         if initial_convergence == True:
 233             tries = 240
 234         else:
 235             if restarting != None:
 236                 tries = 60
 237             else:
 238                 tries = 1
 239         router_compare_json_output(
 240             rname, "show ip route ospf json", "show_ip_route.json", tries
 241         )
 242
 243         # Check that all adjacencies are up and running (except when there's
 244         # an OSPF instance that is shutting down).
 245         if exiting == None:
 246             tries = 240
 247             router_compare_json_output(
 248                 rname, "show ip ospf neighbor json", "show_ip_ospf_neighbor.json", tries
 249             )
 250
 251         # Check the OSPF RIB and LSDB.
 252         # In the restarting router, wait up to one minute for the LSDB to converge.
 253         if exiting != rname:
 254             if initial_convergence == True or restarting == rname:
 255                 tries = 240
 256             else:
 257                 tries = 1
 258             router_compare_json_output(
 259                 rname, "show ip ospf database json", "show_ip_ospf_database.json", tries
 260             )
 261             router_compare_json_output(
 262                 rname, "show ip ospf route json", "show_ip_ospf_route.json", tries
 263             )
 264
 265
 266 def ensure_gr_is_in_zebra(rname):
 267     retry = True
 268     retry_times = 10
 269     tgen = get_topogen()
 270
 271     while retry and retry_times > 0:
 272         out = tgen.net[rname].cmd(
 273             'vtysh -c "show zebra client" | grep "Client: ospf$" -A 40 | grep "Capabilities "'
 274         )
 275
 276         if "Graceful Restart" not in out:
 277             sleep(2)
 278             retry_times -= 1
 279         else:
 280             retry = False
 281
 282     assertmsg = "%s does not appear to have Graceful Restart setup" % rname
 283     assert not retry and retry_times > 0, assertmsg
 284
 285
 286 #
 287 # Test initial network convergence
 288 #
 289 def test_initial_convergence():
 290     logger.info("Test: verify initial network convergence")
 291     tgen = get_topogen()
 292
 293     # Skip if previous fatal error condition is raised
 294     if tgen.routers_have_failure():
 295         pytest.skip(tgen.errors)
 296
 297     check_routers(initial_convergence=True)
 298
 299
 300 #
 301 # Test rt1 performing a graceful restart
 302 #
 303 def test_gr_rt1():
 304     logger.info("Test: verify rt1 performing a graceful restart")
 305     tgen = get_topogen()
 306
 307     # Skip if previous fatal error condition is raised
 308     if tgen.routers_have_failure():
 309         pytest.skip(tgen.errors)
 310
 311     tgen.net["rt1"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 312     expect_grace_lsa(restarting="1.1.1.1", area="0.0.0.1", helper="rt2")
 313     ensure_gr_is_in_zebra("rt1")
 314     kill_router_daemons(tgen, "rt1", ["ospfd"], save_config=False)
 315     check_routers(exiting="rt1")
 316
 317     start_router_daemons(tgen, "rt1", ["ospfd"])
 318     check_routers(restarting="rt1")
 319
 320
 321 #
 322 # Test rt2 performing a graceful restart
 323 #
 324 def test_gr_rt2():
 325     logger.info("Test: verify rt2 performing a graceful restart")
 326     tgen = get_topogen()
 327
 328     # Skip if previous fatal error condition is raised
 329     if tgen.routers_have_failure():
 330         pytest.skip(tgen.errors)
 331
 332     tgen.net["rt2"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 333     expect_grace_lsa(restarting="2.2.2.2", area="0.0.0.1", helper="rt1")
 334     expect_grace_lsa(restarting="2.2.2.2", area="0.0.0.0", helper="rt3")
 335     ensure_gr_is_in_zebra("rt2")
 336     kill_router_daemons(tgen, "rt2", ["ospfd"], save_config=False)
 337     check_routers(exiting="rt2")
 338
 339     start_router_daemons(tgen, "rt2", ["ospfd"])
 340     check_routers(restarting="rt2")
 341
 342
 343 #
 344 # Test rt3 performing a graceful restart
 345 #
 346 def test_gr_rt3():
 347     logger.info("Test: verify rt3 performing a graceful restart")
 348     tgen = get_topogen()
 349
 350     # Skip if previous fatal error condition is raised
 351     if tgen.routers_have_failure():
 352         pytest.skip(tgen.errors)
 353
 354     tgen.net["rt3"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 355     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt2")
 356     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt4")
 357     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt6")
 358     ensure_gr_is_in_zebra("rt3")
 359     kill_router_daemons(tgen, "rt3", ["ospfd"], save_config=False)
 360     check_routers(exiting="rt3")
 361
 362     start_router_daemons(tgen, "rt3", ["ospfd"])
 363     check_routers(restarting="rt3")
 364
 365
 366 #
 367 # Test rt4 performing a graceful restart
 368 #
 369 def test_gr_rt4():
 370     logger.info("Test: verify rt4 performing a graceful restart")
 371     tgen = get_topogen()
 372
 373     # Skip if previous fatal error condition is raised
 374     if tgen.routers_have_failure():
 375         pytest.skip(tgen.errors)
 376
 377     tgen.net["rt4"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 378     expect_grace_lsa(restarting="4.4.4.4", area="0.0.0.0", helper="rt3")
 379     expect_grace_lsa(restarting="4.4.4.4", area="0.0.0.2", helper="rt5")
 380     ensure_gr_is_in_zebra("rt4")
 381     kill_router_daemons(tgen, "rt4", ["ospfd"], save_config=False)
 382     check_routers(exiting="rt4")
 383
 384     start_router_daemons(tgen, "rt4", ["ospfd"])
 385     check_routers(restarting="rt4")
 386
 387
 388 #
 389 # Test rt5 performing a graceful restart
 390 #
 391 def test_gr_rt5():
 392     logger.info("Test: verify rt5 performing a graceful restart")
 393     tgen = get_topogen()
 394
 395     # Skip if previous fatal error condition is raised
 396     if tgen.routers_have_failure():
 397         pytest.skip(tgen.errors)
 398
 399     tgen.net["rt5"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 400     expect_grace_lsa(restarting="5.5.5.5", area="0.0.0.2", helper="rt4")
 401     ensure_gr_is_in_zebra("rt5")
 402     kill_router_daemons(tgen, "rt5", ["ospfd"], save_config=False)
 403     check_routers(exiting="rt5")
 404
 405     start_router_daemons(tgen, "rt5", ["ospfd"])
 406     check_routers(restarting="rt5")
 407
 408
 409 #
 410 # Test rt6 performing a graceful restart
 411 #
 412 def test_gr_rt6():
 413     logger.info("Test: verify rt6 performing a graceful restart")
 414     tgen = get_topogen()
 415
 416     # Skip if previous fatal error condition is raised
 417     if tgen.routers_have_failure():
 418         pytest.skip(tgen.errors)
 419
 420     tgen.net["rt6"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 421     expect_grace_lsa(restarting="6.6.6.6", area="0.0.0.0", helper="rt3")
 422     expect_grace_lsa(restarting="6.6.6.6", area="0.0.0.3", helper="rt7")
 423     ensure_gr_is_in_zebra("rt6")
 424     kill_router_daemons(tgen, "rt6", ["ospfd"], save_config=False)
 425     check_routers(exiting="rt6")
 426
 427     start_router_daemons(tgen, "rt6", ["ospfd"])
 428     check_routers(restarting="rt6")
 429
 430
 431 #
 432 # Test rt7 performing a graceful restart
 433 #
 434 def test_gr_rt7():
 435     logger.info("Test: verify rt7 performing a graceful restart")
 436     tgen = get_topogen()
 437
 438     # Skip if previous fatal error condition is raised
 439     if tgen.routers_have_failure():
 440         pytest.skip(tgen.errors)
 441
 442     tgen.net["rt7"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 443     expect_grace_lsa(restarting="7.7.7.7", area="0.0.0.3", helper="rt6")
 444     ensure_gr_is_in_zebra("rt7")
 445     kill_router_daemons(tgen, "rt7", ["ospfd"], save_config=False)
 446     check_routers(exiting="rt7")
 447
 448     start_router_daemons(tgen, "rt7", ["ospfd"])
 449     check_routers(restarting="rt7")
 450
 451
 452 # Memory leak test template
 453 def test_memory_leak():
 454     "Run the memory leak test and report results."
 455     tgen = get_topogen()
 456     if not tgen.is_memleak_enabled():
 457         pytest.skip("Memory leak test/report is disabled")
 458
 459     tgen.report_memory_leaks()
 460
 461
 462 if __name__ == "__main__":
 463     args = ["-s"] + sys.argv[1:]
 464     sys.exit(pytest.main(args))