tests/topotests/ospf_gr_topo1/test_ospf_gr_topo1.py

   1 #!/usr/bin/env python
   2 # SPDX-License-Identifier: ISC
   3
   4 #
   5 # test_ospf_gr_topo1.py
   6 # Part of NetDEF Topology Tests
   7 #
   8 # Copyright (c) 2021 by
   9 # Network Device Education Foundation, Inc. ("NetDEF")
  10 #
  11
  12 """
  13 test_ospf_gr_topo1.py:
  14
  15              +---------+
  16              |   RT1   |
  17              | 1.1.1.1 |
  18              +---------+
  19                   |eth-rt2
  20                   |
  21                   |10.0.1.0/24
  22                   |
  23                   |eth-rt1
  24              +---------+
  25              |   RT2   |
  26              | 2.2.2.2 |
  27              +---------+
  28                   |eth-rt3
  29                   |
  30                   |10.0.2.0/24
  31                   |
  32                   |eth-rt2
  33              +---------+
  34              |   RT3   |
  35              | 3.3.3.3 |
  36              +---------+
  37           eth-rt4|  |eth-rt6
  38                  |  |
  39      10.0.3.0/24 |  |  10.0.4.0/24
  40        +---------+  +--------+
  41        |                     |
  42        |eth-rt3              |eth-rt3
  43   +---------+           +---------+
  44   |   RT4   |           |   RT6   |
  45   | 4.4.4.4 |           | 6.6.6.6 |
  46   +---------+           +---------+
  47        |eth-rt5              |eth-rt7
  48        |                     |
  49        |10.0.5.0/24          |10.0.6.0/24
  50        |                     |
  51        |eth-rt4              |eth-rt6
  52   +---------+           +---------+
  53   |   RT5   |           |   RT7   |
  54   | 5.5.5.5 |           | 7.7.7.7 |
  55   +---------+           +---------+
  56 """
  57
  58 import os
  59 import sys
  60 import pytest
  61 import json
  62 from time import sleep
  63 from functools import partial
  64
  65 # Save the Current Working Directory to find configuration files.
  66 CWD = os.path.dirname(os.path.realpath(__file__))
  67 sys.path.append(os.path.join(CWD, "../"))
  68
  69 # pylint: disable=C0413
  70 # Import topogen and topotest helpers
  71 from lib import topotest
  72 from lib.topogen import Topogen, TopoRouter, get_topogen
  73 from lib.topolog import logger
  74 from lib.common_config import (
  75     kill_router_daemons,
  76     start_router_daemons,
  77 )
  78
  79 # Required to instantiate the topology builder class.
  80
  81 pytestmark = [pytest.mark.ospfd]
  82
  83 # Global multi-dimensional dictionary containing all expected outputs
  84 outputs = {}
  85
  86
  87 def build_topo(tgen):
  88     #
  89     # Define FRR Routers
  90     #
  91     for router in ["rt1", "rt2", "rt3", "rt4", "rt5", "rt6", "rt7"]:
  92         tgen.add_router(router)
  93
  94     #
  95     # Define connections
  96     #
  97     switch = tgen.add_switch("s1")
  98     switch.add_link(tgen.gears["rt1"], nodeif="eth-rt2")
  99     switch.add_link(tgen.gears["rt2"], nodeif="eth-rt1")
 100
 101     switch = tgen.add_switch("s2")
 102     switch.add_link(tgen.gears["rt1"], nodeif="stub1")
 103
 104     switch = tgen.add_switch("s3")
 105     switch.add_link(tgen.gears["rt2"], nodeif="eth-rt3")
 106     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt2")
 107
 108     switch = tgen.add_switch("s4")
 109     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt4")
 110     switch.add_link(tgen.gears["rt4"], nodeif="eth-rt3")
 111
 112     switch = tgen.add_switch("s5")
 113     switch.add_link(tgen.gears["rt3"], nodeif="eth-rt6")
 114     switch.add_link(tgen.gears["rt6"], nodeif="eth-rt3")
 115
 116     switch = tgen.add_switch("s6")
 117     switch.add_link(tgen.gears["rt4"], nodeif="eth-rt5")
 118     switch.add_link(tgen.gears["rt5"], nodeif="eth-rt4")
 119
 120     switch = tgen.add_switch("s7")
 121     switch.add_link(tgen.gears["rt6"], nodeif="eth-rt7")
 122     switch.add_link(tgen.gears["rt7"], nodeif="eth-rt6")
 123
 124     switch = tgen.add_switch("s8")
 125     switch.add_link(tgen.gears["rt7"], nodeif="stub1")
 126
 127
 128 def setup_module(mod):
 129     "Sets up the pytest environment"
 130     tgen = Topogen(build_topo, mod.__name__)
 131     tgen.start_topology()
 132
 133     router_list = tgen.routers()
 134
 135     # For all registered routers, load the zebra configuration file
 136     for rname, router in router_list.items():
 137         router.load_config(
 138             TopoRouter.RD_ZEBRA, os.path.join(CWD, "{}/zebra.conf".format(rname))
 139         )
 140         router.load_config(
 141             TopoRouter.RD_OSPF, os.path.join(CWD, "{}/ospfd.conf".format(rname))
 142         )
 143
 144     tgen.start_router()
 145
 146
 147 def teardown_module(mod):
 148     "Teardown the pytest environment"
 149     tgen = get_topogen()
 150
 151     # This function tears down the whole topology.
 152     tgen.stop_topology()
 153
 154
 155 def router_compare_json_output(rname, command, reference, tries):
 156     "Compare router JSON output"
 157
 158     logger.info('Comparing router "%s" "%s" output', rname, command)
 159
 160     tgen = get_topogen()
 161     filename = "{}/{}/{}".format(CWD, rname, reference)
 162     expected = json.loads(open(filename).read())
 163
 164     test_func = partial(topotest.router_json_cmp, tgen.gears[rname], command, expected)
 165     _, diff = topotest.run_and_expect(test_func, None, count=tries, wait=0.5)
 166     assertmsg = '"{}" JSON output mismatches the expected result'.format(rname)
 167     assert diff is None, assertmsg
 168
 169
 170 def expect_grace_lsa(restarting, area, helper):
 171     """
 172     Check if the given helper neighbor has already received a Grace-LSA from
 173     the router performing a graceful restart.
 174     """
 175     tgen = get_topogen()
 176
 177     logger.info(
 178         "'{}': checking if a Grace-LSA was received from '{}'".format(
 179             helper, restarting
 180         )
 181     )
 182     test_func = partial(
 183         topotest.router_json_cmp,
 184         tgen.gears[helper],
 185         "show ip ospf database opaque-link json",
 186         {
 187             "linkLocalOpaqueLsa": {
 188                 "areas": {
 189                     area: [
 190                         {
 191                             "advertisingRouter": restarting,
 192                             "opaqueType": "Grace-LSA",
 193                         }
 194                     ]
 195                 }
 196             }
 197         },
 198     )
 199     _, result = topotest.run_and_expect(test_func, None, count=60, wait=1)
 200     assertmsg = '"{}" didn\'t receive a Grace-LSA from "{}"'.format(helper, restarting)
 201
 202     assert result is None, assertmsg
 203
 204
 205 def check_routers(initial_convergence=False, exiting=None, restarting=None):
 206     for rname in ["rt1", "rt2", "rt3", "rt4", "rt5", "rt6", "rt7"]:
 207         # Check the RIB first, which should be preserved across restarts in
 208         # all routers of the routing domain.
 209         # If we are not on initial convergence *but* we are checking
 210         # after a restart.  Looking in the zebra rib for installed
 211         # is a recipe for test failure.  Why?  because if we are restarting
 212         # then ospf is in the process of establishing neighbors and passing
 213         # new routes to zebra.  Zebra will not mark the route as installed
 214         # when it receives a replacement from ospf until it has finished
 215         # processing it.  Let's give it a few seconds to allow this to happen
 216         # under load.
 217         if initial_convergence == True:
 218             tries = 240
 219         else:
 220             if restarting != None:
 221                 tries = 60
 222             else:
 223                 tries = 10
 224         router_compare_json_output(
 225             rname, "show ip route ospf json", "show_ip_route.json", tries
 226         )
 227
 228         # Check that all adjacencies are up and running (except when there's
 229         # an OSPF instance that is shutting down).
 230         if exiting == None:
 231             tries = 240
 232             router_compare_json_output(
 233                 rname, "show ip ospf neighbor json", "show_ip_ospf_neighbor.json", tries
 234             )
 235
 236         # Check the OSPF RIB and LSDB.
 237         # In the restarting router, wait up to one minute for the LSDB to converge.
 238         if exiting != rname:
 239             if initial_convergence == True or restarting == rname:
 240                 tries = 240
 241             else:
 242                 tries = 10
 243             router_compare_json_output(
 244                 rname, "show ip ospf database json", "show_ip_ospf_database.json", tries
 245             )
 246             router_compare_json_output(
 247                 rname, "show ip ospf route json", "show_ip_ospf_route.json", tries
 248             )
 249
 250
 251 def ensure_gr_is_in_zebra(rname):
 252     retry = True
 253     retry_times = 10
 254     tgen = get_topogen()
 255
 256     while retry and retry_times > 0:
 257         out = tgen.net[rname].cmd(
 258             'vtysh -c "show zebra client" | grep "Client: ospf$" -A 40 | grep "Capabilities "'
 259         )
 260
 261         if "Graceful Restart" not in out:
 262             sleep(2)
 263             retry_times -= 1
 264         else:
 265             retry = False
 266
 267     assertmsg = "%s does not appear to have Graceful Restart setup" % rname
 268     assert not retry and retry_times > 0, assertmsg
 269
 270
 271 #
 272 # Test initial network convergence
 273 #
 274 def test_initial_convergence():
 275     logger.info("Test: verify initial network convergence")
 276     tgen = get_topogen()
 277
 278     # Skip if previous fatal error condition is raised
 279     if tgen.routers_have_failure():
 280         pytest.skip(tgen.errors)
 281
 282     check_routers(initial_convergence=True)
 283
 284
 285 #
 286 # Test rt1 performing a graceful restart
 287 #
 288 def test_gr_rt1():
 289     logger.info("Test: verify rt1 performing a graceful restart")
 290     tgen = get_topogen()
 291
 292     # Skip if previous fatal error condition is raised
 293     if tgen.routers_have_failure():
 294         pytest.skip(tgen.errors)
 295
 296     tgen.net["rt1"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 297     expect_grace_lsa(restarting="1.1.1.1", area="0.0.0.1", helper="rt2")
 298     ensure_gr_is_in_zebra("rt1")
 299     kill_router_daemons(tgen, "rt1", ["ospfd"], save_config=False)
 300     check_routers(exiting="rt1")
 301
 302     start_router_daemons(tgen, "rt1", ["ospfd"])
 303     check_routers(restarting="rt1")
 304
 305
 306 #
 307 # Test rt2 performing a graceful restart
 308 #
 309 def test_gr_rt2():
 310     logger.info("Test: verify rt2 performing a graceful restart")
 311     tgen = get_topogen()
 312
 313     # Skip if previous fatal error condition is raised
 314     if tgen.routers_have_failure():
 315         pytest.skip(tgen.errors)
 316
 317     tgen.net["rt2"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 318     expect_grace_lsa(restarting="2.2.2.2", area="0.0.0.1", helper="rt1")
 319     expect_grace_lsa(restarting="2.2.2.2", area="0.0.0.0", helper="rt3")
 320     ensure_gr_is_in_zebra("rt2")
 321     kill_router_daemons(tgen, "rt2", ["ospfd"], save_config=False)
 322     check_routers(exiting="rt2")
 323
 324     start_router_daemons(tgen, "rt2", ["ospfd"])
 325     check_routers(restarting="rt2")
 326
 327
 328 #
 329 # Test rt3 performing a graceful restart
 330 #
 331 def test_gr_rt3():
 332     logger.info("Test: verify rt3 performing a graceful restart")
 333     tgen = get_topogen()
 334
 335     # Skip if previous fatal error condition is raised
 336     if tgen.routers_have_failure():
 337         pytest.skip(tgen.errors)
 338
 339     tgen.net["rt3"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 340     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt2")
 341     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt4")
 342     expect_grace_lsa(restarting="3.3.3.3", area="0.0.0.0", helper="rt6")
 343     ensure_gr_is_in_zebra("rt3")
 344     kill_router_daemons(tgen, "rt3", ["ospfd"], save_config=False)
 345     check_routers(exiting="rt3")
 346
 347     start_router_daemons(tgen, "rt3", ["ospfd"])
 348     check_routers(restarting="rt3")
 349
 350
 351 #
 352 # Test rt4 performing a graceful restart
 353 #
 354 def test_gr_rt4():
 355     logger.info("Test: verify rt4 performing a graceful restart")
 356     tgen = get_topogen()
 357
 358     # Skip if previous fatal error condition is raised
 359     if tgen.routers_have_failure():
 360         pytest.skip(tgen.errors)
 361
 362     tgen.net["rt4"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 363     expect_grace_lsa(restarting="4.4.4.4", area="0.0.0.0", helper="rt3")
 364     expect_grace_lsa(restarting="4.4.4.4", area="0.0.0.2", helper="rt5")
 365     ensure_gr_is_in_zebra("rt4")
 366     kill_router_daemons(tgen, "rt4", ["ospfd"], save_config=False)
 367     check_routers(exiting="rt4")
 368
 369     start_router_daemons(tgen, "rt4", ["ospfd"])
 370     check_routers(restarting="rt4")
 371
 372
 373 #
 374 # Test rt5 performing a graceful restart
 375 #
 376 def test_gr_rt5():
 377     logger.info("Test: verify rt5 performing a graceful restart")
 378     tgen = get_topogen()
 379
 380     # Skip if previous fatal error condition is raised
 381     if tgen.routers_have_failure():
 382         pytest.skip(tgen.errors)
 383
 384     tgen.net["rt5"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 385     expect_grace_lsa(restarting="5.5.5.5", area="0.0.0.2", helper="rt4")
 386     ensure_gr_is_in_zebra("rt5")
 387     kill_router_daemons(tgen, "rt5", ["ospfd"], save_config=False)
 388     check_routers(exiting="rt5")
 389
 390     start_router_daemons(tgen, "rt5", ["ospfd"])
 391     check_routers(restarting="rt5")
 392
 393
 394 #
 395 # Test rt6 performing a graceful restart
 396 #
 397 def test_gr_rt6():
 398     logger.info("Test: verify rt6 performing a graceful restart")
 399     tgen = get_topogen()
 400
 401     # Skip if previous fatal error condition is raised
 402     if tgen.routers_have_failure():
 403         pytest.skip(tgen.errors)
 404
 405     tgen.net["rt6"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 406     expect_grace_lsa(restarting="6.6.6.6", area="0.0.0.0", helper="rt3")
 407     expect_grace_lsa(restarting="6.6.6.6", area="0.0.0.3", helper="rt7")
 408     ensure_gr_is_in_zebra("rt6")
 409     kill_router_daemons(tgen, "rt6", ["ospfd"], save_config=False)
 410     check_routers(exiting="rt6")
 411
 412     start_router_daemons(tgen, "rt6", ["ospfd"])
 413     check_routers(restarting="rt6")
 414
 415
 416 #
 417 # Test rt7 performing a graceful restart
 418 #
 419 def test_gr_rt7():
 420     logger.info("Test: verify rt7 performing a graceful restart")
 421     tgen = get_topogen()
 422
 423     # Skip if previous fatal error condition is raised
 424     if tgen.routers_have_failure():
 425         pytest.skip(tgen.errors)
 426
 427     tgen.net["rt7"].cmd('vtysh -c "graceful-restart prepare ip ospf"')
 428     expect_grace_lsa(restarting="7.7.7.7", area="0.0.0.3", helper="rt6")
 429     ensure_gr_is_in_zebra("rt7")
 430     kill_router_daemons(tgen, "rt7", ["ospfd"], save_config=False)
 431     check_routers(exiting="rt7")
 432
 433     start_router_daemons(tgen, "rt7", ["ospfd"])
 434     check_routers(restarting="rt7")
 435
 436
 437 # Memory leak test template
 438 def test_memory_leak():
 439     "Run the memory leak test and report results."
 440     tgen = get_topogen()
 441     if not tgen.is_memleak_enabled():
 442         pytest.skip("Memory leak test/report is disabled")
 443
 444     tgen.report_memory_leaks()
 445
 446
 447 if __name__ == "__main__":
 448     args = ["-s"] + sys.argv[1:]
 449     sys.exit(pytest.main(args))