tests/topotests/ospfapi/test_ospf_clientapi.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 # SPDX-License-Identifier: GPL-2.0-or-later
   4 #
   5 # Copyright (c) 2021-2022, LabN Consulting, L.L.C.
   6 #
   7
   8 """
   9 test_ospf_clientapi.py: Test the OSPF client API.
  10 """
  11
  12 import logging
  13 import os
  14 import re
  15 import signal
  16 import subprocess
  17 import sys
  18 import time
  19 from datetime import datetime, timedelta
  20
  21 import pytest
  22 from lib.common_config import (
  23     kill_router_daemons,
  24     retry,
  25     run_frr_cmd,
  26     shutdown_bringup_interface,
  27     start_router_daemons,
  28     step,
  29 )
  30 from lib.micronet import Timeout, comm_error
  31 from lib.topogen import Topogen, TopoRouter
  32 from lib.topotest import interface_set_status, json_cmp
  33
  34 pytestmark = [pytest.mark.ospfd]
  35
  36 CWD = os.path.dirname(os.path.realpath(__file__))
  37 TESTDIR = os.path.abspath(CWD)
  38
  39 CLIENTDIR = os.path.abspath(os.path.join(CWD, "../../../ospfclient"))
  40 if not os.path.exists(CLIENTDIR):
  41     CLIENTDIR = os.path.join(CWD, "/usr/lib/frr")
  42
  43 assert os.path.exists(
  44     os.path.join(CLIENTDIR, "ospfclient.py")
  45 ), "can't locate ospfclient.py"
  46
  47
  48 # ----------
  49 # Test Setup
  50 # ----------
  51
  52 #
  53 #  r1 - r2
  54 #  |    |
  55 #  r4 - r3
  56 #
  57
  58
  59 @pytest.fixture(scope="function", name="tgen")
  60 def _tgen(request):
  61     "Setup/Teardown the environment and provide tgen argument to tests"
  62     nrouters = request.param
  63     topodef = {f"sw{i}": (f"r{i}", f"r{i+1}") for i in range(1, nrouters)}
  64     if nrouters == 4:
  65         topodef["sw4"] = ("r4", "r1")
  66
  67     tgen = Topogen(topodef, request.module.__name__)
  68     tgen.start_topology()
  69
  70     router_list = tgen.routers()
  71     for _, router in router_list.items():
  72         router.load_config(TopoRouter.RD_ZEBRA, "zebra.conf")
  73         router.load_config(TopoRouter.RD_OSPF, "ospfd.conf")
  74         router.net.daemons_options["ospfd"] = "--apiserver"
  75
  76     tgen.start_router()
  77
  78     yield tgen
  79
  80     tgen.stop_topology()
  81
  82
  83 # Fixture that executes before each test
  84 @pytest.fixture(autouse=True)
  85 def skip_on_failure(tgen):
  86     if tgen.routers_have_failure():
  87         pytest.skip("skipped because of previous test failure")
  88
  89
  90 # ------------
  91 # Test Utility
  92 # ------------
  93
  94
  95 @retry(retry_timeout=45)
  96 def verify_ospf_database(tgen, dut, input_dict, cmd="show ip ospf database json"):
  97     del tgen
  98     show_ospf_json = run_frr_cmd(dut, cmd, isjson=True)
  99     if not bool(show_ospf_json):
 100         return "ospf is not running"
 101     result = json_cmp(show_ospf_json, input_dict)
 102     return str(result) if result else None
 103
 104
 105 def myreadline(f):
 106     buf = b""
 107     while True:
 108         # logging.debug("READING 1 CHAR")
 109         c = f.read(1)
 110         if not c:
 111             return buf if buf else None
 112         buf += c
 113         # logging.debug("READ CHAR: '%s'", c)
 114         if c == b"\n":
 115             return buf
 116
 117
 118 def _wait_output(p, regex, maxwait=120):
 119     timeout = Timeout(maxwait)
 120     while not timeout.is_expired():
 121         # line = p.stdout.readline()
 122         line = myreadline(p.stdout)
 123         if not line:
 124             assert None, "EOF waiting for '{}'".format(regex)
 125         line = line.decode("utf-8")
 126         line = line.rstrip()
 127         if line:
 128             logging.debug("GOT LINE: '%s'", line)
 129         m = re.search(regex, line)
 130         if m:
 131             return m
 132     assert None, "Failed to get output matching '{}' withint {} actual {}s".format(
 133         regex, maxwait, timeout.elapsed()
 134     )
 135
 136
 137 # -----
 138 # Tests
 139 # -----
 140
 141
 142 def _test_reachability(tgen, testbin):
 143     waitlist = [
 144         "1.0.0.0,2.0.0.0,4.0.0.0",
 145         "2.0.0.0,4.0.0.0",
 146         "1.0.0.0,2.0.0.0,4.0.0.0",
 147     ]
 148     r2 = tgen.gears["r2"]
 149     r3 = tgen.gears["r3"]
 150     r4 = tgen.gears["r4"]
 151
 152     wait_args = [f"--wait={x}" for x in waitlist]
 153
 154     p = None
 155     try:
 156         step("reachable: check for initial reachability")
 157         p = r3.popen(
 158             ["/usr/bin/timeout", "120", testbin, "-v", *wait_args],
 159             encoding=None,  # don't buffer
 160             stdin=subprocess.DEVNULL,
 161             stdout=subprocess.PIPE,
 162             stderr=subprocess.STDOUT,
 163         )
 164         _wait_output(p, "SUCCESS: {}".format(waitlist[0]))
 165
 166         step("reachable: check for modified reachability")
 167         interface_set_status(r2, "r2-eth0", False)
 168         interface_set_status(r4, "r4-eth1", False)
 169         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 170
 171         step("reachable: check for restored reachability")
 172         interface_set_status(r2, "r2-eth0", True)
 173         interface_set_status(r4, "r4-eth1", True)
 174         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
 175     except Exception as error:
 176         logging.error("ERROR: %s", error)
 177         raise
 178     finally:
 179         if p:
 180             p.terminate()
 181             p.wait()
 182
 183
 184 @pytest.mark.parametrize("tgen", [4], indirect=True)
 185 def test_ospf_reachability(tgen):
 186     testbin = os.path.join(TESTDIR, "ctester.py")
 187     rc, o, e = tgen.gears["r2"].net.cmd_status([testbin, "--help"])
 188     logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
 189     _test_reachability(tgen, testbin)
 190
 191
 192 def _test_router_id(tgen, testbin):
 193     r1 = tgen.gears["r1"]
 194     waitlist = [
 195         "1.0.0.0",
 196         "1.1.1.1",
 197         "1.0.0.0",
 198     ]
 199
 200     mon_args = [f"--monitor={x}" for x in waitlist]
 201
 202     p = None
 203     try:
 204         step("router id: check for initial router id")
 205         p = r1.popen(
 206             ["/usr/bin/timeout", "120", testbin, "-v", *mon_args],
 207             encoding=None,  # don't buffer
 208             stdin=subprocess.DEVNULL,
 209             stdout=subprocess.PIPE,
 210             stderr=subprocess.STDOUT,
 211         )
 212         _wait_output(p, "SUCCESS: {}".format(waitlist[0]))
 213
 214         step("router id: check for modified router id")
 215         r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 1.1.1.1")
 216         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 217
 218         step("router id: check for restored router id")
 219         r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 1.0.0.0")
 220         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
 221     except Exception as error:
 222         logging.error("ERROR: %s", error)
 223         raise
 224     finally:
 225         if p:
 226             p.terminate()
 227             p.wait()
 228
 229
 230 @pytest.mark.parametrize("tgen", [2], indirect=True)
 231 def test_ospf_router_id(tgen):
 232     testbin = os.path.join(TESTDIR, "ctester.py")
 233     rc, o, e = tgen.gears["r1"].net.cmd_status([testbin, "--help"])
 234     logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
 235     _test_router_id(tgen, testbin)
 236
 237
 238 def _test_add_data(tgen, apibin):
 239     "Test adding opaque data to domain"
 240
 241     r1 = tgen.gears["r1"]
 242
 243     step("add opaque: add opaque link local")
 244
 245     p = None
 246     try:
 247         p = r1.popen([apibin, "-v", "add,9,10.0.1.1,230,2,00000202"])
 248         input_dict = {
 249             "routerId": "1.0.0.0",
 250             "areas": {
 251                 "1.2.3.4": {
 252                     "linkLocalOpaqueLsa": [
 253                         {
 254                             "lsId": "230.0.0.2",
 255                             "advertisedRouter": "1.0.0.0",
 256                             "sequenceNumber": "80000001",
 257                         }
 258                     ],
 259                 }
 260             },
 261         }
 262         # Wait for it to show up
 263         assert verify_ospf_database(tgen, r1, input_dict) is None
 264
 265         input_dict = {
 266             "linkLocalOpaqueLsa": {
 267                 "areas": {
 268                     "1.2.3.4": [
 269                         {
 270                             "linkStateId": "230.0.0.2",
 271                             "advertisingRouter": "1.0.0.0",
 272                             "lsaSeqNumber": "80000001",
 273                             "opaqueData": "00000202",
 274                         },
 275                     ],
 276                 }
 277             },
 278         }
 279         # verify content
 280         json_cmd = "show ip ospf da opaque-link json"
 281         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 282
 283         step("reset client, add opaque area, verify link local flushing")
 284
 285         p.send_signal(signal.SIGINT)
 286         time.sleep(2)
 287         p.wait()
 288         p = None
 289         p = r1.popen([apibin, "-v", "add,10,1.2.3.4,231,1,00010101"])
 290         input_dict = {
 291             "routerId": "1.0.0.0",
 292             "areas": {
 293                 "1.2.3.4": {
 294                     "linkLocalOpaqueLsa": [
 295                         {
 296                             "lsId": "230.0.0.2",
 297                             "advertisedRouter": "1.0.0.0",
 298                             "sequenceNumber": "80000001",
 299                             "lsaAge": 3600,
 300                         }
 301                     ],
 302                     "areaLocalOpaqueLsa": [
 303                         {
 304                             "lsId": "231.0.0.1",
 305                             "advertisedRouter": "1.0.0.0",
 306                             "sequenceNumber": "80000001",
 307                         },
 308                     ],
 309                 }
 310             },
 311         }
 312         # Wait for it to show up
 313         assert verify_ospf_database(tgen, r1, input_dict) is None
 314
 315         input_dict = {
 316             "areaLocalOpaqueLsa": {
 317                 "areas": {
 318                     "1.2.3.4": [
 319                         {
 320                             "linkStateId": "231.0.0.1",
 321                             "advertisingRouter": "1.0.0.0",
 322                             "lsaSeqNumber": "80000001",
 323                             "opaqueData": "00010101",
 324                         },
 325                     ],
 326                 }
 327             },
 328         }
 329         # verify content
 330         json_cmd = "show ip ospf da opaque-area json"
 331         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 332
 333         step("reset client, add opaque AS, verify area flushing")
 334
 335         p.send_signal(signal.SIGINT)
 336         time.sleep(2)
 337         p.wait()
 338         p = None
 339
 340         p = r1.popen([apibin, "-v", "add,11,232,3,deadbeaf01234567"])
 341         input_dict = {
 342             "routerId": "1.0.0.0",
 343             "areas": {
 344                 "1.2.3.4": {
 345                     "areaLocalOpaqueLsa": [
 346                         {
 347                             "lsId": "231.0.0.1",
 348                             "advertisedRouter": "1.0.0.0",
 349                             "sequenceNumber": "80000001",
 350                             "lsaAge": 3600,
 351                         },
 352                     ],
 353                 }
 354             },
 355             "asExternalOpaqueLsa": [
 356                 {
 357                     "lsId": "232.0.0.3",
 358                     "advertisedRouter": "1.0.0.0",
 359                     "sequenceNumber": "80000001",
 360                 },
 361             ],
 362         }
 363         # Wait for it to show up
 364         assert verify_ospf_database(tgen, r1, input_dict) is None
 365
 366         input_dict = {
 367             "asExternalOpaqueLsa": [
 368                 {
 369                     "linkStateId": "232.0.0.3",
 370                     "advertisingRouter": "1.0.0.0",
 371                     "lsaSeqNumber": "80000001",
 372                     "opaqueData": "deadbeaf01234567",
 373                 },
 374             ]
 375         }
 376         # verify content
 377         json_cmd = "show ip ospf da opaque-as json"
 378         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 379
 380         step("stop client, verify AS flushing")
 381
 382         p.send_signal(signal.SIGINT)
 383         time.sleep(2)
 384         p.wait()
 385         p = None
 386
 387         input_dict = {
 388             "routerId": "1.0.0.0",
 389             "asExternalOpaqueLsa": [
 390                 {
 391                     "lsId": "232.0.0.3",
 392                     "advertisedRouter": "1.0.0.0",
 393                     "sequenceNumber": "80000001",
 394                     "lsaAge": 3600,
 395                 },
 396             ],
 397         }
 398         # Wait for it to be flushed
 399         assert verify_ospf_database(tgen, r1, input_dict) is None
 400
 401         step("start client adding opaque domain, verify new sequence number and data")
 402
 403         # Originate it again
 404         p = r1.popen([apibin, "-v", "add,11,232,3,ebadf00d"])
 405         input_dict = {
 406             "routerId": "1.0.0.0",
 407             "asExternalOpaqueLsa": [
 408                 {
 409                     "lsId": "232.0.0.3",
 410                     "advertisedRouter": "1.0.0.0",
 411                     "sequenceNumber": "80000002",
 412                 },
 413             ],
 414         }
 415         assert verify_ospf_database(tgen, r1, input_dict) is None
 416
 417         input_dict = {
 418             "asExternalOpaqueLsa": [
 419                 {
 420                     "linkStateId": "232.0.0.3",
 421                     "advertisingRouter": "1.0.0.0",
 422                     "lsaSeqNumber": "80000002",
 423                     "opaqueData": "ebadf00d",
 424                 },
 425             ]
 426         }
 427         # verify content
 428         json_cmd = "show ip ospf da opaque-as json"
 429         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 430
 431         logging.debug("sending interrupt to writer api client")
 432         p.send_signal(signal.SIGINT)
 433         time.sleep(2)
 434         p.wait()
 435         p = None
 436
 437     except Exception:
 438         if p:
 439             p.terminate()
 440             if p.wait():
 441                 comm_error(p)
 442             p = None
 443         raise
 444     finally:
 445         if p:
 446             logging.debug("cleanup: sending interrupt to writer api client")
 447             p.terminate()
 448             p.wait()
 449
 450
 451 @pytest.mark.parametrize("tgen", [2], indirect=True)
 452 def test_ospf_opaque_add_data3(tgen):
 453     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
 454     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
 455     logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
 456     _test_add_data(tgen, apibin)
 457
 458
 459 def _test_opaque_add_del(tgen, apibin):
 460     "Test adding opaque data to domain"
 461
 462     r1 = tgen.gears["r1"]
 463     r2 = tgen.gears["r2"]
 464
 465     p = None
 466     pread = None
 467     # Log to our stdin, stderr
 468     pout = open(os.path.join(r1.net.logdir, "r1/add-del.log"), "a+")
 469     try:
 470         step("reachable: check for add notification")
 471         pread = r2.popen(
 472             ["/usr/bin/timeout", "120", apibin, "-v", "--logtag=READER", "wait,120"],
 473             encoding=None,  # don't buffer
 474             stdin=subprocess.DEVNULL,
 475             stdout=subprocess.PIPE,
 476             stderr=subprocess.STDOUT,
 477         )
 478         p = r1.popen(
 479             [
 480                 apibin,
 481                 "-v",
 482                 "add,9,10.0.1.1,230,1",
 483                 "add,9,10.0.1.1,230,2,00000202",
 484                 "wait,1",
 485                 "add,10,1.2.3.4,231,1",
 486                 "add,10,1.2.3.4,231,2,0102030405060708",
 487                 "wait,1",
 488                 "add,11,232,1",
 489                 "add,11,232,2,ebadf00d",
 490                 "wait,20",
 491                 "del,9,10.0.1.1,230,2,0",
 492                 "del,10,1.2.3.4,231,2,1",
 493                 "del,11,232,1,1",
 494             ]
 495         )
 496         add_input_dict = {
 497             "areas": {
 498                 "1.2.3.4": {
 499                     "linkLocalOpaqueLsa": [
 500                         {
 501                             "lsId": "230.0.0.1",
 502                             "advertisedRouter": "1.0.0.0",
 503                             "sequenceNumber": "80000001",
 504                             "checksum": "76bf",
 505                         },
 506                         {
 507                             "lsId": "230.0.0.2",
 508                             "advertisedRouter": "1.0.0.0",
 509                             "sequenceNumber": "80000001",
 510                             "checksum": "8aa2",
 511                         },
 512                     ],
 513                     "linkLocalOpaqueLsaCount": 2,
 514                     "areaLocalOpaqueLsa": [
 515                         {
 516                             "lsId": "231.0.0.1",
 517                             "advertisedRouter": "1.0.0.0",
 518                             "sequenceNumber": "80000001",
 519                             "checksum": "5bd8",
 520                         },
 521                         {
 522                             "lsId": "231.0.0.2",
 523                             "advertisedRouter": "1.0.0.0",
 524                             "sequenceNumber": "80000001",
 525                             "checksum": "7690",
 526                         },
 527                     ],
 528                     "areaLocalOpaqueLsaCount": 2,
 529                 },
 530             },
 531             "asExternalOpaqueLsa": [
 532                 {
 533                     "lsId": "232.0.0.1",
 534                     "advertisedRouter": "1.0.0.0",
 535                     "sequenceNumber": "80000001",
 536                     "checksum": "5ed5",
 537                 },
 538                 {
 539                     "lsId": "232.0.0.2",
 540                     "advertisedRouter": "1.0.0.0",
 541                     "sequenceNumber": "80000001",
 542                     "checksum": "d9bd",
 543                 },
 544             ],
 545             "asExternalOpaqueLsaCount": 2,
 546         }
 547
 548         step("reachable: check for add LSAs")
 549         json_cmd = "show ip ospf da json"
 550         assert verify_ospf_database(tgen, r1, add_input_dict, json_cmd) is None
 551         assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
 552
 553         numcs = 3
 554         json_cmds = [
 555             "show ip ospf da opaque-link json",
 556             "show ip ospf da opaque-area json",
 557             "show ip ospf da opaque-as json",
 558         ]
 559         add_detail_input_dict = [
 560             {
 561                 "linkLocalOpaqueLsa": {
 562                     "areas": {
 563                         "1.2.3.4": [
 564                             {
 565                                 "linkStateId": "230.0.0.1",
 566                                 "advertisingRouter": "1.0.0.0",
 567                                 "lsaSeqNumber": "80000001",
 568                                 "checksum": "76bf",
 569                                 "length": 20,
 570                                 "opaqueDataLength": 0,
 571                             },
 572                             {
 573                                 "linkStateId": "230.0.0.2",
 574                                 "advertisingRouter": "1.0.0.0",
 575                                 "lsaSeqNumber": "80000001",
 576                                 "checksum": "8aa2",
 577                                 "length": 24,
 578                                 "opaqueId": 2,
 579                                 "opaqueDataLength": 4,
 580                             },
 581                         ]
 582                     }
 583                 }
 584             },
 585             {
 586                 "areaLocalOpaqueLsa": {
 587                     "areas": {
 588                         "1.2.3.4": [
 589                             {
 590                                 "linkStateId": "231.0.0.1",
 591                                 "advertisingRouter": "1.0.0.0",
 592                                 "lsaSeqNumber": "80000001",
 593                                 "checksum": "5bd8",
 594                                 "length": 20,
 595                                 "opaqueDataLength": 0,
 596                             },
 597                             {
 598                                 "linkStateId": "231.0.0.2",
 599                                 "advertisingRouter": "1.0.0.0",
 600                                 "lsaSeqNumber": "80000001",
 601                                 "checksum": "7690",
 602                                 "length": 28,
 603                                 "opaqueDataLength": 8,
 604                             },
 605                         ],
 606                     },
 607                 },
 608             },
 609             {
 610                 "asExternalOpaqueLsa": [
 611                     {
 612                         "linkStateId": "232.0.0.1",
 613                         "advertisingRouter": "1.0.0.0",
 614                         "lsaSeqNumber": "80000001",
 615                         "checksum": "5ed5",
 616                         "length": 20,
 617                         "opaqueDataLength": 0,
 618                     },
 619                     {
 620                         "linkStateId": "232.0.0.2",
 621                         "advertisingRouter": "1.0.0.0",
 622                         "lsaSeqNumber": "80000001",
 623                         "checksum": "d9bd",
 624                         "length": 24,
 625                         "opaqueDataLength": 4,
 626                     },
 627                 ],
 628             },
 629         ]
 630         i = 0
 631         while i < numcs:
 632             step("reachable: check for add LSA details: %s" % json_cmds[i])
 633             assert (
 634                 verify_ospf_database(tgen, r1, add_detail_input_dict[i], json_cmds[i])
 635                 is None
 636             )
 637             assert (
 638                 verify_ospf_database(tgen, r2, add_detail_input_dict[i], json_cmds[i])
 639                 is None
 640             )
 641             i += 1
 642
 643         # Wait for add notification
 644         # RECV: LSA update msg for LSA 232.0.0.3 in area 0.0.0.0 seq 0x80000001 len 24 age 9
 645
 646         ls_ids = [
 647             "230.0.0.1",
 648             "230.0.0.2",
 649             "231.0.0.1",
 650             "231.0.0.2",
 651             "232.0.0.1",
 652             "232.0.0.2",
 653         ]
 654         for ls_id in ls_ids:
 655             step("reachable: check for API add notification: %s" % ls_id)
 656             waitfor = "RECV:.*update msg.*LSA {}.*age ([0-9]+)".format(ls_id)
 657             _ = _wait_output(pread, waitfor)
 658
 659         del_input_dict = {
 660             "areas": {
 661                 "1.2.3.4": {
 662                     "linkLocalOpaqueLsa": [
 663                         {
 664                             "lsId": "230.0.0.1",
 665                             "advertisedRouter": "1.0.0.0",
 666                             "sequenceNumber": "80000001",
 667                             "checksum": "76bf",
 668                         },
 669                         {
 670                             "lsId": "230.0.0.2",
 671                             "advertisedRouter": "1.0.0.0",
 672                             "lsaAge": 3600,
 673                             "sequenceNumber": "80000001",
 674                             "checksum": "8aa2",
 675                         },
 676                     ],
 677                     "linkLocalOpaqueLsaCount": 2,
 678                     "areaLocalOpaqueLsa": [
 679                         {
 680                             "lsId": "231.0.0.1",
 681                             "advertisedRouter": "1.0.0.0",
 682                             "sequenceNumber": "80000001",
 683                             "checksum": "5bd8",
 684                         },
 685                         {
 686                             "lsId": "231.0.0.2",
 687                             "advertisedRouter": "1.0.0.0",
 688                             "lsaAge": 3600,
 689                             "sequenceNumber": "80000002",
 690                             "checksum": "4fe2",
 691                         },
 692                     ],
 693                     "areaLocalOpaqueLsaCount": 2,
 694                 },
 695             },
 696             "asExternalOpaqueLsa": [
 697                 {
 698                     "lsId": "232.0.0.1",
 699                     "advertisedRouter": "1.0.0.0",
 700                     "lsaAge": 3600,
 701                     "sequenceNumber": "80000001",
 702                     "checksum": "5ed5",
 703                 },
 704                 {
 705                     "lsId": "232.0.0.2",
 706                     "advertisedRouter": "1.0.0.0",
 707                     "sequenceNumber": "80000001",
 708                     "checksum": "d9bd",
 709                 },
 710             ],
 711             "asExternalOpaqueLsaCount": 2,
 712         }
 713
 714         step("reachable: check for explicit withdrawal LSAs")
 715         json_cmd = "show ip ospf da json"
 716         assert verify_ospf_database(tgen, r1, del_input_dict, json_cmd) is None
 717         assert verify_ospf_database(tgen, r2, del_input_dict, json_cmd) is None
 718
 719         del_detail_input_dict = [
 720             {
 721                 "linkLocalOpaqueLsa": {
 722                     "areas": {
 723                         "1.2.3.4": [
 724                             {
 725                                 "linkStateId": "230.0.0.1",
 726                                 "advertisingRouter": "1.0.0.0",
 727                                 "lsaSeqNumber": "80000001",
 728                                 "checksum": "76bf",
 729                                 "length": 20,
 730                                 "opaqueDataLength": 0,
 731                             },
 732                             {
 733                                 "linkStateId": "230.0.0.2",
 734                                 "advertisingRouter": "1.0.0.0",
 735                                 "lsaAge": 3600,
 736                                 "lsaSeqNumber": "80000001",
 737                                 "checksum": "8aa2",
 738                                 "length": 24,
 739                                 "opaqueId": 2,
 740                                 "opaqueDataLength": 4,
 741                             },
 742                         ]
 743                     }
 744                 }
 745             },
 746             {
 747                 "areaLocalOpaqueLsa": {
 748                     "areas": {
 749                         "1.2.3.4": [
 750                             {
 751                                 "linkStateId": "231.0.0.1",
 752                                 "advertisingRouter": "1.0.0.0",
 753                                 "lsaSeqNumber": "80000001",
 754                                 "checksum": "5bd8",
 755                                 "length": 20,
 756                                 "opaqueDataLength": 0,
 757                             },
 758                             {
 759                                 "lsaAge": 3600,
 760                                 "linkStateId": "231.0.0.2",
 761                                 "advertisingRouter": "1.0.0.0",
 762                                 "lsaSeqNumber": "80000002",
 763                                 "checksum": "4fe2",
 764                                 # data removed
 765                                 "length": 20,
 766                                 "opaqueDataLength": 0,
 767                             },
 768                         ],
 769                     },
 770                 },
 771             },
 772             {
 773                 "asExternalOpaqueLsa": [
 774                     {
 775                         "linkStateId": "232.0.0.1",
 776                         "advertisingRouter": "1.0.0.0",
 777                         "lsaAge": 3600,
 778                         "lsaSeqNumber": "80000001",
 779                         "checksum": "5ed5",
 780                         "length": 20,
 781                         "opaqueDataLength": 0,
 782                     },
 783                     {
 784                         "linkStateId": "232.0.0.2",
 785                         "advertisingRouter": "1.0.0.0",
 786                         "lsaSeqNumber": "80000001",
 787                         "checksum": "d9bd",
 788                         "length": 24,
 789                         "opaqueDataLength": 4,
 790                     },
 791                 ],
 792             },
 793         ]
 794         i = 0
 795         while i < numcs:
 796             step("reachable: check for delete LSA details: %s" % json_cmds[i])
 797             assert (
 798                 verify_ospf_database(tgen, r1, del_detail_input_dict[i], json_cmds[i])
 799                 is None
 800             )
 801             assert (
 802                 verify_ospf_database(tgen, r2, del_detail_input_dict[i], json_cmds[i])
 803                 is None
 804             )
 805             i += 1
 806
 807         p.terminate()
 808         if p.wait():
 809             comm_error(p)
 810
 811         del_detail_input_dict = [
 812             {
 813                 "linkLocalOpaqueLsa": {
 814                     "areas": {
 815                         "1.2.3.4": [
 816                             {
 817                                 "linkStateId": "230.0.0.1",
 818                                 "advertisingRouter": "1.0.0.0",
 819                                 "lsaAge": 3600,
 820                                 "lsaSeqNumber": "80000001",
 821                                 "checksum": "76bf",
 822                                 "length": 20,
 823                                 "opaqueDataLength": 0,
 824                             },
 825                             {
 826                                 "linkStateId": "230.0.0.2",
 827                                 "advertisingRouter": "1.0.0.0",
 828                                 "lsaAge": 3600,
 829                                 "lsaSeqNumber": "80000001",
 830                                 "checksum": "8aa2",
 831                                 "length": 24,
 832                                 "opaqueId": 2,
 833                                 "opaqueDataLength": 4,
 834                             },
 835                         ]
 836                     }
 837                 }
 838             },
 839             {
 840                 "areaLocalOpaqueLsa": {
 841                     "areas": {
 842                         "1.2.3.4": [
 843                             {
 844                                 "lsaAge": 3600,
 845                                 "linkStateId": "231.0.0.1",
 846                                 "advertisingRouter": "1.0.0.0",
 847                                 "lsaSeqNumber": "80000001",
 848                                 "checksum": "5bd8",
 849                                 "length": 20,
 850                                 "opaqueDataLength": 0,
 851                             },
 852                             {
 853                                 "lsaAge": 3600,
 854                                 "linkStateId": "231.0.0.2",
 855                                 "advertisingRouter": "1.0.0.0",
 856                                 "lsaSeqNumber": "80000002",
 857                                 "checksum": "4fe2",
 858                                 # data removed
 859                                 "length": 20,
 860                                 "opaqueDataLength": 0,
 861                             },
 862                         ],
 863                     },
 864                 },
 865             },
 866             {
 867                 "asExternalOpaqueLsa": [
 868                     {
 869                         "linkStateId": "232.0.0.1",
 870                         "advertisingRouter": "1.0.0.0",
 871                         "lsaAge": 3600,
 872                         "lsaSeqNumber": "80000001",
 873                         "checksum": "5ed5",
 874                         "length": 20,
 875                         "opaqueDataLength": 0,
 876                     },
 877                     {
 878                         "linkStateId": "232.0.0.2",
 879                         "advertisingRouter": "1.0.0.0",
 880                         "lsaAge": 3600,
 881                         "lsaSeqNumber": "80000001",
 882                         "checksum": "d9bd",
 883                         "length": 24,
 884                         "opaqueDataLength": 4,
 885                     },
 886                 ],
 887             },
 888         ]
 889         i = 0
 890         while i < numcs:
 891             step(
 892                 "reachable: check for post API shutdown delete LSA details: %s"
 893                 % json_cmds[i]
 894             )
 895             assert (
 896                 verify_ospf_database(tgen, r1, del_detail_input_dict[i], json_cmds[i])
 897                 is None
 898             )
 899             assert (
 900                 verify_ospf_database(tgen, r2, del_detail_input_dict[i], json_cmds[i])
 901                 is None
 902             )
 903             i += 1
 904
 905         # step("reachable: check for flush/age out")
 906         # # Wait for max age notification
 907         # waitfor = "RECV:.*update msg.*LSA {}.*age 3600".format(ls_id)
 908         # _wait_output(pread, waitfor)
 909         ls_ids = [
 910             "230.0.0.2",
 911             "231.0.0.2",
 912             "232.0.0.1",
 913             "230.0.0.1",
 914             "231.0.0.1",
 915             "232.0.0.2",
 916         ]
 917         for ls_id in ls_ids:
 918             step("reachable: check for API delete notification: %s" % ls_id)
 919             waitfor = "RECV:.*delete msg.*LSA {}.*age".format(ls_id)
 920             _ = _wait_output(pread, waitfor)
 921     except Exception:
 922         if p:
 923             p.terminate()
 924             if p.wait():
 925                 comm_error(p)
 926             p = None
 927         raise
 928     finally:
 929         if pread:
 930             pread.terminate()
 931             pread.wait()
 932         if p:
 933             p.terminate()
 934             p.wait()
 935
 936
 937 @pytest.mark.parametrize("tgen", [2], indirect=True)
 938 def test_ospf_opaque_delete_data3(tgen):
 939     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
 940     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
 941     logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
 942     _test_opaque_add_del(tgen, apibin)
 943
 944
 945 def _test_opaque_add_restart_add(tgen, apibin):
 946     "Test adding an opaque LSA and then restarting ospfd"
 947
 948     r1 = tgen.gears["r1"]
 949     r2 = tgen.gears["r2"]
 950
 951     p = None
 952     pread = None
 953     # Log to our stdin, stderr
 954     pout = open(os.path.join(r1.net.logdir, "r1/add-del.log"), "a+")
 955     try:
 956         step("reachable: check for add notification")
 957         pread = r2.popen(
 958             ["/usr/bin/timeout", "120", apibin, "-v", "--logtag=READER", "wait,120"],
 959             encoding=None,  # don't buffer
 960             stdin=subprocess.DEVNULL,
 961             stdout=subprocess.PIPE,
 962             stderr=subprocess.STDOUT,
 963         )
 964         p = r1.popen(
 965             [
 966                 apibin,
 967                 "-v",
 968                 "add,10,1.2.3.4,231,1",  # seq = 80000001
 969                 "wait, 5",
 970                 "add,10,1.2.3.4,231,1,feedaceebeef",  # seq = 80000002
 971                 "wait, 5",
 972             ]
 973         )
 974         add_input_dict = {
 975             "areas": {
 976                 "1.2.3.4": {
 977                     "areaLocalOpaqueLsa": [
 978                         {
 979                             "lsId": "231.0.0.1",
 980                             "advertisedRouter": "1.0.0.0",
 981                             "sequenceNumber": "80000002",
 982                             "checksum": "cd26",
 983                         },
 984                     ],
 985                     "areaLocalOpaqueLsaCount": 1,
 986                 },
 987             },
 988         }
 989         step("Wait for the Opaque LSA to be distributed")
 990         json_cmd = "show ip ospf da json"
 991         assert verify_ospf_database(tgen, r1, add_input_dict, json_cmd) is None
 992         assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
 993
 994         step("Shutdown the interface on r1 to isolate it for r2")
 995         shutdown_bringup_interface(tgen, "r1", "r1-eth0", False)
 996
 997         time.sleep(2)
 998         step("Reset the client")
 999         p.send_signal(signal.SIGINT)
1000         time.sleep(2)
1001         p.wait()
1002         p = None
1003
1004         # Verify the OLD LSA is still there unchanged on R2
1005         assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
1006
1007         step("Kill ospfd on R1")
1008         kill_router_daemons(tgen, "r1", ["ospfd"])
1009         time.sleep(2)
1010
1011         step("Bring ospfd on R1 back up")
1012         start_router_daemons(tgen, "r1", ["ospfd"])
1013
1014         # This will start off with sequence num 80000001
1015         # But should advance to 80000003 when we reestablish with r2
1016         p = r1.popen(
1017             [
1018                 apibin,
1019                 "-v",
1020                 "add,10,1.2.3.4,231,1,feedaceecafebeef",  # seq=80000001
1021                 "wait, 5",
1022             ]
1023         )
1024
1025         # verify the old value on r2 doesn't change yet
1026         time.sleep(2)
1027         assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
1028
1029         json_cmd = "show ip ospf da opaque-area json"
1030         new_add_input_dict = {
1031             "areaLocalOpaqueLsa": {
1032                 "areas": {
1033                     "1.2.3.4": [
1034                         {
1035                             "linkStateId": "231.0.0.1",
1036                             "advertisingRouter": "1.0.0.0",
1037                             "lsaSeqNumber": "80000001",
1038                             "checksum": "b07a",
1039                             "length": 28,
1040                             "opaqueDataLength": 8,
1041                         },
1042                     ],
1043                 },
1044             },
1045         }
1046         # verify new value with initial seq number on r1
1047         assert verify_ospf_database(tgen, r1, new_add_input_dict, json_cmd) is None
1048
1049         step("Bring the interface on r1 back up for connection to r2")
1050         shutdown_bringup_interface(tgen, "r1", "r1-eth0", True)
1051
1052         step("Verify area opaque LSA refresh")
1053
1054         # Update the expected value to sequence number rev and new checksum
1055         update_dict = new_add_input_dict["areaLocalOpaqueLsa"]["areas"]["1.2.3.4"][0]
1056         update_dict["lsaSeqNumber"] = "80000003"
1057         update_dict["checksum"] = "cb27"
1058
1059         # should settle on the same value now.
1060         assert verify_ospf_database(tgen, r1, new_add_input_dict, json_cmd) is None
1061         assert verify_ospf_database(tgen, r2, new_add_input_dict, json_cmd) is None
1062
1063         step("Shutdown the interface on r1 to isolate it for r2")
1064         shutdown_bringup_interface(tgen, "r1", "r1-eth0", False)
1065
1066         time.sleep(2)
1067         step("Reset the client")
1068         p.send_signal(signal.SIGINT)
1069         time.sleep(2)
1070         p.wait()
1071         p = None
1072
1073         step("Kill ospfd on R1")
1074         kill_router_daemons(tgen, "r1", ["ospfd"])
1075         time.sleep(2)
1076
1077         step("Bring ospfd on R1 back up")
1078         start_router_daemons(tgen, "r1", ["ospfd"])
1079
1080         step("Bring the interface on r1 back up for connection to r2")
1081         shutdown_bringup_interface(tgen, "r1", "r1-eth0", True)
1082
1083         step("Verify area opaque LSA Purging")
1084         json_cmd = "show ip ospf da opaque-area json"
1085         add_detail_input_dict = {
1086             "areaLocalOpaqueLsa": {
1087                 "areas": {
1088                     "1.2.3.4": [
1089                         {
1090                             "lsaAge": 3600,
1091                             "linkStateId": "231.0.0.1",
1092                             "advertisingRouter": "1.0.0.0",
1093                             "lsaSeqNumber": "80000003",
1094                             "checksum": "cb27",
1095                             "length": 28,
1096                             "opaqueDataLength": 8,
1097                         },
1098                     ],
1099                 },
1100             },
1101         }
1102         assert verify_ospf_database(tgen, r1, add_detail_input_dict, json_cmd) is None
1103         assert verify_ospf_database(tgen, r2, add_detail_input_dict, json_cmd) is None
1104         step("Verify Area Opaque LSA removal after timeout (60 seconds)")
1105         time.sleep(60)
1106         json_cmd = "show ip ospf da opaque-area json"
1107         timeout_detail_input_dict = {
1108             "areaLocalOpaqueLsa": {
1109                 "areas": {
1110                     "1.2.3.4": [],
1111                 },
1112             },
1113         }
1114         assert (
1115             verify_ospf_database(tgen, r1, timeout_detail_input_dict, json_cmd) is None
1116         )
1117         assert (
1118             verify_ospf_database(tgen, r2, timeout_detail_input_dict, json_cmd) is None
1119         )
1120
1121     except Exception:
1122         if p:
1123             p.terminate()
1124             if p.wait():
1125                 comm_error(p)
1126             p = None
1127         raise
1128     finally:
1129         if pread:
1130             pread.terminate()
1131             pread.wait()
1132         if p:
1133             p.terminate()
1134             p.wait()
1135
1136
1137 @pytest.mark.parametrize("tgen", [2], indirect=True)
1138 def test_ospf_opaque_restart(tgen):
1139     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
1140     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
1141     logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
1142     _test_opaque_add_restart_add(tgen, apibin)
1143
1144
1145 if __name__ == "__main__":
1146     args = ["-s"] + sys.argv[1:]
1147     sys.exit(pytest.main(args))