]> git.proxmox.com Git - mirror_frr.git/blobdiff - tests/topotests/ospfapi/test_ospf_clientapi.py
Merge pull request #12798 from donaldsharp/rib_match_multicast
[mirror_frr.git] / tests / topotests / ospfapi / test_ospf_clientapi.py
index 631c8025b57ccf948f6a99cdaffb611c6bbfe30f..b01c96226ef29e1c20ad5d97e57afe19b0368d4f 100644 (file)
@@ -1,22 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
 #
 # Copyright (c) 2021-2022, LabN Consulting, L.L.C.
 #
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; see the file COPYING; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
-#
 
 """
 test_ospf_clientapi.py: Test the OSPF client API.
@@ -34,7 +21,7 @@ from datetime import datetime, timedelta
 import pytest
 
 from lib.common_config import retry, run_frr_cmd, step
-from lib.micronet import comm_error
+from lib.micronet import Timeout, comm_error
 from lib.topogen import Topogen, TopoRouter
 from lib.topotest import interface_set_status, json_cmp
 
@@ -56,15 +43,20 @@ assert os.path.exists(
 # Test Setup
 # ----------
 
+#
+#  r1 - r2
+#  |    |
+#  r4 - r3
+#
+
 
 @pytest.fixture(scope="function", name="tgen")
 def _tgen(request):
     "Setup/Teardown the environment and provide tgen argument to tests"
     nrouters = request.param
-    if nrouters == 1:
-        topodef = {"sw1:": ("r1",)}
-    else:
-        topodef = {f"sw{i}": (f"r{i}", f"r{i+1}") for i in range(1, nrouters)}
+    topodef = {f"sw{i}": (f"r{i}", f"r{i+1}") for i in range(1, nrouters)}
+    if nrouters == 4:
+        topodef["sw4"] = ("r4", "r1")
 
     tgen = Topogen(topodef, request.module.__name__)
     tgen.start_topology()
@@ -107,23 +99,23 @@ def verify_ospf_database(tgen, dut, input_dict, cmd="show ip ospf database json"
 def myreadline(f):
     buf = b""
     while True:
-        # logging.info("READING 1 CHAR")
+        # logging.debug("READING 1 CHAR")
         c = f.read(1)
         if not c:
             return buf if buf else None
         buf += c
-        # logging.info("READ CHAR: '%s'", c)
+        # logging.debug("READ CHAR: '%s'", c)
         if c == b"\n":
             return buf
 
 
-def _wait_output(p, regex, timeout=120):
-    retry_until = datetime.now() + timedelta(seconds=timeout)
-    while datetime.now() < retry_until:
+def _wait_output(p, regex, maxwait=120):
+    timeout = Timeout(maxwait)
+    while not timeout.is_expired():
         # line = p.stdout.readline()
         line = myreadline(p.stdout)
         if not line:
-            assert None, "Timeout waiting for '{}'".format(regex)
+            assert None, "EOF waiting for '{}'".format(regex)
         line = line.decode("utf-8")
         line = line.rstrip()
         if line:
@@ -131,7 +123,9 @@ def _wait_output(p, regex, timeout=120):
         m = re.search(regex, line)
         if m:
             return m
-    assert None, "Failed to get output withint {}s".format(timeout)
+    assert None, "Failed to get output matching '{}' withint {} actual {}s".format(
+        regex, maxwait, timeout.elapsed()
+    )
 
 
 # -----
@@ -141,12 +135,13 @@ def _wait_output(p, regex, timeout=120):
 
 def _test_reachability(tgen, testbin):
     waitlist = [
-        "192.168.0.1,192.168.0.2,192.168.0.4",
-        "192.168.0.2,192.168.0.4",
-        "192.168.0.1,192.168.0.2,192.168.0.4",
+        "1.0.0.0,2.0.0.0,4.0.0.0",
+        "2.0.0.0,4.0.0.0",
+        "1.0.0.0,2.0.0.0,4.0.0.0",
     ]
     r2 = tgen.gears["r2"]
     r3 = tgen.gears["r3"]
+    r4 = tgen.gears["r4"]
 
     wait_args = [f"--wait={x}" for x in waitlist]
 
@@ -164,10 +159,12 @@ def _test_reachability(tgen, testbin):
 
         step("reachable: check for modified reachability")
         interface_set_status(r2, "r2-eth0", False)
+        interface_set_status(r4, "r4-eth1", False)
         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 
         step("reachable: check for restored reachability")
         interface_set_status(r2, "r2-eth0", True)
+        interface_set_status(r4, "r4-eth1", True)
         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
     except Exception as error:
         logging.error("ERROR: %s", error)
@@ -182,16 +179,16 @@ def _test_reachability(tgen, testbin):
 def test_ospf_reachability(tgen):
     testbin = os.path.join(TESTDIR, "ctester.py")
     rc, o, e = tgen.gears["r2"].net.cmd_status([testbin, "--help"])
-    logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
+    logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
     _test_reachability(tgen, testbin)
 
 
 def _test_router_id(tgen, testbin):
     r1 = tgen.gears["r1"]
     waitlist = [
-        "192.168.0.1",
+        "1.0.0.0",
         "1.1.1.1",
-        "192.168.0.1",
+        "1.0.0.0",
     ]
 
     mon_args = [f"--monitor={x}" for x in waitlist]
@@ -213,7 +210,7 @@ def _test_router_id(tgen, testbin):
         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 
         step("router id: check for restored router id")
-        r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 192.168.0.1")
+        r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 1.0.0.0")
         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
     except Exception as error:
         logging.error("ERROR: %s", error)
@@ -228,7 +225,7 @@ def _test_router_id(tgen, testbin):
 def test_ospf_router_id(tgen):
     testbin = os.path.join(TESTDIR, "ctester.py")
     rc, o, e = tgen.gears["r1"].net.cmd_status([testbin, "--help"])
-    logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
+    logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
     _test_router_id(tgen, testbin)
 
 
@@ -243,13 +240,13 @@ def _test_add_data(tgen, apibin):
     try:
         p = r1.popen([apibin, "-v", "add,9,10.0.1.1,230,2,00000202"])
         input_dict = {
-            "routerId": "192.168.0.1",
+            "routerId": "1.0.0.0",
             "areas": {
                 "1.2.3.4": {
                     "linkLocalOpaqueLsa": [
                         {
                             "lsId": "230.0.0.2",
-                            "advertisedRouter": "192.168.0.1",
+                            "advertisedRouter": "1.0.0.0",
                             "sequenceNumber": "80000001",
                         }
                     ],
@@ -265,7 +262,7 @@ def _test_add_data(tgen, apibin):
                     "1.2.3.4": [
                         {
                             "linkStateId": "230.0.0.2",
-                            "advertisingRouter": "192.168.0.1",
+                            "advertisingRouter": "1.0.0.0",
                             "lsaSeqNumber": "80000001",
                             "opaqueData": "00000202",
                         },
@@ -285,13 +282,13 @@ def _test_add_data(tgen, apibin):
         p = None
         p = r1.popen([apibin, "-v", "add,10,1.2.3.4,231,1,00010101"])
         input_dict = {
-            "routerId": "192.168.0.1",
+            "routerId": "1.0.0.0",
             "areas": {
                 "1.2.3.4": {
                     "linkLocalOpaqueLsa": [
                         {
                             "lsId": "230.0.0.2",
-                            "advertisedRouter": "192.168.0.1",
+                            "advertisedRouter": "1.0.0.0",
                             "sequenceNumber": "80000001",
                             "lsaAge": 3600,
                         }
@@ -299,7 +296,7 @@ def _test_add_data(tgen, apibin):
                     "areaLocalOpaqueLsa": [
                         {
                             "lsId": "231.0.0.1",
-                            "advertisedRouter": "192.168.0.1",
+                            "advertisedRouter": "1.0.0.0",
                             "sequenceNumber": "80000001",
                         },
                     ],
@@ -315,7 +312,7 @@ def _test_add_data(tgen, apibin):
                     "1.2.3.4": [
                         {
                             "linkStateId": "231.0.0.1",
-                            "advertisingRouter": "192.168.0.1",
+                            "advertisingRouter": "1.0.0.0",
                             "lsaSeqNumber": "80000001",
                             "opaqueData": "00010101",
                         },
@@ -336,13 +333,13 @@ def _test_add_data(tgen, apibin):
 
         p = r1.popen([apibin, "-v", "add,11,232,3,deadbeaf01234567"])
         input_dict = {
-            "routerId": "192.168.0.1",
+            "routerId": "1.0.0.0",
             "areas": {
                 "1.2.3.4": {
                     "areaLocalOpaqueLsa": [
                         {
                             "lsId": "231.0.0.1",
-                            "advertisedRouter": "192.168.0.1",
+                            "advertisedRouter": "1.0.0.0",
                             "sequenceNumber": "80000001",
                             "lsaAge": 3600,
                         },
@@ -352,7 +349,7 @@ def _test_add_data(tgen, apibin):
             "asExternalOpaqueLsa": [
                 {
                     "lsId": "232.0.0.3",
-                    "advertisedRouter": "192.168.0.1",
+                    "advertisedRouter": "1.0.0.0",
                     "sequenceNumber": "80000001",
                 },
             ],
@@ -364,7 +361,7 @@ def _test_add_data(tgen, apibin):
             "asExternalOpaqueLsa": [
                 {
                     "linkStateId": "232.0.0.3",
-                    "advertisingRouter": "192.168.0.1",
+                    "advertisingRouter": "1.0.0.0",
                     "lsaSeqNumber": "80000001",
                     "opaqueData": "deadbeaf01234567",
                 },
@@ -382,11 +379,11 @@ def _test_add_data(tgen, apibin):
         p = None
 
         input_dict = {
-            "routerId": "192.168.0.1",
+            "routerId": "1.0.0.0",
             "asExternalOpaqueLsa": [
                 {
                     "lsId": "232.0.0.3",
-                    "advertisedRouter": "192.168.0.1",
+                    "advertisedRouter": "1.0.0.0",
                     "sequenceNumber": "80000001",
                     "lsaAge": 3600,
                 },
@@ -400,11 +397,11 @@ def _test_add_data(tgen, apibin):
         # Originate it again
         p = r1.popen([apibin, "-v", "add,11,232,3,ebadf00d"])
         input_dict = {
-            "routerId": "192.168.0.1",
+            "routerId": "1.0.0.0",
             "asExternalOpaqueLsa": [
                 {
                     "lsId": "232.0.0.3",
-                    "advertisedRouter": "192.168.0.1",
+                    "advertisedRouter": "1.0.0.0",
                     "sequenceNumber": "80000002",
                 },
             ],
@@ -415,7 +412,7 @@ def _test_add_data(tgen, apibin):
             "asExternalOpaqueLsa": [
                 {
                     "linkStateId": "232.0.0.3",
-                    "advertisingRouter": "192.168.0.1",
+                    "advertisingRouter": "1.0.0.0",
                     "lsaSeqNumber": "80000002",
                     "opaqueData": "ebadf00d",
                 },
@@ -425,6 +422,7 @@ def _test_add_data(tgen, apibin):
         json_cmd = "show ip ospf da opaque-as json"
         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 
+        logging.debug("sending interrupt to writer api client")
         p.send_signal(signal.SIGINT)
         time.sleep(2)
         p.wait()
@@ -439,6 +437,7 @@ def _test_add_data(tgen, apibin):
         raise
     finally:
         if p:
+            logging.debug("cleanup: sending interrupt to writer api client")
             p.terminate()
             p.wait()
 
@@ -447,7 +446,7 @@ def _test_add_data(tgen, apibin):
 def test_ospf_opaque_add_data3(tgen):
     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
-    logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
+    logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
     _test_add_data(tgen, apibin)
 
 
@@ -459,37 +458,460 @@ def _test_opaque_add_del(tgen, apibin):
 
     p = None
     pread = None
+    # Log to our stdin, stderr
+    pout = open(os.path.join(r1.net.logdir, "r1/add-del.log"), "a+")
     try:
         step("reachable: check for add notification")
         pread = r2.popen(
-            ["/usr/bin/timeout", "120", apibin, "-v"],
+            ["/usr/bin/timeout", "120", apibin, "-v", "--logtag=READER", "wait,120"],
             encoding=None,  # don't buffer
             stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
         )
-        p = r1.popen([apibin, "-v", "add,11,232,3,ebadf00d"])
+        p = r1.popen(
+            [
+                apibin,
+                "-v",
+                "add,9,10.0.1.1,230,1",
+                "add,9,10.0.1.1,230,2,00000202",
+                "wait,1",
+                "add,10,1.2.3.4,231,1",
+                "add,10,1.2.3.4,231,2,0102030405060708",
+                "wait,1",
+                "add,11,232,1",
+                "add,11,232,2,ebadf00d",
+                "wait,20",
+                "del,9,10.0.1.1,230,2,0",
+                "del,10,1.2.3.4,231,2,1",
+                "del,11,232,1,1",
+            ]
+        )
+        add_input_dict = {
+            "areas": {
+                "1.2.3.4": {
+                    "linkLocalOpaqueLsa": [
+                        {
+                            "lsId": "230.0.0.1",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "76bf",
+                        },
+                        {
+                            "lsId": "230.0.0.2",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "8aa2",
+                        },
+                    ],
+                    "linkLocalOpaqueLsaCount": 2,
+                    "areaLocalOpaqueLsa": [
+                        {
+                            "lsId": "231.0.0.1",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "5bd8",
+                        },
+                        {
+                            "lsId": "231.0.0.2",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "7690",
+                        },
+                    ],
+                    "areaLocalOpaqueLsaCount": 2,
+                },
+            },
+            "asExternalOpaqueLsa": [
+                {
+                    "lsId": "232.0.0.1",
+                    "advertisedRouter": "1.0.0.0",
+                    "sequenceNumber": "80000001",
+                    "checksum": "5ed5",
+                },
+                {
+                    "lsId": "232.0.0.2",
+                    "advertisedRouter": "1.0.0.0",
+                    "sequenceNumber": "80000001",
+                    "checksum": "d9bd",
+                },
+            ],
+            "asExternalOpaqueLsaCount": 2,
+        }
+
+        step("reachable: check for add LSAs")
+        json_cmd = "show ip ospf da json"
+        assert verify_ospf_database(tgen, r1, add_input_dict, json_cmd) is None
+        assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
+
+        numcs = 3
+        json_cmds = [
+            "show ip ospf da opaque-link json",
+            "show ip ospf da opaque-area json",
+            "show ip ospf da opaque-as json",
+        ]
+        add_detail_input_dict = [
+            {
+                "linkLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "linkStateId": "230.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "76bf",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "linkStateId": "230.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "8aa2",
+                                "length": 24,
+                                "opaqueId": 2,
+                                "opaqueDataLength": 4,
+                            },
+                        ]
+                    }
+                }
+            },
+            {
+                "areaLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "linkStateId": "231.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "5bd8",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "linkStateId": "231.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "7690",
+                                "length": 28,
+                                "opaqueDataLength": 8,
+                            },
+                        ],
+                    },
+                },
+            },
+            {
+                "asExternalOpaqueLsa": [
+                    {
+                        "linkStateId": "232.0.0.1",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "5ed5",
+                        "length": 20,
+                        "opaqueDataLength": 0,
+                    },
+                    {
+                        "linkStateId": "232.0.0.2",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "d9bd",
+                        "length": 24,
+                        "opaqueDataLength": 4,
+                    },
+                ],
+            },
+        ]
+        i = 0
+        while i < numcs:
+            step("reachable: check for add LSA details: %s" % json_cmds[i])
+            assert (
+                verify_ospf_database(tgen, r1, add_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            assert (
+                verify_ospf_database(tgen, r2, add_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            i += 1
 
         # Wait for add notification
         # RECV: LSA update msg for LSA 232.0.0.3 in area 0.0.0.0 seq 0x80000001 len 24 age 9
 
-        ls_id = "232.0.0.3"
-        waitfor = "RECV:.*update msg.*LSA {}.*age ([0-9]+)".format(ls_id)
-        _ = _wait_output(pread, waitfor)
+        ls_ids = [
+            "230.0.0.1",
+            "230.0.0.2",
+            "231.0.0.1",
+            "231.0.0.2",
+            "232.0.0.1",
+            "232.0.0.2",
+        ]
+        for ls_id in ls_ids:
+            step("reachable: check for API add notification: %s" % ls_id)
+            waitfor = "RECV:.*update msg.*LSA {}.*age ([0-9]+)".format(ls_id)
+            _ = _wait_output(pread, waitfor)
+
+        del_input_dict = {
+            "areas": {
+                "1.2.3.4": {
+                    "linkLocalOpaqueLsa": [
+                        {
+                            "lsId": "230.0.0.1",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "76bf",
+                        },
+                        {
+                            "lsId": "230.0.0.2",
+                            "advertisedRouter": "1.0.0.0",
+                            "lsaAge": 3600,
+                            "sequenceNumber": "80000001",
+                            "checksum": "8aa2",
+                        },
+                    ],
+                    "linkLocalOpaqueLsaCount": 2,
+                    "areaLocalOpaqueLsa": [
+                        {
+                            "lsId": "231.0.0.1",
+                            "advertisedRouter": "1.0.0.0",
+                            "sequenceNumber": "80000001",
+                            "checksum": "5bd8",
+                        },
+                        {
+                            "lsId": "231.0.0.2",
+                            "advertisedRouter": "1.0.0.0",
+                            "lsaAge": 3600,
+                            "sequenceNumber": "80000002",
+                            "checksum": "4fe2",
+                        },
+                    ],
+                    "areaLocalOpaqueLsaCount": 2,
+                },
+            },
+            "asExternalOpaqueLsa": [
+                {
+                    "lsId": "232.0.0.1",
+                    "advertisedRouter": "1.0.0.0",
+                    "lsaAge": 3600,
+                    "sequenceNumber": "80000001",
+                    "checksum": "5ed5",
+                },
+                {
+                    "lsId": "232.0.0.2",
+                    "advertisedRouter": "1.0.0.0",
+                    "sequenceNumber": "80000001",
+                    "checksum": "d9bd",
+                },
+            ],
+            "asExternalOpaqueLsaCount": 2,
+        }
+
+        step("reachable: check for explicit withdrawal LSAs")
+        json_cmd = "show ip ospf da json"
+        assert verify_ospf_database(tgen, r1, del_input_dict, json_cmd) is None
+        assert verify_ospf_database(tgen, r2, del_input_dict, json_cmd) is None
+
+        del_detail_input_dict = [
+            {
+                "linkLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "linkStateId": "230.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "76bf",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "linkStateId": "230.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaAge": 3600,
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "8aa2",
+                                "length": 24,
+                                "opaqueId": 2,
+                                "opaqueDataLength": 4,
+                            },
+                        ]
+                    }
+                }
+            },
+            {
+                "areaLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "linkStateId": "231.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "5bd8",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "lsaAge": 3600,
+                                "linkStateId": "231.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000002",
+                                "checksum": "4fe2",
+                                # data removed
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                        ],
+                    },
+                },
+            },
+            {
+                "asExternalOpaqueLsa": [
+                    {
+                        "linkStateId": "232.0.0.1",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaAge": 3600,
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "5ed5",
+                        "length": 20,
+                        "opaqueDataLength": 0,
+                    },
+                    {
+                        "linkStateId": "232.0.0.2",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "d9bd",
+                        "length": 24,
+                        "opaqueDataLength": 4,
+                    },
+                ],
+            },
+        ]
+        i = 0
+        while i < numcs:
+            step("reachable: check for delete LSA details: %s" % json_cmds[i])
+            assert (
+                verify_ospf_database(tgen, r1, del_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            assert (
+                verify_ospf_database(tgen, r2, del_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            i += 1
 
         p.terminate()
         if p.wait():
             comm_error(p)
 
+        del_detail_input_dict = [
+            {
+                "linkLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "linkStateId": "230.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaAge": 3600,
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "76bf",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "linkStateId": "230.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaAge": 3600,
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "8aa2",
+                                "length": 24,
+                                "opaqueId": 2,
+                                "opaqueDataLength": 4,
+                            },
+                        ]
+                    }
+                }
+            },
+            {
+                "areaLocalOpaqueLsa": {
+                    "areas": {
+                        "1.2.3.4": [
+                            {
+                                "lsaAge": 3600,
+                                "linkStateId": "231.0.0.1",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000001",
+                                "checksum": "5bd8",
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                            {
+                                "lsaAge": 3600,
+                                "linkStateId": "231.0.0.2",
+                                "advertisingRouter": "1.0.0.0",
+                                "lsaSeqNumber": "80000002",
+                                "checksum": "4fe2",
+                                # data removed
+                                "length": 20,
+                                "opaqueDataLength": 0,
+                            },
+                        ],
+                    },
+                },
+            },
+            {
+                "asExternalOpaqueLsa": [
+                    {
+                        "linkStateId": "232.0.0.1",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaAge": 3600,
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "5ed5",
+                        "length": 20,
+                        "opaqueDataLength": 0,
+                    },
+                    {
+                        "linkStateId": "232.0.0.2",
+                        "advertisingRouter": "1.0.0.0",
+                        "lsaAge": 3600,
+                        "lsaSeqNumber": "80000001",
+                        "checksum": "d9bd",
+                        "length": 24,
+                        "opaqueDataLength": 4,
+                    },
+                ],
+            },
+        ]
+        i = 0
+        while i < numcs:
+            step(
+                "reachable: check for post API shutdown delete LSA details: %s"
+                % json_cmds[i]
+            )
+            assert (
+                verify_ospf_database(tgen, r1, del_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            assert (
+                verify_ospf_database(tgen, r2, del_detail_input_dict[i], json_cmds[i])
+                is None
+            )
+            i += 1
+
         # step("reachable: check for flush/age out")
         # # Wait for max age notification
         # waitfor = "RECV:.*update msg.*LSA {}.*age 3600".format(ls_id)
         # _wait_output(pread, waitfor)
-
-        step("reachable: check for delete")
-        # Wait for delete notification
-        waitfor = "RECV:.*delete msg.*LSA {}.*".format(ls_id)
-        _wait_output(pread, waitfor)
+        ls_ids = [
+            "230.0.0.2",
+            "231.0.0.2",
+            "232.0.0.1",
+            "230.0.0.1",
+            "231.0.0.1",
+            "232.0.0.2",
+        ]
+        for ls_id in ls_ids:
+            step("reachable: check for API delete notification: %s" % ls_id)
+            waitfor = "RECV:.*delete msg.*LSA {}.*age".format(ls_id)
+            _ = _wait_output(pread, waitfor)
     except Exception:
         if p:
             p.terminate()
@@ -510,7 +932,7 @@ def _test_opaque_add_del(tgen, apibin):
 def test_ospf_opaque_delete_data3(tgen):
     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
-    logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
+    logging.debug("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
     _test_opaque_add_del(tgen, apibin)