tests/topotests/ospfapi/test_ospf_clientapi.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 #
   4 # Copyright (c) 2021-2022, LabN Consulting, L.L.C.
   5 #
   6 # This program is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU General Public License
   8 # as published by the Free Software Foundation; either version 2
   9 # of the License, or (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with this program; see the file COPYING; if not, write to the Free Software
  18 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  19 #
  20
  21 """
  22 test_ospf_clientapi.py: Test the OSPF client API.
  23 """
  24
  25 import logging
  26 import os
  27 import re
  28 import signal
  29 import subprocess
  30 import sys
  31 import time
  32 from datetime import datetime, timedelta
  33
  34 import pytest
  35
  36 from lib.common_config import retry, run_frr_cmd, step
  37 from lib.micronet import comm_error
  38 from lib.topogen import Topogen, TopoRouter
  39 from lib.topotest import interface_set_status, json_cmp
  40
  41 pytestmark = [pytest.mark.ospfd]
  42
  43 CWD = os.path.dirname(os.path.realpath(__file__))
  44 TESTDIR = os.path.abspath(CWD)
  45
  46 CLIENTDIR = os.path.abspath(os.path.join(CWD, "../../../ospfclient"))
  47 if not os.path.exists(CLIENTDIR):
  48     CLIENTDIR = os.path.join(CWD, "/usr/lib/frr")
  49
  50 assert os.path.exists(
  51     os.path.join(CLIENTDIR, "ospfclient.py")
  52 ), "can't locate ospfclient.py"
  53
  54
  55 # ----------
  56 # Test Setup
  57 # ----------
  58
  59
  60 @pytest.fixture(scope="function", name="tgen")
  61 def _tgen(request):
  62     "Setup/Teardown the environment and provide tgen argument to tests"
  63     nrouters = request.param
  64     if nrouters == 1:
  65         topodef = {"sw1:": ("r1",)}
  66     else:
  67         topodef = {f"sw{i}": (f"r{i}", f"r{i+1}") for i in range(1, nrouters)}
  68
  69     tgen = Topogen(topodef, request.module.__name__)
  70     tgen.start_topology()
  71
  72     router_list = tgen.routers()
  73     for _, router in router_list.items():
  74         router.load_config(TopoRouter.RD_ZEBRA, "zebra.conf")
  75         router.load_config(TopoRouter.RD_OSPF, "ospfd.conf")
  76         router.net.daemons_options["ospfd"] = "--apiserver"
  77
  78     tgen.start_router()
  79
  80     yield tgen
  81
  82     tgen.stop_topology()
  83
  84
  85 # Fixture that executes before each test
  86 @pytest.fixture(autouse=True)
  87 def skip_on_failure(tgen):
  88     if tgen.routers_have_failure():
  89         pytest.skip("skipped because of previous test failure")
  90
  91
  92 # ------------
  93 # Test Utility
  94 # ------------
  95
  96
  97 @retry(retry_timeout=45)
  98 def verify_ospf_database(tgen, dut, input_dict, cmd="show ip ospf database json"):
  99     del tgen
 100     show_ospf_json = run_frr_cmd(dut, cmd, isjson=True)
 101     if not bool(show_ospf_json):
 102         return "ospf is not running"
 103     result = json_cmp(show_ospf_json, input_dict)
 104     return str(result) if result else None
 105
 106
 107 def myreadline(f):
 108     buf = b""
 109     while True:
 110         # logging.info("READING 1 CHAR")
 111         c = f.read(1)
 112         if not c:
 113             return buf if buf else None
 114         buf += c
 115         # logging.info("READ CHAR: '%s'", c)
 116         if c == b"\n":
 117             return buf
 118
 119
 120 def _wait_output(p, regex, timeout=120):
 121     retry_until = datetime.now() + timedelta(seconds=timeout)
 122     while datetime.now() < retry_until:
 123         # line = p.stdout.readline()
 124         line = myreadline(p.stdout)
 125         if not line:
 126             assert None, "Timeout waiting for '{}'".format(regex)
 127         line = line.decode("utf-8")
 128         line = line.rstrip()
 129         if line:
 130             logging.debug("GOT LINE: '%s'", line)
 131         m = re.search(regex, line)
 132         if m:
 133             return m
 134     assert None, "Failed to get output withint {}s".format(timeout)
 135
 136
 137 # -----
 138 # Tests
 139 # -----
 140
 141
 142 def _test_reachability(tgen, testbin):
 143     waitlist = [
 144         "192.168.0.1,192.168.0.2,192.168.0.4",
 145         "192.168.0.2,192.168.0.4",
 146         "192.168.0.1,192.168.0.2,192.168.0.4",
 147     ]
 148     r2 = tgen.gears["r2"]
 149     r3 = tgen.gears["r3"]
 150
 151     wait_args = [f"--wait={x}" for x in waitlist]
 152
 153     p = None
 154     try:
 155         step("reachable: check for initial reachability")
 156         p = r3.popen(
 157             ["/usr/bin/timeout", "120", testbin, "-v", *wait_args],
 158             encoding=None,  # don't buffer
 159             stdin=subprocess.DEVNULL,
 160             stdout=subprocess.PIPE,
 161             stderr=subprocess.STDOUT,
 162         )
 163         _wait_output(p, "SUCCESS: {}".format(waitlist[0]))
 164
 165         step("reachable: check for modified reachability")
 166         interface_set_status(r2, "r2-eth0", False)
 167         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 168
 169         step("reachable: check for restored reachability")
 170         interface_set_status(r2, "r2-eth0", True)
 171         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
 172     except Exception as error:
 173         logging.error("ERROR: %s", error)
 174         raise
 175     finally:
 176         if p:
 177             p.terminate()
 178             p.wait()
 179
 180
 181 @pytest.mark.parametrize("tgen", [4], indirect=True)
 182 def test_ospf_reachability(tgen):
 183     testbin = os.path.join(TESTDIR, "ctester.py")
 184     rc, o, e = tgen.gears["r2"].net.cmd_status([testbin, "--help"])
 185     logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
 186     _test_reachability(tgen, testbin)
 187
 188
 189 def _test_router_id(tgen, testbin):
 190     r1 = tgen.gears["r1"]
 191     waitlist = [
 192         "192.168.0.1",
 193         "1.1.1.1",
 194         "192.168.0.1",
 195     ]
 196
 197     mon_args = [f"--monitor={x}" for x in waitlist]
 198
 199     p = None
 200     try:
 201         step("router id: check for initial router id")
 202         p = r1.popen(
 203             ["/usr/bin/timeout", "120", testbin, "-v", *mon_args],
 204             encoding=None,  # don't buffer
 205             stdin=subprocess.DEVNULL,
 206             stdout=subprocess.PIPE,
 207             stderr=subprocess.STDOUT,
 208         )
 209         _wait_output(p, "SUCCESS: {}".format(waitlist[0]))
 210
 211         step("router id: check for modified router id")
 212         r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 1.1.1.1")
 213         _wait_output(p, "SUCCESS: {}".format(waitlist[1]))
 214
 215         step("router id: check for restored router id")
 216         r1.vtysh_multicmd("conf t\nrouter ospf\nospf router-id 192.168.0.1")
 217         _wait_output(p, "SUCCESS: {}".format(waitlist[2]))
 218     except Exception as error:
 219         logging.error("ERROR: %s", error)
 220         raise
 221     finally:
 222         if p:
 223             p.terminate()
 224             p.wait()
 225
 226
 227 @pytest.mark.parametrize("tgen", [2], indirect=True)
 228 def test_ospf_router_id(tgen):
 229     testbin = os.path.join(TESTDIR, "ctester.py")
 230     rc, o, e = tgen.gears["r1"].net.cmd_status([testbin, "--help"])
 231     logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", testbin, rc, o, e)
 232     _test_router_id(tgen, testbin)
 233
 234
 235 def _test_add_data(tgen, apibin):
 236     "Test adding opaque data to domain"
 237
 238     r1 = tgen.gears["r1"]
 239
 240     step("add opaque: add opaque link local")
 241
 242     p = None
 243     try:
 244         p = r1.popen([apibin, "-v", "add,9,10.0.1.1,230,2,00000202"])
 245         input_dict = {
 246             "routerId": "192.168.0.1",
 247             "areas": {
 248                 "1.2.3.4": {
 249                     "linkLocalOpaqueLsa": [
 250                         {
 251                             "lsId": "230.0.0.2",
 252                             "advertisedRouter": "192.168.0.1",
 253                             "sequenceNumber": "80000001",
 254                         }
 255                     ],
 256                 }
 257             },
 258         }
 259         # Wait for it to show up
 260         assert verify_ospf_database(tgen, r1, input_dict) is None
 261
 262         input_dict = {
 263             "linkLocalOpaqueLsa": {
 264                 "areas": {
 265                     "1.2.3.4": [
 266                         {
 267                             "linkStateId": "230.0.0.2",
 268                             "advertisingRouter": "192.168.0.1",
 269                             "lsaSeqNumber": "80000001",
 270                             "opaqueData": "00000202",
 271                         },
 272                     ],
 273                 }
 274             },
 275         }
 276         # verify content
 277         json_cmd = "show ip ospf da opaque-link json"
 278         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 279
 280         step("reset client, add opaque area, verify link local flushing")
 281
 282         p.send_signal(signal.SIGINT)
 283         time.sleep(2)
 284         p.wait()
 285         p = None
 286         p = r1.popen([apibin, "-v", "add,10,1.2.3.4,231,1,00010101"])
 287         input_dict = {
 288             "routerId": "192.168.0.1",
 289             "areas": {
 290                 "1.2.3.4": {
 291                     "linkLocalOpaqueLsa": [
 292                         {
 293                             "lsId": "230.0.0.2",
 294                             "advertisedRouter": "192.168.0.1",
 295                             "sequenceNumber": "80000001",
 296                             "lsaAge": 3600,
 297                         }
 298                     ],
 299                     "areaLocalOpaqueLsa": [
 300                         {
 301                             "lsId": "231.0.0.1",
 302                             "advertisedRouter": "192.168.0.1",
 303                             "sequenceNumber": "80000001",
 304                         },
 305                     ],
 306                 }
 307             },
 308         }
 309         # Wait for it to show up
 310         assert verify_ospf_database(tgen, r1, input_dict) is None
 311
 312         input_dict = {
 313             "areaLocalOpaqueLsa": {
 314                 "areas": {
 315                     "1.2.3.4": [
 316                         {
 317                             "linkStateId": "231.0.0.1",
 318                             "advertisingRouter": "192.168.0.1",
 319                             "lsaSeqNumber": "80000001",
 320                             "opaqueData": "00010101",
 321                         },
 322                     ],
 323                 }
 324             },
 325         }
 326         # verify content
 327         json_cmd = "show ip ospf da opaque-area json"
 328         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 329
 330         step("reset client, add opaque AS, verify area flushing")
 331
 332         p.send_signal(signal.SIGINT)
 333         time.sleep(2)
 334         p.wait()
 335         p = None
 336
 337         p = r1.popen([apibin, "-v", "add,11,232,3,deadbeaf01234567"])
 338         input_dict = {
 339             "routerId": "192.168.0.1",
 340             "areas": {
 341                 "1.2.3.4": {
 342                     "areaLocalOpaqueLsa": [
 343                         {
 344                             "lsId": "231.0.0.1",
 345                             "advertisedRouter": "192.168.0.1",
 346                             "sequenceNumber": "80000001",
 347                             "lsaAge": 3600,
 348                         },
 349                     ],
 350                 }
 351             },
 352             "asExternalOpaqueLsa": [
 353                 {
 354                     "lsId": "232.0.0.3",
 355                     "advertisedRouter": "192.168.0.1",
 356                     "sequenceNumber": "80000001",
 357                 },
 358             ],
 359         }
 360         # Wait for it to show up
 361         assert verify_ospf_database(tgen, r1, input_dict) is None
 362
 363         input_dict = {
 364             "asExternalOpaqueLsa": [
 365                 {
 366                     "linkStateId": "232.0.0.3",
 367                     "advertisingRouter": "192.168.0.1",
 368                     "lsaSeqNumber": "80000001",
 369                     "opaqueData": "deadbeaf01234567",
 370                 },
 371             ]
 372         }
 373         # verify content
 374         json_cmd = "show ip ospf da opaque-as json"
 375         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 376
 377         step("stop client, verify AS flushing")
 378
 379         p.send_signal(signal.SIGINT)
 380         time.sleep(2)
 381         p.wait()
 382         p = None
 383
 384         input_dict = {
 385             "routerId": "192.168.0.1",
 386             "asExternalOpaqueLsa": [
 387                 {
 388                     "lsId": "232.0.0.3",
 389                     "advertisedRouter": "192.168.0.1",
 390                     "sequenceNumber": "80000001",
 391                     "lsaAge": 3600,
 392                 },
 393             ],
 394         }
 395         # Wait for it to be flushed
 396         assert verify_ospf_database(tgen, r1, input_dict) is None
 397
 398         step("start client adding opaque domain, verify new sequence number and data")
 399
 400         # Originate it again
 401         p = r1.popen([apibin, "-v", "add,11,232,3,ebadf00d"])
 402         input_dict = {
 403             "routerId": "192.168.0.1",
 404             "asExternalOpaqueLsa": [
 405                 {
 406                     "lsId": "232.0.0.3",
 407                     "advertisedRouter": "192.168.0.1",
 408                     "sequenceNumber": "80000002",
 409                 },
 410             ],
 411         }
 412         assert verify_ospf_database(tgen, r1, input_dict) is None
 413
 414         input_dict = {
 415             "asExternalOpaqueLsa": [
 416                 {
 417                     "linkStateId": "232.0.0.3",
 418                     "advertisingRouter": "192.168.0.1",
 419                     "lsaSeqNumber": "80000002",
 420                     "opaqueData": "ebadf00d",
 421                 },
 422             ]
 423         }
 424         # verify content
 425         json_cmd = "show ip ospf da opaque-as json"
 426         assert verify_ospf_database(tgen, r1, input_dict, json_cmd) is None
 427
 428         p.send_signal(signal.SIGINT)
 429         time.sleep(2)
 430         p.wait()
 431         p = None
 432
 433     except Exception:
 434         if p:
 435             p.terminate()
 436             if p.wait():
 437                 comm_error(p)
 438             p = None
 439         raise
 440     finally:
 441         if p:
 442             p.terminate()
 443             p.wait()
 444
 445
 446 @pytest.mark.parametrize("tgen", [2], indirect=True)
 447 def test_ospf_opaque_add_data3(tgen):
 448     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
 449     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
 450     logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
 451     _test_add_data(tgen, apibin)
 452
 453
 454 def _test_opaque_add_del(tgen, apibin):
 455     "Test adding opaque data to domain"
 456
 457     r1 = tgen.gears["r1"]
 458     r2 = tgen.gears["r2"]
 459
 460     p = None
 461     pread = None
 462     try:
 463         step("reachable: check for add notification")
 464         pread = r2.popen(
 465             ["/usr/bin/timeout", "120", apibin, "-v"],
 466             encoding=None,  # don't buffer
 467             stdin=subprocess.DEVNULL,
 468             stdout=subprocess.PIPE,
 469             stderr=subprocess.STDOUT,
 470         )
 471         p = r1.popen(
 472             [
 473                 apibin,
 474                 "-v",
 475                 "add,9,10.0.1.1,230,2,00000202",
 476                 "add,10,1.2.3.4,231,1,00010101",
 477                 "wait,1",
 478                 "add,10,1.2.3.4,231,2",
 479                 "add,11,232,3,ebadf00d",
 480                 "wait,20",
 481                 "del,10,1.2.3.4,231,1",
 482                 "del,10,1.2.3.4,231,2",
 483             ]
 484         )
 485
 486         add_input_dict = {
 487             "areas": {
 488                 "1.2.3.4": {
 489                     "linkLocalOpaqueLsa": [
 490                         {
 491                             "lsId": "230.0.0.2",
 492                             "advertisedRouter": "192.168.0.1",
 493                             "sequenceNumber": "80000001",
 494                             "checksum": "8142",
 495                         }
 496                     ],
 497                     "areaLocalOpaqueLsa": [
 498                         {
 499                             "lsId": "231.0.0.1",
 500                             "advertisedRouter": "192.168.0.1",
 501                             "sequenceNumber": "80000001",
 502                             "checksum": "695a",
 503                         },
 504                         {
 505                             "lsId": "231.0.0.2",
 506                             "advertisedRouter": "192.168.0.1",
 507                             "sequenceNumber": "80000001",
 508                             "checksum": "4881",
 509                         },
 510                     ],
 511                 }
 512             },
 513             "asExternalOpaqueLsa": [
 514                 {
 515                     "lsId": "232.0.0.3",
 516                     "advertisedRouter": "192.168.0.1",
 517                     "sequenceNumber": "80000001",
 518                     "checksum": "c666",
 519                 }
 520             ],
 521         }
 522
 523         step("reachable: check for add LSAs")
 524         json_cmd = "show ip ospf da json"
 525         assert verify_ospf_database(tgen, r1, add_input_dict, json_cmd) is None
 526         assert verify_ospf_database(tgen, r2, add_input_dict, json_cmd) is None
 527
 528         numcs = 3
 529         json_cmds = [
 530             "show ip ospf da opaque-link json",
 531             "show ip ospf da opaque-area json",
 532             "show ip ospf da opaque-as json",
 533         ]
 534         add_detail_input_dict = [
 535             {
 536                 "linkLocalOpaqueLsa": {
 537                     "areas": {
 538                         "1.2.3.4": [
 539                             {
 540                                 "linkStateId": "230.0.0.2",
 541                                 "advertisingRouter": "192.168.0.1",
 542                                 "lsaSeqNumber": "80000001",
 543                                 "checksum": "8142",
 544                                 "length": 24,
 545                                 "opaqueId": 2,
 546                                 "opaqueDataLength": 4,
 547                             }
 548                         ]
 549                     }
 550                 }
 551             },
 552             {
 553                 "areaLocalOpaqueLsa": {
 554                     "areas": {
 555                         "1.2.3.4": [
 556                             {
 557                                 "linkStateId": "231.0.0.1",
 558                                 "advertisingRouter": "192.168.0.1",
 559                                 "lsaSeqNumber": "80000001",
 560                                 "checksum": "695a",
 561                                 "length": 24,
 562                                 "opaqueDataLength": 4,
 563                             },
 564                             {
 565                                 "linkStateId": "231.0.0.2",
 566                                 "advertisingRouter": "192.168.0.1",
 567                                 "lsaSeqNumber": "80000001",
 568                                 "checksum": "4881",
 569                                 "length": 20,
 570                                 "opaqueDataLength": 0,
 571                             },
 572                         ]
 573                     }
 574                 }
 575             },
 576             {
 577                 "asExternalOpaqueLsa": [
 578                     {
 579                         "linkStateId": "232.0.0.3",
 580                         "advertisingRouter": "192.168.0.1",
 581                         "lsaSeqNumber": "80000001",
 582                         "checksum": "c666",
 583                         "length": 24,
 584                         "opaqueDataLength": 4,
 585                     }
 586                 ]
 587             },
 588         ]
 589         i = 0
 590         while i < numcs:
 591             step("reachable: check for add LSA details: %s" % json_cmds[i])
 592             assert (
 593                 verify_ospf_database(tgen, r1, add_detail_input_dict[i], json_cmds[i])
 594                 is None
 595             )
 596             assert (
 597                 verify_ospf_database(tgen, r2, add_detail_input_dict[i], json_cmds[i])
 598                 is None
 599             )
 600             i += 1
 601
 602         # Wait for add notification
 603         # RECV: LSA update msg for LSA 232.0.0.3 in area 0.0.0.0 seq 0x80000001 len 24 age 9
 604
 605         step("reachable: check for API add notifications")
 606         ls_ids = ["230.0.0.2", "231.0.0.1", "231.0.0.2", "232.0.0.3"]
 607         for ls_id in ls_ids:
 608             waitfor = "RECV:.*update msg.*LSA {}.*age ([0-9]+)".format(ls_id)
 609             _ = _wait_output(pread, waitfor)
 610
 611         del_input_dict = {
 612             "areas": {
 613                 "1.2.3.4": {
 614                     "linkLocalOpaqueLsa": [
 615                         {
 616                             "lsId": "230.0.0.2",
 617                             "advertisedRouter": "192.168.0.1",
 618                             "sequenceNumber": "80000001",
 619                             "checksum": "8142",
 620                         }
 621                     ],
 622                     "areaLocalOpaqueLsa": [
 623                         {
 624                             "lsaAge": 3600,
 625                             "lsId": "231.0.0.1",
 626                             "advertisedRouter": "192.168.0.1",
 627                             "sequenceNumber": "80000001",
 628                             "checksum": "695a",
 629                         },
 630                         {
 631                             "lsaAge": 3600,
 632                             "lsId": "231.0.0.2",
 633                             "advertisedRouter": "192.168.0.1",
 634                             "sequenceNumber": "80000001",
 635                             "checksum": "4881",
 636                         },
 637                     ],
 638                 }
 639             },
 640             "asExternalOpaqueLsa": [
 641                 {
 642                     "lsId": "232.0.0.3",
 643                     "advertisedRouter": "192.168.0.1",
 644                     "sequenceNumber": "80000001",
 645                     "checksum": "c666",
 646                 }
 647             ],
 648         }
 649
 650         step("reachable: check for explicit withdrawal LSAs")
 651         json_cmd = "show ip ospf da json"
 652         assert verify_ospf_database(tgen, r1, del_input_dict, json_cmd) is None
 653         assert verify_ospf_database(tgen, r2, del_input_dict, json_cmd) is None
 654
 655         p.terminate()
 656         if p.wait():
 657             comm_error(p)
 658         del_input_dict = {
 659             "areas": {
 660                 "1.2.3.4": {
 661                     "linkLocalOpaqueLsa": [
 662                         {
 663                             "lsaAge": 3600,
 664                             "lsId": "230.0.0.2",
 665                             "advertisedRouter": "192.168.0.1",
 666                             "sequenceNumber": "80000001",
 667                             "checksum": "8142",
 668                         }
 669                     ],
 670                     "areaLocalOpaqueLsa": [
 671                         {
 672                             "lsaAge": 3600,
 673                             "lsId": "231.0.0.1",
 674                             "advertisedRouter": "192.168.0.1",
 675                             "sequenceNumber": "80000001",
 676                             "checksum": "695a",
 677                         },
 678                         {
 679                             "lsaAge": 3600,
 680                             "lsId": "231.0.0.2",
 681                             "advertisedRouter": "192.168.0.1",
 682                             "sequenceNumber": "80000001",
 683                             "checksum": "4881",
 684                         },
 685                     ],
 686                 }
 687             },
 688             "asExternalOpaqueLsa": [
 689                 {
 690                     "lsaAge": 3600,
 691                     "lsId": "232.0.0.3",
 692                     "advertisedRouter": "192.168.0.1",
 693                     "sequenceNumber": "80000001",
 694                     "checksum": "c666",
 695                 }
 696             ],
 697         }
 698
 699         step("reachable: check for implicit withdrawal LSAs")
 700         json_cmd = "show ip ospf da json"
 701         assert verify_ospf_database(tgen, r1, del_input_dict, json_cmd) is None
 702         assert verify_ospf_database(tgen, r2, del_input_dict, json_cmd) is None
 703
 704         # step("reachable: check for flush/age out")
 705         # # Wait for max age notification
 706         # waitfor = "RECV:.*update msg.*LSA {}.*age 3600".format(ls_id)
 707         # _wait_output(pread, waitfor)
 708
 709         step("reachable: check for API delete notifications")
 710         ls_ids = ["231.0.0.1", "231.0.0.2", "230.0.0.2", "232.0.0.3"]
 711         for ls_id in ls_ids:
 712             waitfor = "RECV:.*delete msg.*LSA {}.*".format(ls_id)
 713             _ = _wait_output(pread, waitfor)
 714     except Exception:
 715         if p:
 716             p.terminate()
 717             if p.wait():
 718                 comm_error(p)
 719             p = None
 720         raise
 721     finally:
 722         if pread:
 723             pread.terminate()
 724             pread.wait()
 725         if p:
 726             p.terminate()
 727             p.wait()
 728
 729
 730 @pytest.mark.parametrize("tgen", [2], indirect=True)
 731 def test_ospf_opaque_delete_data3(tgen):
 732     apibin = os.path.join(CLIENTDIR, "ospfclient.py")
 733     rc, o, e = tgen.gears["r2"].net.cmd_status([apibin, "--help"])
 734     logging.info("%s --help: rc: %s stdout: '%s' stderr: '%s'", apibin, rc, o, e)
 735     _test_opaque_add_del(tgen, apibin)
 736
 737
 738 if __name__ == "__main__":
 739     args = ["-s"] + sys.argv[1:]
 740     sys.exit(pytest.main(args))