tests/topotests/conftest.py

   1 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   2 """
   3 Topotest conftest.py file.
   4 """
   5 # pylint: disable=consider-using-f-string
   6
   7 import glob
   8 import os
   9 import re
  10 import resource
  11 import subprocess
  12 import sys
  13 import time
  14
  15 import lib.fixtures
  16 import pytest
  17 from lib.micronet_compat import ConfigOptionsProxy, Mininet
  18 from lib.topogen import diagnose_env, get_topogen
  19 from lib.topolog import logger
  20 from lib.topotest import json_cmp_result
  21 from munet import cli
  22 from munet.base import Commander, proc_error
  23 from munet.cleanup import cleanup_current, cleanup_previous
  24 from munet.testing.util import pause_test
  25
  26 from lib import topolog, topotest
  27
  28
  29 def pytest_addoption(parser):
  30     """
  31     Add topology-only option to the topology tester. This option makes pytest
  32     only run the setup_module() to setup the topology without running any tests.
  33     """
  34     parser.addoption(
  35         "--asan-abort",
  36         action="store_true",
  37         help="Configure address sanitizer to abort process on error",
  38     )
  39
  40     parser.addoption(
  41         "--cli-on-error",
  42         action="store_true",
  43         help="Mininet cli on test failure",
  44     )
  45
  46     parser.addoption(
  47         "--gdb-breakpoints",
  48         metavar="SYMBOL[,SYMBOL...]",
  49         help="Comma-separated list of functions to set gdb breakpoints on",
  50     )
  51
  52     parser.addoption(
  53         "--gdb-daemons",
  54         metavar="DAEMON[,DAEMON...]",
  55         help="Comma-separated list of daemons to spawn gdb on, or 'all'",
  56     )
  57
  58     parser.addoption(
  59         "--gdb-routers",
  60         metavar="ROUTER[,ROUTER...]",
  61         help="Comma-separated list of routers to spawn gdb on, or 'all'",
  62     )
  63
  64     parser.addoption(
  65         "--logd",
  66         action="append",
  67         metavar="DAEMON[,ROUTER[,...]",
  68         help=(
  69             "Tail-F of DAEMON log file. Specify routers in comma-separated list after "
  70             "daemon to limit to a subset of routers"
  71         ),
  72     )
  73
  74     parser.addoption(
  75         "--pause",
  76         action="store_true",
  77         help="Pause after each test",
  78     )
  79
  80     parser.addoption(
  81         "--pause-at-end",
  82         action="store_true",
  83         help="Pause before taking munet down",
  84     )
  85
  86     parser.addoption(
  87         "--pause-on-error",
  88         action="store_true",
  89         help="Do not pause after (disables default when --shell or -vtysh given)",
  90     )
  91
  92     parser.addoption(
  93         "--no-pause-on-error",
  94         dest="pause_on_error",
  95         action="store_false",
  96         help="Do not pause after (disables default when --shell or -vtysh given)",
  97     )
  98
  99     parser.addoption(
 100         "--pcap",
 101         default="",
 102         metavar="NET[,NET...]",
 103         help="Comma-separated list of networks to capture packets on, or 'all'",
 104     )
 105
 106     rundir_help = "directory for running in and log files"
 107     parser.addini("rundir", rundir_help, default="/tmp/topotests")
 108     parser.addoption("--rundir", metavar="DIR", help=rundir_help)
 109
 110     parser.addoption(
 111         "--shell",
 112         metavar="ROUTER[,ROUTER...]",
 113         help="Comma-separated list of routers to spawn shell on, or 'all'",
 114     )
 115
 116     parser.addoption(
 117         "--shell-on-error",
 118         action="store_true",
 119         help="Spawn shell on all routers on test failure",
 120     )
 121
 122     parser.addoption(
 123         "--strace-daemons",
 124         metavar="DAEMON[,DAEMON...]",
 125         help="Comma-separated list of daemons to strace, or 'all'",
 126     )
 127
 128     parser.addoption(
 129         "--topology-only",
 130         action="store_true",
 131         default=False,
 132         help="Only set up this topology, don't run tests",
 133     )
 134
 135     parser.addoption(
 136         "--valgrind-extra",
 137         action="store_true",
 138         help="Generate suppression file, and enable more precise (slower) valgrind checks",
 139     )
 140
 141     parser.addoption(
 142         "--valgrind-memleaks",
 143         action="store_true",
 144         help="Run all daemons under valgrind for memleak detection",
 145     )
 146
 147     parser.addoption(
 148         "--vtysh",
 149         metavar="ROUTER[,ROUTER...]",
 150         help="Comma-separated list of routers to spawn vtysh on, or 'all'",
 151     )
 152
 153     parser.addoption(
 154         "--vtysh-on-error",
 155         action="store_true",
 156         help="Spawn vtysh on all routers on test failure",
 157     )
 158
 159
 160 def check_for_memleaks():
 161     assert topotest.g_pytest_config.option.valgrind_memleaks
 162
 163     leaks = []
 164     tgen = get_topogen()  # pylint: disable=redefined-outer-name
 165     latest = []
 166     existing = []
 167     if tgen is not None:
 168         logdir = tgen.logdir
 169         if hasattr(tgen, "valgrind_existing_files"):
 170             existing = tgen.valgrind_existing_files
 171         latest = glob.glob(os.path.join(logdir, "*.valgrind.*"))
 172         latest = [x for x in latest if "core" not in x]
 173
 174     daemons = set()
 175     for vfile in latest:
 176         if vfile in existing:
 177             continue
 178         # do not consider memleaks from parent fork (i.e., owned by root)
 179         if os.stat(vfile).st_uid == 0:
 180             existing.append(vfile)  # do not check again
 181             logger.debug("Skipping valgrind file %s owned by root", vfile)
 182             continue
 183         logger.debug("Checking valgrind file %s not owned by root", vfile)
 184         with open(vfile, encoding="ascii") as vf:
 185             vfcontent = vf.read()
 186             match = re.search(r"ERROR SUMMARY: (\d+) errors", vfcontent)
 187             if match:
 188                 existing.append(vfile)  # have summary don't check again
 189             if match and match.group(1) != "0":
 190                 emsg = "{} in {}".format(match.group(1), vfile)
 191                 leaks.append(emsg)
 192                 daemon = re.match(r".*\.valgrind\.(.*)\.\d+", vfile).group(1)
 193                 daemons.add("{}({})".format(daemon, match.group(1)))
 194
 195     if tgen is not None:
 196         tgen.valgrind_existing_files = existing
 197
 198     if leaks:
 199         logger.error("valgrind memleaks found:\n\t%s", "\n\t".join(leaks))
 200         pytest.fail("valgrind memleaks found for daemons: " + " ".join(daemons))
 201
 202
 203 @pytest.fixture(autouse=True, scope="module")
 204 def module_check_memtest(request):
 205     yield
 206     if request.config.option.valgrind_memleaks:
 207         if get_topogen() is not None:
 208             check_for_memleaks()
 209
 210
 211 def pytest_runtest_logstart(nodeid, location):
 212     # location is (filename, lineno, testname)
 213     topolog.logstart(nodeid, location, topotest.g_pytest_config.option.rundir)
 214
 215
 216 def pytest_runtest_logfinish(nodeid, location):
 217     # location is (filename, lineno, testname)
 218     topolog.logfinish(nodeid, location)
 219
 220
 221 @pytest.hookimpl(hookwrapper=True)
 222 def pytest_runtest_call(item: pytest.Item) -> None:
 223     "Hook the function that is called to execute the test."
 224
 225     # For topology only run the CLI then exit
 226     if item.config.option.topology_only:
 227         get_topogen().cli()
 228         pytest.exit("exiting after --topology-only")
 229
 230     # Let the default pytest_runtest_call execute the test function
 231     yield
 232
 233     # Check for leaks if requested
 234     if item.config.option.valgrind_memleaks:
 235         check_for_memleaks()
 236
 237
 238 def pytest_assertrepr_compare(op, left, right):
 239     """
 240     Show proper assertion error message for json_cmp results.
 241     """
 242     del op
 243
 244     json_result = left
 245     if not isinstance(json_result, json_cmp_result):
 246         json_result = right
 247         if not isinstance(json_result, json_cmp_result):
 248             return None
 249
 250     return json_result.gen_report()
 251
 252
 253 def pytest_configure(config):
 254     """
 255     Assert that the environment is correctly configured, and get extra config.
 256     """
 257     topotest.g_pytest_config = ConfigOptionsProxy(config)
 258
 259     if config.getoption("--collect-only"):
 260         return
 261
 262     if "PYTEST_XDIST_WORKER" not in os.environ:
 263         os.environ["PYTEST_XDIST_MODE"] = config.getoption("dist", "no")
 264         os.environ["PYTEST_TOPOTEST_WORKER"] = ""
 265         is_xdist = os.environ["PYTEST_XDIST_MODE"] != "no"
 266         is_worker = False
 267     else:
 268         os.environ["PYTEST_TOPOTEST_WORKER"] = os.environ["PYTEST_XDIST_WORKER"]
 269         is_xdist = True
 270         is_worker = True
 271
 272     resource.setrlimit(
 273         resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
 274     )
 275     # -----------------------------------------------------
 276     # Set some defaults for the pytest.ini [pytest] section
 277     # ---------------------------------------------------
 278
 279     rundir = config.option.rundir
 280     if not rundir:
 281         rundir = config.getini("rundir")
 282     if not rundir:
 283         rundir = "/tmp/topotests"
 284     config.option.rundir = rundir
 285
 286     if not config.getoption("--junitxml"):
 287         config.option.xmlpath = os.path.join(rundir, "topotests.xml")
 288     xmlpath = config.option.xmlpath
 289
 290     # Save an existing topotest.xml
 291     if os.path.exists(xmlpath):
 292         fmtime = time.localtime(os.path.getmtime(xmlpath))
 293         suffix = "-" + time.strftime("%Y%m%d%H%M%S", fmtime)
 294         commander = Commander("pytest")
 295         mv_path = commander.get_exec_path("mv")
 296         commander.cmd_status([mv_path, xmlpath, xmlpath + suffix])
 297
 298     # Set the log_file (exec) to inside the rundir if not specified
 299     if not config.getoption("--log-file") and not config.getini("log_file"):
 300         config.option.log_file = os.path.join(rundir, "exec.log")
 301
 302     # Turn on live logging if user specified verbose and the config has a CLI level set
 303     if config.getoption("--verbose") and not is_xdist and not config.getini("log_cli"):
 304         if config.getoption("--log-cli-level", None) is None:
 305             # By setting the CLI option to the ini value it enables log_cli=1
 306             cli_level = config.getini("log_cli_level")
 307             if cli_level is not None:
 308                 config.option.log_cli_level = cli_level
 309
 310     have_tmux = bool(os.getenv("TMUX", ""))
 311     have_screen = not have_tmux and bool(os.getenv("STY", ""))
 312     have_xterm = not have_tmux and not have_screen and bool(os.getenv("DISPLAY", ""))
 313     have_windows = have_tmux or have_screen or have_xterm
 314     have_windows_pause = have_tmux or have_xterm
 315     xdist_no_windows = is_xdist and not is_worker and not have_windows_pause
 316
 317     def assert_feature_windows(b, feature):
 318         if b and xdist_no_windows:
 319             pytest.exit(
 320                 "{} use requires byobu/TMUX/XTerm under dist {}".format(
 321                     feature, os.environ["PYTEST_XDIST_MODE"]
 322                 )
 323             )
 324         elif b and not is_xdist and not have_windows:
 325             pytest.exit("{} use requires byobu/TMUX/SCREEN/XTerm".format(feature))
 326
 327     #
 328     # Check for window capability if given options that require window
 329     #
 330     assert_feature_windows(config.option.gdb_routers, "GDB")
 331     assert_feature_windows(config.option.gdb_daemons, "GDB")
 332     assert_feature_windows(config.option.cli_on_error, "--cli-on-error")
 333     assert_feature_windows(config.option.shell, "--shell")
 334     assert_feature_windows(config.option.shell_on_error, "--shell-on-error")
 335     assert_feature_windows(config.option.vtysh, "--vtysh")
 336     assert_feature_windows(config.option.vtysh_on_error, "--vtysh-on-error")
 337
 338     if config.option.topology_only and is_xdist:
 339         pytest.exit("Cannot use --topology-only with distributed test mode")
 340
 341     # Check environment now that we have config
 342     if not diagnose_env(rundir):
 343         pytest.exit("environment has errors, please read the logs in %s" % rundir)
 344
 345
 346 @pytest.fixture(autouse=True, scope="session")
 347 def setup_session_auto():
 348     if "PYTEST_TOPOTEST_WORKER" not in os.environ:
 349         is_worker = False
 350     elif not os.environ["PYTEST_TOPOTEST_WORKER"]:
 351         is_worker = False
 352     else:
 353         is_worker = True
 354
 355     logger.debug("Before the run (is_worker: %s)", is_worker)
 356     if not is_worker:
 357         cleanup_previous()
 358     yield
 359     if not is_worker:
 360         cleanup_current()
 361     logger.debug("After the run (is_worker: %s)", is_worker)
 362
 363
 364 def pytest_runtest_setup(item):
 365     module = item.parent.module
 366     script_dir = os.path.abspath(os.path.dirname(module.__file__))
 367     os.environ["PYTEST_TOPOTEST_SCRIPTDIR"] = script_dir
 368
 369
 370 def pytest_runtest_makereport(item, call):
 371     "Log all assert messages to default logger with error level"
 372
 373     pause = bool(item.config.getoption("--pause"))
 374     title = "unset"
 375
 376     if call.excinfo is None:
 377         error = False
 378     else:
 379         parent = item.parent
 380         modname = parent.module.__name__
 381
 382         # Treat skips as non errors, don't pause after
 383         if call.excinfo.typename == "Skipped":
 384             pause = False
 385             error = False
 386             logger.info(
 387                 'test skipped at "{}/{}": {}'.format(
 388                     modname, item.name, call.excinfo.value
 389                 )
 390             )
 391         else:
 392             error = True
 393             # Handle assert failures
 394             parent._previousfailed = item  # pylint: disable=W0212
 395             logger.error(
 396                 'test failed at "{}/{}": {}'.format(
 397                     modname, item.name, call.excinfo.value
 398                 )
 399             )
 400             title = "{}/{}".format(modname, item.name)
 401
 402             # We want to pause, if requested, on any error not just test cases
 403             # (e.g., call.when == "setup")
 404             if not pause:
 405                 pause = item.config.option.pause_on_error or item.config.option.pause
 406
 407             # (topogen) Set topology error to avoid advancing in the test.
 408             tgen = get_topogen()  # pylint: disable=redefined-outer-name
 409             if tgen is not None:
 410                 # This will cause topogen to report error on `routers_have_failure`.
 411                 tgen.set_error("{}/{}".format(modname, item.name))
 412
 413     commander = Commander("pytest")
 414     isatty = sys.stdout.isatty()
 415     error_cmd = None
 416
 417     if error and item.config.option.vtysh_on_error:
 418         error_cmd = commander.get_exec_path(["vtysh"])
 419     elif error and item.config.option.shell_on_error:
 420         error_cmd = os.getenv("SHELL", commander.get_exec_path(["bash"]))
 421
 422     if error_cmd:
 423         is_tmux = bool(os.getenv("TMUX", ""))
 424         is_screen = not is_tmux and bool(os.getenv("STY", ""))
 425         is_xterm = not is_tmux and not is_screen and bool(os.getenv("DISPLAY", ""))
 426
 427         channel = None
 428         win_info = None
 429         wait_for_channels = []
 430         wait_for_procs = []
 431         # Really would like something better than using this global here.
 432         # Not all tests use topogen though so get_topogen() won't work.
 433         for node in Mininet.g_mnet_inst.hosts.values():
 434             pause = True
 435
 436             if is_tmux:
 437                 channel = (
 438                     "{}-{}".format(os.getpid(), Commander.tmux_wait_gen)
 439                     if not isatty
 440                     else None
 441                 )
 442                 Commander.tmux_wait_gen += 1
 443                 wait_for_channels.append(channel)
 444
 445             pane_info = node.run_in_window(
 446                 error_cmd,
 447                 new_window=win_info is None,
 448                 background=True,
 449                 title="{} ({})".format(title, node.name),
 450                 name=title,
 451                 tmux_target=win_info,
 452                 wait_for=channel,
 453             )
 454             if is_tmux:
 455                 if win_info is None:
 456                     win_info = pane_info
 457             elif is_xterm:
 458                 assert isinstance(pane_info, subprocess.Popen)
 459                 wait_for_procs.append(pane_info)
 460
 461         # Now wait on any channels
 462         for channel in wait_for_channels:
 463             logger.debug("Waiting on TMUX channel %s", channel)
 464             commander.cmd_raises([commander.get_exec_path("tmux"), "wait", channel])
 465         for p in wait_for_procs:
 466             logger.debug("Waiting on TMUX xterm process %s", p)
 467             o, e = p.communicate()
 468             if p.wait():
 469                 logger.warning("xterm proc failed: %s:", proc_error(p, o, e))
 470
 471     if error and item.config.option.cli_on_error:
 472         # Really would like something better than using this global here.
 473         # Not all tests use topogen though so get_topogen() won't work.
 474         if Mininet.g_mnet_inst:
 475             cli.cli(Mininet.g_mnet_inst, title=title, background=False)
 476         else:
 477             logger.error("Could not launch CLI b/c no mininet exists yet")
 478
 479     if pause and isatty:
 480         pause_test()
 481
 482
 483 #
 484 # Add common fixtures available to all tests as parameters
 485 #
 486
 487 tgen = pytest.fixture(lib.fixtures.tgen)
 488 topo = pytest.fixture(lib.fixtures.topo)