tests/helpers/python/frrtest.py

   1 #
   2 # Test helpers for FRR
   3 #
   4 # Copyright (C) 2017 by David Lamparter & Christian Franke,
   5 #                       Open Source Routing / NetDEF Inc.
   6 #
   7 # This file is part of FRRouting (FRR)
   8 #
   9 # FRR is free software; you can redistribute it and/or modify it
  10 # under the terms of the GNU General Public License as published by the
  11 # Free Software Foundation; either version 2, or (at your option) any
  12 # later version.
  13 #
  14 # FRR is distributed in the hope that it will be useful, but
  15 # WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 # General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with FRR; see the file COPYING.  If not, write to the Free
  21 # Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
  22 # 02111-1307, USA.
  23 #
  24
  25 import subprocess
  26 import sys
  27 import re
  28 import inspect
  29 import os
  30 import difflib
  31
  32 import frrsix
  33
  34 #
  35 # These are the gritty internals of the TestMultiOut implementation.
  36 # See below for the definition of actual TestMultiOut tests.
  37 #
  38
  39 srcbase = os.path.abspath(inspect.getsourcefile(frrsix))
  40 for i in range(0, 3):
  41     srcbase = os.path.dirname(srcbase)
  42
  43
  44 def binpath(srcpath):
  45     return os.path.relpath(os.path.abspath(srcpath), srcbase)
  46
  47
  48 class MultiTestFailure(Exception):
  49     pass
  50
  51
  52 class MetaTestMultiOut(type):
  53     def __getattr__(cls, name):
  54         if name.startswith("_"):
  55             raise AttributeError
  56
  57         internal_name = "_{}".format(name)
  58         if internal_name not in dir(cls):
  59             raise AttributeError
  60
  61         def registrar(*args, **kwargs):
  62             cls._add_test(getattr(cls, internal_name), *args, **kwargs)
  63
  64         return registrar
  65
  66
  67 @frrsix.add_metaclass(MetaTestMultiOut)
  68 class _TestMultiOut(object):
  69     def _run_tests(self):
  70         if "tests_run" in dir(self.__class__) and self.tests_run:
  71             return
  72         self.__class__.tests_run = True
  73         basedir = os.path.dirname(inspect.getsourcefile(type(self)))
  74         program = os.path.join(basedir, self.program)
  75         proc = subprocess.Popen([binpath(program)], stdout=subprocess.PIPE)
  76         self.output, _ = proc.communicate("")
  77         self.exitcode = proc.wait()
  78
  79         self.__class__.testresults = {}
  80         for test in self.tests:
  81             try:
  82                 test(self)
  83             except MultiTestFailure:
  84                 self.testresults[test] = sys.exc_info()
  85             else:
  86                 self.testresults[test] = None
  87
  88     def _exit_cleanly(self):
  89         if self.exitcode != 0:
  90             raise MultiTestFailure("Program did not terminate with exit code 0")
  91
  92     @classmethod
  93     def _add_test(cls, method, *args, **kwargs):
  94         if "tests" not in dir(cls):
  95             setattr(cls, "tests", [])
  96             if method is not cls._exit_cleanly:
  97                 cls._add_test(cls._exit_cleanly)
  98
  99         def matchfunction(self):
 100             method(self, *args, **kwargs)
 101
 102         cls.tests.append(matchfunction)
 103
 104         def testfunction(self):
 105             self._run_tests()
 106             result = self.testresults[matchfunction]
 107             if result is not None:
 108                 frrsix.reraise(*result)
 109
 110         testname = re.sub(r"[^A-Za-z0-9]", "_", "%r%r" % (args, kwargs))
 111         testname = re.sub(r"__*", "_", testname)
 112         testname = testname.strip("_")
 113         if not testname:
 114             testname = method.__name__.strip("_")
 115         if "test_%s" % testname in dir(cls):
 116             index = 2
 117             while "test_%s_%d" % (testname, index) in dir(cls):
 118                 index += 1
 119             testname = "%s_%d" % (testname, index)
 120         setattr(cls, "test_%s" % testname, testfunction)
 121
 122
 123 #
 124 # This class houses the actual TestMultiOut tests types.
 125 # If you want to add a new test type, you probably do it here.
 126 #
 127 # Say you want to add a test type called foobarlicious. Then define
 128 # a function _foobarlicious here that takes self and the test arguments
 129 # when called. That function should check the output in self.output
 130 # to see whether it matches the expectation of foobarlicious with the
 131 # given arguments and should then adjust self.output according to how
 132 # much output it consumed.
 133 # If the output doesn't meet the expectations, MultiTestFailure can be
 134 # raised, however that should only be done after self.output has been
 135 # modified according to consumed content.
 136 #
 137
 138 re_okfail = re.compile(r"(?:[3[12]m|^)?(?P<ret>OK|failed)".encode("utf8"), re.MULTILINE)
 139
 140
 141 class TestMultiOut(_TestMultiOut):
 142     def _onesimple(self, line):
 143         if type(line) is str:
 144             line = line.encode("utf8")
 145         idx = self.output.find(line)
 146         if idx != -1:
 147             self.output = self.output[idx + len(line) :]
 148         else:
 149             raise MultiTestFailure("%r could not be found" % line)
 150
 151     def _okfail(self, line, okfail=re_okfail):
 152         self._onesimple(line)
 153
 154         m = okfail.search(self.output)
 155         if m is None:
 156             raise MultiTestFailure("OK/fail not found")
 157         self.output = self.output[m.end() :]
 158
 159         if m.group("ret") != "OK".encode("utf8"):
 160             raise MultiTestFailure("Test output indicates failure")
 161
 162
 163 #
 164 # This class implements a test comparing the output of a program against
 165 # an existing reference output
 166 #
 167
 168
 169 class TestRefMismatch(Exception):
 170     def __init__(self, _test, outtext, reftext):
 171         self.outtext = outtext
 172         self.reftext = reftext
 173
 174     def __str__(self):
 175         rv = "Expected output and actual output differ:\n"
 176         rv += "\n".join(
 177             difflib.unified_diff(
 178                 self.reftext.splitlines(),
 179                 self.outtext.splitlines(),
 180                 "outtext",
 181                 "reftext",
 182                 lineterm="",
 183             )
 184         )
 185         return rv
 186
 187
 188 class TestExitNonzero(Exception):
 189     pass
 190
 191
 192 class TestRefOut(object):
 193     def test_refout(self):
 194         basedir = os.path.dirname(inspect.getsourcefile(type(self)))
 195         program = os.path.join(basedir, self.program)
 196
 197         if getattr(self, "built_refin", False):
 198             refin = binpath(program) + ".in"
 199         else:
 200             refin = program + ".in"
 201         if getattr(self, "built_refout", False):
 202             refout = binpath(program) + ".refout"
 203         else:
 204             refout = program + ".refout"
 205
 206         intext = ""
 207         if os.path.exists(refin):
 208             with open(refin, "rb") as f:
 209                 intext = f.read()
 210         with open(refout, "rb") as f:
 211             reftext = f.read()
 212
 213         proc = subprocess.Popen(
 214             [binpath(program)], stdin=subprocess.PIPE, stdout=subprocess.PIPE
 215         )
 216         outtext, _ = proc.communicate(intext)
 217
 218         # Get rid of newline problems (Windows vs Unix Style)
 219         outtext_str = outtext.decode("utf8").replace("\r\n", "\n").replace("\r", "\n")
 220         reftext_str = reftext.decode("utf8").replace("\r\n", "\n").replace("\r", "\n")
 221
 222         if outtext_str != reftext_str:
 223             raise TestRefMismatch(self, outtext_str, reftext_str)
 224         if proc.wait() != 0:
 225             raise TestExitNonzero(self)