lib: collect shutdown errors, add assert on single router shutdown

[mirror_frr.git] / tests / topotests / lib / topotest.py

diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py
index 007e23814e0c4cd353c6130e7d49dc6009319aad..655a05cbd40fd09e8aed380fbae2a92b8a8e1af6 100644 (file)
--- a/tests/topotests/lib/topotest.py
+++ b/tests/topotests/lib/topotest.py
@@ -59,6 +59,24 @@ class json_cmp_result(object):
         "Returns True if there were errors, otherwise False."
         return len(self.errors) > 0
 
+def get_test_logdir(node=None, init=False):
+    """
+    Return the current test log directory based on PYTEST_CURRENT_TEST
+    environment variable.
+    Optional paramters:
+    node:  when set, adds the node specific log directory to the init dir
+    init:  when set, initializes the log directory and fixes path permissions
+    """
+    cur_test = os.environ['PYTEST_CURRENT_TEST']
+
+    ret = '/tmp/topotests/' + cur_test[0:cur_test.find(".py")].replace('/','.')
+    if node != None:
+        dir = ret + "/" + node
+    if init:
+        os.system('mkdir -p ' + dir)
+        os.system('chmod -R go+rw /tmp/topotests')
+    return ret
+
 def json_diff(d1, d2):
     """
     Returns a string with the difference between JSON data.
@@ -462,7 +480,7 @@ class Router(Node):
 
     def __init__(self, name, **params):
         super(Router, self).__init__(name, **params)
-        self.logdir = params.get('logdir', '/tmp')
+        self.logdir = params.get('logdir', get_test_logdir(name, True))
         self.daemondir = None
         self.hasmpls = False
         self.routertype = 'frr'
@@ -470,6 +488,7 @@ class Router(Node):
                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0,
                         'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0}
         self.daemons_options = {'zebra': ''}
+        self.reportCores = True
 
     def _config_frr(self, **params):
         "Configure FRR binaries"
@@ -539,11 +558,14 @@ class Router(Node):
         set_sysctl(self, 'net.ipv4.ip_forward', 0)
         set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0)
         super(Router, self).terminate()
-    def stopRouter(self, wait=True):
+        os.system('chmod -R go+rw /tmp/topotests')
+
+    def stopRouter(self, wait=True, assertOnError=True):
         # Stop Running Quagga or FRR Daemons
         rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
+        errors = ""
         if re.search(r"No such file or directory", rundaemons):
-            return
+            return errors
         if rundaemons is not None:
             numRunning = 0
             for d in StringIO.StringIO(rundaemons):
@@ -571,7 +593,10 @@ class Router(Node):
                         self.waitOutput()
                     self.cmd('rm -- {}'.format(d.rstrip()))
         if wait:
-                self.checkRouterCores()
+                errors = self.checkRouterCores(reportOnce=True)
+                if assertOnError and len(errors) > 0:
+                    assert "Errors found - details follow:" == 0, errors
+        return errors
 
     def removeIPs(self):
         for interface in self.intfNames():
@@ -660,6 +685,9 @@ class Router(Node):
         # Starts actual daemons without init (ie restart)
         # cd to per node directory
         self.cmd('cd {}/{}'.format(self.logdir, self.name))
+        self.cmd('umask 000')
+        #Re-enable to allow for report per run
+        self.reportCores = True
         # Start Zebra first
         if self.daemons['zebra'] == 1:
             zebra_path = os.path.join(self.daemondir, 'zebra')
@@ -692,7 +720,11 @@ class Router(Node):
     def getLog(self, log, daemon):
         return self.cmd('cat {}/{}/{}.{}'.format(self.logdir, self.name, daemon, log))
 
-    def checkRouterCores(self, reportLeaks=True):
+    def checkRouterCores(self, reportLeaks=True, reportOnce=False):
+        if reportOnce and not self.reportCores:
+            return
+        reportMade = False
+        traces = ""
         for daemon in self.daemons:
             if (self.daemons[daemon] == 1):
                 # Look for core file
@@ -705,17 +737,26 @@ class Router(Node):
                     ], shell=True)
                     sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
                     sys.stderr.write("%s" % backtrace)
+                    traces = traces + "\n%s: %s crashed. Core file found - Backtrace follows:\n%s" % (self.name, daemon, backtrace)
+                    reportMade = True
                 elif reportLeaks:
                     log = self.getStdErr(daemon)
                     if "memstats" in log:
                         sys.stderr.write("%s: %s has memory leaks:\n" % (self.name, daemon))
+                        traces = traces + "\n%s: %s has memory leaks:\n" % (self.name, daemon)
                         log = re.sub("core_handler: ", "", log)
                         log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n  ## \1", log)
                         log = re.sub("memstats:  ", "    ", log)
                         sys.stderr.write(log)
+                        reportMade = True
                 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
                 if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon):
                     sys.stderr.write("%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon))
+                    traces = traces + "\n%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon)
+                    reportMade = True
+        if reportMade:
+            self.reportCores = False
+        return traces
 
     def checkRouterRunning(self):
         "Check if router daemons are running and collect crashinfo they don't run"