tests/topotests/analyze.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 #
   4 # July 9 2021, Christian Hopps <chopps@labn.net>
   5 #
   6 # Copyright (c) 2021, LabN Consulting, L.L.C.
   7 #
   8 # This program is free software; you can redistribute it and/or
   9 # modify it under the terms of the GNU General Public License
  10 # as published by the Free Software Foundation; either version 2
  11 # of the License, or (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License along
  19 # with this program; see the file COPYING; if not, write to the Free Software
  20 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  21 #
  22 import argparse
  23 import glob
  24 import logging
  25 import os
  26 import re
  27 import subprocess
  28 import sys
  29 from collections import OrderedDict
  30
  31 import xmltodict
  32
  33
  34 def get_summary(results):
  35     ntest = int(results["@tests"])
  36     nfail = int(results["@failures"])
  37     nerror = int(results["@errors"])
  38     nskip = int(results["@skipped"])
  39     npass = ntest - nfail - nskip - nerror
  40     return ntest, npass, nfail, nerror, nskip
  41
  42
  43 def print_summary(results, args):
  44     ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
  45     for group in results:
  46         _ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
  47         if args.verbose:
  48             print(
  49                 f"Group: {group} Total: {_ntest} PASSED: {_npass}"
  50                 " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}"
  51             )
  52         ntest += _ntest
  53         npass += _npass
  54         nfail += _nfail
  55         nerror += _nerror
  56         nskip += _nskip
  57     print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
  58
  59
  60 def get_global_testcase(results):
  61     for group in results:
  62         for testcase in results[group]["testcase"]:
  63             if "@file" not in testcase:
  64                 return testcase
  65     return None
  66
  67
  68 def get_filtered(tfilters, results, args):
  69     if isinstance(tfilters, str) or tfilters is None:
  70         tfilters = [tfilters]
  71     found_files = OrderedDict()
  72     for group in results:
  73         if isinstance(results[group]["testcase"], list):
  74             tlist = results[group]["testcase"]
  75         else:
  76             tlist = [results[group]["testcase"]]
  77         for testcase in tlist:
  78             for tfilter in tfilters:
  79                 if tfilter is None:
  80                     if (
  81                         "failure" not in testcase
  82                         and "error" not in testcase
  83                         and "skipped" not in testcase
  84                     ):
  85                         break
  86                 elif tfilter in testcase:
  87                     break
  88             else:
  89                 continue
  90             # cname = testcase["@classname"]
  91             fname = testcase.get("@file", "")
  92             cname = testcase.get("@classname", "")
  93             if not fname and not cname:
  94                 name = testcase.get("@name", "")
  95                 if not name:
  96                     continue
  97                 # If we had a failure at the module level we could be here.
  98                 fname = name.replace(".", "/") + ".py"
  99                 tcname = fname
 100             else:
 101                 if not fname:
 102                     fname = cname.replace(".", "/") + ".py"
 103                 if args.files_only or "@name" not in testcase:
 104                     tcname = fname
 105                 else:
 106                     tcname = fname + "::" + testcase["@name"]
 107             found_files[tcname] = testcase
 108     return found_files
 109
 110
 111 def dump_testcase(testcase):
 112     expand_keys = ("failure", "error", "skipped")
 113
 114     s = ""
 115     for key, val in testcase.items():
 116         if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
 117             s += "{}: {}\n".format(key, val)
 118         else:
 119             for k2, v2 in val.items():
 120                 s += "{}: {}\n".format(k2, v2)
 121     return s
 122
 123
 124 def main():
 125     parser = argparse.ArgumentParser()
 126     parser.add_argument(
 127         "-A",
 128         "--save",
 129         action="store_true",
 130         help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist",
 131     )
 132     parser.add_argument(
 133         "-F",
 134         "--files-only",
 135         action="store_true",
 136         help="print test file names rather than individual full testcase names",
 137     )
 138     parser.add_argument(
 139         "-S",
 140         "--select",
 141         default="fe",
 142         help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.",
 143     )
 144     parser.add_argument(
 145         "-r",
 146         "--results",
 147         help="xml results file or directory containing xml results file",
 148     )
 149     parser.add_argument("--rundir", help=argparse.SUPPRESS)
 150     parser.add_argument(
 151         "-E",
 152         "--enumerate",
 153         action="store_true",
 154         help="enumerate each item (results scoped)",
 155     )
 156     parser.add_argument("-T", "--test", help="print testcase at enumeration")
 157     parser.add_argument(
 158         "--errmsg", action="store_true", help="print testcase error message"
 159     )
 160     parser.add_argument(
 161         "--errtext", action="store_true", help="print testcase error text"
 162     )
 163     parser.add_argument("--time", action="store_true", help="print testcase run times")
 164
 165     parser.add_argument("-s", "--summary", action="store_true", help="print summary")
 166     parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
 167     args = parser.parse_args()
 168
 169     if args.save and args.results and not os.path.exists(args.results):
 170         if not os.path.exists("/tmp/topotests"):
 171             logging.critical('No "/tmp/topotests" directory to save')
 172             sys.exit(1)
 173         subprocess.run(["mv", "/tmp/topotests", args.results])
 174         # # Old location for results
 175         # if os.path.exists("/tmp/topotests.xml", args.results):
 176         #     subprocess.run(["mv", "/tmp/topotests.xml", args.results])
 177
 178     assert (
 179         args.test is None or not args.files_only
 180     ), "Can't have both --files and --test"
 181
 182     results = {}
 183     ttfiles = []
 184     if args.rundir:
 185         basedir = os.path.realpath(args.rundir)
 186         os.chdir(basedir)
 187
 188         newfiles = glob.glob("tt-group-*/topotests.xml")
 189         if newfiles:
 190             ttfiles.extend(newfiles)
 191         if os.path.exists("topotests.xml"):
 192             ttfiles.append("topotests.xml")
 193     else:
 194         if args.results:
 195             if os.path.exists(os.path.join(args.results, "topotests.xml")):
 196                 args.results = os.path.join(args.results, "topotests.xml")
 197             if not os.path.exists(args.results):
 198                 logging.critical("%s doesn't exist", args.results)
 199                 sys.exit(1)
 200             ttfiles = [args.results]
 201
 202         if not ttfiles and os.path.exists("/tmp/topotests.xml"):
 203             ttfiles.append("/tmp/topotests.xml")
 204
 205     for f in ttfiles:
 206         m = re.match(r"tt-group-(\d+)/topotests.xml", f)
 207         group = int(m.group(1)) if m else 0
 208         with open(f) as xml_file:
 209             results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]
 210
 211     filters = []
 212     if "e" in args.select:
 213         filters.append("error")
 214     if "f" in args.select:
 215         filters.append("failure")
 216     if "s" in args.select:
 217         filters.append("skipped")
 218     if "p" in args.select:
 219         filters.append(None)
 220
 221     found_files = get_filtered(filters, results, args)
 222     if found_files:
 223         if args.test is not None:
 224             if args.test == "all":
 225                 keys = found_files.keys()
 226             else:
 227                 keys = [list(found_files.keys())[int(args.test)]]
 228             for key in keys:
 229                 testcase = found_files[key]
 230                 if args.errtext:
 231                     if "error" in testcase:
 232                         errmsg = testcase["error"]["#text"]
 233                     elif "failure" in testcase:
 234                         errmsg = testcase["failure"]["#text"]
 235                     else:
 236                         errmsg = "none found"
 237                     s = "{}: {}".format(key, errmsg)
 238                 elif args.time:
 239                     text = testcase["@time"]
 240                     s = "{}: {}".format(text, key)
 241                 elif args.errmsg:
 242                     if "error" in testcase:
 243                         errmsg = testcase["error"]["@message"]
 244                     elif "failure" in testcase:
 245                         errmsg = testcase["failure"]["@message"]
 246                     else:
 247                         errmsg = "none found"
 248                     s = "{}: {}".format(key, errmsg)
 249                 else:
 250                     s = dump_testcase(testcase)
 251                 print(s)
 252         elif filters:
 253             if args.enumerate:
 254                 print(
 255                     "\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
 256                 )
 257             else:
 258                 print("\n".join(found_files))
 259
 260     if args.summary:
 261         print_summary(results, args)
 262
 263
 264 if __name__ == "__main__":
 265     main()