tests/topotests/analyze.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 # SPDX-License-Identifier: GPL-2.0-or-later
   4 #
   5 # July 9 2021, Christian Hopps <chopps@labn.net>
   6 #
   7 # Copyright (c) 2021, LabN Consulting, L.L.C.
   8 #
   9 import argparse
  10 import glob
  11 import logging
  12 import os
  13 import re
  14 import subprocess
  15 import sys
  16 from collections import OrderedDict
  17
  18 import xmltodict
  19
  20
  21 def get_summary(results):
  22     ntest = int(results["@tests"])
  23     nfail = int(results["@failures"])
  24     nerror = int(results["@errors"])
  25     nskip = int(results["@skipped"])
  26     npass = ntest - nfail - nskip - nerror
  27     return ntest, npass, nfail, nerror, nskip
  28
  29
  30 def print_summary(results, args):
  31     ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
  32     for group in results:
  33         _ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
  34         if args.verbose:
  35             print(
  36                 f"Group: {group} Total: {_ntest} PASSED: {_npass}"
  37                 " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}"
  38             )
  39         ntest += _ntest
  40         npass += _npass
  41         nfail += _nfail
  42         nerror += _nerror
  43         nskip += _nskip
  44     print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
  45
  46
  47 def get_global_testcase(results):
  48     for group in results:
  49         for testcase in results[group]["testcase"]:
  50             if "@file" not in testcase:
  51                 return testcase
  52     return None
  53
  54
  55 def get_filtered(tfilters, results, args):
  56     if isinstance(tfilters, str) or tfilters is None:
  57         tfilters = [tfilters]
  58     found_files = OrderedDict()
  59     for group in results:
  60         if isinstance(results[group]["testcase"], list):
  61             tlist = results[group]["testcase"]
  62         else:
  63             tlist = [results[group]["testcase"]]
  64         for testcase in tlist:
  65             for tfilter in tfilters:
  66                 if tfilter is None:
  67                     if (
  68                         "failure" not in testcase
  69                         and "error" not in testcase
  70                         and "skipped" not in testcase
  71                     ):
  72                         break
  73                 elif tfilter in testcase:
  74                     break
  75             else:
  76                 continue
  77             # cname = testcase["@classname"]
  78             fname = testcase.get("@file", "")
  79             cname = testcase.get("@classname", "")
  80             if not fname and not cname:
  81                 name = testcase.get("@name", "")
  82                 if not name:
  83                     continue
  84                 # If we had a failure at the module level we could be here.
  85                 fname = name.replace(".", "/") + ".py"
  86                 tcname = fname
  87             else:
  88                 if not fname:
  89                     fname = cname.replace(".", "/") + ".py"
  90                 if args.files_only or "@name" not in testcase:
  91                     tcname = fname
  92                 else:
  93                     tcname = fname + "::" + testcase["@name"]
  94             found_files[tcname] = testcase
  95     return found_files
  96
  97
  98 def dump_testcase(testcase):
  99     expand_keys = ("failure", "error", "skipped")
 100
 101     s = ""
 102     for key, val in testcase.items():
 103         if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
 104             s += "{}: {}\n".format(key, val)
 105         elif isinstance(val, list):
 106             for k2, v2 in enumerate(val):
 107                 s += "{}: {}\n".format(k2, v2)
 108         else:
 109             for k2, v2 in val.items():
 110                 s += "{}: {}\n".format(k2, v2)
 111     return s
 112
 113
 114 def main():
 115     parser = argparse.ArgumentParser()
 116     parser.add_argument(
 117         "-A",
 118         "--save",
 119         action="store_true",
 120         help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist",
 121     )
 122     parser.add_argument(
 123         "-F",
 124         "--files-only",
 125         action="store_true",
 126         help="print test file names rather than individual full testcase names",
 127     )
 128     parser.add_argument(
 129         "-S",
 130         "--select",
 131         default="fe",
 132         help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.",
 133     )
 134     parser.add_argument(
 135         "-r",
 136         "--results",
 137         help="xml results file or directory containing xml results file",
 138     )
 139     parser.add_argument("--rundir", help=argparse.SUPPRESS)
 140     parser.add_argument(
 141         "-E",
 142         "--enumerate",
 143         action="store_true",
 144         help="enumerate each item (results scoped)",
 145     )
 146     parser.add_argument("-T", "--test", help="print testcase at enumeration")
 147     parser.add_argument(
 148         "--errmsg", action="store_true", help="print testcase error message"
 149     )
 150     parser.add_argument(
 151         "--errtext", action="store_true", help="print testcase error text"
 152     )
 153     parser.add_argument("--time", action="store_true", help="print testcase run times")
 154
 155     parser.add_argument("-s", "--summary", action="store_true", help="print summary")
 156     parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
 157     args = parser.parse_args()
 158
 159     if args.save and args.results and not os.path.exists(args.results):
 160         if not os.path.exists("/tmp/topotests"):
 161             logging.critical('No "/tmp/topotests" directory to save')
 162             sys.exit(1)
 163         subprocess.run(["mv", "/tmp/topotests", args.results])
 164         # # Old location for results
 165         # if os.path.exists("/tmp/topotests.xml", args.results):
 166         #     subprocess.run(["mv", "/tmp/topotests.xml", args.results])
 167
 168     assert (
 169         args.test is None or not args.files_only
 170     ), "Can't have both --files and --test"
 171
 172     results = {}
 173     ttfiles = []
 174     if args.rundir:
 175         basedir = os.path.realpath(args.rundir)
 176         os.chdir(basedir)
 177
 178         newfiles = glob.glob("tt-group-*/topotests.xml")
 179         if newfiles:
 180             ttfiles.extend(newfiles)
 181         if os.path.exists("topotests.xml"):
 182             ttfiles.append("topotests.xml")
 183     else:
 184         if args.results:
 185             if os.path.exists(os.path.join(args.results, "topotests.xml")):
 186                 args.results = os.path.join(args.results, "topotests.xml")
 187             if not os.path.exists(args.results):
 188                 logging.critical("%s doesn't exist", args.results)
 189                 sys.exit(1)
 190             ttfiles = [args.results]
 191         elif os.path.exists("/tmp/topotests/topotests.xml"):
 192             ttfiles.append("/tmp/topotests/topotests.xml")
 193
 194         if not ttfiles:
 195             if os.path.exists("/tmp/topotests.xml"):
 196                 ttfiles.append("/tmp/topotests.xml")
 197
 198     for f in ttfiles:
 199         m = re.match(r"tt-group-(\d+)/topotests.xml", f)
 200         group = int(m.group(1)) if m else 0
 201         with open(f) as xml_file:
 202             results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]
 203
 204     filters = []
 205     if "e" in args.select:
 206         filters.append("error")
 207     if "f" in args.select:
 208         filters.append("failure")
 209     if "s" in args.select:
 210         filters.append("skipped")
 211     if "p" in args.select:
 212         filters.append(None)
 213
 214     found_files = get_filtered(filters, results, args)
 215     if found_files:
 216         if args.test is not None:
 217             if args.test == "all":
 218                 keys = found_files.keys()
 219             else:
 220                 keys = [list(found_files.keys())[int(args.test)]]
 221             for key in keys:
 222                 testcase = found_files[key]
 223                 if args.errtext:
 224                     if "error" in testcase:
 225                         errmsg = testcase["error"]["#text"]
 226                     elif "failure" in testcase:
 227                         errmsg = testcase["failure"]["#text"]
 228                     else:
 229                         errmsg = "none found"
 230                     s = "{}: {}".format(key, errmsg)
 231                 elif args.time:
 232                     text = testcase["@time"]
 233                     s = "{}: {}".format(text, key)
 234                 elif args.errmsg:
 235                     if "error" in testcase:
 236                         errmsg = testcase["error"]["@message"]
 237                     elif "failure" in testcase:
 238                         errmsg = testcase["failure"]["@message"]
 239                     else:
 240                         errmsg = "none found"
 241                     s = "{}: {}".format(key, errmsg)
 242                 else:
 243                     s = dump_testcase(testcase)
 244                 print(s)
 245         elif filters:
 246             if args.enumerate:
 247                 print(
 248                     "\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
 249                 )
 250             else:
 251                 print("\n".join(found_files))
 252
 253     if args.summary:
 254         print_summary(results, args)
 255
 256
 257 if __name__ == "__main__":
 258     main()