tests/topotests/analyze.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 #
   4 # July 9 2021, Christian Hopps <chopps@labn.net>
   5 #
   6 # Copyright (c) 2021, LabN Consulting, L.L.C.
   7 #
   8 # This program is free software; you can redistribute it and/or
   9 # modify it under the terms of the GNU General Public License
  10 # as published by the Free Software Foundation; either version 2
  11 # of the License, or (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License along
  19 # with this program; see the file COPYING; if not, write to the Free Software
  20 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  21 #
  22 import argparse
  23 import glob
  24 import logging
  25 import os
  26 import re
  27 import subprocess
  28 import sys
  29 from collections import OrderedDict
  30
  31 import xmltodict
  32
  33
  34 def get_summary(results):
  35     ntest = int(results["@tests"])
  36     nfail = int(results["@failures"])
  37     nerror = int(results["@errors"])
  38     nskip = int(results["@skipped"])
  39     npass = ntest - nfail - nskip - nerror
  40     return ntest, npass, nfail, nerror, nskip
  41
  42
  43 def print_summary(results, args):
  44     ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
  45     for group in results:
  46         _ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
  47         if args.verbose:
  48             print(
  49                 f"Group: {group} Total: {_ntest} PASSED: {_npass}"
  50                 " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}"
  51             )
  52         ntest += _ntest
  53         npass += _npass
  54         nfail += _nfail
  55         nerror += _nerror
  56         nskip += _nskip
  57     print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
  58
  59
  60 def get_global_testcase(results):
  61     for group in results:
  62         for testcase in results[group]["testcase"]:
  63             if "@file" not in testcase:
  64                 return testcase
  65     return None
  66
  67
  68 def get_filtered(tfilters, results, args):
  69     if isinstance(tfilters, str) or tfilters is None:
  70         tfilters = [tfilters]
  71     found_files = OrderedDict()
  72     for group in results:
  73         if isinstance(results[group]["testcase"], list):
  74             tlist = results[group]["testcase"]
  75         else:
  76             tlist = [results[group]["testcase"]]
  77         for testcase in tlist:
  78             for tfilter in tfilters:
  79                 if tfilter is None:
  80                     if (
  81                         "failure" not in testcase
  82                         and "error" not in testcase
  83                         and "skipped" not in testcase
  84                     ):
  85                         break
  86                 elif tfilter in testcase:
  87                     break
  88             else:
  89                 continue
  90             # cname = testcase["@classname"]
  91             fname = testcase.get("@file", "")
  92             cname = testcase.get("@classname", "")
  93             if not fname and not cname:
  94                 name = testcase.get("@name", "")
  95                 if not name:
  96                     continue
  97                 # If we had a failure at the module level we could be here.
  98                 fname = name.replace(".", "/") + ".py"
  99                 tcname = fname
 100             else:
 101                 if not fname:
 102                     fname = cname.replace(".", "/") + ".py"
 103                 if args.files_only or "@name" not in testcase:
 104                     tcname = fname
 105                 else:
 106                     tcname = fname + "::" + testcase["@name"]
 107             found_files[tcname] = testcase
 108     return found_files
 109
 110
 111 def dump_testcase(testcase):
 112     expand_keys = ("failure", "error", "skipped")
 113
 114     s = ""
 115     for key, val in testcase.items():
 116         if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
 117             s += "{}: {}\n".format(key, val)
 118         elif isinstance(val, list):
 119             for k2, v2 in enumerate(val):
 120                 s += "{}: {}\n".format(k2, v2)
 121         else:
 122             for k2, v2 in val.items():
 123                 s += "{}: {}\n".format(k2, v2)
 124     return s
 125
 126
 127 def main():
 128     parser = argparse.ArgumentParser()
 129     parser.add_argument(
 130         "-A",
 131         "--save",
 132         action="store_true",
 133         help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist",
 134     )
 135     parser.add_argument(
 136         "-F",
 137         "--files-only",
 138         action="store_true",
 139         help="print test file names rather than individual full testcase names",
 140     )
 141     parser.add_argument(
 142         "-S",
 143         "--select",
 144         default="fe",
 145         help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.",
 146     )
 147     parser.add_argument(
 148         "-r",
 149         "--results",
 150         help="xml results file or directory containing xml results file",
 151     )
 152     parser.add_argument("--rundir", help=argparse.SUPPRESS)
 153     parser.add_argument(
 154         "-E",
 155         "--enumerate",
 156         action="store_true",
 157         help="enumerate each item (results scoped)",
 158     )
 159     parser.add_argument("-T", "--test", help="print testcase at enumeration")
 160     parser.add_argument(
 161         "--errmsg", action="store_true", help="print testcase error message"
 162     )
 163     parser.add_argument(
 164         "--errtext", action="store_true", help="print testcase error text"
 165     )
 166     parser.add_argument("--time", action="store_true", help="print testcase run times")
 167
 168     parser.add_argument("-s", "--summary", action="store_true", help="print summary")
 169     parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
 170     args = parser.parse_args()
 171
 172     if args.save and args.results and not os.path.exists(args.results):
 173         if not os.path.exists("/tmp/topotests"):
 174             logging.critical('No "/tmp/topotests" directory to save')
 175             sys.exit(1)
 176         subprocess.run(["mv", "/tmp/topotests", args.results])
 177         # # Old location for results
 178         # if os.path.exists("/tmp/topotests.xml", args.results):
 179         #     subprocess.run(["mv", "/tmp/topotests.xml", args.results])
 180
 181     assert (
 182         args.test is None or not args.files_only
 183     ), "Can't have both --files and --test"
 184
 185     results = {}
 186     ttfiles = []
 187     if args.rundir:
 188         basedir = os.path.realpath(args.rundir)
 189         os.chdir(basedir)
 190
 191         newfiles = glob.glob("tt-group-*/topotests.xml")
 192         if newfiles:
 193             ttfiles.extend(newfiles)
 194         if os.path.exists("topotests.xml"):
 195             ttfiles.append("topotests.xml")
 196     else:
 197         if args.results:
 198             if os.path.exists(os.path.join(args.results, "topotests.xml")):
 199                 args.results = os.path.join(args.results, "topotests.xml")
 200             if not os.path.exists(args.results):
 201                 logging.critical("%s doesn't exist", args.results)
 202                 sys.exit(1)
 203             ttfiles = [args.results]
 204         elif os.path.exists("/tmp/topotests/topotests.xml"):
 205             ttfiles.append("/tmp/topotests/topotests.xml")
 206
 207         if not ttfiles:
 208             if os.path.exists("/tmp/topotests.xml"):
 209                 ttfiles.append("/tmp/topotests.xml")
 210
 211     for f in ttfiles:
 212         m = re.match(r"tt-group-(\d+)/topotests.xml", f)
 213         group = int(m.group(1)) if m else 0
 214         with open(f) as xml_file:
 215             results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]
 216
 217     filters = []
 218     if "e" in args.select:
 219         filters.append("error")
 220     if "f" in args.select:
 221         filters.append("failure")
 222     if "s" in args.select:
 223         filters.append("skipped")
 224     if "p" in args.select:
 225         filters.append(None)
 226
 227     found_files = get_filtered(filters, results, args)
 228     if found_files:
 229         if args.test is not None:
 230             if args.test == "all":
 231                 keys = found_files.keys()
 232             else:
 233                 keys = [list(found_files.keys())[int(args.test)]]
 234             for key in keys:
 235                 testcase = found_files[key]
 236                 if args.errtext:
 237                     if "error" in testcase:
 238                         errmsg = testcase["error"]["#text"]
 239                     elif "failure" in testcase:
 240                         errmsg = testcase["failure"]["#text"]
 241                     else:
 242                         errmsg = "none found"
 243                     s = "{}: {}".format(key, errmsg)
 244                 elif args.time:
 245                     text = testcase["@time"]
 246                     s = "{}: {}".format(text, key)
 247                 elif args.errmsg:
 248                     if "error" in testcase:
 249                         errmsg = testcase["error"]["@message"]
 250                     elif "failure" in testcase:
 251                         errmsg = testcase["failure"]["@message"]
 252                     else:
 253                         errmsg = "none found"
 254                     s = "{}: {}".format(key, errmsg)
 255                 else:
 256                     s = dump_testcase(testcase)
 257                 print(s)
 258         elif filters:
 259             if args.enumerate:
 260                 print(
 261                     "\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
 262                 )
 263             else:
 264                 print("\n".join(found_files))
 265
 266     if args.summary:
 267         print_summary(results, args)
 268
 269
 270 if __name__ == "__main__":
 271     main()