tests/topotests/analyze.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
   3 #
   4 # July 9 2021, Christian Hopps <chopps@labn.net>
   5 #
   6 # Copyright (c) 2021, LabN Consulting, L.L.C.
   7 #
   8 # This program is free software; you can redistribute it and/or
   9 # modify it under the terms of the GNU General Public License
  10 # as published by the Free Software Foundation; either version 2
  11 # of the License, or (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License along
  19 # with this program; see the file COPYING; if not, write to the Free Software
  20 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  21 #
  22 import argparse
  23 import glob
  24 import logging
  25 import os
  26 import re
  27 import subprocess
  28 import sys
  29 from collections import OrderedDict
  30
  31 import xmltodict
  32
  33
  34 def get_summary(results):
  35     ntest = int(results["@tests"])
  36     nfail = int(results["@failures"])
  37     nerror = int(results["@errors"])
  38     nskip = int(results["@skipped"])
  39     npass = ntest - nfail - nskip - nerror
  40     return ntest, npass, nfail, nerror, nskip
  41
  42
  43 def print_summary(results, args):
  44     ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
  45     for group in results:
  46         _ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
  47         if args.verbose:
  48             print(
  49                 f"Group: {group} Total: {_ntest} PASSED: {_npass}"
  50                 " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}"
  51             )
  52         ntest += _ntest
  53         npass += _npass
  54         nfail += _nfail
  55         nerror += _nerror
  56         nskip += _nskip
  57     print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
  58
  59
  60 def get_global_testcase(results):
  61     for group in results:
  62         for testcase in results[group]["testcase"]:
  63             if "@file" not in testcase:
  64                 return testcase
  65     return None
  66
  67
  68 def get_filtered(tfilters, results, args):
  69     if isinstance(tfilters, str) or tfilters is None:
  70         tfilters = [tfilters]
  71     found_files = OrderedDict()
  72     for group in results:
  73         if isinstance(results[group]["testcase"], list):
  74             tlist = results[group]["testcase"]
  75         else:
  76             tlist = [results[group]["testcase"]]
  77         for testcase in tlist:
  78             for tfilter in tfilters:
  79                 if tfilter is None:
  80                     if (
  81                         "failure" not in testcase
  82                         and "error" not in testcase
  83                         and "skipped" not in testcase
  84                     ):
  85                         break
  86                 elif tfilter in testcase:
  87                     break
  88             else:
  89                 continue
  90             # cname = testcase["@classname"]
  91             fname = testcase.get("@file", "")
  92             cname = testcase.get("@classname", "")
  93             if not fname and not cname:
  94                 name = testcase.get("@name", "")
  95                 if not name:
  96                     continue
  97                 # If we had a failure at the module level we could be here.
  98                 fname = name.replace(".", "/") + ".py"
  99                 tcname = fname
 100             else:
 101                 if not fname:
 102                     fname = cname.replace(".", "/") + ".py"
 103                 if args.files_only or "@name" not in testcase:
 104                     tcname = fname
 105                 else:
 106                     tcname = fname + "::" + testcase["@name"]
 107             found_files[tcname] = testcase
 108     return found_files
 109
 110
 111 def dump_testcase(testcase):
 112     expand_keys = ("failure", "error", "skipped")
 113
 114     s = ""
 115     for key, val in testcase.items():
 116         if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
 117             s += "{}: {}\n".format(key, val)
 118         else:
 119             for k2, v2 in val.items():
 120                 s += "{}: {}\n".format(k2, v2)
 121     return s
 122
 123
 124 def main():
 125     parser = argparse.ArgumentParser()
 126     parser.add_argument(
 127         "-A",
 128         "--save",
 129         action="store_true",
 130         help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist",
 131     )
 132     parser.add_argument(
 133         "-F",
 134         "--files-only",
 135         action="store_true",
 136         help="print test file names rather than individual full testcase names",
 137     )
 138     parser.add_argument(
 139         "-S",
 140         "--select",
 141         default="fe",
 142         help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.",
 143     )
 144     parser.add_argument(
 145         "-r",
 146         "--results",
 147         help="xml results file or directory containing xml results file",
 148     )
 149     parser.add_argument("--rundir", help=argparse.SUPPRESS)
 150     parser.add_argument(
 151         "-E",
 152         "--enumerate",
 153         action="store_true",
 154         help="enumerate each item (results scoped)",
 155     )
 156     parser.add_argument("-T", "--test", help="print testcase at enumeration")
 157     parser.add_argument(
 158         "--errmsg", action="store_true", help="print testcase error message"
 159     )
 160     parser.add_argument(
 161         "--errtext", action="store_true", help="print testcase error text"
 162     )
 163     parser.add_argument("--time", action="store_true", help="print testcase run times")
 164
 165     parser.add_argument("-s", "--summary", action="store_true", help="print summary")
 166     parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
 167     args = parser.parse_args()
 168
 169     if args.save and args.results and not os.path.exists(args.results):
 170         if not os.path.exists("/tmp/topotests"):
 171             logging.critical('No "/tmp/topotests" directory to save')
 172             sys.exit(1)
 173         subprocess.run(["mv", "/tmp/topotests", args.results])
 174         # # Old location for results
 175         # if os.path.exists("/tmp/topotests.xml", args.results):
 176         #     subprocess.run(["mv", "/tmp/topotests.xml", args.results])
 177
 178     assert (
 179         args.test is None or not args.files_only
 180     ), "Can't have both --files and --test"
 181
 182     results = {}
 183     ttfiles = []
 184     if args.rundir:
 185         basedir = os.path.realpath(args.rundir)
 186         os.chdir(basedir)
 187
 188         newfiles = glob.glob("tt-group-*/topotests.xml")
 189         if newfiles:
 190             ttfiles.extend(newfiles)
 191         if os.path.exists("topotests.xml"):
 192             ttfiles.append("topotests.xml")
 193     else:
 194         if args.results:
 195             if os.path.exists(os.path.join(args.results, "topotests.xml")):
 196                 args.results = os.path.join(args.results, "topotests.xml")
 197             if not os.path.exists(args.results):
 198                 logging.critical("%s doesn't exist", args.results)
 199                 sys.exit(1)
 200             ttfiles = [args.results]
 201         elif os.path.exists("/tmp/topotests/topotests.xml"):
 202             ttfiles.append("/tmp/topotests/topotests.xml")
 203
 204         if not ttfiles:
 205             if os.path.exists("/tmp/topotests.xml"):
 206                 ttfiles.append("/tmp/topotests.xml")
 207
 208     for f in ttfiles:
 209         m = re.match(r"tt-group-(\d+)/topotests.xml", f)
 210         group = int(m.group(1)) if m else 0
 211         with open(f) as xml_file:
 212             results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]
 213
 214     filters = []
 215     if "e" in args.select:
 216         filters.append("error")
 217     if "f" in args.select:
 218         filters.append("failure")
 219     if "s" in args.select:
 220         filters.append("skipped")
 221     if "p" in args.select:
 222         filters.append(None)
 223
 224     found_files = get_filtered(filters, results, args)
 225     if found_files:
 226         if args.test is not None:
 227             if args.test == "all":
 228                 keys = found_files.keys()
 229             else:
 230                 keys = [list(found_files.keys())[int(args.test)]]
 231             for key in keys:
 232                 testcase = found_files[key]
 233                 if args.errtext:
 234                     if "error" in testcase:
 235                         errmsg = testcase["error"]["#text"]
 236                     elif "failure" in testcase:
 237                         errmsg = testcase["failure"]["#text"]
 238                     else:
 239                         errmsg = "none found"
 240                     s = "{}: {}".format(key, errmsg)
 241                 elif args.time:
 242                     text = testcase["@time"]
 243                     s = "{}: {}".format(text, key)
 244                 elif args.errmsg:
 245                     if "error" in testcase:
 246                         errmsg = testcase["error"]["@message"]
 247                     elif "failure" in testcase:
 248                         errmsg = testcase["failure"]["@message"]
 249                     else:
 250                         errmsg = "none found"
 251                     s = "{}: {}".format(key, errmsg)
 252                 else:
 253                     s = dump_testcase(testcase)
 254                 print(s)
 255         elif filters:
 256             if args.enumerate:
 257                 print(
 258                     "\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
 259                 )
 260             else:
 261                 print("\n".join(found_files))
 262
 263     if args.summary:
 264         print_summary(results, args)
 265
 266
 267 if __name__ == "__main__":
 268     main()