[mirror_frr.git] / tests / topotests / analyze.py

#!/usr/bin/env python3
# -*- coding: utf-8 eval: (blacken-mode 1) -*-
#
# July 9 2021, Christian Hopps <chopps@labn.net>
#
# Copyright (c) 2021, LabN Consulting, L.L.C.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; see the file COPYING; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
import argparse
import glob
import logging
import os
import re
import subprocess
import sys
from collections import OrderedDict

import xmltodict


def get_summary(results):
    ntest = int(results["@tests"])
    nfail = int(results["@failures"])
    nerror = int(results["@errors"])
    nskip = int(results["@skipped"])
    npass = ntest - nfail - nskip - nerror
    return ntest, npass, nfail, nerror, nskip


def print_summary(results, args):
    ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
    for group in results:
        _ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
        if args.verbose:
            print(f"Group: {group} Total: {_ntest} PASSED: {_npass}"
                  " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}")
        ntest += _ntest
        npass += _npass
        nfail += _nfail
        nerror += _nerror
        nskip += _nskip
    print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")


def get_global_testcase(results):
    for group in results:
        for testcase in results[group]["testcase"]:
            if "@file" not in testcase:
                return testcase
    return None


def get_filtered(tfilters, results, args):
    if isinstance(tfilters, str) or tfilters is None:
        tfilters = [tfilters]
    found_files = OrderedDict()
    for group in results:
        if isinstance(results[group]["testcase"], list):
            tlist = results[group]["testcase"]
        else:
            tlist = [results[group]["testcase"]]
        for testcase in tlist:
            for tfilter in tfilters:
                if tfilter is None:
                    if (
                        "failure" not in testcase
                        and "error" not in testcase
                        and "skipped" not in testcase
                    ):
                        break
                elif tfilter in testcase:
                    break
            else:
                continue
            #cname = testcase["@classname"]
            fname = testcase.get("@file", "")
            cname = testcase.get("@classname", "")
            if not fname and not cname:
                name = testcase.get("@name", "")
                if not name:
                    continue
                # If we had a failure at the module level we could be here.
                fname = name.replace(".", "/") + ".py"
                tcname = fname
            else:
                if not fname:
                    fname = cname.replace(".", "/") + ".py"
                if args.files_only or "@name" not in testcase:
                    tcname = fname
                else:
                    tcname = fname + "::" + testcase["@name"]
            found_files[tcname] = testcase
    return found_files


def dump_testcase(testcase):
    expand_keys = ("failure", "error", "skipped")

    s = ""
    for key, val in testcase.items():
        if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
            s += "{}: {}\n".format(key, val)
        else:
            for k2, v2 in val.items():
                s += "{}: {}\n".format(k2, v2)
    return s


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-A", "--save", action="store_true", help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist")
    parser.add_argument("-F", "--files-only", action="store_true", help="print test file names rather than individual full testcase names")
    parser.add_argument("-S", "--select", default="fe", help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.")
    parser.add_argument("-r", "--results", help="xml results file or directory containing xml results file")
    parser.add_argument("--rundir", help=argparse.SUPPRESS)
    parser.add_argument(
        "-E",
        "--enumerate",
        action="store_true",
        help="enumerate each item (results scoped)",
    )
    parser.add_argument(
        "-T", "--test", help="print testcase at enumeration"
    )
    parser.add_argument(
        "--errmsg", action="store_true", help="print testcase error message"
    )
    parser.add_argument(
        "--errtext", action="store_true", help="print testcase error text"
    )
    parser.add_argument(
        "--time", action="store_true", help="print testcase run times"
    )

    parser.add_argument("-s", "--summary", action="store_true", help="print summary")
    parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
    args = parser.parse_args()

    if args.save and args.results and not os.path.exists(args.results):
        if not os.path.exists("/tmp/topotests"):
            logging.critical("No \"/tmp/topotests\" directory to save")
            sys.exit(1)
        subprocess.run(["mv", "/tmp/topotests", args.results])
        # # Old location for results
        # if os.path.exists("/tmp/topotests.xml", args.results):
        #     subprocess.run(["mv", "/tmp/topotests.xml", args.results])

    assert args.test is None or not args.files_only, "Can't have both --files and --test"

    results = {}
    ttfiles = []
    if args.rundir:
        basedir = os.path.realpath(args.rundir)
        os.chdir(basedir)

        newfiles = glob.glob("tt-group-*/topotests.xml")
        if newfiles:
            ttfiles.extend(newfiles)
        if os.path.exists("topotests.xml"):
            ttfiles.append("topotests.xml")
    else:
        if args.results:
            if os.path.exists(os.path.join(args.results, "topotests.xml")):
                args.results = os.path.join(args.results, "topotests.xml")
            if not os.path.exists(args.results):
                logging.critical("%s doesn't exist", args.results)
                sys.exit(1)
            ttfiles = [args.results]

        if not ttfiles and os.path.exists("/tmp/topotests.xml"):
            ttfiles.append("/tmp/topotests.xml")

    for f in ttfiles:
        m = re.match(r"tt-group-(\d+)/topotests.xml", f)
        group = int(m.group(1)) if m else 0
        with open(f) as xml_file:
            results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]

    filters = []
    if "e" in args.select:
        filters.append("error")
    if "f" in args.select:
        filters.append("failure")
    if "s" in args.select:
        filters.append("skipped")
    if "p" in args.select:
        filters.append(None)

    found_files = get_filtered(filters, results, args)
    if found_files:
        if args.test is not None:
            if args.test == "all":
                keys = found_files.keys()
            else:
                keys = [list(found_files.keys())[int(args.test)]]
            for key in keys:
                testcase = found_files[key]
                if args.errtext:
                    if "error" in testcase:
                        errmsg = testcase["error"]["#text"]
                    elif "failure" in testcase:
                        errmsg = testcase["failure"]["#text"]
                    else:
                        errmsg = "none found"
                    s = "{}: {}".format(key, errmsg)
                elif args.time:
                    text = testcase["@time"]
                    s = "{}: {}".format(text, key)
                elif args.errmsg:
                    if "error" in testcase:
                        errmsg = testcase["error"]["@message"]
                    elif "failure" in testcase:
                        errmsg = testcase["failure"]["@message"]
                    else:
                        errmsg = "none found"
                    s = "{}: {}".format(key, errmsg)
                else:
                    s = dump_testcase(testcase)
                print(s)
        elif filters:
            if args.enumerate:
                print(
                    "\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
                )
            else:
                print("\n".join(found_files))

    if args.summary:
        print_summary(results, args)


if __name__ == "__main__":
    main()
Commit	Line	Data
49549fe2 CH	1	#!/usr/bin/env python3
	2	# -- coding: utf-8 eval: (blacken-mode 1) --
	3	#
	4	# July 9 2021, Christian Hopps <chopps@labn.net>
	5	#
	6	# Copyright (c) 2021, LabN Consulting, L.L.C.
	7	#
	8	# This program is free software; you can redistribute it and/or
	9	# modify it under the terms of the GNU General Public License
	10	# as published by the Free Software Foundation; either version 2
	11	# of the License, or (at your option) any later version.
	12	#
	13	# This program is distributed in the hope that it will be useful,
	14	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	16	# GNU General Public License for more details.
	17	#
	18	# You should have received a copy of the GNU General Public License along
	19	# with this program; see the file COPYING; if not, write to the Free Software
	20	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	21	#
	22	import argparse
	23	import glob
49549fe2 CH	24	import logging
49549fe2 CH	25	import os
49549fe2 CH	26	import re
	27	import subprocess
	28	import sys
	29	from collections import OrderedDict
	30
	31	import xmltodict
	32
	33
	34	def get_summary(results):
	35	ntest = int(results["@tests"])
	36	nfail = int(results["@failures"])
	37	nerror = int(results["@errors"])
	38	nskip = int(results["@skipped"])
	39	npass = ntest - nfail - nskip - nerror
	40	return ntest, npass, nfail, nerror, nskip
	41
	42
	43	def print_summary(results, args):
	44	ntest, npass, nfail, nerror, nskip = (0, 0, 0, 0, 0)
	45	for group in results:
	46	_ntest, _npass, _nfail, _nerror, _nskip = get_summary(results[group])
	47	if args.verbose:
	48	print(f"Group: {group} Total: {_ntest} PASSED: {_npass}"
	49	" FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}")
	50	ntest += _ntest
	51	npass += _npass
	52	nfail += _nfail
	53	nerror += _nerror
	54	nskip += _nskip
	55	print(f"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
	56
	57
	58	def get_global_testcase(results):
	59	for group in results:
	60	for testcase in results[group]["testcase"]:
	61	if "@file" not in testcase:
	62	return testcase
	63	return None
	64
	65
	66	def get_filtered(tfilters, results, args):
	67	if isinstance(tfilters, str) or tfilters is None:
	68	tfilters = [tfilters]
	69	found_files = OrderedDict()
	70	for group in results:
	71	if isinstance(results[group]["testcase"], list):
	72	tlist = results[group]["testcase"]
	73	else:
	74	tlist = [results[group]["testcase"]]
	75	for testcase in tlist:
	76	for tfilter in tfilters:
	77	if tfilter is None:
	78	if (
	79	"failure" not in testcase
	80	and "error" not in testcase
	81	and "skipped" not in testcase
	82	):
	83	break
	84	elif tfilter in testcase:
	85	break
	86	else:
	87	continue
	88	#cname = testcase["@classname"]
	89	fname = testcase.get("@file", "")
90	cname = testcase.get("@classname", "")
91	if not fname and not cname:
92	name = testcase.get("@name", "")
93	if not name:
94	continue
95	# If we had a failure at the module level we could be here.
96	fname = name.replace(".", "/") + ".py"
97	tcname = fname
98	else:
99	if not fname:
100	fname = cname.replace(".", "/") + ".py"
101	if args.files_only or "@name" not in testcase:
102	tcname = fname
103	else:
104	tcname = fname + "::" + testcase["@name"]
105	found_files[tcname] = testcase
106	return found_files
107
108
109
110	def dump_testcase(testcase):
111	expand_keys = ("failure", "error", "skipped")
112
113	s = ""
114	for key, val in testcase.items():
115	if isinstance(val, str) or isinstance(val, float) or isinstance(val, int):
116	s += "{}: {}\n".format(key, val)
117	else:
118	for k2, v2 in val.items():
119	s += "{}: {}\n".format(k2, v2)
120	return s
121
122
123	def main():
124	parser = argparse.ArgumentParser()
125	parser.add_argument("-A", "--save", action="store_true", help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist")
126	parser.add_argument("-F", "--files-only", action="store_true", help="print test file names rather than individual full testcase names")
127	parser.add_argument("-S", "--select", default="fe", help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.")
128	parser.add_argument("-r", "--results", help="xml results file or directory containing xml results file")
129	parser.add_argument("--rundir", help=argparse.SUPPRESS)
130	parser.add_argument(
131	"-E",
132	"--enumerate",
133	action="store_true",
134	help="enumerate each item (results scoped)",
135	)
136	parser.add_argument(
137	"-T", "--test", help="print testcase at enumeration"
138	)
139	parser.add_argument(
140	"--errmsg", action="store_true", help="print testcase error message"
141	)
142	parser.add_argument(
143	"--errtext", action="store_true", help="print testcase error text"
144	)
145	parser.add_argument(
146	"--time", action="store_true", help="print testcase run times"
147	)
148
149	parser.add_argument("-s", "--summary", action="store_true", help="print summary")
150	parser.add_argument("-v", "--verbose", action="store_true", help="be verbose")
151	args = parser.parse_args()
152
153	if args.save and args.results and not os.path.exists(args.results):
154	if not os.path.exists("/tmp/topotests"):
155	logging.critical("No \"/tmp/topotests\" directory to save")
156	sys.exit(1)
157	subprocess.run(["mv", "/tmp/topotests", args.results])
158	# # Old location for results
159	# if os.path.exists("/tmp/topotests.xml", args.results):
160	# subprocess.run(["mv", "/tmp/topotests.xml", args.results])
161
162	assert args.test is None or not args.files_only, "Can't have both --files and --test"
163
164	results = {}
165	ttfiles = []
166	if args.rundir:
167	basedir = os.path.realpath(args.rundir)
168	os.chdir(basedir)
169
170	newfiles = glob.glob("tt-group-*/topotests.xml")
171	if newfiles:
172	ttfiles.extend(newfiles)
173	if os.path.exists("topotests.xml"):
174	ttfiles.append("topotests.xml")
175	else:
176	if args.results:
177	if os.path.exists(os.path.join(args.results, "topotests.xml")):
178	args.results = os.path.join(args.results, "topotests.xml")
179	if not os.path.exists(args.results):
180	logging.critical("%s doesn't exist", args.results)
181	sys.exit(1)
182	ttfiles = [args.results]
183
184	if not ttfiles and os.path.exists("/tmp/topotests.xml"):
185	ttfiles.append("/tmp/topotests.xml")
186
187	for f in ttfiles:
188	m = re.match(r"tt-group-(\d+)/topotests.xml", f)
189	group = int(m.group(1)) if m else 0
190	with open(f) as xml_file:
191	results[group] = xmltodict.parse(xml_file.read())["testsuites"]["testsuite"]
192
193	filters = []
194	if "e" in args.select:
195	filters.append("error")
196	if "f" in args.select:
197	filters.append("failure")
198	if "s" in args.select:
199	filters.append("skipped")
200	if "p" in args.select:
201	filters.append(None)
202
203	found_files = get_filtered(filters, results, args)
204	if found_files:
205	if args.test is not None:
206	if args.test == "all":
207	keys = found_files.keys()
208	else:
209	keys = [list(found_files.keys())[int(args.test)]]
210	for key in keys:
211	testcase = found_files[key]
212	if args.errtext:
213	if "error" in testcase:
214	errmsg = testcase["error"]["#text"]
215	elif "failure" in testcase:
216	errmsg = testcase["failure"]["#text"]
217	else:
218	errmsg = "none found"
219	s = "{}: {}".format(key, errmsg)
220	elif args.time:
221	text = testcase["@time"]
222	s = "{}: {}".format(text, key)
223	elif args.errmsg:
224	if "error" in testcase:
225	errmsg = testcase["error"]["@message"]
226	elif "failure" in testcase:
227	errmsg = testcase["failure"]["@message"]
228	else:
229	errmsg = "none found"
230	s = "{}: {}".format(key, errmsg)
231	else:
232	s = dump_testcase(testcase)
233	print(s)
234	elif filters:
235	if args.enumerate:
236	print(
237	"\n".join(["{} {}".format(i, x) for i, x in enumerate(found_files)])
238	)
239	else:
240	print("\n".join(found_files))
241
242	if args.summary:
243	print_summary(results, args)
244
245
246	if __name__ == "__main__":
247	main()