]>
git.proxmox.com Git - mirror_frr.git/blob - tests/topotests/analyze.py
2 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
4 # July 9 2021, Christian Hopps <chopps@labn.net>
6 # Copyright (c) 2021, LabN Consulting, L.L.C.
8 # This program is free software; you can redistribute it and/or
9 # modify it under the terms of the GNU General Public License
10 # as published by the Free Software Foundation; either version 2
11 # of the License, or (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License along
19 # with this program; see the file COPYING; if not, write to the Free Software
20 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
29 from collections
import OrderedDict
34 def get_summary(results
):
35 ntest
= int(results
["@tests"])
36 nfail
= int(results
["@failures"])
37 nerror
= int(results
["@errors"])
38 nskip
= int(results
["@skipped"])
39 npass
= ntest
- nfail
- nskip
- nerror
40 return ntest
, npass
, nfail
, nerror
, nskip
43 def print_summary(results
, args
):
44 ntest
, npass
, nfail
, nerror
, nskip
= (0, 0, 0, 0, 0)
46 _ntest
, _npass
, _nfail
, _nerror
, _nskip
= get_summary(results
[group
])
49 f
"Group: {group} Total: {_ntest} PASSED: {_npass}"
50 " FAIL: {_nfail} ERROR: {_nerror} SKIP: {_nskip}"
57 print(f
"Total: {ntest} PASSED: {npass} FAIL: {nfail} ERROR: {nerror} SKIP: {nskip}")
60 def get_global_testcase(results
):
62 for testcase
in results
[group
]["testcase"]:
63 if "@file" not in testcase
:
68 def get_filtered(tfilters
, results
, args
):
69 if isinstance(tfilters
, str) or tfilters
is None:
71 found_files
= OrderedDict()
73 if isinstance(results
[group
]["testcase"], list):
74 tlist
= results
[group
]["testcase"]
76 tlist
= [results
[group
]["testcase"]]
77 for testcase
in tlist
:
78 for tfilter
in tfilters
:
81 "failure" not in testcase
82 and "error" not in testcase
83 and "skipped" not in testcase
86 elif tfilter
in testcase
:
90 # cname = testcase["@classname"]
91 fname
= testcase
.get("@file", "")
92 cname
= testcase
.get("@classname", "")
93 if not fname
and not cname
:
94 name
= testcase
.get("@name", "")
97 # If we had a failure at the module level we could be here.
98 fname
= name
.replace(".", "/") + ".py"
102 fname
= cname
.replace(".", "/") + ".py"
103 if args
.files_only
or "@name" not in testcase
:
106 tcname
= fname
+ "::" + testcase
["@name"]
107 found_files
[tcname
] = testcase
111 def dump_testcase(testcase
):
112 expand_keys
= ("failure", "error", "skipped")
115 for key
, val
in testcase
.items():
116 if isinstance(val
, str) or isinstance(val
, float) or isinstance(val
, int):
117 s
+= "{}: {}\n".format(key
, val
)
118 elif isinstance(val
, list):
119 for k2
, v2
in enumerate(val
):
120 s
+= "{}: {}\n".format(k2
, v2
)
122 for k2
, v2
in val
.items():
123 s
+= "{}: {}\n".format(k2
, v2
)
128 parser
= argparse
.ArgumentParser()
133 help="Save /tmp/topotests{,.xml} in --rundir if --rundir does not yet exist",
139 help="print test file names rather than individual full testcase names",
145 help="select results combination of letters: 'e'rrored 'f'ailed 'p'assed 's'kipped.",
150 help="xml results file or directory containing xml results file",
152 parser
.add_argument("--rundir", help=argparse
.SUPPRESS
)
157 help="enumerate each item (results scoped)",
159 parser
.add_argument("-T", "--test", help="print testcase at enumeration")
161 "--errmsg", action
="store_true", help="print testcase error message"
164 "--errtext", action
="store_true", help="print testcase error text"
166 parser
.add_argument("--time", action
="store_true", help="print testcase run times")
168 parser
.add_argument("-s", "--summary", action
="store_true", help="print summary")
169 parser
.add_argument("-v", "--verbose", action
="store_true", help="be verbose")
170 args
= parser
.parse_args()
172 if args
.save
and args
.results
and not os
.path
.exists(args
.results
):
173 if not os
.path
.exists("/tmp/topotests"):
174 logging
.critical('No "/tmp/topotests" directory to save')
176 subprocess
.run(["mv", "/tmp/topotests", args
.results
])
177 # # Old location for results
178 # if os.path.exists("/tmp/topotests.xml", args.results):
179 # subprocess.run(["mv", "/tmp/topotests.xml", args.results])
182 args
.test
is None or not args
.files_only
183 ), "Can't have both --files and --test"
188 basedir
= os
.path
.realpath(args
.rundir
)
191 newfiles
= glob
.glob("tt-group-*/topotests.xml")
193 ttfiles
.extend(newfiles
)
194 if os
.path
.exists("topotests.xml"):
195 ttfiles
.append("topotests.xml")
198 if os
.path
.exists(os
.path
.join(args
.results
, "topotests.xml")):
199 args
.results
= os
.path
.join(args
.results
, "topotests.xml")
200 if not os
.path
.exists(args
.results
):
201 logging
.critical("%s doesn't exist", args
.results
)
203 ttfiles
= [args
.results
]
204 elif os
.path
.exists("/tmp/topotests/topotests.xml"):
205 ttfiles
.append("/tmp/topotests/topotests.xml")
208 if os
.path
.exists("/tmp/topotests.xml"):
209 ttfiles
.append("/tmp/topotests.xml")
212 m
= re
.match(r
"tt-group-(\d+)/topotests.xml", f
)
213 group
= int(m
.group(1)) if m
else 0
214 with
open(f
) as xml_file
:
215 results
[group
] = xmltodict
.parse(xml_file
.read())["testsuites"]["testsuite"]
218 if "e" in args
.select
:
219 filters
.append("error")
220 if "f" in args
.select
:
221 filters
.append("failure")
222 if "s" in args
.select
:
223 filters
.append("skipped")
224 if "p" in args
.select
:
227 found_files
= get_filtered(filters
, results
, args
)
229 if args
.test
is not None:
230 if args
.test
== "all":
231 keys
= found_files
.keys()
233 keys
= [list(found_files
.keys())[int(args
.test
)]]
235 testcase
= found_files
[key
]
237 if "error" in testcase
:
238 errmsg
= testcase
["error"]["#text"]
239 elif "failure" in testcase
:
240 errmsg
= testcase
["failure"]["#text"]
242 errmsg
= "none found"
243 s
= "{}: {}".format(key
, errmsg
)
245 text
= testcase
["@time"]
246 s
= "{}: {}".format(text
, key
)
248 if "error" in testcase
:
249 errmsg
= testcase
["error"]["@message"]
250 elif "failure" in testcase
:
251 errmsg
= testcase
["failure"]["@message"]
253 errmsg
= "none found"
254 s
= "{}: {}".format(key
, errmsg
)
256 s
= dump_testcase(testcase
)
261 "\n".join(["{} {}".format(i
, x
) for i
, x
in enumerate(found_files
)])
264 print("\n".join(found_files
))
267 print_summary(results
, args
)
270 if __name__
== "__main__":