]>
Commit | Line | Data |
---|---|---|
acddc0ed | 1 | # SPDX-License-Identifier: GPL-2.0-or-later |
36a8fdfd DL |
2 | # FRR ELF xref extractor |
3 | # | |
4 | # Copyright (C) 2020 David Lamparter for NetDEF, Inc. | |
36a8fdfd DL |
5 | |
6 | import sys | |
7 | import os | |
8 | import struct | |
9 | import re | |
10 | import traceback | |
d40aee77 DL |
11 | |
12 | json_dump_args = {} | |
13 | ||
14 | try: | |
15 | import ujson as json | |
16 | ||
17 | json_dump_args["escape_forward_slashes"] = False | |
18 | except ImportError: | |
19 | import json | |
20 | ||
36a8fdfd DL |
21 | import argparse |
22 | ||
23 | from clippy.uidhash import uidhash | |
24 | from clippy.elf import * | |
9eebf97e | 25 | from clippy import frr_top_src, CmdAttr |
36a8fdfd | 26 | from tiabwarfo import FieldApplicator |
89cb86ae | 27 | from xref2vtysh import CommandEntry |
36a8fdfd DL |
28 | |
29 | try: | |
00f0c399 | 30 | with open(os.path.join(frr_top_src, "python", "xrefstructs.json"), "r") as fd: |
36a8fdfd DL |
31 | xrefstructs = json.load(fd) |
32 | except FileNotFoundError: | |
00f0c399 DL |
33 | sys.stderr.write( |
34 | """ | |
36a8fdfd DL |
35 | The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole |
36 | tool available) could not be found. It should be included with the sources. | |
00f0c399 DL |
37 | """ |
38 | ) | |
36a8fdfd DL |
39 | sys.exit(1) |
40 | ||
41 | # constants, need to be kept in sync manually... | |
42 | ||
5163a1c5 | 43 | XREFT_EVENTSCHED = 0x100 |
36a8fdfd DL |
44 | XREFT_LOGMSG = 0x200 |
45 | XREFT_DEFUN = 0x300 | |
46 | XREFT_INSTALL_ELEMENT = 0x301 | |
47 | ||
48 | # LOG_* | |
49 | priovals = {} | |
00f0c399 | 50 | prios = ["0", "1", "2", "E", "W", "N", "I", "D"] |
36a8fdfd DL |
51 | |
52 | ||
53 | class XrelfoJson(object): | |
54 | def dump(self): | |
55 | pass | |
56 | ||
57 | def check(self, wopt): | |
58 | yield from [] | |
59 | ||
60 | def to_dict(self, refs): | |
61 | pass | |
62 | ||
00f0c399 | 63 | |
36a8fdfd | 64 | class Xref(ELFDissectStruct, XrelfoJson): |
00f0c399 DL |
65 | struct = "xref" |
66 | fieldrename = {"type": "typ"} | |
36a8fdfd DL |
67 | containers = {} |
68 | ||
69 | def __init__(self, *args, **kwargs): | |
70 | super().__init__(*args, **kwargs) | |
71 | ||
72 | self._container = None | |
73 | if self.xrefdata: | |
74 | self.xrefdata.ref_from(self, self.typ) | |
75 | ||
76 | def container(self): | |
77 | if self._container is None: | |
78 | if self.typ in self.containers: | |
00f0c399 | 79 | self._container = self.container_of(self.containers[self.typ], "xref") |
36a8fdfd DL |
80 | return self._container |
81 | ||
82 | def check(self, *args, **kwargs): | |
83 | if self._container: | |
84 | yield from self._container.check(*args, **kwargs) | |
85 | ||
86 | ||
87 | class Xrefdata(ELFDissectStruct): | |
00f0c399 | 88 | struct = "xrefdata" |
36a8fdfd DL |
89 | |
90 | # uid is all zeroes in the data loaded from ELF | |
00f0c399 | 91 | fieldrename = {"uid": "_uid"} |
36a8fdfd DL |
92 | |
93 | def ref_from(self, xref, typ): | |
94 | self.xref = xref | |
95 | ||
96 | @property | |
97 | def uid(self): | |
98 | if self.hashstr is None: | |
99 | return None | |
100 | return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1) | |
101 | ||
00f0c399 | 102 | |
36a8fdfd DL |
103 | class XrefPtr(ELFDissectStruct): |
104 | fields = [ | |
00f0c399 | 105 | ("xref", "P", Xref), |
36a8fdfd DL |
106 | ] |
107 | ||
00f0c399 | 108 | |
36a8fdfd | 109 | class XrefThreadSched(ELFDissectStruct, XrelfoJson): |
00f0c399 DL |
110 | struct = "xref_threadsched" |
111 | ||
112 | ||
5163a1c5 | 113 | Xref.containers[XREFT_EVENTSCHED] = XrefThreadSched |
36a8fdfd | 114 | |
00f0c399 | 115 | |
36a8fdfd | 116 | class XrefLogmsg(ELFDissectStruct, XrelfoJson): |
00f0c399 | 117 | struct = "xref_logmsg" |
36a8fdfd DL |
118 | |
119 | def _warn_fmt(self, text): | |
00f0c399 DL |
120 | lines = text.split("\n") |
121 | yield ( | |
122 | (self.xref.file, self.xref.line), | |
123 | "%s:%d: %s (in %s())%s\n" | |
124 | % ( | |
125 | self.xref.file, | |
126 | self.xref.line, | |
127 | lines[0], | |
128 | self.xref.func, | |
129 | "".join(["\n" + l for l in lines[1:]]), | |
130 | ), | |
131 | ) | |
36a8fdfd | 132 | |
2621bb8b | 133 | fmt_regexes = [ |
00f0c399 DL |
134 | (re.compile(r"([\n\t]+)"), "error: log message contains tab or newline"), |
135 | # (re.compile(r'^(\s+)'), 'warning: log message starts with whitespace'), | |
136 | ( | |
137 | re.compile(r"^((?:warn(?:ing)?|error):\s*)", re.I), | |
138 | "warning: log message starts with severity", | |
139 | ), | |
36a8fdfd | 140 | ] |
2621bb8b | 141 | arg_regexes = [ |
00f0c399 DL |
142 | # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)" |
143 | ( | |
144 | re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)"), | |
145 | "cleanup: replace inet_ntop(AF_INET, ...) with %pI4", | |
146 | lambda s: True, | |
147 | ), | |
148 | ( | |
149 | re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)"), | |
150 | "cleanup: replace inet_ntop(AF_INET6, ...) with %pI6", | |
151 | lambda s: True, | |
152 | ), | |
153 | ( | |
6c7bc494 DL |
154 | # string split-up here is to not trigger "inet_ntoa forbidden" |
155 | re.compile(r"((?<![\?:] )inet_" + r"ntoa)"), | |
156 | "cleanup: replace inet_" + "ntoa(...) with %pI4", | |
00f0c399 DL |
157 | lambda s: True, |
158 | ), | |
159 | ( | |
160 | re.compile(r"((?<![\?:] )ipaddr2str)"), | |
161 | "cleanup: replace ipaddr2str(...) with %pIA", | |
162 | lambda s: True, | |
163 | ), | |
164 | ( | |
165 | re.compile(r"((?<![\?:] )prefix2str)"), | |
166 | "cleanup: replace prefix2str(...) with %pFX", | |
167 | lambda s: True, | |
168 | ), | |
169 | ( | |
170 | re.compile(r"((?<![\?:] )prefix_mac2str)"), | |
171 | "cleanup: replace prefix_mac2str(...) with %pEA", | |
172 | lambda s: True, | |
173 | ), | |
174 | ( | |
175 | re.compile(r"((?<![\?:] )sockunion2str)"), | |
176 | "cleanup: replace sockunion2str(...) with %pSU", | |
177 | lambda s: True, | |
178 | ), | |
179 | # (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ), | |
2621bb8b | 180 | ] |
36a8fdfd DL |
181 | |
182 | def check(self, wopt): | |
2621bb8b DL |
183 | def fmt_msg(rex, itext): |
184 | if sys.stderr.isatty(): | |
185 | items = rex.split(itext) | |
186 | out = [] | |
187 | for i, text in enumerate(items): | |
188 | if (i % 2) == 1: | |
00f0c399 | 189 | out.append("\033[41;37;1m%s\033[m" % repr(text)[1:-1]) |
2621bb8b DL |
190 | else: |
191 | out.append(repr(text)[1:-1]) | |
192 | ||
00f0c399 | 193 | excerpt = "".join(out) |
2621bb8b DL |
194 | else: |
195 | excerpt = repr(itext)[1:-1] | |
196 | return excerpt | |
197 | ||
36a8fdfd | 198 | if wopt.Wlog_format: |
2621bb8b | 199 | for rex, msg in self.fmt_regexes: |
36a8fdfd DL |
200 | if not rex.search(self.fmtstring): |
201 | continue | |
202 | ||
2621bb8b DL |
203 | excerpt = fmt_msg(rex, self.fmtstring) |
204 | yield from self._warn_fmt('%s: "%s"' % (msg, excerpt)) | |
36a8fdfd | 205 | |
2621bb8b DL |
206 | if wopt.Wlog_args: |
207 | for rex, msg, cond in self.arg_regexes: | |
208 | if not cond(self): | |
209 | continue | |
210 | if not rex.search(self.args): | |
211 | continue | |
36a8fdfd | 212 | |
2621bb8b | 213 | excerpt = fmt_msg(rex, self.args) |
00f0c399 DL |
214 | yield from self._warn_fmt( |
215 | '%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt) | |
216 | ) | |
36a8fdfd DL |
217 | |
218 | def dump(self): | |
00f0c399 DL |
219 | print( |
220 | "%-60s %s%s %-25s [EC %d] %s" | |
221 | % ( | |
222 | "%s:%d %s()" % (self.xref.file, self.xref.line, self.xref.func), | |
223 | prios[self.priority & 7], | |
224 | priovals.get(self.priority & 0x30, " "), | |
225 | self.xref.xrefdata.uid, | |
226 | self.ec, | |
227 | self.fmtstring, | |
228 | ) | |
229 | ) | |
36a8fdfd DL |
230 | |
231 | def to_dict(self, xrelfo): | |
00f0c399 | 232 | jsobj = dict([(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]) |
36a8fdfd | 233 | if self.ec != 0: |
00f0c399 DL |
234 | jsobj["ec"] = self.ec |
235 | jsobj["fmtstring"] = self.fmtstring | |
236 | jsobj["args"] = self.args | |
237 | jsobj["priority"] = self.priority & 7 | |
238 | jsobj["type"] = "logmsg" | |
239 | jsobj["binary"] = self._elfsect._elfwrap.orig_filename | |
36a8fdfd DL |
240 | |
241 | if self.priority & 0x10: | |
00f0c399 | 242 | jsobj.setdefault("flags", []).append("errno") |
36a8fdfd | 243 | if self.priority & 0x20: |
00f0c399 DL |
244 | jsobj.setdefault("flags", []).append("getaddrinfo") |
245 | ||
246 | xrelfo["refs"].setdefault(self.xref.xrefdata.uid, []).append(jsobj) | |
36a8fdfd | 247 | |
36a8fdfd DL |
248 | |
249 | Xref.containers[XREFT_LOGMSG] = XrefLogmsg | |
250 | ||
00f0c399 | 251 | |
36a8fdfd | 252 | class CmdElement(ELFDissectStruct, XrelfoJson): |
00f0c399 | 253 | struct = "cmd_element" |
36a8fdfd | 254 | |
36a8fdfd DL |
255 | def __init__(self, *args, **kwargs): |
256 | super().__init__(*args, **kwargs) | |
257 | ||
258 | def to_dict(self, xrelfo): | |
00f0c399 DL |
259 | jsobj = ( |
260 | xrelfo["cli"] | |
261 | .setdefault(self.name, {}) | |
262 | .setdefault(self._elfsect._elfwrap.orig_filename, {}) | |
263 | ) | |
264 | ||
265 | jsobj.update( | |
266 | { | |
267 | "string": self.string, | |
268 | "doc": self.doc, | |
269 | } | |
270 | ) | |
9eebf97e | 271 | if self.attr: |
00f0c399 | 272 | jsobj["attr"] = attr = self.attr |
9eebf97e DL |
273 | for attrname in CmdAttr.__members__: |
274 | val = CmdAttr[attrname] | |
275 | if attr & val: | |
00f0c399 | 276 | jsobj.setdefault("attrs", []).append(attrname.lower()) |
9eebf97e | 277 | attr &= ~val |
36a8fdfd | 278 | |
00f0c399 DL |
279 | jsobj["defun"] = dict( |
280 | [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]] | |
281 | ) | |
282 | ||
36a8fdfd DL |
283 | |
284 | Xref.containers[XREFT_DEFUN] = CmdElement | |
285 | ||
00f0c399 | 286 | |
36a8fdfd | 287 | class XrefInstallElement(ELFDissectStruct, XrelfoJson): |
00f0c399 | 288 | struct = "xref_install_element" |
36a8fdfd DL |
289 | |
290 | def to_dict(self, xrelfo): | |
00f0c399 DL |
291 | jsobj = ( |
292 | xrelfo["cli"] | |
293 | .setdefault(self.cmd_element.name, {}) | |
294 | .setdefault(self._elfsect._elfwrap.orig_filename, {}) | |
295 | ) | |
296 | nodes = jsobj.setdefault("nodes", []) | |
297 | ||
298 | nodes.append( | |
299 | { | |
300 | "node": self.node_type, | |
301 | "install": dict( | |
302 | [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]] | |
303 | ), | |
304 | } | |
305 | ) | |
36a8fdfd | 306 | |
36a8fdfd DL |
307 | |
308 | Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement | |
309 | ||
310 | # shove in field defs | |
311 | fieldapply = FieldApplicator(xrefstructs) | |
312 | fieldapply.add(Xref) | |
313 | fieldapply.add(Xrefdata) | |
314 | fieldapply.add(XrefLogmsg) | |
315 | fieldapply.add(XrefThreadSched) | |
316 | fieldapply.add(CmdElement) | |
317 | fieldapply.add(XrefInstallElement) | |
318 | fieldapply() | |
319 | ||
320 | ||
321 | class Xrelfo(dict): | |
322 | def __init__(self): | |
00f0c399 DL |
323 | super().__init__( |
324 | { | |
325 | "refs": {}, | |
326 | "cli": {}, | |
327 | } | |
328 | ) | |
36a8fdfd DL |
329 | self._xrefs = [] |
330 | ||
331 | def load_file(self, filename): | |
332 | orig_filename = filename | |
00f0c399 DL |
333 | if filename.endswith(".la") or filename.endswith(".lo"): |
334 | with open(filename, "r") as fd: | |
36a8fdfd DL |
335 | for line in fd: |
336 | line = line.strip() | |
00f0c399 | 337 | if line.startswith("#") or line == "" or "=" not in line: |
36a8fdfd DL |
338 | continue |
339 | ||
00f0c399 DL |
340 | var, val = line.split("=", 1) |
341 | if var not in ["library_names", "pic_object"]: | |
36a8fdfd DL |
342 | continue |
343 | if val.startswith("'") or val.startswith('"'): | |
344 | val = val[1:-1] | |
345 | ||
00f0c399 | 346 | if var == "pic_object": |
36a8fdfd DL |
347 | filename = os.path.join(os.path.dirname(filename), val) |
348 | break | |
349 | ||
350 | val = val.strip().split()[0] | |
00f0c399 | 351 | filename = os.path.join(os.path.dirname(filename), ".libs", val) |
36a8fdfd DL |
352 | break |
353 | else: | |
00f0c399 DL |
354 | raise ValueError( |
355 | 'could not process libtool file "%s"' % orig_filename | |
356 | ) | |
36a8fdfd DL |
357 | |
358 | while True: | |
00f0c399 | 359 | with open(filename, "rb") as fd: |
36a8fdfd DL |
360 | hdr = fd.read(4) |
361 | ||
00f0c399 | 362 | if hdr == b"\x7fELF": |
36a8fdfd DL |
363 | self.load_elf(filename, orig_filename) |
364 | return | |
365 | ||
00f0c399 | 366 | if hdr[:2] == b"#!": |
36a8fdfd | 367 | path, name = os.path.split(filename) |
00f0c399 | 368 | filename = os.path.join(path, ".libs", name) |
36a8fdfd DL |
369 | continue |
370 | ||
00f0c399 DL |
371 | if hdr[:1] == b"{": |
372 | with open(filename, "r") as fd: | |
36a8fdfd DL |
373 | self.load_json(fd) |
374 | return | |
375 | ||
00f0c399 | 376 | raise ValueError("cannot determine file type for %s" % (filename)) |
36a8fdfd DL |
377 | |
378 | def load_elf(self, filename, orig_filename): | |
379 | edf = ELFDissectFile(filename) | |
380 | edf.orig_filename = orig_filename | |
381 | ||
00f0c399 | 382 | note = edf._elffile.find_note("FRRouting", "XREF") |
36a8fdfd | 383 | if note is not None: |
00f0c399 | 384 | endian = ">" if edf._elffile.bigendian else "<" |
36a8fdfd DL |
385 | mem = edf._elffile[note] |
386 | if edf._elffile.elfclass == 64: | |
00f0c399 | 387 | start, end = struct.unpack(endian + "QQ", mem) |
36a8fdfd DL |
388 | start += note.start |
389 | end += note.start + 8 | |
390 | else: | |
00f0c399 | 391 | start, end = struct.unpack(endian + "II", mem) |
36a8fdfd DL |
392 | start += note.start |
393 | end += note.start + 4 | |
394 | ||
395 | ptrs = edf.iter_data(XrefPtr, slice(start, end)) | |
396 | ||
397 | else: | |
00f0c399 | 398 | xrefarray = edf.get_section("xref_array") |
36a8fdfd | 399 | if xrefarray is None: |
00f0c399 | 400 | raise ValueError("file has neither xref note nor xref_array section") |
36a8fdfd DL |
401 | |
402 | ptrs = xrefarray.iter_data(XrefPtr) | |
403 | ||
404 | for ptr in ptrs: | |
405 | if ptr.xref is None: | |
00f0c399 | 406 | print("NULL xref") |
36a8fdfd DL |
407 | continue |
408 | self._xrefs.append(ptr.xref) | |
409 | ||
410 | container = ptr.xref.container() | |
411 | if container is None: | |
412 | continue | |
413 | container.to_dict(self) | |
414 | ||
415 | return edf | |
416 | ||
417 | def load_json(self, fd): | |
418 | data = json.load(fd) | |
00f0c399 DL |
419 | for uid, items in data["refs"].items(): |
420 | myitems = self["refs"].setdefault(uid, []) | |
36a8fdfd DL |
421 | for item in items: |
422 | if item in myitems: | |
423 | continue | |
424 | myitems.append(item) | |
425 | ||
00f0c399 DL |
426 | for cmd, items in data["cli"].items(): |
427 | self["cli"].setdefault(cmd, {}).update(items) | |
36a8fdfd DL |
428 | |
429 | return data | |
430 | ||
431 | def check(self, checks): | |
432 | for xref in self._xrefs: | |
433 | yield from xref.check(checks) | |
434 | ||
00f0c399 | 435 | |
36a8fdfd | 436 | def main(): |
00f0c399 DL |
437 | argp = argparse.ArgumentParser(description="FRR xref ELF extractor") |
438 | argp.add_argument("-o", dest="output", type=str, help="write JSON output") | |
439 | argp.add_argument("--out-by-file", type=str, help="write by-file JSON output") | |
440 | argp.add_argument("-c", dest="vtysh_cmds", type=str, help="write vtysh_cmd.c") | |
441 | argp.add_argument("-Wlog-format", action="store_const", const=True) | |
442 | argp.add_argument("-Wlog-args", action="store_const", const=True) | |
443 | argp.add_argument("-Werror", action="store_const", const=True) | |
444 | argp.add_argument("--profile", action="store_const", const=True) | |
445 | argp.add_argument( | |
446 | "binaries", | |
447 | metavar="BINARY", | |
448 | nargs="+", | |
449 | type=str, | |
450 | help="files to read (ELF files or libtool objects)", | |
451 | ) | |
36a8fdfd DL |
452 | args = argp.parse_args() |
453 | ||
454 | if args.profile: | |
455 | import cProfile | |
00f0c399 DL |
456 | |
457 | cProfile.runctx("_main(args)", globals(), {"args": args}, sort="cumtime") | |
36a8fdfd DL |
458 | else: |
459 | _main(args) | |
460 | ||
00f0c399 | 461 | |
36a8fdfd DL |
462 | def _main(args): |
463 | errors = 0 | |
464 | xrelfo = Xrelfo() | |
465 | ||
466 | for fn in args.binaries: | |
467 | try: | |
468 | xrelfo.load_file(fn) | |
469 | except: | |
470 | errors += 1 | |
00f0c399 | 471 | sys.stderr.write("while processing %s:\n" % (fn)) |
36a8fdfd DL |
472 | traceback.print_exc() |
473 | ||
474 | for option in dir(args): | |
00f0c399 | 475 | if option.startswith("W") and option != "Werror": |
36a8fdfd | 476 | checks = sorted(xrelfo.check(args)) |
00f0c399 | 477 | sys.stderr.write("".join([c[-1] for c in checks])) |
13f9aea3 DL |
478 | |
479 | if args.Werror and len(checks) > 0: | |
480 | errors += 1 | |
36a8fdfd DL |
481 | break |
482 | ||
00f0c399 | 483 | refs = xrelfo["refs"] |
36a8fdfd DL |
484 | |
485 | counts = {} | |
486 | for k, v in refs.items(): | |
00f0c399 | 487 | strs = set([i["fmtstring"] for i in v]) |
36a8fdfd | 488 | if len(strs) != 1: |
00f0c399 | 489 | print("\033[31;1m%s\033[m" % k) |
36a8fdfd DL |
490 | counts[k] = len(v) |
491 | ||
492 | out = xrelfo | |
493 | outbyfile = {} | |
494 | for uid, locs in refs.items(): | |
495 | for loc in locs: | |
00f0c399 | 496 | filearray = outbyfile.setdefault(loc["file"], []) |
36a8fdfd | 497 | loc = dict(loc) |
00f0c399 | 498 | del loc["file"] |
36a8fdfd DL |
499 | filearray.append(loc) |
500 | ||
501 | for k in outbyfile.keys(): | |
00f0c399 | 502 | outbyfile[k] = sorted(outbyfile[k], key=lambda x: x["line"]) |
36a8fdfd DL |
503 | |
504 | if errors: | |
505 | sys.exit(1) | |
506 | ||
507 | if args.output: | |
00f0c399 | 508 | with open(args.output + ".tmp", "w") as fd: |
d40aee77 | 509 | json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args) |
00f0c399 | 510 | os.rename(args.output + ".tmp", args.output) |
36a8fdfd DL |
511 | |
512 | if args.out_by_file: | |
00f0c399 | 513 | with open(args.out_by_file + ".tmp", "w") as fd: |
d40aee77 | 514 | json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args) |
00f0c399 | 515 | os.rename(args.out_by_file + ".tmp", args.out_by_file) |
36a8fdfd | 516 | |
89cb86ae | 517 | if args.vtysh_cmds: |
00f0c399 | 518 | with open(args.vtysh_cmds + ".tmp", "w") as fd: |
89cb86ae | 519 | CommandEntry.run(out, fd) |
00f0c399 | 520 | os.rename(args.vtysh_cmds + ".tmp", args.vtysh_cmds) |
89cb86ae DL |
521 | if args.Werror and CommandEntry.warn_counter: |
522 | sys.exit(1) | |
523 | ||
524 | ||
00f0c399 | 525 | if __name__ == "__main__": |
36a8fdfd | 526 | main() |