python/xrelfo.py

   1 # SPDX-License-Identifier: GPL-2.0-or-later
   2 # FRR ELF xref extractor
   3 #
   4 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   5
   6 import sys
   7 import os
   8 import struct
   9 import re
  10 import traceback
  11
  12 json_dump_args = {}
  13
  14 try:
  15     import ujson as json
  16
  17     json_dump_args["escape_forward_slashes"] = False
  18 except ImportError:
  19     import json
  20
  21 import argparse
  22
  23 from clippy.uidhash import uidhash
  24 from clippy.elf import *
  25 from clippy import frr_top_src, CmdAttr
  26 from tiabwarfo import FieldApplicator
  27 from xref2vtysh import CommandEntry
  28
  29 try:
  30     with open(os.path.join(frr_top_src, "python", "xrefstructs.json"), "r") as fd:
  31         xrefstructs = json.load(fd)
  32 except FileNotFoundError:
  33     sys.stderr.write(
  34         """
  35 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  36 tool available) could not be found.  It should be included with the sources.
  37 """
  38     )
  39     sys.exit(1)
  40
  41 # constants, need to be kept in sync manually...
  42
  43 XREFT_THREADSCHED = 0x100
  44 XREFT_LOGMSG = 0x200
  45 XREFT_DEFUN = 0x300
  46 XREFT_INSTALL_ELEMENT = 0x301
  47
  48 # LOG_*
  49 priovals = {}
  50 prios = ["0", "1", "2", "E", "W", "N", "I", "D"]
  51
  52
  53 class XrelfoJson(object):
  54     def dump(self):
  55         pass
  56
  57     def check(self, wopt):
  58         yield from []
  59
  60     def to_dict(self, refs):
  61         pass
  62
  63
  64 class Xref(ELFDissectStruct, XrelfoJson):
  65     struct = "xref"
  66     fieldrename = {"type": "typ"}
  67     containers = {}
  68
  69     def __init__(self, *args, **kwargs):
  70         super().__init__(*args, **kwargs)
  71
  72         self._container = None
  73         if self.xrefdata:
  74             self.xrefdata.ref_from(self, self.typ)
  75
  76     def container(self):
  77         if self._container is None:
  78             if self.typ in self.containers:
  79                 self._container = self.container_of(self.containers[self.typ], "xref")
  80         return self._container
  81
  82     def check(self, *args, **kwargs):
  83         if self._container:
  84             yield from self._container.check(*args, **kwargs)
  85
  86
  87 class Xrefdata(ELFDissectStruct):
  88     struct = "xrefdata"
  89
  90     # uid is all zeroes in the data loaded from ELF
  91     fieldrename = {"uid": "_uid"}
  92
  93     def ref_from(self, xref, typ):
  94         self.xref = xref
  95
  96     @property
  97     def uid(self):
  98         if self.hashstr is None:
  99             return None
 100         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 101
 102
 103 class XrefPtr(ELFDissectStruct):
 104     fields = [
 105         ("xref", "P", Xref),
 106     ]
 107
 108
 109 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 110     struct = "xref_threadsched"
 111
 112
 113 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 114
 115
 116 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 117     struct = "xref_logmsg"
 118
 119     def _warn_fmt(self, text):
 120         lines = text.split("\n")
 121         yield (
 122             (self.xref.file, self.xref.line),
 123             "%s:%d: %s (in %s())%s\n"
 124             % (
 125                 self.xref.file,
 126                 self.xref.line,
 127                 lines[0],
 128                 self.xref.func,
 129                 "".join(["\n" + l for l in lines[1:]]),
 130             ),
 131         )
 132
 133     fmt_regexes = [
 134         (re.compile(r"([\n\t]+)"), "error: log message contains tab or newline"),
 135         #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 136         (
 137             re.compile(r"^((?:warn(?:ing)?|error):\s*)", re.I),
 138             "warning: log message starts with severity",
 139         ),
 140     ]
 141     arg_regexes = [
 142         # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 143         (
 144             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)"),
 145             "cleanup: replace inet_ntop(AF_INET, ...) with %pI4",
 146             lambda s: True,
 147         ),
 148         (
 149             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)"),
 150             "cleanup: replace inet_ntop(AF_INET6, ...) with %pI6",
 151             lambda s: True,
 152         ),
 153         (
 154             # string split-up here is to not trigger "inet_ntoa forbidden"
 155             re.compile(r"((?<![\?:] )inet_" + r"ntoa)"),
 156             "cleanup: replace inet_" + "ntoa(...) with %pI4",
 157             lambda s: True,
 158         ),
 159         (
 160             re.compile(r"((?<![\?:] )ipaddr2str)"),
 161             "cleanup: replace ipaddr2str(...) with %pIA",
 162             lambda s: True,
 163         ),
 164         (
 165             re.compile(r"((?<![\?:] )prefix2str)"),
 166             "cleanup: replace prefix2str(...) with %pFX",
 167             lambda s: True,
 168         ),
 169         (
 170             re.compile(r"((?<![\?:] )prefix_mac2str)"),
 171             "cleanup: replace prefix_mac2str(...) with %pEA",
 172             lambda s: True,
 173         ),
 174         (
 175             re.compile(r"((?<![\?:] )sockunion2str)"),
 176             "cleanup: replace sockunion2str(...) with %pSU",
 177             lambda s: True,
 178         ),
 179         #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 180     ]
 181
 182     def check(self, wopt):
 183         def fmt_msg(rex, itext):
 184             if sys.stderr.isatty():
 185                 items = rex.split(itext)
 186                 out = []
 187                 for i, text in enumerate(items):
 188                     if (i % 2) == 1:
 189                         out.append("\033[41;37;1m%s\033[m" % repr(text)[1:-1])
 190                     else:
 191                         out.append(repr(text)[1:-1])
 192
 193                 excerpt = "".join(out)
 194             else:
 195                 excerpt = repr(itext)[1:-1]
 196             return excerpt
 197
 198         if wopt.Wlog_format:
 199             for rex, msg in self.fmt_regexes:
 200                 if not rex.search(self.fmtstring):
 201                     continue
 202
 203                 excerpt = fmt_msg(rex, self.fmtstring)
 204                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 205
 206         if wopt.Wlog_args:
 207             for rex, msg, cond in self.arg_regexes:
 208                 if not cond(self):
 209                     continue
 210                 if not rex.search(self.args):
 211                     continue
 212
 213                 excerpt = fmt_msg(rex, self.args)
 214                 yield from self._warn_fmt(
 215                     '%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt)
 216                 )
 217
 218     def dump(self):
 219         print(
 220             "%-60s %s%s %-25s [EC %d] %s"
 221             % (
 222                 "%s:%d %s()" % (self.xref.file, self.xref.line, self.xref.func),
 223                 prios[self.priority & 7],
 224                 priovals.get(self.priority & 0x30, " "),
 225                 self.xref.xrefdata.uid,
 226                 self.ec,
 227                 self.fmtstring,
 228             )
 229         )
 230
 231     def to_dict(self, xrelfo):
 232         jsobj = dict([(i, getattr(self.xref, i)) for i in ["file", "line", "func"]])
 233         if self.ec != 0:
 234             jsobj["ec"] = self.ec
 235         jsobj["fmtstring"] = self.fmtstring
 236         jsobj["args"] = self.args
 237         jsobj["priority"] = self.priority & 7
 238         jsobj["type"] = "logmsg"
 239         jsobj["binary"] = self._elfsect._elfwrap.orig_filename
 240
 241         if self.priority & 0x10:
 242             jsobj.setdefault("flags", []).append("errno")
 243         if self.priority & 0x20:
 244             jsobj.setdefault("flags", []).append("getaddrinfo")
 245
 246         xrelfo["refs"].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 247
 248
 249 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 250
 251
 252 class CmdElement(ELFDissectStruct, XrelfoJson):
 253     struct = "cmd_element"
 254
 255     def __init__(self, *args, **kwargs):
 256         super().__init__(*args, **kwargs)
 257
 258     def to_dict(self, xrelfo):
 259         jsobj = (
 260             xrelfo["cli"]
 261             .setdefault(self.name, {})
 262             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 263         )
 264
 265         jsobj.update(
 266             {
 267                 "string": self.string,
 268                 "doc": self.doc,
 269             }
 270         )
 271         if self.attr:
 272             jsobj["attr"] = attr = self.attr
 273             for attrname in CmdAttr.__members__:
 274                 val = CmdAttr[attrname]
 275                 if attr & val:
 276                     jsobj.setdefault("attrs", []).append(attrname.lower())
 277                     attr &= ~val
 278
 279         jsobj["defun"] = dict(
 280             [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 281         )
 282
 283
 284 Xref.containers[XREFT_DEFUN] = CmdElement
 285
 286
 287 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 288     struct = "xref_install_element"
 289
 290     def to_dict(self, xrelfo):
 291         jsobj = (
 292             xrelfo["cli"]
 293             .setdefault(self.cmd_element.name, {})
 294             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 295         )
 296         nodes = jsobj.setdefault("nodes", [])
 297
 298         nodes.append(
 299             {
 300                 "node": self.node_type,
 301                 "install": dict(
 302                     [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 303                 ),
 304             }
 305         )
 306
 307
 308 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 309
 310 # shove in field defs
 311 fieldapply = FieldApplicator(xrefstructs)
 312 fieldapply.add(Xref)
 313 fieldapply.add(Xrefdata)
 314 fieldapply.add(XrefLogmsg)
 315 fieldapply.add(XrefThreadSched)
 316 fieldapply.add(CmdElement)
 317 fieldapply.add(XrefInstallElement)
 318 fieldapply()
 319
 320
 321 class Xrelfo(dict):
 322     def __init__(self):
 323         super().__init__(
 324             {
 325                 "refs": {},
 326                 "cli": {},
 327             }
 328         )
 329         self._xrefs = []
 330
 331     def load_file(self, filename):
 332         orig_filename = filename
 333         if filename.endswith(".la") or filename.endswith(".lo"):
 334             with open(filename, "r") as fd:
 335                 for line in fd:
 336                     line = line.strip()
 337                     if line.startswith("#") or line == "" or "=" not in line:
 338                         continue
 339
 340                     var, val = line.split("=", 1)
 341                     if var not in ["library_names", "pic_object"]:
 342                         continue
 343                     if val.startswith("'") or val.startswith('"'):
 344                         val = val[1:-1]
 345
 346                     if var == "pic_object":
 347                         filename = os.path.join(os.path.dirname(filename), val)
 348                         break
 349
 350                     val = val.strip().split()[0]
 351                     filename = os.path.join(os.path.dirname(filename), ".libs", val)
 352                     break
 353                 else:
 354                     raise ValueError(
 355                         'could not process libtool file "%s"' % orig_filename
 356                     )
 357
 358         while True:
 359             with open(filename, "rb") as fd:
 360                 hdr = fd.read(4)
 361
 362             if hdr == b"\x7fELF":
 363                 self.load_elf(filename, orig_filename)
 364                 return
 365
 366             if hdr[:2] == b"#!":
 367                 path, name = os.path.split(filename)
 368                 filename = os.path.join(path, ".libs", name)
 369                 continue
 370
 371             if hdr[:1] == b"{":
 372                 with open(filename, "r") as fd:
 373                     self.load_json(fd)
 374                 return
 375
 376             raise ValueError("cannot determine file type for %s" % (filename))
 377
 378     def load_elf(self, filename, orig_filename):
 379         edf = ELFDissectFile(filename)
 380         edf.orig_filename = orig_filename
 381
 382         note = edf._elffile.find_note("FRRouting", "XREF")
 383         if note is not None:
 384             endian = ">" if edf._elffile.bigendian else "<"
 385             mem = edf._elffile[note]
 386             if edf._elffile.elfclass == 64:
 387                 start, end = struct.unpack(endian + "QQ", mem)
 388                 start += note.start
 389                 end += note.start + 8
 390             else:
 391                 start, end = struct.unpack(endian + "II", mem)
 392                 start += note.start
 393                 end += note.start + 4
 394
 395             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 396
 397         else:
 398             xrefarray = edf.get_section("xref_array")
 399             if xrefarray is None:
 400                 raise ValueError("file has neither xref note nor xref_array section")
 401
 402             ptrs = xrefarray.iter_data(XrefPtr)
 403
 404         for ptr in ptrs:
 405             if ptr.xref is None:
 406                 print("NULL xref")
 407                 continue
 408             self._xrefs.append(ptr.xref)
 409
 410             container = ptr.xref.container()
 411             if container is None:
 412                 continue
 413             container.to_dict(self)
 414
 415         return edf
 416
 417     def load_json(self, fd):
 418         data = json.load(fd)
 419         for uid, items in data["refs"].items():
 420             myitems = self["refs"].setdefault(uid, [])
 421             for item in items:
 422                 if item in myitems:
 423                     continue
 424                 myitems.append(item)
 425
 426         for cmd, items in data["cli"].items():
 427             self["cli"].setdefault(cmd, {}).update(items)
 428
 429         return data
 430
 431     def check(self, checks):
 432         for xref in self._xrefs:
 433             yield from xref.check(checks)
 434
 435
 436 def main():
 437     argp = argparse.ArgumentParser(description="FRR xref ELF extractor")
 438     argp.add_argument("-o", dest="output", type=str, help="write JSON output")
 439     argp.add_argument("--out-by-file", type=str, help="write by-file JSON output")
 440     argp.add_argument("-c", dest="vtysh_cmds", type=str, help="write vtysh_cmd.c")
 441     argp.add_argument("-Wlog-format", action="store_const", const=True)
 442     argp.add_argument("-Wlog-args", action="store_const", const=True)
 443     argp.add_argument("-Werror", action="store_const", const=True)
 444     argp.add_argument("--profile", action="store_const", const=True)
 445     argp.add_argument(
 446         "binaries",
 447         metavar="BINARY",
 448         nargs="+",
 449         type=str,
 450         help="files to read (ELF files or libtool objects)",
 451     )
 452     args = argp.parse_args()
 453
 454     if args.profile:
 455         import cProfile
 456
 457         cProfile.runctx("_main(args)", globals(), {"args": args}, sort="cumtime")
 458     else:
 459         _main(args)
 460
 461
 462 def _main(args):
 463     errors = 0
 464     xrelfo = Xrelfo()
 465
 466     for fn in args.binaries:
 467         try:
 468             xrelfo.load_file(fn)
 469         except:
 470             errors += 1
 471             sys.stderr.write("while processing %s:\n" % (fn))
 472             traceback.print_exc()
 473
 474     for option in dir(args):
 475         if option.startswith("W") and option != "Werror":
 476             checks = sorted(xrelfo.check(args))
 477             sys.stderr.write("".join([c[-1] for c in checks]))
 478
 479             if args.Werror and len(checks) > 0:
 480                 errors += 1
 481             break
 482
 483     refs = xrelfo["refs"]
 484
 485     counts = {}
 486     for k, v in refs.items():
 487         strs = set([i["fmtstring"] for i in v])
 488         if len(strs) != 1:
 489             print("\033[31;1m%s\033[m" % k)
 490         counts[k] = len(v)
 491
 492     out = xrelfo
 493     outbyfile = {}
 494     for uid, locs in refs.items():
 495         for loc in locs:
 496             filearray = outbyfile.setdefault(loc["file"], [])
 497             loc = dict(loc)
 498             del loc["file"]
 499             filearray.append(loc)
 500
 501     for k in outbyfile.keys():
 502         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x["line"])
 503
 504     if errors:
 505         sys.exit(1)
 506
 507     if args.output:
 508         with open(args.output + ".tmp", "w") as fd:
 509             json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args)
 510         os.rename(args.output + ".tmp", args.output)
 511
 512     if args.out_by_file:
 513         with open(args.out_by_file + ".tmp", "w") as fd:
 514             json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args)
 515         os.rename(args.out_by_file + ".tmp", args.out_by_file)
 516
 517     if args.vtysh_cmds:
 518         with open(args.vtysh_cmds + ".tmp", "w") as fd:
 519             CommandEntry.run(out, fd)
 520         os.rename(args.vtysh_cmds + ".tmp", args.vtysh_cmds)
 521         if args.Werror and CommandEntry.warn_counter:
 522             sys.exit(1)
 523
 524
 525 if __name__ == "__main__":
 526     main()