python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24
  25 json_dump_args = {}
  26
  27 try:
  28     import ujson as json
  29
  30     json_dump_args["escape_forward_slashes"] = False
  31 except ImportError:
  32     import json
  33
  34 import argparse
  35
  36 from clippy.uidhash import uidhash
  37 from clippy.elf import *
  38 from clippy import frr_top_src, CmdAttr
  39 from tiabwarfo import FieldApplicator
  40 from xref2vtysh import CommandEntry
  41
  42 try:
  43     with open(os.path.join(frr_top_src, "python", "xrefstructs.json"), "r") as fd:
  44         xrefstructs = json.load(fd)
  45 except FileNotFoundError:
  46     sys.stderr.write(
  47         """
  48 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  49 tool available) could not be found.  It should be included with the sources.
  50 """
  51     )
  52     sys.exit(1)
  53
  54 # constants, need to be kept in sync manually...
  55
  56 XREFT_THREADSCHED = 0x100
  57 XREFT_LOGMSG = 0x200
  58 XREFT_DEFUN = 0x300
  59 XREFT_INSTALL_ELEMENT = 0x301
  60
  61 # LOG_*
  62 priovals = {}
  63 prios = ["0", "1", "2", "E", "W", "N", "I", "D"]
  64
  65
  66 class XrelfoJson(object):
  67     def dump(self):
  68         pass
  69
  70     def check(self, wopt):
  71         yield from []
  72
  73     def to_dict(self, refs):
  74         pass
  75
  76
  77 class Xref(ELFDissectStruct, XrelfoJson):
  78     struct = "xref"
  79     fieldrename = {"type": "typ"}
  80     containers = {}
  81
  82     def __init__(self, *args, **kwargs):
  83         super().__init__(*args, **kwargs)
  84
  85         self._container = None
  86         if self.xrefdata:
  87             self.xrefdata.ref_from(self, self.typ)
  88
  89     def container(self):
  90         if self._container is None:
  91             if self.typ in self.containers:
  92                 self._container = self.container_of(self.containers[self.typ], "xref")
  93         return self._container
  94
  95     def check(self, *args, **kwargs):
  96         if self._container:
  97             yield from self._container.check(*args, **kwargs)
  98
  99
 100 class Xrefdata(ELFDissectStruct):
 101     struct = "xrefdata"
 102
 103     # uid is all zeroes in the data loaded from ELF
 104     fieldrename = {"uid": "_uid"}
 105
 106     def ref_from(self, xref, typ):
 107         self.xref = xref
 108
 109     @property
 110     def uid(self):
 111         if self.hashstr is None:
 112             return None
 113         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 114
 115
 116 class XrefPtr(ELFDissectStruct):
 117     fields = [
 118         ("xref", "P", Xref),
 119     ]
 120
 121
 122 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 123     struct = "xref_threadsched"
 124
 125
 126 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 127
 128
 129 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 130     struct = "xref_logmsg"
 131
 132     def _warn_fmt(self, text):
 133         lines = text.split("\n")
 134         yield (
 135             (self.xref.file, self.xref.line),
 136             "%s:%d: %s (in %s())%s\n"
 137             % (
 138                 self.xref.file,
 139                 self.xref.line,
 140                 lines[0],
 141                 self.xref.func,
 142                 "".join(["\n" + l for l in lines[1:]]),
 143             ),
 144         )
 145
 146     fmt_regexes = [
 147         (re.compile(r"([\n\t]+)"), "error: log message contains tab or newline"),
 148         #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 149         (
 150             re.compile(r"^((?:warn(?:ing)?|error):\s*)", re.I),
 151             "warning: log message starts with severity",
 152         ),
 153     ]
 154     arg_regexes = [
 155         # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 156         (
 157             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)"),
 158             "cleanup: replace inet_ntop(AF_INET, ...) with %pI4",
 159             lambda s: True,
 160         ),
 161         (
 162             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)"),
 163             "cleanup: replace inet_ntop(AF_INET6, ...) with %pI6",
 164             lambda s: True,
 165         ),
 166         (
 167             re.compile(r"((?<![\?:] )inet_ntoa)"),
 168             "cleanup: replace inet_ntoa(...) with %pI4",
 169             lambda s: True,
 170         ),
 171         (
 172             re.compile(r"((?<![\?:] )ipaddr2str)"),
 173             "cleanup: replace ipaddr2str(...) with %pIA",
 174             lambda s: True,
 175         ),
 176         (
 177             re.compile(r"((?<![\?:] )prefix2str)"),
 178             "cleanup: replace prefix2str(...) with %pFX",
 179             lambda s: True,
 180         ),
 181         (
 182             re.compile(r"((?<![\?:] )prefix_mac2str)"),
 183             "cleanup: replace prefix_mac2str(...) with %pEA",
 184             lambda s: True,
 185         ),
 186         (
 187             re.compile(r"((?<![\?:] )sockunion2str)"),
 188             "cleanup: replace sockunion2str(...) with %pSU",
 189             lambda s: True,
 190         ),
 191         #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 192     ]
 193
 194     def check(self, wopt):
 195         def fmt_msg(rex, itext):
 196             if sys.stderr.isatty():
 197                 items = rex.split(itext)
 198                 out = []
 199                 for i, text in enumerate(items):
 200                     if (i % 2) == 1:
 201                         out.append("\033[41;37;1m%s\033[m" % repr(text)[1:-1])
 202                     else:
 203                         out.append(repr(text)[1:-1])
 204
 205                 excerpt = "".join(out)
 206             else:
 207                 excerpt = repr(itext)[1:-1]
 208             return excerpt
 209
 210         if wopt.Wlog_format:
 211             for rex, msg in self.fmt_regexes:
 212                 if not rex.search(self.fmtstring):
 213                     continue
 214
 215                 excerpt = fmt_msg(rex, self.fmtstring)
 216                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 217
 218         if wopt.Wlog_args:
 219             for rex, msg, cond in self.arg_regexes:
 220                 if not cond(self):
 221                     continue
 222                 if not rex.search(self.args):
 223                     continue
 224
 225                 excerpt = fmt_msg(rex, self.args)
 226                 yield from self._warn_fmt(
 227                     '%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt)
 228                 )
 229
 230     def dump(self):
 231         print(
 232             "%-60s %s%s %-25s [EC %d] %s"
 233             % (
 234                 "%s:%d %s()" % (self.xref.file, self.xref.line, self.xref.func),
 235                 prios[self.priority & 7],
 236                 priovals.get(self.priority & 0x30, " "),
 237                 self.xref.xrefdata.uid,
 238                 self.ec,
 239                 self.fmtstring,
 240             )
 241         )
 242
 243     def to_dict(self, xrelfo):
 244         jsobj = dict([(i, getattr(self.xref, i)) for i in ["file", "line", "func"]])
 245         if self.ec != 0:
 246             jsobj["ec"] = self.ec
 247         jsobj["fmtstring"] = self.fmtstring
 248         jsobj["args"] = self.args
 249         jsobj["priority"] = self.priority & 7
 250         jsobj["type"] = "logmsg"
 251         jsobj["binary"] = self._elfsect._elfwrap.orig_filename
 252
 253         if self.priority & 0x10:
 254             jsobj.setdefault("flags", []).append("errno")
 255         if self.priority & 0x20:
 256             jsobj.setdefault("flags", []).append("getaddrinfo")
 257
 258         xrelfo["refs"].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 259
 260
 261 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 262
 263
 264 class CmdElement(ELFDissectStruct, XrelfoJson):
 265     struct = "cmd_element"
 266
 267     def __init__(self, *args, **kwargs):
 268         super().__init__(*args, **kwargs)
 269
 270     def to_dict(self, xrelfo):
 271         jsobj = (
 272             xrelfo["cli"]
 273             .setdefault(self.name, {})
 274             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 275         )
 276
 277         jsobj.update(
 278             {
 279                 "string": self.string,
 280                 "doc": self.doc,
 281             }
 282         )
 283         if self.attr:
 284             jsobj["attr"] = attr = self.attr
 285             for attrname in CmdAttr.__members__:
 286                 val = CmdAttr[attrname]
 287                 if attr & val:
 288                     jsobj.setdefault("attrs", []).append(attrname.lower())
 289                     attr &= ~val
 290
 291         jsobj["defun"] = dict(
 292             [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 293         )
 294
 295
 296 Xref.containers[XREFT_DEFUN] = CmdElement
 297
 298
 299 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 300     struct = "xref_install_element"
 301
 302     def to_dict(self, xrelfo):
 303         jsobj = (
 304             xrelfo["cli"]
 305             .setdefault(self.cmd_element.name, {})
 306             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 307         )
 308         nodes = jsobj.setdefault("nodes", [])
 309
 310         nodes.append(
 311             {
 312                 "node": self.node_type,
 313                 "install": dict(
 314                     [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 315                 ),
 316             }
 317         )
 318
 319
 320 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 321
 322 # shove in field defs
 323 fieldapply = FieldApplicator(xrefstructs)
 324 fieldapply.add(Xref)
 325 fieldapply.add(Xrefdata)
 326 fieldapply.add(XrefLogmsg)
 327 fieldapply.add(XrefThreadSched)
 328 fieldapply.add(CmdElement)
 329 fieldapply.add(XrefInstallElement)
 330 fieldapply()
 331
 332
 333 class Xrelfo(dict):
 334     def __init__(self):
 335         super().__init__(
 336             {
 337                 "refs": {},
 338                 "cli": {},
 339             }
 340         )
 341         self._xrefs = []
 342
 343     def load_file(self, filename):
 344         orig_filename = filename
 345         if filename.endswith(".la") or filename.endswith(".lo"):
 346             with open(filename, "r") as fd:
 347                 for line in fd:
 348                     line = line.strip()
 349                     if line.startswith("#") or line == "" or "=" not in line:
 350                         continue
 351
 352                     var, val = line.split("=", 1)
 353                     if var not in ["library_names", "pic_object"]:
 354                         continue
 355                     if val.startswith("'") or val.startswith('"'):
 356                         val = val[1:-1]
 357
 358                     if var == "pic_object":
 359                         filename = os.path.join(os.path.dirname(filename), val)
 360                         break
 361
 362                     val = val.strip().split()[0]
 363                     filename = os.path.join(os.path.dirname(filename), ".libs", val)
 364                     break
 365                 else:
 366                     raise ValueError(
 367                         'could not process libtool file "%s"' % orig_filename
 368                     )
 369
 370         while True:
 371             with open(filename, "rb") as fd:
 372                 hdr = fd.read(4)
 373
 374             if hdr == b"\x7fELF":
 375                 self.load_elf(filename, orig_filename)
 376                 return
 377
 378             if hdr[:2] == b"#!":
 379                 path, name = os.path.split(filename)
 380                 filename = os.path.join(path, ".libs", name)
 381                 continue
 382
 383             if hdr[:1] == b"{":
 384                 with open(filename, "r") as fd:
 385                     self.load_json(fd)
 386                 return
 387
 388             raise ValueError("cannot determine file type for %s" % (filename))
 389
 390     def load_elf(self, filename, orig_filename):
 391         edf = ELFDissectFile(filename)
 392         edf.orig_filename = orig_filename
 393
 394         note = edf._elffile.find_note("FRRouting", "XREF")
 395         if note is not None:
 396             endian = ">" if edf._elffile.bigendian else "<"
 397             mem = edf._elffile[note]
 398             if edf._elffile.elfclass == 64:
 399                 start, end = struct.unpack(endian + "QQ", mem)
 400                 start += note.start
 401                 end += note.start + 8
 402             else:
 403                 start, end = struct.unpack(endian + "II", mem)
 404                 start += note.start
 405                 end += note.start + 4
 406
 407             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 408
 409         else:
 410             xrefarray = edf.get_section("xref_array")
 411             if xrefarray is None:
 412                 raise ValueError("file has neither xref note nor xref_array section")
 413
 414             ptrs = xrefarray.iter_data(XrefPtr)
 415
 416         for ptr in ptrs:
 417             if ptr.xref is None:
 418                 print("NULL xref")
 419                 continue
 420             self._xrefs.append(ptr.xref)
 421
 422             container = ptr.xref.container()
 423             if container is None:
 424                 continue
 425             container.to_dict(self)
 426
 427         return edf
 428
 429     def load_json(self, fd):
 430         data = json.load(fd)
 431         for uid, items in data["refs"].items():
 432             myitems = self["refs"].setdefault(uid, [])
 433             for item in items:
 434                 if item in myitems:
 435                     continue
 436                 myitems.append(item)
 437
 438         for cmd, items in data["cli"].items():
 439             self["cli"].setdefault(cmd, {}).update(items)
 440
 441         return data
 442
 443     def check(self, checks):
 444         for xref in self._xrefs:
 445             yield from xref.check(checks)
 446
 447
 448 def main():
 449     argp = argparse.ArgumentParser(description="FRR xref ELF extractor")
 450     argp.add_argument("-o", dest="output", type=str, help="write JSON output")
 451     argp.add_argument("--out-by-file", type=str, help="write by-file JSON output")
 452     argp.add_argument("-c", dest="vtysh_cmds", type=str, help="write vtysh_cmd.c")
 453     argp.add_argument("-Wlog-format", action="store_const", const=True)
 454     argp.add_argument("-Wlog-args", action="store_const", const=True)
 455     argp.add_argument("-Werror", action="store_const", const=True)
 456     argp.add_argument("--profile", action="store_const", const=True)
 457     argp.add_argument(
 458         "binaries",
 459         metavar="BINARY",
 460         nargs="+",
 461         type=str,
 462         help="files to read (ELF files or libtool objects)",
 463     )
 464     args = argp.parse_args()
 465
 466     if args.profile:
 467         import cProfile
 468
 469         cProfile.runctx("_main(args)", globals(), {"args": args}, sort="cumtime")
 470     else:
 471         _main(args)
 472
 473
 474 def _main(args):
 475     errors = 0
 476     xrelfo = Xrelfo()
 477
 478     for fn in args.binaries:
 479         try:
 480             xrelfo.load_file(fn)
 481         except:
 482             errors += 1
 483             sys.stderr.write("while processing %s:\n" % (fn))
 484             traceback.print_exc()
 485
 486     for option in dir(args):
 487         if option.startswith("W") and option != "Werror":
 488             checks = sorted(xrelfo.check(args))
 489             sys.stderr.write("".join([c[-1] for c in checks]))
 490
 491             if args.Werror and len(checks) > 0:
 492                 errors += 1
 493             break
 494
 495     refs = xrelfo["refs"]
 496
 497     counts = {}
 498     for k, v in refs.items():
 499         strs = set([i["fmtstring"] for i in v])
 500         if len(strs) != 1:
 501             print("\033[31;1m%s\033[m" % k)
 502         counts[k] = len(v)
 503
 504     out = xrelfo
 505     outbyfile = {}
 506     for uid, locs in refs.items():
 507         for loc in locs:
 508             filearray = outbyfile.setdefault(loc["file"], [])
 509             loc = dict(loc)
 510             del loc["file"]
 511             filearray.append(loc)
 512
 513     for k in outbyfile.keys():
 514         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x["line"])
 515
 516     if errors:
 517         sys.exit(1)
 518
 519     if args.output:
 520         with open(args.output + ".tmp", "w") as fd:
 521             json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args)
 522         os.rename(args.output + ".tmp", args.output)
 523
 524     if args.out_by_file:
 525         with open(args.out_by_file + ".tmp", "w") as fd:
 526             json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args)
 527         os.rename(args.out_by_file + ".tmp", args.out_by_file)
 528
 529     if args.vtysh_cmds:
 530         with open(args.vtysh_cmds + ".tmp", "w") as fd:
 531             CommandEntry.run(out, fd)
 532         os.rename(args.vtysh_cmds + ".tmp", args.vtysh_cmds)
 533         if args.Werror and CommandEntry.warn_counter:
 534             sys.exit(1)
 535
 536
 537 if __name__ == "__main__":
 538     main()