python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24
  25 json_dump_args = {}
  26
  27 try:
  28     import ujson as json
  29
  30     json_dump_args["escape_forward_slashes"] = False
  31 except ImportError:
  32     import json
  33
  34 import argparse
  35
  36 from clippy.uidhash import uidhash
  37 from clippy.elf import *
  38 from clippy import frr_top_src, CmdAttr
  39 from tiabwarfo import FieldApplicator
  40 from xref2vtysh import CommandEntry
  41
  42 try:
  43     with open(os.path.join(frr_top_src, "python", "xrefstructs.json"), "r") as fd:
  44         xrefstructs = json.load(fd)
  45 except FileNotFoundError:
  46     sys.stderr.write(
  47         """
  48 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  49 tool available) could not be found.  It should be included with the sources.
  50 """
  51     )
  52     sys.exit(1)
  53
  54 # constants, need to be kept in sync manually...
  55
  56 XREFT_THREADSCHED = 0x100
  57 XREFT_LOGMSG = 0x200
  58 XREFT_DEFUN = 0x300
  59 XREFT_INSTALL_ELEMENT = 0x301
  60
  61 # LOG_*
  62 priovals = {}
  63 prios = ["0", "1", "2", "E", "W", "N", "I", "D"]
  64
  65
  66 class XrelfoJson(object):
  67     def dump(self):
  68         pass
  69
  70     def check(self, wopt):
  71         yield from []
  72
  73     def to_dict(self, refs):
  74         pass
  75
  76
  77 class Xref(ELFDissectStruct, XrelfoJson):
  78     struct = "xref"
  79     fieldrename = {"type": "typ"}
  80     containers = {}
  81
  82     def __init__(self, *args, **kwargs):
  83         super().__init__(*args, **kwargs)
  84
  85         self._container = None
  86         if self.xrefdata:
  87             self.xrefdata.ref_from(self, self.typ)
  88
  89     def container(self):
  90         if self._container is None:
  91             if self.typ in self.containers:
  92                 self._container = self.container_of(self.containers[self.typ], "xref")
  93         return self._container
  94
  95     def check(self, *args, **kwargs):
  96         if self._container:
  97             yield from self._container.check(*args, **kwargs)
  98
  99
 100 class Xrefdata(ELFDissectStruct):
 101     struct = "xrefdata"
 102
 103     # uid is all zeroes in the data loaded from ELF
 104     fieldrename = {"uid": "_uid"}
 105
 106     def ref_from(self, xref, typ):
 107         self.xref = xref
 108
 109     @property
 110     def uid(self):
 111         if self.hashstr is None:
 112             return None
 113         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 114
 115
 116 class XrefPtr(ELFDissectStruct):
 117     fields = [
 118         ("xref", "P", Xref),
 119     ]
 120
 121
 122 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 123     struct = "xref_threadsched"
 124
 125
 126 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 127
 128
 129 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 130     struct = "xref_logmsg"
 131
 132     def _warn_fmt(self, text):
 133         lines = text.split("\n")
 134         yield (
 135             (self.xref.file, self.xref.line),
 136             "%s:%d: %s (in %s())%s\n"
 137             % (
 138                 self.xref.file,
 139                 self.xref.line,
 140                 lines[0],
 141                 self.xref.func,
 142                 "".join(["\n" + l for l in lines[1:]]),
 143             ),
 144         )
 145
 146     fmt_regexes = [
 147         (re.compile(r"([\n\t]+)"), "error: log message contains tab or newline"),
 148         #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 149         (
 150             re.compile(r"^((?:warn(?:ing)?|error):\s*)", re.I),
 151             "warning: log message starts with severity",
 152         ),
 153     ]
 154     arg_regexes = [
 155         # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 156         (
 157             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)"),
 158             "cleanup: replace inet_ntop(AF_INET, ...) with %pI4",
 159             lambda s: True,
 160         ),
 161         (
 162             re.compile(r"((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)"),
 163             "cleanup: replace inet_ntop(AF_INET6, ...) with %pI6",
 164             lambda s: True,
 165         ),
 166         (
 167             # string split-up here is to not trigger "inet_ntoa forbidden"
 168             re.compile(r"((?<![\?:] )inet_" + r"ntoa)"),
 169             "cleanup: replace inet_" + "ntoa(...) with %pI4",
 170             lambda s: True,
 171         ),
 172         (
 173             re.compile(r"((?<![\?:] )ipaddr2str)"),
 174             "cleanup: replace ipaddr2str(...) with %pIA",
 175             lambda s: True,
 176         ),
 177         (
 178             re.compile(r"((?<![\?:] )prefix2str)"),
 179             "cleanup: replace prefix2str(...) with %pFX",
 180             lambda s: True,
 181         ),
 182         (
 183             re.compile(r"((?<![\?:] )prefix_mac2str)"),
 184             "cleanup: replace prefix_mac2str(...) with %pEA",
 185             lambda s: True,
 186         ),
 187         (
 188             re.compile(r"((?<![\?:] )sockunion2str)"),
 189             "cleanup: replace sockunion2str(...) with %pSU",
 190             lambda s: True,
 191         ),
 192         #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 193     ]
 194
 195     def check(self, wopt):
 196         def fmt_msg(rex, itext):
 197             if sys.stderr.isatty():
 198                 items = rex.split(itext)
 199                 out = []
 200                 for i, text in enumerate(items):
 201                     if (i % 2) == 1:
 202                         out.append("\033[41;37;1m%s\033[m" % repr(text)[1:-1])
 203                     else:
 204                         out.append(repr(text)[1:-1])
 205
 206                 excerpt = "".join(out)
 207             else:
 208                 excerpt = repr(itext)[1:-1]
 209             return excerpt
 210
 211         if wopt.Wlog_format:
 212             for rex, msg in self.fmt_regexes:
 213                 if not rex.search(self.fmtstring):
 214                     continue
 215
 216                 excerpt = fmt_msg(rex, self.fmtstring)
 217                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 218
 219         if wopt.Wlog_args:
 220             for rex, msg, cond in self.arg_regexes:
 221                 if not cond(self):
 222                     continue
 223                 if not rex.search(self.args):
 224                     continue
 225
 226                 excerpt = fmt_msg(rex, self.args)
 227                 yield from self._warn_fmt(
 228                     '%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt)
 229                 )
 230
 231     def dump(self):
 232         print(
 233             "%-60s %s%s %-25s [EC %d] %s"
 234             % (
 235                 "%s:%d %s()" % (self.xref.file, self.xref.line, self.xref.func),
 236                 prios[self.priority & 7],
 237                 priovals.get(self.priority & 0x30, " "),
 238                 self.xref.xrefdata.uid,
 239                 self.ec,
 240                 self.fmtstring,
 241             )
 242         )
 243
 244     def to_dict(self, xrelfo):
 245         jsobj = dict([(i, getattr(self.xref, i)) for i in ["file", "line", "func"]])
 246         if self.ec != 0:
 247             jsobj["ec"] = self.ec
 248         jsobj["fmtstring"] = self.fmtstring
 249         jsobj["args"] = self.args
 250         jsobj["priority"] = self.priority & 7
 251         jsobj["type"] = "logmsg"
 252         jsobj["binary"] = self._elfsect._elfwrap.orig_filename
 253
 254         if self.priority & 0x10:
 255             jsobj.setdefault("flags", []).append("errno")
 256         if self.priority & 0x20:
 257             jsobj.setdefault("flags", []).append("getaddrinfo")
 258
 259         xrelfo["refs"].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 260
 261
 262 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 263
 264
 265 class CmdElement(ELFDissectStruct, XrelfoJson):
 266     struct = "cmd_element"
 267
 268     def __init__(self, *args, **kwargs):
 269         super().__init__(*args, **kwargs)
 270
 271     def to_dict(self, xrelfo):
 272         jsobj = (
 273             xrelfo["cli"]
 274             .setdefault(self.name, {})
 275             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 276         )
 277
 278         jsobj.update(
 279             {
 280                 "string": self.string,
 281                 "doc": self.doc,
 282             }
 283         )
 284         if self.attr:
 285             jsobj["attr"] = attr = self.attr
 286             for attrname in CmdAttr.__members__:
 287                 val = CmdAttr[attrname]
 288                 if attr & val:
 289                     jsobj.setdefault("attrs", []).append(attrname.lower())
 290                     attr &= ~val
 291
 292         jsobj["defun"] = dict(
 293             [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 294         )
 295
 296
 297 Xref.containers[XREFT_DEFUN] = CmdElement
 298
 299
 300 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 301     struct = "xref_install_element"
 302
 303     def to_dict(self, xrelfo):
 304         jsobj = (
 305             xrelfo["cli"]
 306             .setdefault(self.cmd_element.name, {})
 307             .setdefault(self._elfsect._elfwrap.orig_filename, {})
 308         )
 309         nodes = jsobj.setdefault("nodes", [])
 310
 311         nodes.append(
 312             {
 313                 "node": self.node_type,
 314                 "install": dict(
 315                     [(i, getattr(self.xref, i)) for i in ["file", "line", "func"]]
 316                 ),
 317             }
 318         )
 319
 320
 321 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 322
 323 # shove in field defs
 324 fieldapply = FieldApplicator(xrefstructs)
 325 fieldapply.add(Xref)
 326 fieldapply.add(Xrefdata)
 327 fieldapply.add(XrefLogmsg)
 328 fieldapply.add(XrefThreadSched)
 329 fieldapply.add(CmdElement)
 330 fieldapply.add(XrefInstallElement)
 331 fieldapply()
 332
 333
 334 class Xrelfo(dict):
 335     def __init__(self):
 336         super().__init__(
 337             {
 338                 "refs": {},
 339                 "cli": {},
 340             }
 341         )
 342         self._xrefs = []
 343
 344     def load_file(self, filename):
 345         orig_filename = filename
 346         if filename.endswith(".la") or filename.endswith(".lo"):
 347             with open(filename, "r") as fd:
 348                 for line in fd:
 349                     line = line.strip()
 350                     if line.startswith("#") or line == "" or "=" not in line:
 351                         continue
 352
 353                     var, val = line.split("=", 1)
 354                     if var not in ["library_names", "pic_object"]:
 355                         continue
 356                     if val.startswith("'") or val.startswith('"'):
 357                         val = val[1:-1]
 358
 359                     if var == "pic_object":
 360                         filename = os.path.join(os.path.dirname(filename), val)
 361                         break
 362
 363                     val = val.strip().split()[0]
 364                     filename = os.path.join(os.path.dirname(filename), ".libs", val)
 365                     break
 366                 else:
 367                     raise ValueError(
 368                         'could not process libtool file "%s"' % orig_filename
 369                     )
 370
 371         while True:
 372             with open(filename, "rb") as fd:
 373                 hdr = fd.read(4)
 374
 375             if hdr == b"\x7fELF":
 376                 self.load_elf(filename, orig_filename)
 377                 return
 378
 379             if hdr[:2] == b"#!":
 380                 path, name = os.path.split(filename)
 381                 filename = os.path.join(path, ".libs", name)
 382                 continue
 383
 384             if hdr[:1] == b"{":
 385                 with open(filename, "r") as fd:
 386                     self.load_json(fd)
 387                 return
 388
 389             raise ValueError("cannot determine file type for %s" % (filename))
 390
 391     def load_elf(self, filename, orig_filename):
 392         edf = ELFDissectFile(filename)
 393         edf.orig_filename = orig_filename
 394
 395         note = edf._elffile.find_note("FRRouting", "XREF")
 396         if note is not None:
 397             endian = ">" if edf._elffile.bigendian else "<"
 398             mem = edf._elffile[note]
 399             if edf._elffile.elfclass == 64:
 400                 start, end = struct.unpack(endian + "QQ", mem)
 401                 start += note.start
 402                 end += note.start + 8
 403             else:
 404                 start, end = struct.unpack(endian + "II", mem)
 405                 start += note.start
 406                 end += note.start + 4
 407
 408             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 409
 410         else:
 411             xrefarray = edf.get_section("xref_array")
 412             if xrefarray is None:
 413                 raise ValueError("file has neither xref note nor xref_array section")
 414
 415             ptrs = xrefarray.iter_data(XrefPtr)
 416
 417         for ptr in ptrs:
 418             if ptr.xref is None:
 419                 print("NULL xref")
 420                 continue
 421             self._xrefs.append(ptr.xref)
 422
 423             container = ptr.xref.container()
 424             if container is None:
 425                 continue
 426             container.to_dict(self)
 427
 428         return edf
 429
 430     def load_json(self, fd):
 431         data = json.load(fd)
 432         for uid, items in data["refs"].items():
 433             myitems = self["refs"].setdefault(uid, [])
 434             for item in items:
 435                 if item in myitems:
 436                     continue
 437                 myitems.append(item)
 438
 439         for cmd, items in data["cli"].items():
 440             self["cli"].setdefault(cmd, {}).update(items)
 441
 442         return data
 443
 444     def check(self, checks):
 445         for xref in self._xrefs:
 446             yield from xref.check(checks)
 447
 448
 449 def main():
 450     argp = argparse.ArgumentParser(description="FRR xref ELF extractor")
 451     argp.add_argument("-o", dest="output", type=str, help="write JSON output")
 452     argp.add_argument("--out-by-file", type=str, help="write by-file JSON output")
 453     argp.add_argument("-c", dest="vtysh_cmds", type=str, help="write vtysh_cmd.c")
 454     argp.add_argument("-Wlog-format", action="store_const", const=True)
 455     argp.add_argument("-Wlog-args", action="store_const", const=True)
 456     argp.add_argument("-Werror", action="store_const", const=True)
 457     argp.add_argument("--profile", action="store_const", const=True)
 458     argp.add_argument(
 459         "binaries",
 460         metavar="BINARY",
 461         nargs="+",
 462         type=str,
 463         help="files to read (ELF files or libtool objects)",
 464     )
 465     args = argp.parse_args()
 466
 467     if args.profile:
 468         import cProfile
 469
 470         cProfile.runctx("_main(args)", globals(), {"args": args}, sort="cumtime")
 471     else:
 472         _main(args)
 473
 474
 475 def _main(args):
 476     errors = 0
 477     xrelfo = Xrelfo()
 478
 479     for fn in args.binaries:
 480         try:
 481             xrelfo.load_file(fn)
 482         except:
 483             errors += 1
 484             sys.stderr.write("while processing %s:\n" % (fn))
 485             traceback.print_exc()
 486
 487     for option in dir(args):
 488         if option.startswith("W") and option != "Werror":
 489             checks = sorted(xrelfo.check(args))
 490             sys.stderr.write("".join([c[-1] for c in checks]))
 491
 492             if args.Werror and len(checks) > 0:
 493                 errors += 1
 494             break
 495
 496     refs = xrelfo["refs"]
 497
 498     counts = {}
 499     for k, v in refs.items():
 500         strs = set([i["fmtstring"] for i in v])
 501         if len(strs) != 1:
 502             print("\033[31;1m%s\033[m" % k)
 503         counts[k] = len(v)
 504
 505     out = xrelfo
 506     outbyfile = {}
 507     for uid, locs in refs.items():
 508         for loc in locs:
 509             filearray = outbyfile.setdefault(loc["file"], [])
 510             loc = dict(loc)
 511             del loc["file"]
 512             filearray.append(loc)
 513
 514     for k in outbyfile.keys():
 515         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x["line"])
 516
 517     if errors:
 518         sys.exit(1)
 519
 520     if args.output:
 521         with open(args.output + ".tmp", "w") as fd:
 522             json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args)
 523         os.rename(args.output + ".tmp", args.output)
 524
 525     if args.out_by_file:
 526         with open(args.out_by_file + ".tmp", "w") as fd:
 527             json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args)
 528         os.rename(args.out_by_file + ".tmp", args.out_by_file)
 529
 530     if args.vtysh_cmds:
 531         with open(args.vtysh_cmds + ".tmp", "w") as fd:
 532             CommandEntry.run(out, fd)
 533         os.rename(args.vtysh_cmds + ".tmp", args.vtysh_cmds)
 534         if args.Werror and CommandEntry.warn_counter:
 535             sys.exit(1)
 536
 537
 538 if __name__ == "__main__":
 539     main()