python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24
  25 json_dump_args = {}
  26
  27 try:
  28     import ujson as json
  29
  30     json_dump_args["escape_forward_slashes"] = False
  31 except ImportError:
  32     import json
  33
  34 import argparse
  35
  36 from clippy.uidhash import uidhash
  37 from clippy.elf import *
  38 from clippy import frr_top_src, CmdAttr
  39 from tiabwarfo import FieldApplicator
  40 from xref2vtysh import CommandEntry
  41
  42 try:
  43     with open(os.path.join(frr_top_src, 'python', 'xrefstructs.json'), 'r') as fd:
  44         xrefstructs = json.load(fd)
  45 except FileNotFoundError:
  46     sys.stderr.write('''
  47 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  48 tool available) could not be found.  It should be included with the sources.
  49 ''')
  50     sys.exit(1)
  51
  52 # constants, need to be kept in sync manually...
  53
  54 XREFT_THREADSCHED = 0x100
  55 XREFT_LOGMSG = 0x200
  56 XREFT_DEFUN = 0x300
  57 XREFT_INSTALL_ELEMENT = 0x301
  58
  59 # LOG_*
  60 priovals = {}
  61 prios = ['0', '1', '2', 'E', 'W', 'N', 'I', 'D']
  62
  63
  64 class XrelfoJson(object):
  65     def dump(self):
  66         pass
  67
  68     def check(self, wopt):
  69         yield from []
  70
  71     def to_dict(self, refs):
  72         pass
  73
  74 class Xref(ELFDissectStruct, XrelfoJson):
  75     struct = 'xref'
  76     fieldrename = {'type': 'typ'}
  77     containers = {}
  78
  79     def __init__(self, *args, **kwargs):
  80         super().__init__(*args, **kwargs)
  81
  82         self._container = None
  83         if self.xrefdata:
  84             self.xrefdata.ref_from(self, self.typ)
  85
  86     def container(self):
  87         if self._container is None:
  88             if self.typ in self.containers:
  89                 self._container = self.container_of(self.containers[self.typ], 'xref')
  90         return self._container
  91
  92     def check(self, *args, **kwargs):
  93         if self._container:
  94             yield from self._container.check(*args, **kwargs)
  95
  96
  97 class Xrefdata(ELFDissectStruct):
  98     struct = 'xrefdata'
  99
 100     # uid is all zeroes in the data loaded from ELF
 101     fieldrename = {'uid': '_uid'}
 102
 103     def ref_from(self, xref, typ):
 104         self.xref = xref
 105
 106     @property
 107     def uid(self):
 108         if self.hashstr is None:
 109             return None
 110         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 111
 112 class XrefPtr(ELFDissectStruct):
 113     fields = [
 114         ('xref', 'P', Xref),
 115     ]
 116
 117 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 118     struct = 'xref_threadsched'
 119 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 120
 121 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 122     struct = 'xref_logmsg'
 123
 124     def _warn_fmt(self, text):
 125         lines = text.split('\n')
 126         yield ((self.xref.file, self.xref.line), '%s:%d: %s (in %s())%s\n' % (self.xref.file, self.xref.line, lines[0], self.xref.func, ''.join(['\n' + l for l in lines[1:]])))
 127
 128     fmt_regexes = [
 129         (re.compile(r'([\n\t]+)'), 'error: log message contains tab or newline'),
 130     #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 131         (re.compile(r'^((?:warn(?:ing)?|error):\s*)', re.I), 'warning: log message starts with severity'),
 132     ]
 133     arg_regexes = [
 134     # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 135         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)'),   'cleanup: replace inet_ntop(AF_INET, ...) with %pI4',  lambda s: True),
 136         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)'), 'cleanup: replace inet_ntop(AF_INET6, ...) with %pI6', lambda s: True),
 137         (re.compile(r'((?<![\?:] )inet_ntoa)'),                               'cleanup: replace inet_ntoa(...) with %pI4',           lambda s: True),
 138         (re.compile(r'((?<![\?:] )ipaddr2str)'),                              'cleanup: replace ipaddr2str(...) with %pIA',          lambda s: True),
 139         (re.compile(r'((?<![\?:] )prefix2str)'),                              'cleanup: replace prefix2str(...) with %pFX',          lambda s: True),
 140         (re.compile(r'((?<![\?:] )prefix_mac2str)'),                          'cleanup: replace prefix_mac2str(...) with %pEA',      lambda s: True),
 141         (re.compile(r'((?<![\?:] )sockunion2str)'),                           'cleanup: replace sockunion2str(...) with %pSU',       lambda s: True),
 142
 143     #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 144     ]
 145
 146     def check(self, wopt):
 147         def fmt_msg(rex, itext):
 148             if sys.stderr.isatty():
 149                 items = rex.split(itext)
 150                 out = []
 151                 for i, text in enumerate(items):
 152                     if (i % 2) == 1:
 153                         out.append('\033[41;37;1m%s\033[m' % repr(text)[1:-1])
 154                     else:
 155                         out.append(repr(text)[1:-1])
 156
 157                 excerpt = ''.join(out)
 158             else:
 159                 excerpt = repr(itext)[1:-1]
 160             return excerpt
 161
 162         if wopt.Wlog_format:
 163             for rex, msg in self.fmt_regexes:
 164                 if not rex.search(self.fmtstring):
 165                     continue
 166
 167                 excerpt = fmt_msg(rex, self.fmtstring)
 168                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 169
 170         if wopt.Wlog_args:
 171             for rex, msg, cond in self.arg_regexes:
 172                 if not cond(self):
 173                     continue
 174                 if not rex.search(self.args):
 175                     continue
 176
 177                 excerpt = fmt_msg(rex, self.args)
 178                 yield from self._warn_fmt('%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt))
 179
 180     def dump(self):
 181         print('%-60s %s%s %-25s [EC %d] %s' % (
 182             '%s:%d %s()' % (self.xref.file, self.xref.line, self.xref.func),
 183             prios[self.priority & 7],
 184             priovals.get(self.priority & 0x30, ' '),
 185             self.xref.xrefdata.uid, self.ec, self.fmtstring))
 186
 187     def to_dict(self, xrelfo):
 188         jsobj = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 189         if self.ec != 0:
 190             jsobj['ec'] = self.ec
 191         jsobj['fmtstring'] = self.fmtstring
 192         jsobj['args'] = self.args
 193         jsobj['priority'] = self.priority & 7
 194         jsobj['type'] = 'logmsg'
 195         jsobj['binary'] = self._elfsect._elfwrap.orig_filename
 196
 197         if self.priority & 0x10:
 198             jsobj.setdefault('flags', []).append('errno')
 199         if self.priority & 0x20:
 200             jsobj.setdefault('flags', []).append('getaddrinfo')
 201
 202         xrelfo['refs'].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 203
 204 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 205
 206 class CmdElement(ELFDissectStruct, XrelfoJson):
 207     struct = 'cmd_element'
 208
 209     def __init__(self, *args, **kwargs):
 210         super().__init__(*args, **kwargs)
 211
 212     def to_dict(self, xrelfo):
 213         jsobj = xrelfo['cli'].setdefault(self.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 214
 215         jsobj.update({
 216             'string': self.string,
 217             'doc': self.doc,
 218         })
 219         if self.attr:
 220             jsobj['attr'] = attr = self.attr
 221             for attrname in CmdAttr.__members__:
 222                 val = CmdAttr[attrname]
 223                 if attr & val:
 224                     jsobj.setdefault('attrs', []).append(attrname.lower())
 225                     attr &= ~val
 226
 227         jsobj['defun'] = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 228
 229 Xref.containers[XREFT_DEFUN] = CmdElement
 230
 231 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 232     struct = 'xref_install_element'
 233
 234     def to_dict(self, xrelfo):
 235         jsobj = xrelfo['cli'].setdefault(self.cmd_element.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 236         nodes = jsobj.setdefault('nodes', [])
 237
 238         nodes.append({
 239             'node': self.node_type,
 240             'install': dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]),
 241         })
 242
 243 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 244
 245 # shove in field defs
 246 fieldapply = FieldApplicator(xrefstructs)
 247 fieldapply.add(Xref)
 248 fieldapply.add(Xrefdata)
 249 fieldapply.add(XrefLogmsg)
 250 fieldapply.add(XrefThreadSched)
 251 fieldapply.add(CmdElement)
 252 fieldapply.add(XrefInstallElement)
 253 fieldapply()
 254
 255
 256 class Xrelfo(dict):
 257     def __init__(self):
 258         super().__init__({
 259             'refs': {},
 260             'cli': {},
 261         })
 262         self._xrefs = []
 263
 264     def load_file(self, filename):
 265         orig_filename = filename
 266         if filename.endswith('.la') or filename.endswith('.lo'):
 267             with open(filename, 'r') as fd:
 268                 for line in fd:
 269                     line = line.strip()
 270                     if line.startswith('#') or line == '' or '=' not in line:
 271                         continue
 272
 273                     var, val = line.split('=', 1)
 274                     if var not in ['library_names', 'pic_object']:
 275                         continue
 276                     if val.startswith("'") or val.startswith('"'):
 277                         val = val[1:-1]
 278
 279                     if var == 'pic_object':
 280                         filename = os.path.join(os.path.dirname(filename), val)
 281                         break
 282
 283                     val = val.strip().split()[0]
 284                     filename = os.path.join(os.path.dirname(filename), '.libs', val)
 285                     break
 286                 else:
 287                     raise ValueError('could not process libtool file "%s"' % orig_filename)
 288
 289         while True:
 290             with open(filename, 'rb') as fd:
 291                 hdr = fd.read(4)
 292
 293             if hdr == b'\x7fELF':
 294                 self.load_elf(filename, orig_filename)
 295                 return
 296
 297             if hdr[:2] == b'#!':
 298                 path, name = os.path.split(filename)
 299                 filename = os.path.join(path, '.libs', name)
 300                 continue
 301
 302             if hdr[:1] == b'{':
 303                 with open(filename, 'r') as fd:
 304                     self.load_json(fd)
 305                 return
 306
 307             raise ValueError('cannot determine file type for %s' % (filename))
 308
 309     def load_elf(self, filename, orig_filename):
 310         edf = ELFDissectFile(filename)
 311         edf.orig_filename = orig_filename
 312
 313         note = edf._elffile.find_note('FRRouting', 'XREF')
 314         if note is not None:
 315             endian = '>' if edf._elffile.bigendian else '<'
 316             mem = edf._elffile[note]
 317             if edf._elffile.elfclass == 64:
 318                 start, end = struct.unpack(endian + 'QQ', mem)
 319                 start += note.start
 320                 end += note.start + 8
 321             else:
 322                 start, end = struct.unpack(endian + 'II', mem)
 323                 start += note.start
 324                 end += note.start + 4
 325
 326             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 327
 328         else:
 329             xrefarray = edf.get_section('xref_array')
 330             if xrefarray is None:
 331                 raise ValueError('file has neither xref note nor xref_array section')
 332
 333             ptrs = xrefarray.iter_data(XrefPtr)
 334
 335         for ptr in ptrs:
 336             if ptr.xref is None:
 337                 print('NULL xref')
 338                 continue
 339             self._xrefs.append(ptr.xref)
 340
 341             container = ptr.xref.container()
 342             if container is None:
 343                 continue
 344             container.to_dict(self)
 345
 346         return edf
 347
 348     def load_json(self, fd):
 349         data = json.load(fd)
 350         for uid, items in data['refs'].items():
 351             myitems = self['refs'].setdefault(uid, [])
 352             for item in items:
 353                 if item in myitems:
 354                     continue
 355                 myitems.append(item)
 356
 357         for cmd, items in data['cli'].items():
 358             self['cli'].setdefault(cmd, {}).update(items)
 359
 360         return data
 361
 362     def check(self, checks):
 363         for xref in self._xrefs:
 364             yield from xref.check(checks)
 365
 366 def main():
 367     argp = argparse.ArgumentParser(description = 'FRR xref ELF extractor')
 368     argp.add_argument('-o', dest='output', type=str, help='write JSON output')
 369     argp.add_argument('--out-by-file',     type=str, help='write by-file JSON output')
 370     argp.add_argument('-c', dest='vtysh_cmds', type=str, help='write vtysh_cmd.c')
 371     argp.add_argument('-Wlog-format',      action='store_const', const=True)
 372     argp.add_argument('-Wlog-args',        action='store_const', const=True)
 373     argp.add_argument('-Werror',           action='store_const', const=True)
 374     argp.add_argument('--profile',         action='store_const', const=True)
 375     argp.add_argument('binaries', metavar='BINARY', nargs='+', type=str, help='files to read (ELF files or libtool objects)')
 376     args = argp.parse_args()
 377
 378     if args.profile:
 379         import cProfile
 380         cProfile.runctx('_main(args)', globals(), {'args': args}, sort='cumtime')
 381     else:
 382         _main(args)
 383
 384 def _main(args):
 385     errors = 0
 386     xrelfo = Xrelfo()
 387
 388     for fn in args.binaries:
 389         try:
 390             xrelfo.load_file(fn)
 391         except:
 392             errors += 1
 393             sys.stderr.write('while processing %s:\n' % (fn))
 394             traceback.print_exc()
 395
 396     for option in dir(args):
 397         if option.startswith('W') and option != 'Werror':
 398             checks = sorted(xrelfo.check(args))
 399             sys.stderr.write(''.join([c[-1] for c in checks]))
 400
 401             if args.Werror and len(checks) > 0:
 402                 errors += 1
 403             break
 404
 405
 406     refs = xrelfo['refs']
 407
 408     counts = {}
 409     for k, v in refs.items():
 410         strs = set([i['fmtstring'] for i in v])
 411         if len(strs) != 1:
 412             print('\033[31;1m%s\033[m' % k)
 413         counts[k] = len(v)
 414
 415     out = xrelfo
 416     outbyfile = {}
 417     for uid, locs in refs.items():
 418         for loc in locs:
 419             filearray = outbyfile.setdefault(loc['file'], [])
 420             loc = dict(loc)
 421             del loc['file']
 422             filearray.append(loc)
 423
 424     for k in outbyfile.keys():
 425         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x['line'])
 426
 427     if errors:
 428         sys.exit(1)
 429
 430     if args.output:
 431         with open(args.output + '.tmp', 'w') as fd:
 432             json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args)
 433         os.rename(args.output + '.tmp', args.output)
 434
 435     if args.out_by_file:
 436         with open(args.out_by_file + '.tmp', 'w') as fd:
 437             json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args)
 438         os.rename(args.out_by_file + '.tmp', args.out_by_file)
 439
 440     if args.vtysh_cmds:
 441         with open(args.vtysh_cmds + '.tmp', 'w') as fd:
 442             CommandEntry.run(out, fd)
 443         os.rename(args.vtysh_cmds + '.tmp', args.vtysh_cmds)
 444         if args.Werror and CommandEntry.warn_counter:
 445             sys.exit(1)
 446
 447
 448 if __name__ == '__main__':
 449     main()