python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24 import json
  25 import argparse
  26
  27 from clippy.uidhash import uidhash
  28 from clippy.elf import *
  29 from clippy import frr_top_src
  30 from tiabwarfo import FieldApplicator
  31
  32 try:
  33     with open(os.path.join(frr_top_src, 'python', 'xrefstructs.json'), 'r') as fd:
  34         xrefstructs = json.load(fd)
  35 except FileNotFoundError:
  36     sys.stderr.write('''
  37 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  38 tool available) could not be found.  It should be included with the sources.
  39 ''')
  40     sys.exit(1)
  41
  42 # constants, need to be kept in sync manually...
  43
  44 XREFT_THREADSCHED = 0x100
  45 XREFT_LOGMSG = 0x200
  46 XREFT_DEFUN = 0x300
  47 XREFT_INSTALL_ELEMENT = 0x301
  48
  49 # LOG_*
  50 priovals = {}
  51 prios = ['0', '1', '2', 'E', 'W', 'N', 'I', 'D']
  52
  53
  54 class XrelfoJson(object):
  55     def dump(self):
  56         pass
  57
  58     def check(self, wopt):
  59         yield from []
  60
  61     def to_dict(self, refs):
  62         pass
  63
  64 class Xref(ELFDissectStruct, XrelfoJson):
  65     struct = 'xref'
  66     fieldrename = {'type': 'typ'}
  67     containers = {}
  68
  69     def __init__(self, *args, **kwargs):
  70         super().__init__(*args, **kwargs)
  71
  72         self._container = None
  73         if self.xrefdata:
  74             self.xrefdata.ref_from(self, self.typ)
  75
  76     def container(self):
  77         if self._container is None:
  78             if self.typ in self.containers:
  79                 self._container = self.container_of(self.containers[self.typ], 'xref')
  80         return self._container
  81
  82     def check(self, *args, **kwargs):
  83         if self._container:
  84             yield from self._container.check(*args, **kwargs)
  85
  86
  87 class Xrefdata(ELFDissectStruct):
  88     struct = 'xrefdata'
  89
  90     # uid is all zeroes in the data loaded from ELF
  91     fieldrename = {'uid': '_uid'}
  92
  93     def ref_from(self, xref, typ):
  94         self.xref = xref
  95
  96     @property
  97     def uid(self):
  98         if self.hashstr is None:
  99             return None
 100         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 101
 102 class XrefPtr(ELFDissectStruct):
 103     fields = [
 104         ('xref', 'P', Xref),
 105     ]
 106
 107 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 108     struct = 'xref_threadsched'
 109 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 110
 111 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 112     struct = 'xref_logmsg'
 113
 114     def _warn_fmt(self, text):
 115         lines = text.split('\n')
 116         yield ((self.xref.file, self.xref.line), '%s:%d: %s (in %s())%s\n' % (self.xref.file, self.xref.line, lines[0], self.xref.func, ''.join(['\n' + l for l in lines[1:]])))
 117
 118     fmt_regexes = [
 119         (re.compile(r'([\n\t]+)'), 'error: log message contains tab or newline'),
 120     #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 121         (re.compile(r'^((?:warn(?:ing)?|error):\s*)', re.I), 'warning: log message starts with severity'),
 122     ]
 123     arg_regexes = [
 124     # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 125         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)'),   'cleanup: replace inet_ntop(AF_INET, ...) with %pI4',  lambda s: True),
 126         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)'), 'cleanup: replace inet_ntop(AF_INET6, ...) with %pI6', lambda s: True),
 127         (re.compile(r'((?<![\?:] )inet_ntoa)'),                               'cleanup: replace inet_ntoa(...) with %pI4',           lambda s: True),
 128         (re.compile(r'((?<![\?:] )ipaddr2str)'),                              'cleanup: replace ipaddr2str(...) with %pIA',          lambda s: True),
 129         (re.compile(r'((?<![\?:] )prefix2str)'),                              'cleanup: replace prefix2str(...) with %pFX',          lambda s: True),
 130         (re.compile(r'((?<![\?:] )prefix_mac2str)'),                          'cleanup: replace prefix_mac2str(...) with %pEA',      lambda s: True),
 131         (re.compile(r'((?<![\?:] )sockunion2str)'),                           'cleanup: replace sockunion2str(...) with %pSU',       lambda s: True),
 132
 133     #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 134     ]
 135
 136     def check(self, wopt):
 137         def fmt_msg(rex, itext):
 138             if sys.stderr.isatty():
 139                 items = rex.split(itext)
 140                 out = []
 141                 for i, text in enumerate(items):
 142                     if (i % 2) == 1:
 143                         out.append('\033[41;37;1m%s\033[m' % repr(text)[1:-1])
 144                     else:
 145                         out.append(repr(text)[1:-1])
 146
 147                 excerpt = ''.join(out)
 148             else:
 149                 excerpt = repr(itext)[1:-1]
 150             return excerpt
 151
 152         if wopt.Wlog_format:
 153             for rex, msg in self.fmt_regexes:
 154                 if not rex.search(self.fmtstring):
 155                     continue
 156
 157                 excerpt = fmt_msg(rex, self.fmtstring)
 158                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 159
 160         if wopt.Wlog_args:
 161             for rex, msg, cond in self.arg_regexes:
 162                 if not cond(self):
 163                     continue
 164                 if not rex.search(self.args):
 165                     continue
 166
 167                 excerpt = fmt_msg(rex, self.args)
 168                 yield from self._warn_fmt('%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt))
 169
 170     def dump(self):
 171         print('%-60s %s%s %-25s [EC %d] %s' % (
 172             '%s:%d %s()' % (self.xref.file, self.xref.line, self.xref.func),
 173             prios[self.priority & 7],
 174             priovals.get(self.priority & 0x30, ' '),
 175             self.xref.xrefdata.uid, self.ec, self.fmtstring))
 176
 177     def to_dict(self, xrelfo):
 178         jsobj = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 179         if self.ec != 0:
 180             jsobj['ec'] = self.ec
 181         jsobj['fmtstring'] = self.fmtstring
 182         jsobj['args'] = self.args
 183         jsobj['priority'] = self.priority & 7
 184         jsobj['type'] = 'logmsg'
 185         jsobj['binary'] = self._elfsect._elfwrap.orig_filename
 186
 187         if self.priority & 0x10:
 188             jsobj.setdefault('flags', []).append('errno')
 189         if self.priority & 0x20:
 190             jsobj.setdefault('flags', []).append('getaddrinfo')
 191
 192         xrelfo['refs'].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 193
 194 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 195
 196 class CmdElement(ELFDissectStruct, XrelfoJson):
 197     struct = 'cmd_element'
 198
 199     cmd_attrs = { 0: None, 1: 'deprecated', 2: 'hidden'}
 200
 201     def __init__(self, *args, **kwargs):
 202         super().__init__(*args, **kwargs)
 203
 204     def to_dict(self, xrelfo):
 205         jsobj = xrelfo['cli'].setdefault(self.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 206
 207         jsobj.update({
 208             'string': self.string,
 209             'doc': self.doc,
 210             'attr': self.cmd_attrs.get(self.attr, self.attr),
 211         })
 212         if jsobj['attr'] is None:
 213             del jsobj['attr']
 214
 215         jsobj['defun'] = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 216
 217 Xref.containers[XREFT_DEFUN] = CmdElement
 218
 219 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 220     struct = 'xref_install_element'
 221
 222     def to_dict(self, xrelfo):
 223         jsobj = xrelfo['cli'].setdefault(self.cmd_element.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 224         nodes = jsobj.setdefault('nodes', [])
 225
 226         nodes.append({
 227             'node': self.node_type,
 228             'install': dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]),
 229         })
 230
 231 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 232
 233 # shove in field defs
 234 fieldapply = FieldApplicator(xrefstructs)
 235 fieldapply.add(Xref)
 236 fieldapply.add(Xrefdata)
 237 fieldapply.add(XrefLogmsg)
 238 fieldapply.add(XrefThreadSched)
 239 fieldapply.add(CmdElement)
 240 fieldapply.add(XrefInstallElement)
 241 fieldapply()
 242
 243
 244 class Xrelfo(dict):
 245     def __init__(self):
 246         super().__init__({
 247             'refs': {},
 248             'cli': {},
 249         })
 250         self._xrefs = []
 251
 252     def load_file(self, filename):
 253         orig_filename = filename
 254         if filename.endswith('.la') or filename.endswith('.lo'):
 255             with open(filename, 'r') as fd:
 256                 for line in fd:
 257                     line = line.strip()
 258                     if line.startswith('#') or line == '' or '=' not in line:
 259                         continue
 260
 261                     var, val = line.split('=', 1)
 262                     if var not in ['library_names', 'pic_object']:
 263                         continue
 264                     if val.startswith("'") or val.startswith('"'):
 265                         val = val[1:-1]
 266
 267                     if var == 'pic_object':
 268                         filename = os.path.join(os.path.dirname(filename), val)
 269                         break
 270
 271                     val = val.strip().split()[0]
 272                     filename = os.path.join(os.path.dirname(filename), '.libs', val)
 273                     break
 274                 else:
 275                     raise ValueError('could not process libtool file "%s"' % orig_filename)
 276
 277         while True:
 278             with open(filename, 'rb') as fd:
 279                 hdr = fd.read(4)
 280
 281             if hdr == b'\x7fELF':
 282                 self.load_elf(filename, orig_filename)
 283                 return
 284
 285             if hdr[:2] == b'#!':
 286                 path, name = os.path.split(filename)
 287                 filename = os.path.join(path, '.libs', name)
 288                 continue
 289
 290             if hdr[:1] == b'{':
 291                 with open(filename, 'r') as fd:
 292                     self.load_json(fd)
 293                 return
 294
 295             raise ValueError('cannot determine file type for %s' % (filename))
 296
 297     def load_elf(self, filename, orig_filename):
 298         edf = ELFDissectFile(filename)
 299         edf.orig_filename = orig_filename
 300
 301         note = edf._elffile.find_note('FRRouting', 'XREF')
 302         if note is not None:
 303             endian = '>' if edf._elffile.bigendian else '<'
 304             mem = edf._elffile[note]
 305             if edf._elffile.elfclass == 64:
 306                 start, end = struct.unpack(endian + 'QQ', mem)
 307                 start += note.start
 308                 end += note.start + 8
 309             else:
 310                 start, end = struct.unpack(endian + 'II', mem)
 311                 start += note.start
 312                 end += note.start + 4
 313
 314             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 315
 316         else:
 317             xrefarray = edf.get_section('xref_array')
 318             if xrefarray is None:
 319                 raise ValueError('file has neither xref note nor xref_array section')
 320
 321             ptrs = xrefarray.iter_data(XrefPtr)
 322
 323         for ptr in ptrs:
 324             if ptr.xref is None:
 325                 print('NULL xref')
 326                 continue
 327             self._xrefs.append(ptr.xref)
 328
 329             container = ptr.xref.container()
 330             if container is None:
 331                 continue
 332             container.to_dict(self)
 333
 334         return edf
 335
 336     def load_json(self, fd):
 337         data = json.load(fd)
 338         for uid, items in data['refs'].items():
 339             myitems = self['refs'].setdefault(uid, [])
 340             for item in items:
 341                 if item in myitems:
 342                     continue
 343                 myitems.append(item)
 344
 345         for cmd, items in data['cli'].items():
 346             self['cli'].setdefault(cmd, {}).update(items)
 347
 348         return data
 349
 350     def check(self, checks):
 351         for xref in self._xrefs:
 352             yield from xref.check(checks)
 353
 354 def main():
 355     argp = argparse.ArgumentParser(description = 'FRR xref ELF extractor')
 356     argp.add_argument('-o', dest='output', type=str, help='write JSON output')
 357     argp.add_argument('--out-by-file',     type=str, help='write by-file JSON output')
 358     argp.add_argument('-Wlog-format',      action='store_const', const=True)
 359     argp.add_argument('-Wlog-args',        action='store_const', const=True)
 360     argp.add_argument('--profile',         action='store_const', const=True)
 361     argp.add_argument('binaries', metavar='BINARY', nargs='+', type=str, help='files to read (ELF files or libtool objects)')
 362     args = argp.parse_args()
 363
 364     if args.profile:
 365         import cProfile
 366         cProfile.runctx('_main(args)', globals(), {'args': args}, sort='cumtime')
 367     else:
 368         _main(args)
 369
 370 def _main(args):
 371     errors = 0
 372     xrelfo = Xrelfo()
 373
 374     for fn in args.binaries:
 375         try:
 376             xrelfo.load_file(fn)
 377         except:
 378             errors += 1
 379             sys.stderr.write('while processing %s:\n' % (fn))
 380             traceback.print_exc()
 381
 382     for option in dir(args):
 383         if option.startswith('W'):
 384             checks = sorted(xrelfo.check(args))
 385             sys.stderr.write(''.join([c[-1] for c in checks]))
 386             break
 387
 388
 389     refs = xrelfo['refs']
 390
 391     counts = {}
 392     for k, v in refs.items():
 393         strs = set([i['fmtstring'] for i in v])
 394         if len(strs) != 1:
 395             print('\033[31;1m%s\033[m' % k)
 396         counts[k] = len(v)
 397
 398     out = xrelfo
 399     outbyfile = {}
 400     for uid, locs in refs.items():
 401         for loc in locs:
 402             filearray = outbyfile.setdefault(loc['file'], [])
 403             loc = dict(loc)
 404             del loc['file']
 405             filearray.append(loc)
 406
 407     for k in outbyfile.keys():
 408         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x['line'])
 409
 410     if errors:
 411         sys.exit(1)
 412
 413     if args.output:
 414         with open(args.output + '.tmp', 'w') as fd:
 415             json.dump(out, fd, indent=2, sort_keys=True)
 416         os.rename(args.output + '.tmp', args.output)
 417
 418     if args.out_by_file:
 419         with open(args.out_by_file + '.tmp', 'w') as fd:
 420             json.dump(outbyfile, fd, indent=2, sort_keys=True)
 421         os.rename(args.out_by_file + '.tmp', args.out_by_file)
 422
 423 if __name__ == '__main__':
 424     main()