python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24
  25 json_dump_args = {}
  26
  27 try:
  28     import ujson as json
  29
  30     json_dump_args["escape_forward_slashes"] = False
  31 except ImportError:
  32     import json
  33
  34 import argparse
  35
  36 from clippy.uidhash import uidhash
  37 from clippy.elf import *
  38 from clippy import frr_top_src, CmdAttr
  39 from tiabwarfo import FieldApplicator
  40
  41 try:
  42     with open(os.path.join(frr_top_src, 'python', 'xrefstructs.json'), 'r') as fd:
  43         xrefstructs = json.load(fd)
  44 except FileNotFoundError:
  45     sys.stderr.write('''
  46 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  47 tool available) could not be found.  It should be included with the sources.
  48 ''')
  49     sys.exit(1)
  50
  51 # constants, need to be kept in sync manually...
  52
  53 XREFT_THREADSCHED = 0x100
  54 XREFT_LOGMSG = 0x200
  55 XREFT_DEFUN = 0x300
  56 XREFT_INSTALL_ELEMENT = 0x301
  57
  58 # LOG_*
  59 priovals = {}
  60 prios = ['0', '1', '2', 'E', 'W', 'N', 'I', 'D']
  61
  62
  63 class XrelfoJson(object):
  64     def dump(self):
  65         pass
  66
  67     def check(self, wopt):
  68         yield from []
  69
  70     def to_dict(self, refs):
  71         pass
  72
  73 class Xref(ELFDissectStruct, XrelfoJson):
  74     struct = 'xref'
  75     fieldrename = {'type': 'typ'}
  76     containers = {}
  77
  78     def __init__(self, *args, **kwargs):
  79         super().__init__(*args, **kwargs)
  80
  81         self._container = None
  82         if self.xrefdata:
  83             self.xrefdata.ref_from(self, self.typ)
  84
  85     def container(self):
  86         if self._container is None:
  87             if self.typ in self.containers:
  88                 self._container = self.container_of(self.containers[self.typ], 'xref')
  89         return self._container
  90
  91     def check(self, *args, **kwargs):
  92         if self._container:
  93             yield from self._container.check(*args, **kwargs)
  94
  95
  96 class Xrefdata(ELFDissectStruct):
  97     struct = 'xrefdata'
  98
  99     # uid is all zeroes in the data loaded from ELF
 100     fieldrename = {'uid': '_uid'}
 101
 102     def ref_from(self, xref, typ):
 103         self.xref = xref
 104
 105     @property
 106     def uid(self):
 107         if self.hashstr is None:
 108             return None
 109         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 110
 111 class XrefPtr(ELFDissectStruct):
 112     fields = [
 113         ('xref', 'P', Xref),
 114     ]
 115
 116 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 117     struct = 'xref_threadsched'
 118 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 119
 120 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 121     struct = 'xref_logmsg'
 122
 123     def _warn_fmt(self, text):
 124         lines = text.split('\n')
 125         yield ((self.xref.file, self.xref.line), '%s:%d: %s (in %s())%s\n' % (self.xref.file, self.xref.line, lines[0], self.xref.func, ''.join(['\n' + l for l in lines[1:]])))
 126
 127     fmt_regexes = [
 128         (re.compile(r'([\n\t]+)'), 'error: log message contains tab or newline'),
 129     #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 130         (re.compile(r'^((?:warn(?:ing)?|error):\s*)', re.I), 'warning: log message starts with severity'),
 131     ]
 132     arg_regexes = [
 133     # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)"
 134         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)'),   'cleanup: replace inet_ntop(AF_INET, ...) with %pI4',  lambda s: True),
 135         (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)'), 'cleanup: replace inet_ntop(AF_INET6, ...) with %pI6', lambda s: True),
 136         (re.compile(r'((?<![\?:] )inet_ntoa)'),                               'cleanup: replace inet_ntoa(...) with %pI4',           lambda s: True),
 137         (re.compile(r'((?<![\?:] )ipaddr2str)'),                              'cleanup: replace ipaddr2str(...) with %pIA',          lambda s: True),
 138         (re.compile(r'((?<![\?:] )prefix2str)'),                              'cleanup: replace prefix2str(...) with %pFX',          lambda s: True),
 139         (re.compile(r'((?<![\?:] )prefix_mac2str)'),                          'cleanup: replace prefix_mac2str(...) with %pEA',      lambda s: True),
 140         (re.compile(r'((?<![\?:] )sockunion2str)'),                           'cleanup: replace sockunion2str(...) with %pSU',       lambda s: True),
 141
 142     #   (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ),
 143     ]
 144
 145     def check(self, wopt):
 146         def fmt_msg(rex, itext):
 147             if sys.stderr.isatty():
 148                 items = rex.split(itext)
 149                 out = []
 150                 for i, text in enumerate(items):
 151                     if (i % 2) == 1:
 152                         out.append('\033[41;37;1m%s\033[m' % repr(text)[1:-1])
 153                     else:
 154                         out.append(repr(text)[1:-1])
 155
 156                 excerpt = ''.join(out)
 157             else:
 158                 excerpt = repr(itext)[1:-1]
 159             return excerpt
 160
 161         if wopt.Wlog_format:
 162             for rex, msg in self.fmt_regexes:
 163                 if not rex.search(self.fmtstring):
 164                     continue
 165
 166                 excerpt = fmt_msg(rex, self.fmtstring)
 167                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 168
 169         if wopt.Wlog_args:
 170             for rex, msg, cond in self.arg_regexes:
 171                 if not cond(self):
 172                     continue
 173                 if not rex.search(self.args):
 174                     continue
 175
 176                 excerpt = fmt_msg(rex, self.args)
 177                 yield from self._warn_fmt('%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt))
 178
 179     def dump(self):
 180         print('%-60s %s%s %-25s [EC %d] %s' % (
 181             '%s:%d %s()' % (self.xref.file, self.xref.line, self.xref.func),
 182             prios[self.priority & 7],
 183             priovals.get(self.priority & 0x30, ' '),
 184             self.xref.xrefdata.uid, self.ec, self.fmtstring))
 185
 186     def to_dict(self, xrelfo):
 187         jsobj = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 188         if self.ec != 0:
 189             jsobj['ec'] = self.ec
 190         jsobj['fmtstring'] = self.fmtstring
 191         jsobj['args'] = self.args
 192         jsobj['priority'] = self.priority & 7
 193         jsobj['type'] = 'logmsg'
 194         jsobj['binary'] = self._elfsect._elfwrap.orig_filename
 195
 196         if self.priority & 0x10:
 197             jsobj.setdefault('flags', []).append('errno')
 198         if self.priority & 0x20:
 199             jsobj.setdefault('flags', []).append('getaddrinfo')
 200
 201         xrelfo['refs'].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 202
 203 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 204
 205 class CmdElement(ELFDissectStruct, XrelfoJson):
 206     struct = 'cmd_element'
 207
 208     def __init__(self, *args, **kwargs):
 209         super().__init__(*args, **kwargs)
 210
 211     def to_dict(self, xrelfo):
 212         jsobj = xrelfo['cli'].setdefault(self.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 213
 214         jsobj.update({
 215             'string': self.string,
 216             'doc': self.doc,
 217         })
 218         if self.attr:
 219             jsobj['attr'] = attr = self.attr
 220             for attrname in CmdAttr.__members__:
 221                 val = CmdAttr[attrname]
 222                 if attr & val:
 223                     jsobj.setdefault('attrs', []).append(attrname.lower())
 224                     attr &= ~val
 225
 226         jsobj['defun'] = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 227
 228 Xref.containers[XREFT_DEFUN] = CmdElement
 229
 230 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 231     struct = 'xref_install_element'
 232
 233     def to_dict(self, xrelfo):
 234         jsobj = xrelfo['cli'].setdefault(self.cmd_element.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 235         nodes = jsobj.setdefault('nodes', [])
 236
 237         nodes.append({
 238             'node': self.node_type,
 239             'install': dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]),
 240         })
 241
 242 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 243
 244 # shove in field defs
 245 fieldapply = FieldApplicator(xrefstructs)
 246 fieldapply.add(Xref)
 247 fieldapply.add(Xrefdata)
 248 fieldapply.add(XrefLogmsg)
 249 fieldapply.add(XrefThreadSched)
 250 fieldapply.add(CmdElement)
 251 fieldapply.add(XrefInstallElement)
 252 fieldapply()
 253
 254
 255 class Xrelfo(dict):
 256     def __init__(self):
 257         super().__init__({
 258             'refs': {},
 259             'cli': {},
 260         })
 261         self._xrefs = []
 262
 263     def load_file(self, filename):
 264         orig_filename = filename
 265         if filename.endswith('.la') or filename.endswith('.lo'):
 266             with open(filename, 'r') as fd:
 267                 for line in fd:
 268                     line = line.strip()
 269                     if line.startswith('#') or line == '' or '=' not in line:
 270                         continue
 271
 272                     var, val = line.split('=', 1)
 273                     if var not in ['library_names', 'pic_object']:
 274                         continue
 275                     if val.startswith("'") or val.startswith('"'):
 276                         val = val[1:-1]
 277
 278                     if var == 'pic_object':
 279                         filename = os.path.join(os.path.dirname(filename), val)
 280                         break
 281
 282                     val = val.strip().split()[0]
 283                     filename = os.path.join(os.path.dirname(filename), '.libs', val)
 284                     break
 285                 else:
 286                     raise ValueError('could not process libtool file "%s"' % orig_filename)
 287
 288         while True:
 289             with open(filename, 'rb') as fd:
 290                 hdr = fd.read(4)
 291
 292             if hdr == b'\x7fELF':
 293                 self.load_elf(filename, orig_filename)
 294                 return
 295
 296             if hdr[:2] == b'#!':
 297                 path, name = os.path.split(filename)
 298                 filename = os.path.join(path, '.libs', name)
 299                 continue
 300
 301             if hdr[:1] == b'{':
 302                 with open(filename, 'r') as fd:
 303                     self.load_json(fd)
 304                 return
 305
 306             raise ValueError('cannot determine file type for %s' % (filename))
 307
 308     def load_elf(self, filename, orig_filename):
 309         edf = ELFDissectFile(filename)
 310         edf.orig_filename = orig_filename
 311
 312         note = edf._elffile.find_note('FRRouting', 'XREF')
 313         if note is not None:
 314             endian = '>' if edf._elffile.bigendian else '<'
 315             mem = edf._elffile[note]
 316             if edf._elffile.elfclass == 64:
 317                 start, end = struct.unpack(endian + 'QQ', mem)
 318                 start += note.start
 319                 end += note.start + 8
 320             else:
 321                 start, end = struct.unpack(endian + 'II', mem)
 322                 start += note.start
 323                 end += note.start + 4
 324
 325             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 326
 327         else:
 328             xrefarray = edf.get_section('xref_array')
 329             if xrefarray is None:
 330                 raise ValueError('file has neither xref note nor xref_array section')
 331
 332             ptrs = xrefarray.iter_data(XrefPtr)
 333
 334         for ptr in ptrs:
 335             if ptr.xref is None:
 336                 print('NULL xref')
 337                 continue
 338             self._xrefs.append(ptr.xref)
 339
 340             container = ptr.xref.container()
 341             if container is None:
 342                 continue
 343             container.to_dict(self)
 344
 345         return edf
 346
 347     def load_json(self, fd):
 348         data = json.load(fd)
 349         for uid, items in data['refs'].items():
 350             myitems = self['refs'].setdefault(uid, [])
 351             for item in items:
 352                 if item in myitems:
 353                     continue
 354                 myitems.append(item)
 355
 356         for cmd, items in data['cli'].items():
 357             self['cli'].setdefault(cmd, {}).update(items)
 358
 359         return data
 360
 361     def check(self, checks):
 362         for xref in self._xrefs:
 363             yield from xref.check(checks)
 364
 365 def main():
 366     argp = argparse.ArgumentParser(description = 'FRR xref ELF extractor')
 367     argp.add_argument('-o', dest='output', type=str, help='write JSON output')
 368     argp.add_argument('--out-by-file',     type=str, help='write by-file JSON output')
 369     argp.add_argument('-Wlog-format',      action='store_const', const=True)
 370     argp.add_argument('-Wlog-args',        action='store_const', const=True)
 371     argp.add_argument('-Werror',           action='store_const', const=True)
 372     argp.add_argument('--profile',         action='store_const', const=True)
 373     argp.add_argument('binaries', metavar='BINARY', nargs='+', type=str, help='files to read (ELF files or libtool objects)')
 374     args = argp.parse_args()
 375
 376     if args.profile:
 377         import cProfile
 378         cProfile.runctx('_main(args)', globals(), {'args': args}, sort='cumtime')
 379     else:
 380         _main(args)
 381
 382 def _main(args):
 383     errors = 0
 384     xrelfo = Xrelfo()
 385
 386     for fn in args.binaries:
 387         try:
 388             xrelfo.load_file(fn)
 389         except:
 390             errors += 1
 391             sys.stderr.write('while processing %s:\n' % (fn))
 392             traceback.print_exc()
 393
 394     for option in dir(args):
 395         if option.startswith('W') and option != 'Werror':
 396             checks = sorted(xrelfo.check(args))
 397             sys.stderr.write(''.join([c[-1] for c in checks]))
 398
 399             if args.Werror and len(checks) > 0:
 400                 errors += 1
 401             break
 402
 403
 404     refs = xrelfo['refs']
 405
 406     counts = {}
 407     for k, v in refs.items():
 408         strs = set([i['fmtstring'] for i in v])
 409         if len(strs) != 1:
 410             print('\033[31;1m%s\033[m' % k)
 411         counts[k] = len(v)
 412
 413     out = xrelfo
 414     outbyfile = {}
 415     for uid, locs in refs.items():
 416         for loc in locs:
 417             filearray = outbyfile.setdefault(loc['file'], [])
 418             loc = dict(loc)
 419             del loc['file']
 420             filearray.append(loc)
 421
 422     for k in outbyfile.keys():
 423         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x['line'])
 424
 425     if errors:
 426         sys.exit(1)
 427
 428     if args.output:
 429         with open(args.output + '.tmp', 'w') as fd:
 430             json.dump(out, fd, indent=2, sort_keys=True, **json_dump_args)
 431         os.rename(args.output + '.tmp', args.output)
 432
 433     if args.out_by_file:
 434         with open(args.out_by_file + '.tmp', 'w') as fd:
 435             json.dump(outbyfile, fd, indent=2, sort_keys=True, **json_dump_args)
 436         os.rename(args.out_by_file + '.tmp', args.out_by_file)
 437
 438 if __name__ == '__main__':
 439     main()