python/xrelfo.py

   1 # FRR ELF xref extractor
   2 #
   3 # Copyright (C) 2020  David Lamparter for NetDEF, Inc.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License as published by the Free
   7 # Software Foundation; either version 2 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; see the file COPYING; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  18
  19 import sys
  20 import os
  21 import struct
  22 import re
  23 import traceback
  24 import json
  25 import argparse
  26
  27 from clippy.uidhash import uidhash
  28 from clippy.elf import *
  29 from clippy import frr_top_src
  30 from tiabwarfo import FieldApplicator
  31
  32 try:
  33     with open(os.path.join(frr_top_src, 'python', 'xrefstructs.json'), 'r') as fd:
  34         xrefstructs = json.load(fd)
  35 except FileNotFoundError:
  36     sys.stderr.write('''
  37 The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole
  38 tool available) could not be found.  It should be included with the sources.
  39 ''')
  40     sys.exit(1)
  41
  42 # constants, need to be kept in sync manually...
  43
  44 XREFT_THREADSCHED = 0x100
  45 XREFT_LOGMSG = 0x200
  46 XREFT_DEFUN = 0x300
  47 XREFT_INSTALL_ELEMENT = 0x301
  48
  49 # LOG_*
  50 priovals = {}
  51 prios = ['0', '1', '2', 'E', 'W', 'N', 'I', 'D']
  52
  53
  54 class XrelfoJson(object):
  55     def dump(self):
  56         pass
  57
  58     def check(self, wopt):
  59         yield from []
  60
  61     def to_dict(self, refs):
  62         pass
  63
  64 class Xref(ELFDissectStruct, XrelfoJson):
  65     struct = 'xref'
  66     fieldrename = {'type': 'typ'}
  67     containers = {}
  68
  69     def __init__(self, *args, **kwargs):
  70         super().__init__(*args, **kwargs)
  71
  72         self._container = None
  73         if self.xrefdata:
  74             self.xrefdata.ref_from(self, self.typ)
  75
  76     def container(self):
  77         if self._container is None:
  78             if self.typ in self.containers:
  79                 self._container = self.container_of(self.containers[self.typ], 'xref')
  80         return self._container
  81
  82     def check(self, *args, **kwargs):
  83         if self._container:
  84             yield from self._container.check(*args, **kwargs)
  85
  86
  87 class Xrefdata(ELFDissectStruct):
  88     struct = 'xrefdata'
  89
  90     # uid is all zeroes in the data loaded from ELF
  91     fieldrename = {'uid': '_uid'}
  92
  93     def ref_from(self, xref, typ):
  94         self.xref = xref
  95
  96     @property
  97     def uid(self):
  98         if self.hashstr is None:
  99             return None
 100         return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1)
 101
 102 class XrefPtr(ELFDissectStruct):
 103     fields = [
 104         ('xref', 'P', Xref),
 105     ]
 106
 107 class XrefThreadSched(ELFDissectStruct, XrelfoJson):
 108     struct = 'xref_threadsched'
 109 Xref.containers[XREFT_THREADSCHED] = XrefThreadSched
 110
 111 class XrefLogmsg(ELFDissectStruct, XrelfoJson):
 112     struct = 'xref_logmsg'
 113
 114     def _warn_fmt(self, text):
 115         yield ((self.xref.file, self.xref.line), '%s:%d: %s (in %s())\n' % (self.xref.file, self.xref.line, text, self.xref.func))
 116
 117     regexes = [
 118         (re.compile(r'([\n\t]+)'), 'error: log message contains tab or newline'),
 119     #    (re.compile(r'^(\s+)'),   'warning: log message starts with whitespace'),
 120         (re.compile(r'^((?:warn(?:ing)?|error):\s*)', re.I), 'warning: log message starts with severity'),
 121     ]
 122
 123     def check(self, wopt):
 124         if wopt.Wlog_format:
 125             for rex, msg in self.regexes:
 126                 if not rex.search(self.fmtstring):
 127                     continue
 128
 129                 if sys.stderr.isatty():
 130                     items = rex.split(self.fmtstring)
 131                     out = []
 132                     for i, text in enumerate(items):
 133                         if (i % 2) == 1:
 134                             out.append('\033[41;37;1m%s\033[m' % repr(text)[1:-1])
 135                         else:
 136                             out.append(repr(text)[1:-1])
 137
 138                     excerpt = ''.join(out)
 139
 140                 else:
 141                     excerpt = repr(self.fmtstring)[1:-1]
 142
 143                 yield from self._warn_fmt('%s: "%s"' % (msg, excerpt))
 144
 145     def dump(self):
 146         print('%-60s %s%s %-25s [EC %d] %s' % (
 147             '%s:%d %s()' % (self.xref.file, self.xref.line, self.xref.func),
 148             prios[self.priority & 7],
 149             priovals.get(self.priority & 0x30, ' '),
 150             self.xref.xrefdata.uid, self.ec, self.fmtstring))
 151
 152     def to_dict(self, xrelfo):
 153         jsobj = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 154         if self.ec != 0:
 155             jsobj['ec'] = self.ec
 156         jsobj['fmtstring'] = self.fmtstring
 157         jsobj['priority'] = self.priority & 7
 158         jsobj['type'] = 'logmsg'
 159         jsobj['binary'] = self._elfsect._elfwrap.orig_filename
 160
 161         if self.priority & 0x10:
 162             jsobj.setdefault('flags', []).append('errno')
 163         if self.priority & 0x20:
 164             jsobj.setdefault('flags', []).append('getaddrinfo')
 165
 166         xrelfo['refs'].setdefault(self.xref.xrefdata.uid, []).append(jsobj)
 167
 168 Xref.containers[XREFT_LOGMSG] = XrefLogmsg
 169
 170 class CmdElement(ELFDissectStruct, XrelfoJson):
 171     struct = 'cmd_element'
 172
 173     cmd_attrs = { 0: None, 1: 'deprecated', 2: 'hidden'}
 174
 175     def __init__(self, *args, **kwargs):
 176         super().__init__(*args, **kwargs)
 177
 178     def to_dict(self, xrelfo):
 179         jsobj = xrelfo['cli'].setdefault(self.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 180
 181         jsobj.update({
 182             'string': self.string,
 183             'doc': self.doc,
 184             'attr': self.cmd_attrs.get(self.attr, self.attr),
 185         })
 186         if jsobj['attr'] is None:
 187             del jsobj['attr']
 188
 189         jsobj['defun'] = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']])
 190
 191 Xref.containers[XREFT_DEFUN] = CmdElement
 192
 193 class XrefInstallElement(ELFDissectStruct, XrelfoJson):
 194     struct = 'xref_install_element'
 195
 196     def to_dict(self, xrelfo):
 197         jsobj = xrelfo['cli'].setdefault(self.cmd_element.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {})
 198         nodes = jsobj.setdefault('nodes', [])
 199
 200         nodes.append({
 201             'node': self.node_type,
 202             'install': dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]),
 203         })
 204
 205 Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement
 206
 207 # shove in field defs
 208 fieldapply = FieldApplicator(xrefstructs)
 209 fieldapply.add(Xref)
 210 fieldapply.add(Xrefdata)
 211 fieldapply.add(XrefLogmsg)
 212 fieldapply.add(XrefThreadSched)
 213 fieldapply.add(CmdElement)
 214 fieldapply.add(XrefInstallElement)
 215 fieldapply()
 216
 217
 218 class Xrelfo(dict):
 219     def __init__(self):
 220         super().__init__({
 221             'refs': {},
 222             'cli': {},
 223         })
 224         self._xrefs = []
 225
 226     def load_file(self, filename):
 227         orig_filename = filename
 228         if filename.endswith('.la') or filename.endswith('.lo'):
 229             with open(filename, 'r') as fd:
 230                 for line in fd:
 231                     line = line.strip()
 232                     if line.startswith('#') or line == '' or '=' not in line:
 233                         continue
 234
 235                     var, val = line.split('=', 1)
 236                     if var not in ['library_names', 'pic_object']:
 237                         continue
 238                     if val.startswith("'") or val.startswith('"'):
 239                         val = val[1:-1]
 240
 241                     if var == 'pic_object':
 242                         filename = os.path.join(os.path.dirname(filename), val)
 243                         break
 244
 245                     val = val.strip().split()[0]
 246                     filename = os.path.join(os.path.dirname(filename), '.libs', val)
 247                     break
 248                 else:
 249                     raise ValueError('could not process libtool file "%s"' % orig_filename)
 250
 251         while True:
 252             with open(filename, 'rb') as fd:
 253                 hdr = fd.read(4)
 254
 255             if hdr == b'\x7fELF':
 256                 self.load_elf(filename, orig_filename)
 257                 return
 258
 259             if hdr[:2] == b'#!':
 260                 path, name = os.path.split(filename)
 261                 filename = os.path.join(path, '.libs', name)
 262                 continue
 263
 264             if hdr[:1] == b'{':
 265                 with open(filename, 'r') as fd:
 266                     self.load_json(fd)
 267                 return
 268
 269             raise ValueError('cannot determine file type for %s' % (filename))
 270
 271     def load_elf(self, filename, orig_filename):
 272         edf = ELFDissectFile(filename)
 273         edf.orig_filename = orig_filename
 274
 275         note = edf._elffile.find_note('FRRouting', 'XREF')
 276         if note is not None:
 277             endian = '>' if edf._elffile.bigendian else '<'
 278             mem = edf._elffile[note]
 279             if edf._elffile.elfclass == 64:
 280                 start, end = struct.unpack(endian + 'QQ', mem)
 281                 start += note.start
 282                 end += note.start + 8
 283             else:
 284                 start, end = struct.unpack(endian + 'II', mem)
 285                 start += note.start
 286                 end += note.start + 4
 287
 288             ptrs = edf.iter_data(XrefPtr, slice(start, end))
 289
 290         else:
 291             xrefarray = edf.get_section('xref_array')
 292             if xrefarray is None:
 293                 raise ValueError('file has neither xref note nor xref_array section')
 294
 295             ptrs = xrefarray.iter_data(XrefPtr)
 296
 297         for ptr in ptrs:
 298             if ptr.xref is None:
 299                 print('NULL xref')
 300                 continue
 301             self._xrefs.append(ptr.xref)
 302
 303             container = ptr.xref.container()
 304             if container is None:
 305                 continue
 306             container.to_dict(self)
 307
 308         return edf
 309
 310     def load_json(self, fd):
 311         data = json.load(fd)
 312         for uid, items in data['refs'].items():
 313             myitems = self['refs'].setdefault(uid, [])
 314             for item in items:
 315                 if item in myitems:
 316                     continue
 317                 myitems.append(item)
 318
 319         for cmd, items in data['cli'].items():
 320             self['cli'].setdefault(cmd, {}).update(items)
 321
 322         return data
 323
 324     def check(self, checks):
 325         for xref in self._xrefs:
 326             yield from xref.check(checks)
 327
 328 def main():
 329     argp = argparse.ArgumentParser(description = 'FRR xref ELF extractor')
 330     argp.add_argument('-o', dest='output', type=str, help='write JSON output')
 331     argp.add_argument('--out-by-file',     type=str, help='write by-file JSON output')
 332     argp.add_argument('-Wlog-format',      action='store_const', const=True)
 333     argp.add_argument('--profile',         action='store_const', const=True)
 334     argp.add_argument('binaries', metavar='BINARY', nargs='+', type=str, help='files to read (ELF files or libtool objects)')
 335     args = argp.parse_args()
 336
 337     if args.profile:
 338         import cProfile
 339         cProfile.runctx('_main(args)', globals(), {'args': args}, sort='cumtime')
 340     else:
 341         _main(args)
 342
 343 def _main(args):
 344     errors = 0
 345     xrelfo = Xrelfo()
 346
 347     for fn in args.binaries:
 348         try:
 349             xrelfo.load_file(fn)
 350         except:
 351             errors += 1
 352             sys.stderr.write('while processing %s:\n' % (fn))
 353             traceback.print_exc()
 354
 355     for option in dir(args):
 356         if option.startswith('W'):
 357             checks = sorted(xrelfo.check(args))
 358             sys.stderr.write(''.join([c[-1] for c in checks]))
 359             break
 360
 361
 362     refs = xrelfo['refs']
 363
 364     counts = {}
 365     for k, v in refs.items():
 366         strs = set([i['fmtstring'] for i in v])
 367         if len(strs) != 1:
 368             print('\033[31;1m%s\033[m' % k)
 369         counts[k] = len(v)
 370
 371     out = xrelfo
 372     outbyfile = {}
 373     for uid, locs in refs.items():
 374         for loc in locs:
 375             filearray = outbyfile.setdefault(loc['file'], [])
 376             loc = dict(loc)
 377             del loc['file']
 378             filearray.append(loc)
 379
 380     for k in outbyfile.keys():
 381         outbyfile[k] = sorted(outbyfile[k], key=lambda x: x['line'])
 382
 383     if errors:
 384         sys.exit(1)
 385
 386     if args.output:
 387         with open(args.output + '.tmp', 'w') as fd:
 388             json.dump(out, fd, indent=2, sort_keys=True)
 389         os.rename(args.output + '.tmp', args.output)
 390
 391     if args.out_by_file:
 392         with open(args.out_by_file + '.tmp', 'w') as fd:
 393             json.dump(outbyfile, fd, indent=2, sort_keys=True)
 394         os.rename(args.out_by_file + '.tmp', args.out_by_file)
 395
 396 if __name__ == '__main__':
 397     main()