]>
Commit | Line | Data |
---|---|---|
36a8fdfd DL |
1 | # FRR ELF xref extractor |
2 | # | |
3 | # Copyright (C) 2020 David Lamparter for NetDEF, Inc. | |
4 | # | |
5 | # This program is free software; you can redistribute it and/or modify it | |
6 | # under the terms of the GNU General Public License as published by the Free | |
7 | # Software Foundation; either version 2 of the License, or (at your option) | |
8 | # any later version. | |
9 | # | |
10 | # This program is distributed in the hope that it will be useful, but WITHOUT | |
11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | # more details. | |
14 | # | |
15 | # You should have received a copy of the GNU General Public License along | |
16 | # with this program; see the file COPYING; if not, write to the Free Software | |
17 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | |
18 | ||
19 | import sys | |
20 | import os | |
21 | import struct | |
22 | import re | |
23 | import traceback | |
24 | import json | |
25 | import argparse | |
26 | ||
27 | from clippy.uidhash import uidhash | |
28 | from clippy.elf import * | |
29 | from clippy import frr_top_src | |
30 | from tiabwarfo import FieldApplicator | |
31 | ||
32 | try: | |
33 | with open(os.path.join(frr_top_src, 'python', 'xrefstructs.json'), 'r') as fd: | |
34 | xrefstructs = json.load(fd) | |
35 | except FileNotFoundError: | |
36 | sys.stderr.write(''' | |
37 | The "xrefstructs.json" file (created by running tiabwarfo.py with the pahole | |
38 | tool available) could not be found. It should be included with the sources. | |
39 | ''') | |
40 | sys.exit(1) | |
41 | ||
42 | # constants, need to be kept in sync manually... | |
43 | ||
44 | XREFT_THREADSCHED = 0x100 | |
45 | XREFT_LOGMSG = 0x200 | |
46 | XREFT_DEFUN = 0x300 | |
47 | XREFT_INSTALL_ELEMENT = 0x301 | |
48 | ||
49 | # LOG_* | |
50 | priovals = {} | |
51 | prios = ['0', '1', '2', 'E', 'W', 'N', 'I', 'D'] | |
52 | ||
53 | ||
54 | class XrelfoJson(object): | |
55 | def dump(self): | |
56 | pass | |
57 | ||
58 | def check(self, wopt): | |
59 | yield from [] | |
60 | ||
61 | def to_dict(self, refs): | |
62 | pass | |
63 | ||
64 | class Xref(ELFDissectStruct, XrelfoJson): | |
65 | struct = 'xref' | |
66 | fieldrename = {'type': 'typ'} | |
67 | containers = {} | |
68 | ||
69 | def __init__(self, *args, **kwargs): | |
70 | super().__init__(*args, **kwargs) | |
71 | ||
72 | self._container = None | |
73 | if self.xrefdata: | |
74 | self.xrefdata.ref_from(self, self.typ) | |
75 | ||
76 | def container(self): | |
77 | if self._container is None: | |
78 | if self.typ in self.containers: | |
79 | self._container = self.container_of(self.containers[self.typ], 'xref') | |
80 | return self._container | |
81 | ||
82 | def check(self, *args, **kwargs): | |
83 | if self._container: | |
84 | yield from self._container.check(*args, **kwargs) | |
85 | ||
86 | ||
87 | class Xrefdata(ELFDissectStruct): | |
88 | struct = 'xrefdata' | |
89 | ||
90 | # uid is all zeroes in the data loaded from ELF | |
91 | fieldrename = {'uid': '_uid'} | |
92 | ||
93 | def ref_from(self, xref, typ): | |
94 | self.xref = xref | |
95 | ||
96 | @property | |
97 | def uid(self): | |
98 | if self.hashstr is None: | |
99 | return None | |
100 | return uidhash(self.xref.file, self.hashstr, self.hashu32_0, self.hashu32_1) | |
101 | ||
102 | class XrefPtr(ELFDissectStruct): | |
103 | fields = [ | |
104 | ('xref', 'P', Xref), | |
105 | ] | |
106 | ||
107 | class XrefThreadSched(ELFDissectStruct, XrelfoJson): | |
108 | struct = 'xref_threadsched' | |
109 | Xref.containers[XREFT_THREADSCHED] = XrefThreadSched | |
110 | ||
111 | class XrefLogmsg(ELFDissectStruct, XrelfoJson): | |
112 | struct = 'xref_logmsg' | |
113 | ||
114 | def _warn_fmt(self, text): | |
2621bb8b DL |
115 | lines = text.split('\n') |
116 | yield ((self.xref.file, self.xref.line), '%s:%d: %s (in %s())%s\n' % (self.xref.file, self.xref.line, lines[0], self.xref.func, ''.join(['\n' + l for l in lines[1:]]))) | |
36a8fdfd | 117 | |
2621bb8b | 118 | fmt_regexes = [ |
36a8fdfd DL |
119 | (re.compile(r'([\n\t]+)'), 'error: log message contains tab or newline'), |
120 | # (re.compile(r'^(\s+)'), 'warning: log message starts with whitespace'), | |
121 | (re.compile(r'^((?:warn(?:ing)?|error):\s*)', re.I), 'warning: log message starts with severity'), | |
122 | ] | |
2621bb8b DL |
123 | arg_regexes = [ |
124 | # the (?<![\?:] ) avoids warning for x ? inet_ntop(...) : "(bla)" | |
125 | (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET|2)\s*,)'), 'cleanup: replace inet_ntop(AF_INET, ...) with %pI4', lambda s: True), | |
126 | (re.compile(r'((?<![\?:] )inet_ntop\s*\(\s*(?:[AP]F_INET6|10)\s*,)'), 'cleanup: replace inet_ntop(AF_INET6, ...) with %pI6', lambda s: True), | |
127 | (re.compile(r'((?<![\?:] )inet_ntoa)'), 'cleanup: replace inet_ntoa(...) with %pI4', lambda s: True), | |
128 | (re.compile(r'((?<![\?:] )ipaddr2str)'), 'cleanup: replace ipaddr2str(...) with %pIA', lambda s: True), | |
129 | (re.compile(r'((?<![\?:] )prefix2str)'), 'cleanup: replace prefix2str(...) with %pFX', lambda s: True), | |
130 | (re.compile(r'((?<![\?:] )prefix_mac2str)'), 'cleanup: replace prefix_mac2str(...) with %pEA', lambda s: True), | |
131 | (re.compile(r'((?<![\?:] )sockunion2str)'), 'cleanup: replace sockunion2str(...) with %pSU', lambda s: True), | |
132 | ||
133 | # (re.compile(r'^(\s*__(?:func|FUNCTION|PRETTY_FUNCTION)__\s*)'), 'error: debug message starts with __func__', lambda s: (s.priority & 7 == 7) ), | |
134 | ] | |
36a8fdfd DL |
135 | |
136 | def check(self, wopt): | |
2621bb8b DL |
137 | def fmt_msg(rex, itext): |
138 | if sys.stderr.isatty(): | |
139 | items = rex.split(itext) | |
140 | out = [] | |
141 | for i, text in enumerate(items): | |
142 | if (i % 2) == 1: | |
143 | out.append('\033[41;37;1m%s\033[m' % repr(text)[1:-1]) | |
144 | else: | |
145 | out.append(repr(text)[1:-1]) | |
146 | ||
147 | excerpt = ''.join(out) | |
148 | else: | |
149 | excerpt = repr(itext)[1:-1] | |
150 | return excerpt | |
151 | ||
36a8fdfd | 152 | if wopt.Wlog_format: |
2621bb8b | 153 | for rex, msg in self.fmt_regexes: |
36a8fdfd DL |
154 | if not rex.search(self.fmtstring): |
155 | continue | |
156 | ||
2621bb8b DL |
157 | excerpt = fmt_msg(rex, self.fmtstring) |
158 | yield from self._warn_fmt('%s: "%s"' % (msg, excerpt)) | |
36a8fdfd | 159 | |
2621bb8b DL |
160 | if wopt.Wlog_args: |
161 | for rex, msg, cond in self.arg_regexes: | |
162 | if not cond(self): | |
163 | continue | |
164 | if not rex.search(self.args): | |
165 | continue | |
36a8fdfd | 166 | |
2621bb8b DL |
167 | excerpt = fmt_msg(rex, self.args) |
168 | yield from self._warn_fmt('%s:\n\t"%s",\n\t%s' % (msg, repr(self.fmtstring)[1:-1], excerpt)) | |
36a8fdfd DL |
169 | |
170 | def dump(self): | |
171 | print('%-60s %s%s %-25s [EC %d] %s' % ( | |
172 | '%s:%d %s()' % (self.xref.file, self.xref.line, self.xref.func), | |
173 | prios[self.priority & 7], | |
174 | priovals.get(self.priority & 0x30, ' '), | |
175 | self.xref.xrefdata.uid, self.ec, self.fmtstring)) | |
176 | ||
177 | def to_dict(self, xrelfo): | |
178 | jsobj = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]) | |
179 | if self.ec != 0: | |
180 | jsobj['ec'] = self.ec | |
181 | jsobj['fmtstring'] = self.fmtstring | |
2621bb8b | 182 | jsobj['args'] = self.args |
36a8fdfd DL |
183 | jsobj['priority'] = self.priority & 7 |
184 | jsobj['type'] = 'logmsg' | |
185 | jsobj['binary'] = self._elfsect._elfwrap.orig_filename | |
186 | ||
187 | if self.priority & 0x10: | |
188 | jsobj.setdefault('flags', []).append('errno') | |
189 | if self.priority & 0x20: | |
190 | jsobj.setdefault('flags', []).append('getaddrinfo') | |
191 | ||
192 | xrelfo['refs'].setdefault(self.xref.xrefdata.uid, []).append(jsobj) | |
193 | ||
194 | Xref.containers[XREFT_LOGMSG] = XrefLogmsg | |
195 | ||
196 | class CmdElement(ELFDissectStruct, XrelfoJson): | |
197 | struct = 'cmd_element' | |
198 | ||
199 | cmd_attrs = { 0: None, 1: 'deprecated', 2: 'hidden'} | |
200 | ||
201 | def __init__(self, *args, **kwargs): | |
202 | super().__init__(*args, **kwargs) | |
203 | ||
204 | def to_dict(self, xrelfo): | |
205 | jsobj = xrelfo['cli'].setdefault(self.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {}) | |
206 | ||
207 | jsobj.update({ | |
208 | 'string': self.string, | |
209 | 'doc': self.doc, | |
210 | 'attr': self.cmd_attrs.get(self.attr, self.attr), | |
211 | }) | |
212 | if jsobj['attr'] is None: | |
213 | del jsobj['attr'] | |
214 | ||
215 | jsobj['defun'] = dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]) | |
216 | ||
217 | Xref.containers[XREFT_DEFUN] = CmdElement | |
218 | ||
219 | class XrefInstallElement(ELFDissectStruct, XrelfoJson): | |
220 | struct = 'xref_install_element' | |
221 | ||
222 | def to_dict(self, xrelfo): | |
223 | jsobj = xrelfo['cli'].setdefault(self.cmd_element.name, {}).setdefault(self._elfsect._elfwrap.orig_filename, {}) | |
224 | nodes = jsobj.setdefault('nodes', []) | |
225 | ||
226 | nodes.append({ | |
227 | 'node': self.node_type, | |
228 | 'install': dict([(i, getattr(self.xref, i)) for i in ['file', 'line', 'func']]), | |
229 | }) | |
230 | ||
231 | Xref.containers[XREFT_INSTALL_ELEMENT] = XrefInstallElement | |
232 | ||
233 | # shove in field defs | |
234 | fieldapply = FieldApplicator(xrefstructs) | |
235 | fieldapply.add(Xref) | |
236 | fieldapply.add(Xrefdata) | |
237 | fieldapply.add(XrefLogmsg) | |
238 | fieldapply.add(XrefThreadSched) | |
239 | fieldapply.add(CmdElement) | |
240 | fieldapply.add(XrefInstallElement) | |
241 | fieldapply() | |
242 | ||
243 | ||
244 | class Xrelfo(dict): | |
245 | def __init__(self): | |
246 | super().__init__({ | |
247 | 'refs': {}, | |
248 | 'cli': {}, | |
249 | }) | |
250 | self._xrefs = [] | |
251 | ||
252 | def load_file(self, filename): | |
253 | orig_filename = filename | |
254 | if filename.endswith('.la') or filename.endswith('.lo'): | |
255 | with open(filename, 'r') as fd: | |
256 | for line in fd: | |
257 | line = line.strip() | |
258 | if line.startswith('#') or line == '' or '=' not in line: | |
259 | continue | |
260 | ||
261 | var, val = line.split('=', 1) | |
262 | if var not in ['library_names', 'pic_object']: | |
263 | continue | |
264 | if val.startswith("'") or val.startswith('"'): | |
265 | val = val[1:-1] | |
266 | ||
267 | if var == 'pic_object': | |
268 | filename = os.path.join(os.path.dirname(filename), val) | |
269 | break | |
270 | ||
271 | val = val.strip().split()[0] | |
272 | filename = os.path.join(os.path.dirname(filename), '.libs', val) | |
273 | break | |
274 | else: | |
275 | raise ValueError('could not process libtool file "%s"' % orig_filename) | |
276 | ||
277 | while True: | |
278 | with open(filename, 'rb') as fd: | |
279 | hdr = fd.read(4) | |
280 | ||
281 | if hdr == b'\x7fELF': | |
282 | self.load_elf(filename, orig_filename) | |
283 | return | |
284 | ||
285 | if hdr[:2] == b'#!': | |
286 | path, name = os.path.split(filename) | |
287 | filename = os.path.join(path, '.libs', name) | |
288 | continue | |
289 | ||
290 | if hdr[:1] == b'{': | |
291 | with open(filename, 'r') as fd: | |
292 | self.load_json(fd) | |
293 | return | |
294 | ||
295 | raise ValueError('cannot determine file type for %s' % (filename)) | |
296 | ||
297 | def load_elf(self, filename, orig_filename): | |
298 | edf = ELFDissectFile(filename) | |
299 | edf.orig_filename = orig_filename | |
300 | ||
301 | note = edf._elffile.find_note('FRRouting', 'XREF') | |
302 | if note is not None: | |
303 | endian = '>' if edf._elffile.bigendian else '<' | |
304 | mem = edf._elffile[note] | |
305 | if edf._elffile.elfclass == 64: | |
306 | start, end = struct.unpack(endian + 'QQ', mem) | |
307 | start += note.start | |
308 | end += note.start + 8 | |
309 | else: | |
310 | start, end = struct.unpack(endian + 'II', mem) | |
311 | start += note.start | |
312 | end += note.start + 4 | |
313 | ||
314 | ptrs = edf.iter_data(XrefPtr, slice(start, end)) | |
315 | ||
316 | else: | |
317 | xrefarray = edf.get_section('xref_array') | |
318 | if xrefarray is None: | |
319 | raise ValueError('file has neither xref note nor xref_array section') | |
320 | ||
321 | ptrs = xrefarray.iter_data(XrefPtr) | |
322 | ||
323 | for ptr in ptrs: | |
324 | if ptr.xref is None: | |
325 | print('NULL xref') | |
326 | continue | |
327 | self._xrefs.append(ptr.xref) | |
328 | ||
329 | container = ptr.xref.container() | |
330 | if container is None: | |
331 | continue | |
332 | container.to_dict(self) | |
333 | ||
334 | return edf | |
335 | ||
336 | def load_json(self, fd): | |
337 | data = json.load(fd) | |
338 | for uid, items in data['refs'].items(): | |
339 | myitems = self['refs'].setdefault(uid, []) | |
340 | for item in items: | |
341 | if item in myitems: | |
342 | continue | |
343 | myitems.append(item) | |
344 | ||
345 | for cmd, items in data['cli'].items(): | |
346 | self['cli'].setdefault(cmd, {}).update(items) | |
347 | ||
348 | return data | |
349 | ||
350 | def check(self, checks): | |
351 | for xref in self._xrefs: | |
352 | yield from xref.check(checks) | |
353 | ||
354 | def main(): | |
355 | argp = argparse.ArgumentParser(description = 'FRR xref ELF extractor') | |
356 | argp.add_argument('-o', dest='output', type=str, help='write JSON output') | |
357 | argp.add_argument('--out-by-file', type=str, help='write by-file JSON output') | |
358 | argp.add_argument('-Wlog-format', action='store_const', const=True) | |
2621bb8b | 359 | argp.add_argument('-Wlog-args', action='store_const', const=True) |
13f9aea3 | 360 | argp.add_argument('-Werror', action='store_const', const=True) |
36a8fdfd DL |
361 | argp.add_argument('--profile', action='store_const', const=True) |
362 | argp.add_argument('binaries', metavar='BINARY', nargs='+', type=str, help='files to read (ELF files or libtool objects)') | |
363 | args = argp.parse_args() | |
364 | ||
365 | if args.profile: | |
366 | import cProfile | |
367 | cProfile.runctx('_main(args)', globals(), {'args': args}, sort='cumtime') | |
368 | else: | |
369 | _main(args) | |
370 | ||
371 | def _main(args): | |
372 | errors = 0 | |
373 | xrelfo = Xrelfo() | |
374 | ||
375 | for fn in args.binaries: | |
376 | try: | |
377 | xrelfo.load_file(fn) | |
378 | except: | |
379 | errors += 1 | |
380 | sys.stderr.write('while processing %s:\n' % (fn)) | |
381 | traceback.print_exc() | |
382 | ||
383 | for option in dir(args): | |
13f9aea3 | 384 | if option.startswith('W') and option != 'Werror': |
36a8fdfd DL |
385 | checks = sorted(xrelfo.check(args)) |
386 | sys.stderr.write(''.join([c[-1] for c in checks])) | |
13f9aea3 DL |
387 | |
388 | if args.Werror and len(checks) > 0: | |
389 | errors += 1 | |
36a8fdfd DL |
390 | break |
391 | ||
392 | ||
393 | refs = xrelfo['refs'] | |
394 | ||
395 | counts = {} | |
396 | for k, v in refs.items(): | |
397 | strs = set([i['fmtstring'] for i in v]) | |
398 | if len(strs) != 1: | |
399 | print('\033[31;1m%s\033[m' % k) | |
400 | counts[k] = len(v) | |
401 | ||
402 | out = xrelfo | |
403 | outbyfile = {} | |
404 | for uid, locs in refs.items(): | |
405 | for loc in locs: | |
406 | filearray = outbyfile.setdefault(loc['file'], []) | |
407 | loc = dict(loc) | |
408 | del loc['file'] | |
409 | filearray.append(loc) | |
410 | ||
411 | for k in outbyfile.keys(): | |
412 | outbyfile[k] = sorted(outbyfile[k], key=lambda x: x['line']) | |
413 | ||
414 | if errors: | |
415 | sys.exit(1) | |
416 | ||
417 | if args.output: | |
418 | with open(args.output + '.tmp', 'w') as fd: | |
419 | json.dump(out, fd, indent=2, sort_keys=True) | |
420 | os.rename(args.output + '.tmp', args.output) | |
421 | ||
422 | if args.out_by_file: | |
423 | with open(args.out_by_file + '.tmp', 'w') as fd: | |
424 | json.dump(outbyfile, fd, indent=2, sort_keys=True) | |
425 | os.rename(args.out_by_file + '.tmp', args.out_by_file) | |
426 | ||
427 | if __name__ == '__main__': | |
428 | main() |