]>
Commit | Line | Data |
---|---|---|
ba7eb55e | 1 | #!/usr/bin/python3 |
47a3a827 | 2 | # SPDX-License-Identifier: NONE |
ba7eb55e DL |
3 | # |
4 | # 2019 by David Lamparter, placed in public domain | |
5 | # | |
6 | # This tool generates a report of possibly unused symbols in the build. It's | |
7 | # particularly useful for libfrr to find bitrotting functions that aren't even | |
8 | # used anywhere anymore. | |
9 | # | |
10 | # Note that the tool can't distinguish between "a symbol is completely unused" | |
11 | # and "a symbol is used only in its file" since file-internal references are | |
12 | # invisible in nm output. However, the compiler will warn you if a static | |
13 | # symbol is unused. | |
14 | # | |
15 | # This tool is only tested on Linux, it probably needs `nm` from GNU binutils | |
16 | # (as opposed to BSD `nm`). Could use pyelftools instead but that's a lot of | |
17 | # extra work. | |
18 | # | |
19 | # This is a developer tool, please don't put it in any packages :) | |
20 | ||
21 | import sys, os, subprocess | |
22 | import re | |
23 | from collections import namedtuple | |
24 | ||
701a0192 | 25 | sys.path.insert( |
26 | 0, | |
27 | os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "python"), | |
28 | ) | |
ba7eb55e | 29 | |
879a9dc5 | 30 | from makevars import MakeVars |
ba7eb55e | 31 | |
701a0192 | 32 | SymRowBase = namedtuple( |
33 | "SymRow", | |
34 | [ | |
35 | "target", | |
36 | "object", | |
37 | "name", | |
38 | "address", | |
39 | "klass", | |
40 | "typ", | |
41 | "size", | |
42 | "line", | |
43 | "section", | |
44 | "loc", | |
45 | ], | |
46 | ) | |
47 | ||
48 | ||
ba7eb55e | 49 | class SymRow(SymRowBase): |
701a0192 | 50 | """ |
ba7eb55e | 51 | wrapper around a line of `nm` output |
701a0192 | 52 | """ |
53 | ||
54 | lib_re = re.compile(r"/lib[^/]+\.(so|la)$") | |
55 | ||
ba7eb55e | 56 | def is_global(self): |
701a0192 | 57 | return self.klass.isupper() or self.klass in "uvw" |
58 | ||
ba7eb55e DL |
59 | def scope(self): |
60 | if self.lib_re.search(self.target) is None: | |
61 | return self.target | |
62 | # "global" | |
63 | return None | |
64 | ||
65 | def is_export(self): | |
701a0192 | 66 | """ |
ba7eb55e DL |
67 | FRR-specific list of symbols which are considered "externally used" |
68 | ||
69 | e.g. hooks are by design APIs for external use, same for qobj_t_* | |
70 | frr_inet_ntop is here because it's used through an ELF alias to | |
71 | "inet_ntop()" | |
701a0192 | 72 | """ |
73 | if self.name in ["main", "frr_inet_ntop", "_libfrr_version"]: | |
ba7eb55e | 74 | return True |
701a0192 | 75 | if self.name.startswith("_hook_"): |
ba7eb55e | 76 | return True |
701a0192 | 77 | if self.name.startswith("qobj_t_"): |
ba7eb55e DL |
78 | return True |
79 | return False | |
80 | ||
701a0192 | 81 | |
ba7eb55e | 82 | class Symbols(dict): |
701a0192 | 83 | """ |
ba7eb55e | 84 | dict of all symbols in all libs & executables |
701a0192 | 85 | """ |
ba7eb55e | 86 | |
701a0192 | 87 | from_re = re.compile(r"^Symbols from (.*?):$") |
88 | lt_re = re.compile(r"^(.*/)([^/]+)\.l[oa]$") | |
ba7eb55e DL |
89 | |
90 | def __init__(self): | |
91 | super().__init__() | |
92 | ||
93 | class ReportSym(object): | |
94 | def __init__(self, sym): | |
95 | self.sym = sym | |
701a0192 | 96 | |
ba7eb55e | 97 | def __repr__(self): |
701a0192 | 98 | return "<%-25s %-40s [%s]>" % ( |
99 | self.__class__.__name__ + ":", | |
100 | self.sym.name, | |
101 | self.sym.loc, | |
102 | ) | |
103 | ||
ba7eb55e DL |
104 | def __lt__(self, other): |
105 | return self.sym.name.__lt__(other.sym.name) | |
106 | ||
107 | class ReportSymCouldBeStaticAlreadyLocal(ReportSym): | |
701a0192 | 108 | idshort = "Z" |
109 | idlong = "extrastatic" | |
ba7eb55e | 110 | title = "symbol is local to library, but only used in its source file (make static?)" |
701a0192 | 111 | |
ba7eb55e | 112 | class ReportSymCouldBeStatic(ReportSym): |
701a0192 | 113 | idshort = "S" |
114 | idlong = "static" | |
ba7eb55e | 115 | title = "symbol is only used in its source file (make static?)" |
701a0192 | 116 | |
ba7eb55e | 117 | class ReportSymCouldBeLibLocal(ReportSym): |
701a0192 | 118 | idshort = "L" |
119 | idlong = "liblocal" | |
ba7eb55e | 120 | title = "symbol is only used inside of library" |
701a0192 | 121 | |
ba7eb55e | 122 | class ReportSymModuleAPI(ReportSym): |
701a0192 | 123 | idshort = "A" |
124 | idlong = "api" | |
ba7eb55e DL |
125 | title = "symbol (in executable) is referenced externally from a module" |
126 | ||
127 | class Symbol(object): | |
128 | def __init__(self, name): | |
129 | super().__init__() | |
130 | self.name = name | |
131 | self.defs = {} | |
132 | self.refs = [] | |
133 | ||
134 | def process(self, row): | |
135 | scope = row.scope() | |
701a0192 | 136 | if row.section == "*UND*": |
ba7eb55e DL |
137 | self.refs.append(row) |
138 | else: | |
139 | self.defs.setdefault(scope, []).append(row) | |
140 | ||
141 | def evaluate(self, out): | |
701a0192 | 142 | """ |
ba7eb55e DL |
143 | generate output report |
144 | ||
145 | invoked after all object files have been read in, so it can look | |
146 | at inter-object-file relationships | |
701a0192 | 147 | """ |
ba7eb55e DL |
148 | if len(self.defs) == 0: |
149 | out.extsyms.add(self.name) | |
150 | return | |
151 | ||
152 | for scopename, symdefs in self.defs.items(): | |
701a0192 | 153 | common_defs = [ |
154 | symdef for symdef in symdefs if symdef.section == "*COM*" | |
155 | ] | |
156 | proper_defs = [ | |
157 | symdef for symdef in symdefs if symdef.section != "*COM*" | |
158 | ] | |
ba7eb55e DL |
159 | |
160 | if len(proper_defs) > 1: | |
701a0192 | 161 | print(self.name, " DUPLICATE") |
162 | print( | |
163 | "\tD: %s %s" | |
164 | % (scopename, "\n\t\t".join([repr(s) for s in symdefs])) | |
165 | ) | |
ba7eb55e | 166 | for syms in self.refs: |
701a0192 | 167 | print("\tR: %s" % (syms,)) |
ba7eb55e DL |
168 | return |
169 | ||
170 | if len(proper_defs): | |
171 | primary_def = proper_defs[0] | |
172 | elif len(common_defs): | |
173 | # "common" = global variables without initializer; | |
174 | # they can occur in multiple .o files and the linker will | |
175 | # merge them into one variable/storage location. | |
176 | primary_def = common_defs[0] | |
177 | else: | |
178 | # undefined symbol, e.g. libc | |
179 | continue | |
180 | ||
181 | if scopename is not None and len(self.refs) > 0: | |
182 | for ref in self.refs: | |
701a0192 | 183 | if ref.target != primary_def.target and ref.target.endswith( |
184 | ".la" | |
185 | ): | |
ba7eb55e DL |
186 | outobj = out.report.setdefault(primary_def.object, []) |
187 | outobj.append(out.ReportSymModuleAPI(primary_def)) | |
188 | break | |
189 | ||
190 | if len(self.refs) == 0: | |
191 | if primary_def.is_export(): | |
192 | continue | |
193 | outobj = out.report.setdefault(primary_def.object, []) | |
194 | if primary_def.visible: | |
195 | outobj.append(out.ReportSymCouldBeStatic(primary_def)) | |
196 | else: | |
701a0192 | 197 | outobj.append( |
198 | out.ReportSymCouldBeStaticAlreadyLocal(primary_def) | |
199 | ) | |
ba7eb55e DL |
200 | continue |
201 | ||
202 | if scopename is None and primary_def.visible: | |
203 | # lib symbol | |
204 | for ref in self.refs: | |
205 | if ref.target != primary_def.target: | |
206 | break | |
207 | else: | |
208 | outobj = out.report.setdefault(primary_def.object, []) | |
209 | outobj.append(out.ReportSymCouldBeLibLocal(primary_def)) | |
210 | ||
ba7eb55e DL |
211 | def evaluate(self): |
212 | self.extsyms = set() | |
213 | self.report = {} | |
214 | ||
215 | for sym in self.values(): | |
216 | sym.evaluate(self) | |
217 | ||
218 | def load(self, target, files): | |
219 | def libtoolmustdie(fn): | |
220 | m = self.lt_re.match(fn) | |
221 | if m is None: | |
222 | return fn | |
701a0192 | 223 | return m.group(1) + ".libs/" + m.group(2) + ".o" |
ba7eb55e DL |
224 | |
225 | def libtooltargetmustdie(fn): | |
226 | m = self.lt_re.match(fn) | |
227 | if m is None: | |
701a0192 | 228 | a, b = fn.rsplit("/", 1) |
229 | return "%s/.libs/%s" % (a, b) | |
230 | return m.group(1) + ".libs/" + m.group(2) + ".so" | |
ba7eb55e DL |
231 | |
232 | files = list(set([libtoolmustdie(fn) for fn in files])) | |
233 | ||
234 | def parse_nm_output(text): | |
235 | filename = None | |
236 | path_rel_to = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
237 | ||
701a0192 | 238 | for line in text.split("\n"): |
239 | if line.strip() == "": | |
ba7eb55e DL |
240 | continue |
241 | m = self.from_re.match(line) | |
242 | if m is not None: | |
243 | filename = m.group(1) | |
244 | continue | |
701a0192 | 245 | if line.startswith("Name"): |
ba7eb55e DL |
246 | continue |
247 | ||
701a0192 | 248 | items = [i.strip() for i in line.split("|")] |
ba7eb55e | 249 | loc = None |
701a0192 | 250 | if "\t" in items[-1]: |
251 | items[-1], loc = items[-1].split("\t", 1) | |
252 | fn, lno = loc.rsplit(":", 1) | |
ba7eb55e | 253 | fn = os.path.relpath(fn, path_rel_to) |
701a0192 | 254 | loc = "%s:%s" % (fn, lno) |
ba7eb55e | 255 | |
701a0192 | 256 | items[1] = int(items[1] if items[1] != "" else "0", 16) |
257 | items[4] = int(items[4] if items[4] != "" else "0", 16) | |
ba7eb55e DL |
258 | items.append(loc) |
259 | row = SymRow(target, filename, *items) | |
260 | ||
701a0192 | 261 | if row.section == ".group" or row.name == "_GLOBAL_OFFSET_TABLE_": |
ba7eb55e DL |
262 | continue |
263 | if not row.is_global(): | |
264 | continue | |
265 | ||
266 | yield row | |
267 | ||
268 | visible_syms = set() | |
269 | ||
270 | # the actual symbol report uses output from the individual object files | |
271 | # (e.g. lib/.libs/foo.o), but we also read the linked binary (e.g. | |
272 | # lib/.libs/libfrr.so) to determine which symbols are actually visible | |
273 | # in the linked result (this covers ELF "hidden"/"internal" linkage) | |
274 | ||
275 | libfile = libtooltargetmustdie(target) | |
701a0192 | 276 | nmlib = subprocess.Popen( |
277 | ["nm", "-l", "-g", "--defined-only", "-f", "sysv", libfile], | |
278 | stdout=subprocess.PIPE, | |
279 | ) | |
280 | out = nmlib.communicate()[0].decode("US-ASCII") | |
ba7eb55e DL |
281 | |
282 | for row in parse_nm_output(out): | |
283 | visible_syms.add(row.name) | |
284 | ||
701a0192 | 285 | nm = subprocess.Popen( |
286 | ["nm", "-l", "-f", "sysv"] + files, stdout=subprocess.PIPE | |
287 | ) | |
288 | out = nm.communicate()[0].decode("US-ASCII") | |
ba7eb55e DL |
289 | |
290 | for row in parse_nm_output(out): | |
291 | row.visible = row.name in visible_syms | |
292 | sym = self.setdefault(row.name, self.Symbol(row.name)) | |
293 | sym.process(row) | |
294 | ||
295 | ||
296 | def write_html_report(syms): | |
297 | try: | |
298 | import jinja2 | |
299 | except ImportError: | |
701a0192 | 300 | sys.stderr.write("jinja2 could not be imported, not writing HTML report!\n") |
ba7eb55e DL |
301 | return |
302 | ||
303 | self_path = os.path.dirname(os.path.abspath(__file__)) | |
304 | jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(self_path)) | |
701a0192 | 305 | template = jenv.get_template("symalyzer.html") |
ba7eb55e DL |
306 | |
307 | dirgroups = {} | |
308 | for fn, reports in syms.report.items(): | |
701a0192 | 309 | dirname, filename = fn.replace(".libs/", "").rsplit("/", 1) |
ba7eb55e DL |
310 | dirgroups.setdefault(dirname, {})[fn] = reports |
311 | ||
312 | klasses = { | |
701a0192 | 313 | "T": "code / plain old regular function (Text)", |
314 | "D": "global variable, read-write, with nonzero initializer (Data)", | |
315 | "B": "global variable, read-write, with zero initializer (BSS)", | |
316 | "C": "global variable, read-write, with zero initializer (Common)", | |
317 | "R": "global variable, read-only (Rodata)", | |
ba7eb55e DL |
318 | } |
319 | ||
701a0192 | 320 | with open("symalyzer_report.html.tmp", "w") as fd: |
321 | fd.write(template.render(dirgroups=dirgroups, klasses=klasses)) | |
322 | os.rename("symalyzer_report.html.tmp", "symalyzer_report.html") | |
ba7eb55e | 323 | |
701a0192 | 324 | if not os.path.exists("jquery-3.4.1.min.js"): |
325 | url = "https://code.jquery.com/jquery-3.4.1.min.js" | |
ba7eb55e | 326 | sys.stderr.write( |
701a0192 | 327 | "trying to grab a copy of jquery from %s\nif this fails, please get it manually (the HTML output is done.)\n" |
328 | % (url) | |
329 | ) | |
ba7eb55e | 330 | import requests |
701a0192 | 331 | |
332 | r = requests.get("https://code.jquery.com/jquery-3.4.1.min.js") | |
ba7eb55e | 333 | if r.status_code != 200: |
701a0192 | 334 | sys.stderr.write( |
335 | "failed -- please download jquery-3.4.1.min.js and put it next to the HTML report\n" | |
336 | ) | |
ba7eb55e | 337 | else: |
701a0192 | 338 | with open("jquery-3.4.1.min.js.tmp", "w") as fd: |
ba7eb55e | 339 | fd.write(r.text) |
701a0192 | 340 | os.rename("jquery-3.4.1.min.js.tmp", "jquery-3.4.1.min.js") |
341 | sys.stderr.write("done.\n") | |
342 | ||
ba7eb55e DL |
343 | |
344 | def automake_escape(s): | |
701a0192 | 345 | return s.replace(".", "_").replace("/", "_") |
ba7eb55e | 346 | |
701a0192 | 347 | |
348 | if __name__ == "__main__": | |
ba7eb55e DL |
349 | mv = MakeVars() |
350 | ||
701a0192 | 351 | if not (os.path.exists("config.version") and os.path.exists("lib/.libs/libfrr.so")): |
352 | sys.stderr.write( | |
353 | "please execute this script in the root directory of an FRR build tree\n" | |
354 | ) | |
355 | sys.stderr.write("./configure && make need to have completed successfully\n") | |
ba7eb55e DL |
356 | sys.exit(1) |
357 | ||
701a0192 | 358 | amtargets = [ |
359 | "bin_PROGRAMS", | |
360 | "sbin_PROGRAMS", | |
361 | "lib_LTLIBRARIES", | |
362 | "module_LTLIBRARIES", | |
363 | ] | |
ba7eb55e DL |
364 | targets = [] |
365 | ||
366 | mv.getvars(amtargets) | |
367 | for amtarget in amtargets: | |
701a0192 | 368 | targets.extend( |
369 | [item for item in mv[amtarget].strip().split() if item != "tools/ssd"] | |
370 | ) | |
ba7eb55e | 371 | |
701a0192 | 372 | mv.getvars(["%s_LDADD" % automake_escape(t) for t in targets]) |
ba7eb55e DL |
373 | ldobjs = targets[:] |
374 | for t in targets: | |
701a0192 | 375 | ldadd = mv["%s_LDADD" % automake_escape(t)].strip().split() |
ba7eb55e | 376 | for item in ldadd: |
701a0192 | 377 | if item.startswith("-"): |
ba7eb55e | 378 | continue |
701a0192 | 379 | if item.endswith(".a"): |
ba7eb55e DL |
380 | ldobjs.append(item) |
381 | ||
701a0192 | 382 | mv.getvars(["%s_OBJECTS" % automake_escape(o) for o in ldobjs]) |
ba7eb55e DL |
383 | |
384 | syms = Symbols() | |
385 | ||
386 | for t in targets: | |
701a0192 | 387 | objs = mv["%s_OBJECTS" % automake_escape(t)].strip().split() |
388 | ldadd = mv["%s_LDADD" % automake_escape(t)].strip().split() | |
ba7eb55e | 389 | for item in ldadd: |
701a0192 | 390 | if item.startswith("-"): |
ba7eb55e | 391 | continue |
701a0192 | 392 | if item.endswith(".a"): |
393 | objs.extend(mv["%s_OBJECTS" % automake_escape(item)].strip().split()) | |
ba7eb55e | 394 | |
701a0192 | 395 | sys.stderr.write("processing %s...\n" % t) |
ba7eb55e | 396 | sys.stderr.flush() |
701a0192 | 397 | # print(t, '\n\t', objs) |
ba7eb55e DL |
398 | syms.load(t, objs) |
399 | ||
400 | syms.evaluate() | |
401 | ||
402 | for obj, reports in sorted(syms.report.items()): | |
701a0192 | 403 | print("%s:" % obj) |
ba7eb55e | 404 | for report in reports: |
701a0192 | 405 | print("\t%r" % report) |
ba7eb55e DL |
406 | |
407 | write_html_report(syms) |