]>
Commit | Line | Data |
---|---|---|
7c5227af | 1 | #!/usr/bin/env python3 |
24fe1f03 | 2 | |
b1a3f243 | 3 | """Find Kconfig symbols that are referenced but not defined.""" |
24fe1f03 | 4 | |
f175ba17 | 5 | # (c) 2014-2016 Valentin Rothberg <valentinrothberg@gmail.com> |
cc641d55 | 6 | # (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de> |
24fe1f03 | 7 | # |
cc641d55 | 8 | # Licensed under the terms of the GNU GPL License version 2 |
24fe1f03 VR |
9 | |
10 | ||
14390e31 | 11 | import argparse |
1b2c8414 | 12 | import difflib |
24fe1f03 VR |
13 | import os |
14 | import re | |
e2042a8a | 15 | import signal |
f175ba17 | 16 | import subprocess |
b1a3f243 | 17 | import sys |
e2042a8a | 18 | from multiprocessing import Pool, cpu_count |
24fe1f03 | 19 | |
cc641d55 VR |
20 | |
21 | # regex expressions | |
24fe1f03 | 22 | OPERATORS = r"&|\(|\)|\||\!" |
ef3f5543 VR |
23 | SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}" |
24 | DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*" | |
25 | EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+" | |
0bd38ae3 VR |
26 | DEFAULT = r"default\s+.*?(?:if\s.+){,1}" |
27 | STMT = r"^\s*(?:if|select|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR | |
ef3f5543 | 28 | SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")" |
24fe1f03 | 29 | |
cc641d55 | 30 | # regex objects |
24fe1f03 | 31 | REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$") |
ef3f5543 VR |
32 | REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)') |
33 | REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL) | |
cc641d55 | 34 | REGEX_KCONFIG_DEF = re.compile(DEF) |
24fe1f03 VR |
35 | REGEX_KCONFIG_EXPR = re.compile(EXPR) |
36 | REGEX_KCONFIG_STMT = re.compile(STMT) | |
37 | REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$") | |
ef3f5543 | 38 | REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$") |
0bd38ae3 | 39 | REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+") |
e2042a8a | 40 | REGEX_QUOTES = re.compile("(\"(.*?)\")") |
24fe1f03 VR |
41 | |
42 | ||
b1a3f243 VR |
43 | def parse_options(): |
44 | """The user interface of this module.""" | |
14390e31 VR |
45 | usage = "Run this tool to detect Kconfig symbols that are referenced but " \ |
46 | "not defined in Kconfig. If no option is specified, " \ | |
47 | "checkkconfigsymbols defaults to check your current tree. " \ | |
48 | "Please note that specifying commits will 'git reset --hard\' " \ | |
49 | "your current tree! You may save uncommitted changes to avoid " \ | |
50 | "losing data." | |
51 | ||
52 | parser = argparse.ArgumentParser(description=usage) | |
53 | ||
54 | parser.add_argument('-c', '--commit', dest='commit', action='store', | |
55 | default="", | |
56 | help="check if the specified commit (hash) introduces " | |
57 | "undefined Kconfig symbols") | |
58 | ||
59 | parser.add_argument('-d', '--diff', dest='diff', action='store', | |
60 | default="", | |
61 | help="diff undefined symbols between two commits " | |
62 | "(e.g., -d commmit1..commit2)") | |
63 | ||
64 | parser.add_argument('-f', '--find', dest='find', action='store_true', | |
65 | default=False, | |
66 | help="find and show commits that may cause symbols to be " | |
67 | "missing (required to run with --diff)") | |
68 | ||
69 | parser.add_argument('-i', '--ignore', dest='ignore', action='store', | |
70 | default="", | |
71 | help="ignore files matching this Python regex " | |
72 | "(e.g., -i '.*defconfig')") | |
73 | ||
74 | parser.add_argument('-s', '--sim', dest='sim', action='store', default="", | |
75 | help="print a list of max. 10 string-similar symbols") | |
76 | ||
77 | parser.add_argument('--force', dest='force', action='store_true', | |
78 | default=False, | |
79 | help="reset current Git tree even when it's dirty") | |
80 | ||
81 | parser.add_argument('--no-color', dest='color', action='store_false', | |
82 | default=True, | |
83 | help="don't print colored output (default when not " | |
84 | "outputting to a terminal)") | |
85 | ||
86 | args = parser.parse_args() | |
87 | ||
88 | if args.commit and args.diff: | |
b1a3f243 VR |
89 | sys.exit("Please specify only one option at once.") |
90 | ||
14390e31 | 91 | if args.diff and not re.match(r"^[\w\-\.]+\.\.[\w\-\.]+$", args.diff): |
b1a3f243 | 92 | sys.exit("Please specify valid input in the following format: " |
38cbfe4f | 93 | "\'commit1..commit2\'") |
b1a3f243 | 94 | |
14390e31 VR |
95 | if args.commit or args.diff: |
96 | if not args.force and tree_is_dirty(): | |
b1a3f243 VR |
97 | sys.exit("The current Git tree is dirty (see 'git status'). " |
98 | "Running this script may\ndelete important data since it " | |
99 | "calls 'git reset --hard' for some performance\nreasons. " | |
100 | " Please run this script in a clean Git tree or pass " | |
101 | "'--force' if you\nwant to ignore this warning and " | |
102 | "continue.") | |
103 | ||
14390e31 VR |
104 | if args.commit: |
105 | args.find = False | |
a42fa92c | 106 | |
14390e31 | 107 | if args.ignore: |
cf132e4a | 108 | try: |
14390e31 | 109 | re.match(args.ignore, "this/is/just/a/test.c") |
cf132e4a VR |
110 | except: |
111 | sys.exit("Please specify a valid Python regex.") | |
112 | ||
14390e31 | 113 | return args |
b1a3f243 VR |
114 | |
115 | ||
24fe1f03 VR |
116 | def main(): |
117 | """Main function of this module.""" | |
14390e31 | 118 | args = parse_options() |
b1a3f243 | 119 | |
36c79c7f VR |
120 | global COLOR |
121 | COLOR = args.color and sys.stdout.isatty() | |
4c73c088 | 122 | |
14390e31 VR |
123 | if args.sim and not args.commit and not args.diff: |
124 | sims = find_sims(args.sim, args.ignore) | |
1b2c8414 | 125 | if sims: |
7c5227af | 126 | print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) |
1b2c8414 | 127 | else: |
7c5227af | 128 | print("%s: no similar symbols found" % yel("Similar symbols")) |
1b2c8414 VR |
129 | sys.exit(0) |
130 | ||
131 | # dictionary of (un)defined symbols | |
132 | defined = {} | |
133 | undefined = {} | |
134 | ||
14390e31 | 135 | if args.commit or args.diff: |
b1a3f243 VR |
136 | head = get_head() |
137 | ||
138 | # get commit range | |
139 | commit_a = None | |
140 | commit_b = None | |
14390e31 VR |
141 | if args.commit: |
142 | commit_a = args.commit + "~" | |
143 | commit_b = args.commit | |
144 | elif args.diff: | |
145 | split = args.diff.split("..") | |
b1a3f243 VR |
146 | commit_a = split[0] |
147 | commit_b = split[1] | |
148 | undefined_a = {} | |
149 | undefined_b = {} | |
150 | ||
151 | # get undefined items before the commit | |
152 | execute("git reset --hard %s" % commit_a) | |
14390e31 | 153 | undefined_a, _ = check_symbols(args.ignore) |
b1a3f243 VR |
154 | |
155 | # get undefined items for the commit | |
156 | execute("git reset --hard %s" % commit_b) | |
14390e31 | 157 | undefined_b, defined = check_symbols(args.ignore) |
b1a3f243 VR |
158 | |
159 | # report cases that are present for the commit but not before | |
ef3f5543 VR |
160 | for symbol in sorted(undefined_b): |
161 | # symbol has not been undefined before | |
162 | if symbol not in undefined_a: | |
163 | files = sorted(undefined_b.get(symbol)) | |
164 | undefined[symbol] = files | |
165 | # check if there are new files that reference the undefined symbol | |
b1a3f243 | 166 | else: |
ef3f5543 VR |
167 | files = sorted(undefined_b.get(symbol) - |
168 | undefined_a.get(symbol)) | |
b1a3f243 | 169 | if files: |
ef3f5543 | 170 | undefined[symbol] = files |
b1a3f243 VR |
171 | |
172 | # reset to head | |
173 | execute("git reset --hard %s" % head) | |
174 | ||
175 | # default to check the entire tree | |
176 | else: | |
14390e31 | 177 | undefined, defined = check_symbols(args.ignore) |
1b2c8414 VR |
178 | |
179 | # now print the output | |
ef3f5543 VR |
180 | for symbol in sorted(undefined): |
181 | print(red(symbol)) | |
1b2c8414 | 182 | |
ef3f5543 | 183 | files = sorted(undefined.get(symbol)) |
7c5227af | 184 | print("%s: %s" % (yel("Referencing files"), ", ".join(files))) |
1b2c8414 | 185 | |
ef3f5543 | 186 | sims = find_sims(symbol, args.ignore, defined) |
1b2c8414 VR |
187 | sims_out = yel("Similar symbols") |
188 | if sims: | |
7c5227af | 189 | print("%s: %s" % (sims_out, ', '.join(sims))) |
1b2c8414 | 190 | else: |
7c5227af | 191 | print("%s: %s" % (sims_out, "no similar symbols found")) |
1b2c8414 | 192 | |
14390e31 | 193 | if args.find: |
7c5227af | 194 | print("%s:" % yel("Commits changing symbol")) |
ef3f5543 | 195 | commits = find_commits(symbol, args.diff) |
1b2c8414 VR |
196 | if commits: |
197 | for commit in commits: | |
198 | commit = commit.split(" ", 1) | |
7c5227af | 199 | print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) |
1b2c8414 | 200 | else: |
7c5227af | 201 | print("\t- no commit found") |
36c79c7f | 202 | print() # new line |
c7455663 VR |
203 | |
204 | ||
205 | def yel(string): | |
206 | """ | |
207 | Color %string yellow. | |
208 | """ | |
36c79c7f | 209 | return "\033[33m%s\033[0m" % string if COLOR else string |
c7455663 VR |
210 | |
211 | ||
212 | def red(string): | |
213 | """ | |
214 | Color %string red. | |
215 | """ | |
36c79c7f | 216 | return "\033[31m%s\033[0m" % string if COLOR else string |
b1a3f243 VR |
217 | |
218 | ||
219 | def execute(cmd): | |
220 | """Execute %cmd and return stdout. Exit in case of error.""" | |
f175ba17 VR |
221 | try: |
222 | cmdlist = cmd.split(" ") | |
7c5227af VR |
223 | stdout = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT, shell=False) |
224 | stdout = stdout.decode(errors='replace') | |
f175ba17 VR |
225 | except subprocess.CalledProcessError as fail: |
226 | exit("Failed to execute %s\n%s" % (cmd, fail)) | |
b1a3f243 VR |
227 | return stdout |
228 | ||
229 | ||
a42fa92c VR |
230 | def find_commits(symbol, diff): |
231 | """Find commits changing %symbol in the given range of %diff.""" | |
232 | commits = execute("git log --pretty=oneline --abbrev-commit -G %s %s" | |
233 | % (symbol, diff)) | |
1b2c8414 | 234 | return [x for x in commits.split("\n") if x] |
a42fa92c VR |
235 | |
236 | ||
b1a3f243 VR |
237 | def tree_is_dirty(): |
238 | """Return true if the current working tree is dirty (i.e., if any file has | |
239 | been added, deleted, modified, renamed or copied but not committed).""" | |
240 | stdout = execute("git status --porcelain") | |
241 | for line in stdout: | |
242 | if re.findall(r"[URMADC]{1}", line[:2]): | |
243 | return True | |
244 | return False | |
245 | ||
246 | ||
247 | def get_head(): | |
248 | """Return commit hash of current HEAD.""" | |
249 | stdout = execute("git rev-parse HEAD") | |
250 | return stdout.strip('\n') | |
251 | ||
252 | ||
e2042a8a VR |
253 | def partition(lst, size): |
254 | """Partition list @lst into eveni-sized lists of size @size.""" | |
7c5227af | 255 | return [lst[i::size] for i in range(size)] |
e2042a8a VR |
256 | |
257 | ||
258 | def init_worker(): | |
259 | """Set signal handler to ignore SIGINT.""" | |
260 | signal.signal(signal.SIGINT, signal.SIG_IGN) | |
261 | ||
262 | ||
36c79c7f | 263 | def find_sims(symbol, ignore, defined=[]): |
1b2c8414 VR |
264 | """Return a list of max. ten Kconfig symbols that are string-similar to |
265 | @symbol.""" | |
266 | if defined: | |
267 | return sorted(difflib.get_close_matches(symbol, set(defined), 10)) | |
268 | ||
269 | pool = Pool(cpu_count(), init_worker) | |
270 | kfiles = [] | |
271 | for gitfile in get_files(): | |
272 | if REGEX_FILE_KCONFIG.match(gitfile): | |
273 | kfiles.append(gitfile) | |
274 | ||
275 | arglist = [] | |
276 | for part in partition(kfiles, cpu_count()): | |
277 | arglist.append((part, ignore)) | |
278 | ||
279 | for res in pool.map(parse_kconfig_files, arglist): | |
280 | defined.extend(res[0]) | |
281 | ||
282 | return sorted(difflib.get_close_matches(symbol, set(defined), 10)) | |
283 | ||
284 | ||
285 | def get_files(): | |
286 | """Return a list of all files in the current git directory.""" | |
287 | # use 'git ls-files' to get the worklist | |
288 | stdout = execute("git ls-files") | |
289 | if len(stdout) > 0 and stdout[-1] == "\n": | |
290 | stdout = stdout[:-1] | |
291 | ||
292 | files = [] | |
293 | for gitfile in stdout.rsplit("\n"): | |
294 | if ".git" in gitfile or "ChangeLog" in gitfile or \ | |
295 | ".log" in gitfile or os.path.isdir(gitfile) or \ | |
296 | gitfile.startswith("tools/"): | |
297 | continue | |
298 | files.append(gitfile) | |
299 | return files | |
300 | ||
301 | ||
cf132e4a | 302 | def check_symbols(ignore): |
b1a3f243 | 303 | """Find undefined Kconfig symbols and return a dict with the symbol as key |
cf132e4a VR |
304 | and a list of referencing files as value. Files matching %ignore are not |
305 | checked for undefined symbols.""" | |
e2042a8a VR |
306 | pool = Pool(cpu_count(), init_worker) |
307 | try: | |
308 | return check_symbols_helper(pool, ignore) | |
309 | except KeyboardInterrupt: | |
310 | pool.terminate() | |
311 | pool.join() | |
312 | sys.exit(1) | |
313 | ||
314 | ||
315 | def check_symbols_helper(pool, ignore): | |
316 | """Helper method for check_symbols(). Used to catch keyboard interrupts in | |
317 | check_symbols() in order to properly terminate running worker processes.""" | |
24fe1f03 VR |
318 | source_files = [] |
319 | kconfig_files = [] | |
ef3f5543 VR |
320 | defined_symbols = [] |
321 | referenced_symbols = dict() # {file: [symbols]} | |
24fe1f03 | 322 | |
1b2c8414 | 323 | for gitfile in get_files(): |
24fe1f03 VR |
324 | if REGEX_FILE_KCONFIG.match(gitfile): |
325 | kconfig_files.append(gitfile) | |
326 | else: | |
e2042a8a VR |
327 | if ignore and not re.match(ignore, gitfile): |
328 | continue | |
329 | # add source files that do not match the ignore pattern | |
24fe1f03 VR |
330 | source_files.append(gitfile) |
331 | ||
e2042a8a VR |
332 | # parse source files |
333 | arglist = partition(source_files, cpu_count()) | |
334 | for res in pool.map(parse_source_files, arglist): | |
ef3f5543 | 335 | referenced_symbols.update(res) |
24fe1f03 | 336 | |
e2042a8a VR |
337 | # parse kconfig files |
338 | arglist = [] | |
339 | for part in partition(kconfig_files, cpu_count()): | |
340 | arglist.append((part, ignore)) | |
341 | for res in pool.map(parse_kconfig_files, arglist): | |
ef3f5543 VR |
342 | defined_symbols.extend(res[0]) |
343 | referenced_symbols.update(res[1]) | |
344 | defined_symbols = set(defined_symbols) | |
e2042a8a | 345 | |
ef3f5543 | 346 | # inverse mapping of referenced_symbols to dict(symbol: [files]) |
e2042a8a | 347 | inv_map = dict() |
ef3f5543 VR |
348 | for _file, symbols in referenced_symbols.items(): |
349 | for symbol in symbols: | |
350 | inv_map[symbol] = inv_map.get(symbol, set()) | |
351 | inv_map[symbol].add(_file) | |
352 | referenced_symbols = inv_map | |
353 | ||
354 | undefined = {} # {symbol: [files]} | |
355 | for symbol in sorted(referenced_symbols): | |
cc641d55 | 356 | # filter some false positives |
ef3f5543 VR |
357 | if symbol == "FOO" or symbol == "BAR" or \ |
358 | symbol == "FOO_BAR" or symbol == "XXX": | |
cc641d55 | 359 | continue |
ef3f5543 VR |
360 | if symbol not in defined_symbols: |
361 | if symbol.endswith("_MODULE"): | |
cc641d55 | 362 | # avoid false positives for kernel modules |
ef3f5543 | 363 | if symbol[:-len("_MODULE")] in defined_symbols: |
24fe1f03 | 364 | continue |
ef3f5543 VR |
365 | undefined[symbol] = referenced_symbols.get(symbol) |
366 | return undefined, defined_symbols | |
24fe1f03 VR |
367 | |
368 | ||
e2042a8a VR |
369 | def parse_source_files(source_files): |
370 | """Parse each source file in @source_files and return dictionary with source | |
371 | files as keys and lists of references Kconfig symbols as values.""" | |
ef3f5543 | 372 | referenced_symbols = dict() |
e2042a8a | 373 | for sfile in source_files: |
ef3f5543 VR |
374 | referenced_symbols[sfile] = parse_source_file(sfile) |
375 | return referenced_symbols | |
e2042a8a VR |
376 | |
377 | ||
378 | def parse_source_file(sfile): | |
ef3f5543 | 379 | """Parse @sfile and return a list of referenced Kconfig symbols.""" |
24fe1f03 | 380 | lines = [] |
e2042a8a VR |
381 | references = [] |
382 | ||
383 | if not os.path.exists(sfile): | |
384 | return references | |
385 | ||
7c5227af | 386 | with open(sfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03 VR |
387 | lines = stream.readlines() |
388 | ||
389 | for line in lines: | |
36c79c7f | 390 | if "CONFIG_" not in line: |
24fe1f03 | 391 | continue |
ef3f5543 VR |
392 | symbols = REGEX_SOURCE_SYMBOL.findall(line) |
393 | for symbol in symbols: | |
394 | if not REGEX_FILTER_SYMBOLS.search(symbol): | |
24fe1f03 | 395 | continue |
ef3f5543 | 396 | references.append(symbol) |
e2042a8a VR |
397 | |
398 | return references | |
24fe1f03 VR |
399 | |
400 | ||
ef3f5543 VR |
401 | def get_symbols_in_line(line): |
402 | """Return mentioned Kconfig symbols in @line.""" | |
403 | return REGEX_SYMBOL.findall(line) | |
24fe1f03 VR |
404 | |
405 | ||
e2042a8a VR |
406 | def parse_kconfig_files(args): |
407 | """Parse kconfig files and return tuple of defined and references Kconfig | |
408 | symbols. Note, @args is a tuple of a list of files and the @ignore | |
409 | pattern.""" | |
410 | kconfig_files = args[0] | |
411 | ignore = args[1] | |
ef3f5543 VR |
412 | defined_symbols = [] |
413 | referenced_symbols = dict() | |
e2042a8a VR |
414 | |
415 | for kfile in kconfig_files: | |
416 | defined, references = parse_kconfig_file(kfile) | |
ef3f5543 | 417 | defined_symbols.extend(defined) |
e2042a8a VR |
418 | if ignore and re.match(ignore, kfile): |
419 | # do not collect references for files that match the ignore pattern | |
420 | continue | |
ef3f5543 VR |
421 | referenced_symbols[kfile] = references |
422 | return (defined_symbols, referenced_symbols) | |
e2042a8a VR |
423 | |
424 | ||
425 | def parse_kconfig_file(kfile): | |
ef3f5543 | 426 | """Parse @kfile and update symbol definitions and references.""" |
24fe1f03 | 427 | lines = [] |
e2042a8a VR |
428 | defined = [] |
429 | references = [] | |
24fe1f03 VR |
430 | skip = False |
431 | ||
e2042a8a VR |
432 | if not os.path.exists(kfile): |
433 | return defined, references | |
434 | ||
7c5227af | 435 | with open(kfile, "r", encoding='utf-8', errors='replace') as stream: |
24fe1f03 VR |
436 | lines = stream.readlines() |
437 | ||
438 | for i in range(len(lines)): | |
439 | line = lines[i] | |
440 | line = line.strip('\n') | |
cc641d55 | 441 | line = line.split("#")[0] # ignore comments |
24fe1f03 VR |
442 | |
443 | if REGEX_KCONFIG_DEF.match(line): | |
ef3f5543 VR |
444 | symbol_def = REGEX_KCONFIG_DEF.findall(line) |
445 | defined.append(symbol_def[0]) | |
24fe1f03 VR |
446 | skip = False |
447 | elif REGEX_KCONFIG_HELP.match(line): | |
448 | skip = True | |
449 | elif skip: | |
cc641d55 | 450 | # ignore content of help messages |
24fe1f03 VR |
451 | pass |
452 | elif REGEX_KCONFIG_STMT.match(line): | |
e2042a8a | 453 | line = REGEX_QUOTES.sub("", line) |
ef3f5543 | 454 | symbols = get_symbols_in_line(line) |
cc641d55 | 455 | # multi-line statements |
24fe1f03 VR |
456 | while line.endswith("\\"): |
457 | i += 1 | |
458 | line = lines[i] | |
459 | line = line.strip('\n') | |
ef3f5543 VR |
460 | symbols.extend(get_symbols_in_line(line)) |
461 | for symbol in set(symbols): | |
462 | if REGEX_NUMERIC.match(symbol): | |
0bd38ae3 VR |
463 | # ignore numeric values |
464 | continue | |
ef3f5543 | 465 | references.append(symbol) |
e2042a8a VR |
466 | |
467 | return defined, references | |
24fe1f03 VR |
468 | |
469 | ||
470 | if __name__ == "__main__": | |
471 | main() |