Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483#!/usr/bin/env python3# SPDX-License-Identifier: GPL-2.0-only
"""Find Kconfig symbols that are referenced but not defined."""
# (c) 2014-2017 Valentin Rothberg <valentinrothberg@gmail.com># (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>#
import argparseimport difflibimport osimport reimport signalimport subprocessimport sysfrom multiprocessing import Pool, cpu_count
# regex expressionsOPERATORS = r"&|\(|\)|\||\!"SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}"DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*"EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+"DEFAULT = r"default\s+.*?(?:if\s.+){,1}"STMT = r"^\s*(?:if|select|imply|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPRSOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")"
# regex objectsREGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)')REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL)REGEX_KCONFIG_DEF = re.compile(DEF)REGEX_KCONFIG_EXPR = re.compile(EXPR)REGEX_KCONFIG_STMT = re.compile(STMT)REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$")REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")REGEX_QUOTES = re.compile("(\"(.*?)\")")
def parse_options(): """The user interface of this module.""" usage = "Run this tool to detect Kconfig symbols that are referenced but " \ "not defined in Kconfig. If no option is specified, " \ "checkkconfigsymbols defaults to check your current tree. " \ "Please note that specifying commits will 'git reset --hard\' " \ "your current tree! You may save uncommitted changes to avoid " \ "losing data."
parser = argparse.ArgumentParser(description=usage)
parser.add_argument('-c', '--commit', dest='commit', action='store', default="", help="check if the specified commit (hash) introduces " "undefined Kconfig symbols")
parser.add_argument('-d', '--diff', dest='diff', action='store', default="", help="diff undefined symbols between two commits " "(e.g., -d commmit1..commit2)")
parser.add_argument('-f', '--find', dest='find', action='store_true', default=False, help="find and show commits that may cause symbols to be " "missing (required to run with --diff)")
parser.add_argument('-i', '--ignore', dest='ignore', action='store', default="", help="ignore files matching this Python regex " "(e.g., -i '.*defconfig')")
parser.add_argument('-s', '--sim', dest='sim', action='store', default="", help="print a list of max. 10 string-similar symbols")
parser.add_argument('--force', dest='force', action='store_true', default=False, help="reset current Git tree even when it's dirty")
parser.add_argument('--no-color', dest='color', action='store_false', default=True, help="don't print colored output (default when not " "outputting to a terminal)")
args = parser.parse_args()
if args.commit and args.diff: sys.exit("Please specify only one option at once.")
if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff): sys.exit("Please specify valid input in the following format: " "\'commit1..commit2\'")
if args.commit or args.diff: if not args.force and tree_is_dirty(): sys.exit("The current Git tree is dirty (see 'git status'). " "Running this script may\ndelete important data since it " "calls 'git reset --hard' for some performance\nreasons. " " Please run this script in a clean Git tree or pass " "'--force' if you\nwant to ignore this warning and " "continue.")
if args.commit: if args.commit.startswith('HEAD'): sys.exit("The --commit option can't use the HEAD ref")
args.find = False
if args.ignore: try: re.match(args.ignore, "this/is/just/a/test.c") except: sys.exit("Please specify a valid Python regex.")
return args
def print_undefined_symbols(): """Main function of this module.""" args = parse_options()
global COLOR COLOR = args.color and sys.stdout.isatty()
if args.sim and not args.commit and not args.diff: sims = find_sims(args.sim, args.ignore) if sims: print("%s: %s" % (yel("Similar symbols"), ', '.join(sims))) else: print("%s: no similar symbols found" % yel("Similar symbols")) sys.exit(0)
# dictionary of (un)defined symbols defined = {} undefined = {}
if args.commit or args.diff: head = get_head()
# get commit range commit_a = None commit_b = None if args.commit: commit_a = args.commit + "~" commit_b = args.commit elif args.diff: split = args.diff.split("..") commit_a = split[0] commit_b = split[1] undefined_a = {} undefined_b = {}
# get undefined items before the commit reset(commit_a) undefined_a, _ = check_symbols(args.ignore)
# get undefined items for the commit reset(commit_b) undefined_b, defined = check_symbols(args.ignore)
# report cases that are present for the commit but not before for symbol in sorted(undefined_b): # symbol has not been undefined before if symbol not in undefined_a: files = sorted(undefined_b.get(symbol)) undefined[symbol] = files # check if there are new files that reference the undefined symbol else: files = sorted(undefined_b.get(symbol) - undefined_a.get(symbol)) if files: undefined[symbol] = files
# reset to head reset(head)
# default to check the entire tree else: undefined, defined = check_symbols(args.ignore)
# now print the output for symbol in sorted(undefined): print(red(symbol))
files = sorted(undefined.get(symbol)) print("%s: %s" % (yel("Referencing files"), ", ".join(files)))
sims = find_sims(symbol, args.ignore, defined) sims_out = yel("Similar symbols") if sims: print("%s: %s" % (sims_out, ', '.join(sims))) else: print("%s: %s" % (sims_out, "no similar symbols found"))
if args.find: print("%s:" % yel("Commits changing symbol")) commits = find_commits(symbol, args.diff) if commits: for commit in commits: commit = commit.split(" ", 1) print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1])) else: print("\t- no commit found") print() # new line
def reset(commit): """Reset current git tree to %commit.""" execute(["git", "reset", "--hard", commit])
def yel(string): """ Color %string yellow. """ return "\033[33m%s\033[0m" % string if COLOR else string
def red(string): """ Color %string red. """ return "\033[31m%s\033[0m" % string if COLOR else string
def execute(cmd): """Execute %cmd and return stdout. Exit in case of error.""" try: stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False) stdout = stdout.decode(errors='replace') except subprocess.CalledProcessError as fail: exit(fail) return stdout
def find_commits(symbol, diff): """Find commits changing %symbol in the given range of %diff.""" commits = execute(["git", "log", "--pretty=oneline", "--abbrev-commit", "-G", symbol, diff]) return [x for x in commits.split("\n") if x]
def tree_is_dirty(): """Return true if the current working tree is dirty (i.e., if any file has been added, deleted, modified, renamed or copied but not committed).""" stdout = execute(["git", "status", "--porcelain"]) for line in stdout: if re.findall(r"[URMADC]{1}", line[:2]): return True return False
def get_head(): """Return commit hash of current HEAD.""" stdout = execute(["git", "rev-parse", "HEAD"]) return stdout.strip('\n')
def partition(lst, size): """Partition list @lst into eveni-sized lists of size @size.""" return [lst[i::size] for i in range(size)]
def init_worker(): """Set signal handler to ignore SIGINT.""" signal.signal(signal.SIGINT, signal.SIG_IGN)
def find_sims(symbol, ignore, defined=[]): """Return a list of max. ten Kconfig symbols that are string-similar to @symbol.""" if defined: return difflib.get_close_matches(symbol, set(defined), 10)
pool = Pool(cpu_count(), init_worker) kfiles = [] for gitfile in get_files(): if REGEX_FILE_KCONFIG.match(gitfile): kfiles.append(gitfile)
arglist = [] for part in partition(kfiles, cpu_count()): arglist.append((part, ignore))
for res in pool.map(parse_kconfig_files, arglist): defined.extend(res[0])
return difflib.get_close_matches(symbol, set(defined), 10)
def get_files(): """Return a list of all files in the current git directory.""" # use 'git ls-files' to get the worklist stdout = execute(["git", "ls-files"]) if len(stdout) > 0 and stdout[-1] == "\n": stdout = stdout[:-1]
files = [] for gitfile in stdout.rsplit("\n"): if ".git" in gitfile or "ChangeLog" in gitfile or \ ".log" in gitfile or os.path.isdir(gitfile) or \ gitfile.startswith("tools/"): continue files.append(gitfile) return files
def check_symbols(ignore): """Find undefined Kconfig symbols and return a dict with the symbol as key and a list of referencing files as value. Files matching %ignore are not checked for undefined symbols.""" pool = Pool(cpu_count(), init_worker) try: return check_symbols_helper(pool, ignore) except KeyboardInterrupt: pool.terminate() pool.join() sys.exit(1)
def check_symbols_helper(pool, ignore): """Helper method for check_symbols(). Used to catch keyboard interrupts in check_symbols() in order to properly terminate running worker processes.""" source_files = [] kconfig_files = [] defined_symbols = [] referenced_symbols = dict() # {file: [symbols]}
for gitfile in get_files(): if REGEX_FILE_KCONFIG.match(gitfile): kconfig_files.append(gitfile) else: if ignore and re.match(ignore, gitfile): continue # add source files that do not match the ignore pattern source_files.append(gitfile)
# parse source files arglist = partition(source_files, cpu_count()) for res in pool.map(parse_source_files, arglist): referenced_symbols.update(res)
# parse kconfig files arglist = [] for part in partition(kconfig_files, cpu_count()): arglist.append((part, ignore)) for res in pool.map(parse_kconfig_files, arglist): defined_symbols.extend(res[0]) referenced_symbols.update(res[1]) defined_symbols = set(defined_symbols)
# inverse mapping of referenced_symbols to dict(symbol: [files]) inv_map = dict() for _file, symbols in referenced_symbols.items(): for symbol in symbols: inv_map[symbol] = inv_map.get(symbol, set()) inv_map[symbol].add(_file) referenced_symbols = inv_map
undefined = {} # {symbol: [files]} for symbol in sorted(referenced_symbols): # filter some false positives if symbol == "FOO" or symbol == "BAR" or \ symbol == "FOO_BAR" or symbol == "XXX": continue if symbol not in defined_symbols: if symbol.endswith("_MODULE"): # avoid false positives for kernel modules if symbol[:-len("_MODULE")] in defined_symbols: continue undefined[symbol] = referenced_symbols.get(symbol) return undefined, defined_symbols
def parse_source_files(source_files): """Parse each source file in @source_files and return dictionary with source files as keys and lists of references Kconfig symbols as values.""" referenced_symbols = dict() for sfile in source_files: referenced_symbols[sfile] = parse_source_file(sfile) return referenced_symbols
def parse_source_file(sfile): """Parse @sfile and return a list of referenced Kconfig symbols.""" lines = [] references = []
if not os.path.exists(sfile): return references
with open(sfile, "r", encoding='utf-8', errors='replace') as stream: lines = stream.readlines()
for line in lines: if "CONFIG_" not in line: continue symbols = REGEX_SOURCE_SYMBOL.findall(line) for symbol in symbols: if not REGEX_FILTER_SYMBOLS.search(symbol): continue references.append(symbol)
return references
def get_symbols_in_line(line): """Return mentioned Kconfig symbols in @line.""" return REGEX_SYMBOL.findall(line)
def parse_kconfig_files(args): """Parse kconfig files and return tuple of defined and references Kconfig symbols. Note, @args is a tuple of a list of files and the @ignore pattern.""" kconfig_files = args[0] ignore = args[1] defined_symbols = [] referenced_symbols = dict()
for kfile in kconfig_files: defined, references = parse_kconfig_file(kfile) defined_symbols.extend(defined) if ignore and re.match(ignore, kfile): # do not collect references for files that match the ignore pattern continue referenced_symbols[kfile] = references return (defined_symbols, referenced_symbols)
def parse_kconfig_file(kfile): """Parse @kfile and update symbol definitions and references.""" lines = [] defined = [] references = []
if not os.path.exists(kfile): return defined, references
with open(kfile, "r", encoding='utf-8', errors='replace') as stream: lines = stream.readlines()
for i in range(len(lines)): line = lines[i] line = line.strip('\n') line = line.split("#")[0] # ignore comments
if REGEX_KCONFIG_DEF.match(line): symbol_def = REGEX_KCONFIG_DEF.findall(line) defined.append(symbol_def[0]) elif REGEX_KCONFIG_STMT.match(line): line = REGEX_QUOTES.sub("", line) symbols = get_symbols_in_line(line) # multi-line statements while line.endswith("\\"): i += 1 line = lines[i] line = line.strip('\n') symbols.extend(get_symbols_in_line(line)) for symbol in set(symbols): if REGEX_NUMERIC.match(symbol): # ignore numeric values continue references.append(symbol)
return defined, references
def main(): try: print_undefined_symbols() except BrokenPipeError: # Python flushes standard streams on exit; redirect remaining output # to devnull to avoid another BrokenPipeError at shutdown devnull = os.open(os.devnull, os.O_WRONLY) os.dup2(devnull, sys.stdout.fileno()) sys.exit(1) # Python exits with error code 1 on EPIPE
if __name__ == "__main__": main()