Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632#!/usr/bin/env python3# pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.# SPDX-License-Identifier: GPL-2.0
"""Parse ABI documentation and produce results from it."""
from argparse import Namespaceimport loggingimport osimport re
from pprint import pformatfrom random import randrange, seed
# Import Python modules
from abi.helpers import AbiDebug, ABI_DIR
class AbiParser: """Main class to parse ABI files."""
#: Valid tags at Documentation/ABI. TAGS = r"(what|where|date|kernelversion|contact|description|users)"
#: ABI elements that will auto-generate cross-references. XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)"
def __init__(self, directory, logger=None, enable_lineno=False, show_warnings=True, debug=0): """Stores arguments for the class and initialize class vars."""
self.directory = directory self.enable_lineno = enable_lineno self.show_warnings = show_warnings self.debug = debug
if not logger: self.log = logging.getLogger("get_abi") else: self.log = logger
self.data = {} self.what_symbols = {} self.file_refs = {} self.what_refs = {}
# Ignore files that contain such suffixes self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~")
# Regular expressions used on parser self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR) self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I) self.re_valid = re.compile(self.TAGS) self.re_start_spc = re.compile(r"(\s*)(\S.*)") self.re_whitespace = re.compile(r"^\s+")
# Regular used on print self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})") self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])") self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)") self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n") self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst") self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)") self.re_xref_node = re.compile(self.XREF)
def warn(self, fdata, msg, extra=None): """Displays a parse error if warning is enabled."""
if not self.show_warnings: return
msg = f"{fdata.fname}:{fdata.ln}: {msg}" if extra: msg += "\n\t\t" + extra
self.log.warning(msg)
def add_symbol(self, what, fname, ln=None, xref=None): """Create a reference table describing where each 'what' is located."""
if what not in self.what_symbols: self.what_symbols[what] = {"file": {}}
if fname not in self.what_symbols[what]["file"]: self.what_symbols[what]["file"][fname] = []
if ln and ln not in self.what_symbols[what]["file"][fname]: self.what_symbols[what]["file"][fname].append(ln)
if xref: self.what_symbols[what]["xref"] = xref
def _parse_line(self, fdata, line): """Parse a single line of an ABI file."""
new_what = False new_tag = False content = None
match = self.re_tag.match(line) if match: new = match.group(1).lower() sep = match.group(2) content = match.group(3)
match = self.re_valid.search(new) if match: new_tag = match.group(1) else: if fdata.tag == "description": # New "tag" is actually part of description. # Don't consider it a tag new_tag = False elif fdata.tag != "": self.warn(fdata, f"tag '{fdata.tag}' is invalid", line)
if new_tag: # "where" is Invalid, but was a common mistake. Warn if found if new_tag == "where": self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead") new_tag = "what"
if new_tag == "what": fdata.space = None
if content not in self.what_symbols: self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln)
if fdata.tag == "what": fdata.what.append(content.strip("\n")) else: if fdata.key: if "description" not in self.data.get(fdata.key, {}): self.warn(fdata, f"{fdata.key} doesn't have a description")
for w in fdata.what: self.add_symbol(what=w, fname=fdata.fname, ln=fdata.what_ln, xref=fdata.key)
fdata.label = content new_what = True
key = "abi_" + content.lower() fdata.key = self.re_unprintable.sub("_", key).strip("_")
# Avoid duplicated keys but using a defined seed, to make # the namespace identical if there aren't changes at the # ABI symbols seed(42)
while fdata.key in self.data: char = randrange(0, 51) + ord("A") if char > ord("Z"): char += ord("a") - ord("Z") - 1
fdata.key += chr(char)
if fdata.key and fdata.key not in self.data: self.data[fdata.key] = { "what": [content], "file": [fdata.file_ref], "path": fdata.ftype, "line_no": fdata.ln, }
fdata.what = self.data[fdata.key]["what"]
self.what_refs[content] = fdata.key fdata.tag = new_tag fdata.what_ln = fdata.ln
if fdata.nametag["what"]: t = (content, fdata.key) if t not in fdata.nametag["symbols"]: fdata.nametag["symbols"].append(t)
return
if fdata.tag and new_tag: fdata.tag = new_tag
if new_what: fdata.label = ""
if "description" in self.data[fdata.key]: self.data[fdata.key]["description"] += "\n\n"
if fdata.file_ref not in self.data[fdata.key]["file"]: self.data[fdata.key]["file"].append(fdata.file_ref)
if self.debug == AbiDebug.WHAT_PARSING: self.log.debug("what: %s", fdata.what)
if not fdata.what: self.warn(fdata, "'What:' should come first:", line) return
if new_tag == "description": fdata.space = None
if content: sep = sep.replace(":", " ")
c = " " * len(new_tag) + sep + content c = c.expandtabs()
match = self.re_start_spc.match(c) if match: # Preserve initial spaces for the first line fdata.space = match.group(1) content = match.group(2) + "\n"
self.data[fdata.key][fdata.tag] = content
return
# Store any contents before tags at the database if not fdata.tag and "what" in fdata.nametag: fdata.nametag["description"] += line return
if fdata.tag == "description": content = line.expandtabs()
if self.re_whitespace.sub("", content) == "": self.data[fdata.key][fdata.tag] += "\n" return
if fdata.space is None: match = self.re_start_spc.match(content) if match: # Preserve initial spaces for the first line fdata.space = match.group(1)
content = match.group(2) + "\n" else: if content.startswith(fdata.space): content = content[len(fdata.space):]
else: fdata.space = ""
if fdata.tag == "what": w = content.strip("\n") if w: self.data[fdata.key][fdata.tag].append(w) else: self.data[fdata.key][fdata.tag] += content return
content = line.strip() if fdata.tag: if fdata.tag == "what": w = content.strip("\n") if w: self.data[fdata.key][fdata.tag].append(w) else: self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n") return
# Everything else is error if content: self.warn(fdata, "Unexpected content", line)
def parse_readme(self, nametag, fname): """Parse ABI README file."""
nametag["what"] = ["Introduction"] nametag["path"] = "README" with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: for line in fp: match = self.re_tag.match(line) if match: new = match.group(1).lower()
match = self.re_valid.search(new) if match: nametag["description"] += "\n:" + line continue
nametag["description"] += line
def parse_file(self, fname, path, basename): """Parse a single file."""
ref = f"abi_file_{path}_{basename}" ref = self.re_unprintable.sub("_", ref).strip("_")
# Store per-file state into a namespace variable. This will be used # by the per-line parser state machine and by the warning function. fdata = Namespace
fdata.fname = fname fdata.name = basename
pos = fname.find(ABI_DIR) if pos > 0: f = fname[pos:] else: f = fname
fdata.file_ref = (f, ref) self.file_refs[f] = ref
fdata.ln = 0 fdata.what_ln = 0 fdata.tag = "" fdata.label = "" fdata.what = [] fdata.key = None fdata.xrefs = None fdata.space = None fdata.ftype = path.split("/")[0]
fdata.nametag = {} fdata.nametag["what"] = [f"ABI file {path}/{basename}"] fdata.nametag["type"] = "File" fdata.nametag["path"] = fdata.ftype fdata.nametag["file"] = [fdata.file_ref] fdata.nametag["line_no"] = 1 fdata.nametag["description"] = "" fdata.nametag["symbols"] = []
self.data[ref] = fdata.nametag
if self.debug & AbiDebug.WHAT_OPEN: self.log.debug("Opening file %s", fname)
if basename == "README": self.parse_readme(fdata.nametag, fname) return
with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: for line in fp: fdata.ln += 1
self._parse_line(fdata, line)
if "description" in fdata.nametag: fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n")
if fdata.key: if "description" not in self.data.get(fdata.key, {}): self.warn(fdata, f"{fdata.key} doesn't have a description")
for w in fdata.what: self.add_symbol(what=w, fname=fname, xref=fdata.key)
def _parse_abi(self, root=None): """Internal function to parse documentation ABI recursively."""
if not root: root = self.directory
with os.scandir(root) as obj: for entry in obj: name = os.path.join(root, entry.name)
if entry.is_dir(): self._parse_abi(name) continue
if not entry.is_file(): continue
basename = os.path.basename(name)
if basename.startswith("."): continue
if basename.endswith(self.ignore_suffixes): continue
path = self.re_abi_dir.sub("", os.path.dirname(name))
self.parse_file(name, path, basename)
def parse_abi(self, root=None): """Parse documentation ABI."""
self._parse_abi(root)
if self.debug & AbiDebug.DUMP_ABI_STRUCTS: self.log.debug(pformat(self.data))
def desc_txt(self, desc): """Print description as found inside ABI files."""
desc = desc.strip(" \t\n")
return desc + "\n\n"
def xref(self, fname): """ Converts a Documentation/ABI + basename into a ReST cross-reference. """
xref = self.file_refs.get(fname) if not xref: return None else: return xref
def desc_rst(self, desc): """Enrich ReST output by creating cross-references."""
# Remove title markups from the description # Having titles inside ABI files will only work if extra # care would be taken in order to strictly follow the same # level order for each markup. desc = self.re_title_mark.sub("\n\n", "\n" + desc) desc = desc.rstrip(" \t\n").lstrip("\n")
# Python's regex performance for non-compiled expressions is a lot # than Perl, as Perl automatically caches them at their # first usage. Here, we'll need to do the same, as otherwise the # performance penalty is be high
new_desc = "" for d in desc.split("\n"): if d == "": new_desc += "\n" continue
# Use cross-references for doc files where needed d = self.re_doc.sub(r":doc:`/\1`", d)
# Use cross-references for ABI generated docs where needed matches = self.re_abi.findall(d) for m in matches: abi = m[0] + m[1]
xref = self.file_refs.get(abi) if not xref: # This may happen if ABI is on a separate directory, # like parsing ABI testing and symbol is at stable. # The proper solution is to move this part of the code # for it to be inside sphinx/kernel_abi.py self.log.info("Didn't find ABI reference for '%s'", abi) else: new = self.re_escape.sub(r"\\\1", m[1]) d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d)
# Seek for cross reference symbols like /sys/... # Need to be careful to avoid doing it on a code block if d[0] not in [" ", "\t"]: matches = self.re_xref_node.findall(d) for m in matches: # Finding ABI here is more complex due to wildcards xref = self.what_refs.get(m) if xref: new = self.re_escape.sub(r"\\\1", m) d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d)
new_desc += d + "\n"
return new_desc + "\n\n"
def doc(self, output_in_txt=False, show_symbols=True, show_file=True, filter_path=None): """Print ABI at stdout."""
part = None for key, v in sorted(self.data.items(), key=lambda x: (x[1].get("type", ""), x[1].get("what"))):
wtype = v.get("type", "Symbol") file_ref = v.get("file") names = v.get("what", [""])
if wtype == "File": if not show_file: continue else: if not show_symbols: continue
if filter_path: if v.get("path") != filter_path: continue
msg = ""
if wtype != "File": cur_part = names[0] if cur_part.find("/") >= 0: match = self.re_what.match(cur_part) if match: symbol = match.group(1).rstrip("/") cur_part = "Symbols under " + symbol
if cur_part and cur_part != part: part = cur_part msg += part + "\n"+ "-" * len(part) +"\n\n"
msg += f".. _{key}:\n\n"
max_len = 0 for i in range(0, len(names)): # pylint: disable=C0200 names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**"
max_len = max(max_len, len(names[i]))
msg += "+-" + "-" * max_len + "-+\n" for name in names: msg += f"| {name}" + " " * (max_len - len(name)) + " |\n" msg += "+-" + "-" * max_len + "-+\n" msg += "\n"
for ref in file_ref: if wtype == "File": msg += f".. _{ref[1]}:\n\n" else: base = os.path.basename(ref[0]) msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n"
if wtype == "File": msg += names[0] +"\n" + "-" * len(names[0]) +"\n\n"
desc = v.get("description") if not desc and wtype != "File": msg += f"DESCRIPTION MISSING for {names[0]}\n\n"
if desc: if output_in_txt: msg += self.desc_txt(desc) else: msg += self.desc_rst(desc)
symbols = v.get("symbols") if symbols: msg += "Has the following ABI:\n\n"
for w, label in symbols: # Escape special chars from content content = self.re_escape.sub(r"\\\1", w)
msg += f"- :ref:`{content} <{label}>`\n\n"
users = v.get("users") if users and users.strip(" \t\n"): users = users.strip("\n").replace('\n', '\n\t') msg += f"Users:\n\t{users}\n\n"
ln = v.get("line_no", 1)
yield (msg, file_ref[0][0], ln)
def check_issues(self): """Warn about duplicated ABI entries."""
for what, v in self.what_symbols.items(): files = v.get("file") if not files: # Should never happen if the parser works properly self.log.warning("%s doesn't have a file associated", what) continue
if len(files) == 1: continue
f = [] for fname, lines in sorted(files.items()): if not lines: f.append(f"{fname}") elif len(lines) == 1: f.append(f"{fname}:{lines[0]}") else: m = fname + "lines " m += ", ".join(str(x) for x in lines) f.append(m)
self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f))
def search_symbols(self, expr): """ Searches for ABI symbols."""
regex = re.compile(expr, re.I)
found_keys = 0 for t in sorted(self.data.items(), key=lambda x: [0]): v = t[1]
wtype = v.get("type", "") if wtype == "File": continue
for what in v.get("what", [""]): if regex.search(what): found_keys += 1
kernelversion = v.get("kernelversion", "").strip(" \t\n") date = v.get("date", "").strip(" \t\n") contact = v.get("contact", "").strip(" \t\n") users = v.get("users", "").strip(" \t\n") desc = v.get("description", "").strip(" \t\n")
files = [] for f in v.get("file", ()): files.append(f[0])
what = str(found_keys) + ". " + what title_tag = "-" * len(what)
print(f"\n{what}\n{title_tag}\n")
if kernelversion: print(f"Kernel version:\t\t{kernelversion}")
if date: print(f"Date:\t\t\t{date}")
if contact: print(f"Contact:\t\t{contact}")
if users: print(f"Users:\t\t\t{users}")
print("Defined on file(s):\t" + ", ".join(files))
if desc: desc = desc.strip("\n") print(f"\n{desc}\n")
if not found_keys: print(f"Regular expression /{expr}/ not found.")