Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193"""Discovery and filtering of indexable source files."""
from __future__ import annotations
import hashlibimport osimport subprocessfrom dataclasses import dataclass
# Binary, media, and archive formats — nothing to embed.BINARY_EXT = { ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".webp", ".avif", ".tiff", ".icns", ".pdf", ".psd", ".ai", ".sketch", ".zip", ".gz", ".bz2", ".xz", ".tar", ".7z", ".rar", ".dmg", ".pkg", ".deb", ".woff", ".woff2", ".ttf", ".otf", ".eot", ".mp3", ".mp4", ".wav", ".ogg", ".webm", ".mov", ".avi", ".flac", ".m4a", ".so", ".dylib", ".dll", ".exe", ".o", ".a", ".class", ".jar", ".wasm", ".node", ".bin", ".gguf", ".safetensors", ".pyc", ".pyo", ".db", ".sqlite", ".sqlite3", ".mdb",}
# Generated or machine-authored text that carries no design intent.NOISE_EXT = {".lock", ".map", ".snap", ".min.js", ".min.css", ".pb.go", ".pb.py"}
NOISE_NAME = { "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "npm-shrinkwrap.json", "Cargo.lock", "poetry.lock", "uv.lock", "Gemfile.lock", "composer.lock", "go.sum", "flake.lock", "bun.lockb", "pdm.lock",}
# Path segments whose subtrees are dependencies, build output, or vendored code.# Deliberately conservative: a wrongly excluded file is invisible to search with no# error, which is worse than a little noise. Directories that merely *often* hold# generated content (assets, migrations, fixtures) are left in and handled by the# binary, size, and minification filters instead.EXCLUDED_SEGMENTS = { "node_modules", ".yarn", ".pnp", "bower_components", "vendor", "third_party", "thirdparty", "chrome-extensions", "browser-extensions", "dist", "build", "out", "target", "obj", ".next", ".nuxt", ".svelte-kit", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".tox", ".venv", "venv", "site-packages", ".eggs", "coverage", ".nyc_output", "htmlcov", ".git", ".idea", ".vscode", ".gradle", "Pods", "DerivedData", "generated", "__generated__", "__snapshots__",}
# Files this size or larger are almost never hand-written source.MAX_FILE_BYTES = 400_000
# A mean line longer than this means minified or single-line generated output.MAX_MEAN_LINE_LEN = 300
@dataclass(frozen=True)class SourceFile: repo: str # absolute path to the repository root rel: str # path relative to the repository root size: int sha: str # sha256 of file contents
@property def abspath(self) -> str: return os.path.join(self.repo, self.rel)
def read(self) -> tuple[str, str]: """The file's current text, and the sha of that same read.
`sha` is taken during discovery, which on a full build is hours before the indexer reaches the file. Storing the discovery hash next to text read later would record a pair that never existed: an edit landing in that window would be indexed and then hidden behind the hash of the version before it. Hashing the bytes actually used keeps the pair honest, and the cost is a second hash of a file that is being read anyway. """ with open(self.abspath, "rb") as fh: data = fh.read() return data.decode("utf-8"), hashlib.sha256(data).hexdigest()
def _excluded_by_path(rel: str) -> bool: parts = rel.split("/") for seg in parts[:-1]: if seg in EXCLUDED_SEGMENTS or seg.startswith("."): if seg not in (".github", ".claude", ".config"): return True base = parts[-1] if base in NOISE_NAME: return True low = base.lower() if any(low.endswith(ext) for ext in BINARY_EXT): return True if any(low.endswith(ext) for ext in NOISE_EXT): return True return False
def _looks_like_text(data: bytes) -> bool: if b"\0" in data[:8192]: return False sample = data[:8192] if not sample: return False printable = sum(1 for b in sample if b in (9, 10, 13) or 32 <= b < 127) return printable / len(sample) > 0.85
def _looks_minified(text: str) -> bool: lines = text.split("\n") if len(lines) < 2: return len(text) > 2000 return len(text) / len(lines) > MAX_MEAN_LINE_LEN
def git_tracked(repo: str) -> list[str]: """Tracked paths, so .gitignore is honoured without reimplementing it.""" out = subprocess.run( ["git", "-C", repo, "ls-files", "-z"], capture_output=True, check=True, timeout=120, ).stdout return [p.decode("utf-8", "replace") for p in out.split(b"\0") if p]
def discover(repo: str) -> tuple[list[SourceFile], dict[str, int]]: """Return indexable files under `repo`, plus counts of what was skipped and why.""" repo = os.path.abspath(repo) skipped: dict[str, int] = {}
def skip(reason: str) -> None: skipped[reason] = skipped.get(reason, 0) + 1
files: list[SourceFile] = [] for rel in git_tracked(repo): if _excluded_by_path(rel): skip("path") continue path = os.path.join(repo, rel) try: size = os.path.getsize(path) except OSError: skip("missing") continue if size == 0: skip("empty") continue if size > MAX_FILE_BYTES: skip("too-large") continue try: with open(path, "rb") as fh: data = fh.read() except OSError: skip("unreadable") continue if not _looks_like_text(data): skip("binary") continue try: text = data.decode("utf-8") except UnicodeDecodeError: skip("not-utf8") continue if _looks_minified(text): skip("minified") continue files.append( SourceFile( repo=repo, rel=rel, size=size, sha=hashlib.sha256(data).hexdigest(), ) ) return files, skipped
def _is_repo(path: str) -> bool: # In a linked worktree or a submodule, `.git` is a file pointing at the real # git directory rather than being one, so testing for a directory skips them. return os.path.exists(os.path.join(path, ".git"))
def find_repos(root: str) -> list[str]: """`root` itself if it is a repository, otherwise its repository children.""" root = os.path.abspath(root) if _is_repo(root): return [root] found = [] for name in sorted(os.listdir(root)): path = os.path.join(root, name) if os.path.isdir(path) and _is_repo(path): found.append(path) return found