"""Scorer for the retrieval eval: known-answer queries against the live index. Reuses `code_index.embed.embed()` and `code_index.store.Store.search()` — the same path `cli.py cmd_search()` takes — so this measures retrieval as a search actually sees it rather than some parallel implementation of it. uv run python eval/score.py # unscoped uv run python eval/score.py --repo-scoped # each query filtered to its own repo uv run python eval/score.py --repos one,two # only queries for these repos uv run python eval/score.py --exclude-path eval/queries # drop hits into a path Run without CODE_INDEX_DATA set, so it scores the shared index every session searches rather than an empty one built nowhere. The query set comes from `CODE_INDEX_EVAL_QUERIES`, falling back to `eval/queries.example.jsonl` when that path holds nothing — see `queries_path()`. `--repos` restricts the query set to a subset of repositories without touching the query file itself, for comparing indexes that only cover part of the workspace (see `eval/model-comparison.md`). `--exclude-path` drops any hit whose path contains the given substring before ranking, for discounting a file that wins on text similarity for reasons that have nothing to do with the model — the query file itself is indexed and contains every query's own text verbatim, so it is a plausible top hit for any query regardless of retrieval quality. """ from __future__ import annotations import argparse import json import os import sys from collections import defaultdict from code_index import embed from code_index.paths import EVAL_QUERIES, EVAL_QUERIES_EXAMPLE from code_index.store import Hit, Store def queries_path() -> str: """Where to read the query set from. A query set is only worth anything if it names real files in real repositories, which makes it a description of whatever tree it was written against rather than something to commit. So the real one is a local file at `CODE_INDEX_EVAL_QUERIES`, and the committed example — whose corpus is this repository itself — is what a fresh clone scores until one exists. """ return EVAL_QUERIES if os.path.exists(EVAL_QUERIES) else EVAL_QUERIES_EXAMPLE def load_queries(path: str | None = None) -> list[dict]: queries = [] with open(path or queries_path()) as fh: for line in fh: line = line.strip() if not line: continue queries.append(json.loads(line)) return queries def _matches(hit: Hit, query: dict) -> bool: return os.path.basename(hit.repo) == query["repo"] and hit.rel == query["path"] def run( queries: list[dict], store: Store, repo_scoped: bool, k: int = 5, exclude_path: str | None = None, ) -> list[dict]: """Run every query, returning one result record each. A record carries whether `path` showed up in the top 1 / top 5 hits, and, on a miss, what came back instead so a regression is diagnosable without a rerun. `exclude_path` drops any hit whose path contains it before the top-1/top-5 cut, over-fetching first so the excluded hit doesn't just leave a gap. """ results = [] for q in queries: vec = embed.embed([q["query"]], is_query=True)[0] repo = q["repo"] if repo_scoped else None fetch_k = k * 4 if exclude_path else k hits = store.search(vec, k=fetch_k, repo=repo) if exclude_path: hits = [h for h in hits if exclude_path not in h.rel][:k] results.append({ "query": q, "top1": bool(hits) and _matches(hits[0], q), "top5": any(_matches(h, q) for h in hits), "top_hit": f"{os.path.basename(hits[0].repo)}/{hits[0].rel}" if hits else None, }) return results def summarize(results: list[dict]) -> tuple[dict, dict]: """Overall and per-repo top-1 / top-5 hit counts.""" overall = {"n": 0, "top1": 0, "top5": 0} per_repo: dict[str, dict] = defaultdict(lambda: {"n": 0, "top1": 0, "top5": 0}) for r in results: for bucket in (overall, per_repo[r["query"]["repo"]]): bucket["n"] += 1 bucket["top1"] += r["top1"] bucket["top5"] += r["top5"] return overall, per_repo def _rate(hit: int, n: int) -> float: return hit / n if n else 0.0 def print_report(results: list[dict], repo_scoped: bool, k: int = 5) -> None: overall, per_repo = summarize(results) label = "repo-scoped" if repo_scoped else "unscoped" print(f"=== {label} ({overall['n']} queries) ===") print( f"overall: top-1 {overall['top1']}/{overall['n']} ({_rate(overall['top1'], overall['n']):.0%})" f" top-5 {overall['top5']}/{overall['n']} ({_rate(overall['top5'], overall['n']):.0%})" ) for repo in sorted(per_repo): b = per_repo[repo] print( f" {repo:20} top-1 {b['top1']}/{b['n']} ({_rate(b['top1'], b['n']):.0%})" f" top-5 {b['top5']}/{b['n']} ({_rate(b['top5'], b['n']):.0%})" ) misses = [r for r in results if not r["top5"]] if misses: print(f"misses (outside top {k}):") for r in misses: q = r["query"] got = r["top_hit"] or "no hits" print(f" [{q['repo']}] {q['query']!r} -> expected {q['path']}, got {got}") def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Score the index against a known-answer query set") parser.add_argument( "--queries", help=f"path to the query set (default: {queries_path()})", ) parser.add_argument("-k", type=int, default=5, help="how many hits count as top-5") parser.add_argument( "--repo-scoped", action="store_true", help="filter each query's search to its own repo", ) parser.add_argument( "--repos", help="comma-separated repo names to restrict the query set to", ) parser.add_argument( "--exclude-path", help="drop hits whose path contains this substring", ) args = parser.parse_args(argv) path = args.queries or queries_path() print(f"query set: {path}", file=sys.stderr) queries = load_queries(path) if args.repos: wanted = set(args.repos.split(",")) queries = [q for q in queries if q["repo"] in wanted] store = Store(dim=embed.EMBED_DIM) if store.vector_count() == 0: print("index is empty; run `code-index index` first", file=sys.stderr) return 1 embed.start() results = run( queries, store, repo_scoped=args.repo_scoped, k=args.k, exclude_path=args.exclude_path, ) print_report(results, repo_scoped=args.repo_scoped, k=args.k) return 0 if __name__ == "__main__": raise SystemExit(main())