Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176"""Scorer for the retrieval eval: known-answer queries against the live index.
Reuses `code_index.embed.embed()` and `code_index.store.Store.search()` — the samepath `cli.py cmd_search()` takes — so this measures retrieval as a search actuallysees it rather than some parallel implementation of it.
uv run python eval/score.py # unscoped uv run python eval/score.py --repo-scoped # each query filtered to its own repo uv run python eval/score.py --repos one,two # only queries for these repos uv run python eval/score.py --exclude-path eval/queries # drop hits into a path
Run without CODE_INDEX_DATA set, so it scores the shared index every sessionsearches rather than an empty one built nowhere.
The query set comes from `CODE_INDEX_EVAL_QUERIES`, falling back to`eval/queries.example.jsonl` when that path holds nothing — see `queries_path()`.
`--repos` restricts the query set to a subset of repositories without touchingthe query file itself, for comparing indexes that only cover part of theworkspace (see `eval/model-comparison.md`). `--exclude-path` drops any hit whosepath contains the given substring before ranking, for discounting a file thatwins on text similarity for reasons that have nothing to do with the model — thequery file itself is indexed and contains every query's own text verbatim, so itis a plausible top hit for any query regardless of retrieval quality."""
from __future__ import annotations
import argparseimport jsonimport osimport sysfrom collections import defaultdict
from code_index import embedfrom code_index.paths import EVAL_QUERIES, EVAL_QUERIES_EXAMPLEfrom code_index.store import Hit, Store
def queries_path() -> str: """Where to read the query set from.
A query set is only worth anything if it names real files in real repositories, which makes it a description of whatever tree it was written against rather than something to commit. So the real one is a local file at `CODE_INDEX_EVAL_QUERIES`, and the committed example — whose corpus is this repository itself — is what a fresh clone scores until one exists. """ return EVAL_QUERIES if os.path.exists(EVAL_QUERIES) else EVAL_QUERIES_EXAMPLE
def load_queries(path: str | None = None) -> list[dict]: queries = [] with open(path or queries_path()) as fh: for line in fh: line = line.strip() if not line: continue queries.append(json.loads(line)) return queries
def _matches(hit: Hit, query: dict) -> bool: return os.path.basename(hit.repo) == query["repo"] and hit.rel == query["path"]
def run( queries: list[dict], store: Store, repo_scoped: bool, k: int = 5, exclude_path: str | None = None,) -> list[dict]: """Run every query, returning one result record each.
A record carries whether `path` showed up in the top 1 / top 5 hits, and, on a miss, what came back instead so a regression is diagnosable without a rerun.
`exclude_path` drops any hit whose path contains it before the top-1/top-5 cut, over-fetching first so the excluded hit doesn't just leave a gap. """ results = [] for q in queries: vec = embed.embed([q["query"]], is_query=True)[0] repo = q["repo"] if repo_scoped else None fetch_k = k * 4 if exclude_path else k hits = store.search(vec, k=fetch_k, repo=repo) if exclude_path: hits = [h for h in hits if exclude_path not in h.rel][:k] results.append({ "query": q, "top1": bool(hits) and _matches(hits[0], q), "top5": any(_matches(h, q) for h in hits), "top_hit": f"{os.path.basename(hits[0].repo)}/{hits[0].rel}" if hits else None, }) return results
def summarize(results: list[dict]) -> tuple[dict, dict]: """Overall and per-repo top-1 / top-5 hit counts.""" overall = {"n": 0, "top1": 0, "top5": 0} per_repo: dict[str, dict] = defaultdict(lambda: {"n": 0, "top1": 0, "top5": 0}) for r in results: for bucket in (overall, per_repo[r["query"]["repo"]]): bucket["n"] += 1 bucket["top1"] += r["top1"] bucket["top5"] += r["top5"] return overall, per_repo
def _rate(hit: int, n: int) -> float: return hit / n if n else 0.0
def print_report(results: list[dict], repo_scoped: bool, k: int = 5) -> None: overall, per_repo = summarize(results) label = "repo-scoped" if repo_scoped else "unscoped" print(f"=== {label} ({overall['n']} queries) ===") print( f"overall: top-1 {overall['top1']}/{overall['n']} ({_rate(overall['top1'], overall['n']):.0%})" f" top-5 {overall['top5']}/{overall['n']} ({_rate(overall['top5'], overall['n']):.0%})" ) for repo in sorted(per_repo): b = per_repo[repo] print( f" {repo:20} top-1 {b['top1']}/{b['n']} ({_rate(b['top1'], b['n']):.0%})" f" top-5 {b['top5']}/{b['n']} ({_rate(b['top5'], b['n']):.0%})" )
misses = [r for r in results if not r["top5"]] if misses: print(f"misses (outside top {k}):") for r in misses: q = r["query"] got = r["top_hit"] or "no hits" print(f" [{q['repo']}] {q['query']!r} -> expected {q['path']}, got {got}")
def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Score the index against a known-answer query set") parser.add_argument( "--queries", help=f"path to the query set (default: {queries_path()})", ) parser.add_argument("-k", type=int, default=5, help="how many hits count as top-5") parser.add_argument( "--repo-scoped", action="store_true", help="filter each query's search to its own repo", ) parser.add_argument( "--repos", help="comma-separated repo names to restrict the query set to", ) parser.add_argument( "--exclude-path", help="drop hits whose path contains this substring", ) args = parser.parse_args(argv)
path = args.queries or queries_path() print(f"query set: {path}", file=sys.stderr) queries = load_queries(path) if args.repos: wanted = set(args.repos.split(",")) queries = [q for q in queries if q["repo"] in wanted] store = Store(dim=embed.EMBED_DIM) if store.vector_count() == 0: print("index is empty; run `code-index index` first", file=sys.stderr) return 1 embed.start()
results = run( queries, store, repo_scoped=args.repo_scoped, k=args.k, exclude_path=args.exclude_path, ) print_report(results, repo_scoped=args.repo_scoped, k=args.k) return 0
if __name__ == "__main__": raise SystemExit(main())