Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380"""Building and refreshing the index."""
from __future__ import annotations
import bisectimport contextlibimport fcntlimport osimport sysimport time
import numpy as np
from . import embedfrom .chunk import chunk_filefrom .files import discover, find_reposfrom .paths import LOCK_PATH, MODEL_PATHfrom .store import Store, discard_shadow
# Chunks embedded between checkpoints. A full build takes hours and the largest# repository is over half of it, so checkpointing per repository means a crash# costs an hour. A checkpoint writes only the rows added since the last one, so# what this number buys is a bound on the work a crash repeats, not a way of# keeping the cost of checkpointing down.CHECKPOINT_EVERY = 2000
class IndexLocked(RuntimeError): pass
@contextlib.contextmanagerdef _exclusive_lock(): """Refuse to run two builds at once.
Concurrent builds interleave their writes and leave chunk metadata pointing at another run's vectors — a corruption with no error and no way to notice it except by reading results that are subtly wrong. """ os.makedirs(os.path.dirname(LOCK_PATH), exist_ok=True) handle = open(LOCK_PATH, "w") try: try: fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as exc: raise IndexLocked( "another index run holds the lock; wait for it or stop it first" ) from exc handle.write(str(os.getpid())) handle.flush() yield finally: handle.close()
def _progress(done: int, total: int, started: float, label: str) -> None: if total == 0: return elapsed = time.time() - started rate = done / elapsed if elapsed > 0 else 0 eta = (total - done) / rate if rate > 0 else 0 bar_len = 24 filled = int(bar_len * done / total) bar = "#" * filled + "." * (bar_len - filled) sys.stderr.write( f"\r [{bar}] {done}/{total} {label} {rate:5.1f}/s eta {eta / 60:5.1f}m " ) sys.stderr.flush()
def _reusable_state(store: Store, force: bool) -> tuple[dict, dict, np.ndarray | None]: """Previous chunks and vectors that a new run may reuse.
Reuse is abandoned wholesale when the embedding space has changed, or when the matrix is shorter than the metadata claims. The latter means a previous run died between writing its rows and writing its matrix; reusing across that gap would read past the end of the matrix and fail every subsequent run identically. """ if force: return {}, {}, None
old_vecs = store._load_vectors() if old_vecs is None: return {}, {}, None
model_changed = store.get_meta("model") not in (None, os.path.basename(MODEL_PATH)) dim_changed = int(store.get_meta("dim") or old_vecs.shape[1]) != embed.EMBED_DIM if model_changed or dim_changed or old_vecs.shape[1] != embed.EMBED_DIM: print( " embedding model or dimension changed; re-embedding everything", file=sys.stderr, ) return {}, {}, None
old_vecs = np.asarray(old_vecs) n_rows = old_vecs.shape[0] by_file: dict[tuple[str, str], list] = {} sha: dict[tuple[str, str], str] = {} truncated: set[tuple[str, str]] = set() for row in store.all_chunks(): key = (row["repo"], row["rel"]) if row["row"] >= n_rows: truncated.add(key) by_file.setdefault(key, []).append(row) sha[key] = row["sha"]
for key in truncated: by_file.pop(key, None) sha.pop(key, None) if truncated: print( f" {len(truncated)} file(s) had vectors missing from a previous run; " "re-embedding them", file=sys.stderr, ) return by_file, sha, old_vecs
def _records_from( records: list[tuple], extra: list[tuple], start: int) -> list[tuple]: """Records from `start` onward, across the finished repositories and a partial repository's prefix.""" if start < len(records): return records[start:] + extra return extra[start - len(records):]
def _rows_from( blocks: list[np.ndarray], extra: np.ndarray | None, start: int) -> np.ndarray: """Vector rows from `start` onward, across the finished repositories' blocks and a partial repository's prefix.
Joining the whole list to write only its tail would copy the entire matrix at every checkpoint, which is the cost a checkpoint exists to avoid. """ parts = list(blocks) if extra is not None and len(extra): parts.append(extra) tail: list[np.ndarray] = [] pos = 0 for block in parts: end = pos + len(block) if end > start: tail.append(block[max(start - pos, 0):]) pos = end if not tail: return np.zeros((0, embed.EMBED_DIM), dtype=np.float32) return tail[0] if len(tail) == 1 else np.concatenate(tail)
def build(roots: list[str], threads: int | None = None, force: bool = False) -> dict: """Index every git repository under each root.
Files whose contents are unchanged since the last run keep their existing vectors and never reach the embedding server. Work goes into a shadow index every CHECKPOINT_EVERY chunks, so an interrupted run resumes near where it stopped, and a repository that fails is skipped rather than ending the run. The live index is only replaced once the run finishes, so searches keep returning the previous results for the hours a full build takes. """ with _exclusive_lock(): return _build(roots, threads=threads, force=force)
def _build(roots: list[str], threads: int | None, force: bool) -> dict: live = Store(dim=embed.EMBED_DIM)
repos: list[str] = [] for root in roots: repos.extend(find_repos(root)) # Overlapping roots would otherwise index a repository twice, doubling its # chunks and every search result drawn from it. repos = list(dict.fromkeys(repos)) if not repos: raise SystemExit(f"no git repositories found under: {', '.join(roots)}")
if force: discard_shadow() shadow = Store.shadow(dim=embed.EMBED_DIM) if not force and shadow.get_meta("vector_generation") is None: # No interrupted run to carry on from, so the shadow starts as a copy of # the live index and unchanged files keep the vectors they already have. # An interrupted run's shadow is left alone: it is this run's resume point. shadow.replace_from(live)
old_by_file, old_sha, old_vecs = _reusable_state(shadow, force)
# With no index yet there is nothing for the shadow to protect, and holding work # back would mean hours of a first build with nothing to search at all. Decided # once, up front: as soon as an index exists it is a whole index, never a # half-finished one that later checkpoints could shrink. first_build = live.vector_count() == 0
records: list[tuple] = [] vectors_out: list[np.ndarray] = [] now = time.time() totals = {"reused": 0, "changed": 0, "embedded": 0} skipped_total: dict[str, int] = {} failed_repos: list[str] = [] since_checkpoint = 0 # Rows of this run already in the shadow. Everything below this number is # stored; a checkpoint publishes the rest and nothing else. published = 0 seeded = True # the shadow still holds the copy it was seeded with
def checkpoint( note: str, extra_records: list[tuple] | None = None, extra_vecs: np.ndarray | None = None, ) -> None: """Record everything complete so far, optionally including a partial repo.
This writes the shadow, not the live index: it is a resume point for a run that dies, and unless there is no index yet, nothing searchable changes until the run finishes.
Only the rows added since the last checkpoint are written. The first checkpoint of a run is the exception: the shadow still holds the seed it was given, which this run's rows replace rather than follow. """ nonlocal published, seeded extra_records = list(extra_records or []) total = len(records) + len(extra_records) if seeded: shadow.rebuild( records + extra_records, _rows_from(vectors_out, extra_vecs, 0), ) seeded = False elif total > published: shadow.append( _records_from(records, extra_records, published), _rows_from(vectors_out, extra_vecs, published), ) shadow.set_meta("model", os.path.basename(MODEL_PATH)) shadow.set_meta("dim", str(embed.EMBED_DIM)) shadow.set_meta("built_at", time.strftime("%Y-%m-%d %H:%M:%S")) shadow.set_meta("progress", note) if first_build and total > published: live.replace_from(shadow) published = total # The live index keeps its own contents, but `status` still has to be able # to say how far a run has got. live.set_meta("progress", note)
for repo_no, repo in enumerate(repos, 1): name = os.path.basename(repo) try: files, skipped = discover(repo) except Exception as exc: # a broken or unreadable repository is not fatal print(f" [{repo_no}/{len(repos)}] {name:22} SKIPPED: {exc}", file=sys.stderr) failed_repos.append(name) continue for reason, n in skipped.items(): skipped_total[reason] = skipped_total.get(reason, 0) + n
repo_records: list[tuple] = [] repo_vecs_reused: list[tuple[int, int]] = [] # (local row, previous row) pending: list[tuple[int, str]] = [] # (local row, text to embed) # Row counts at which a file ends. A checkpoint may only publish up to one # of these: a file stored with just some of its chunks would match on sha # next run and keep the truncation permanently, losing the rest silently. file_ends: list[int] = [] reused = 0
for sf in files: key = (sf.repo, sf.rel) if old_sha.get(key) == sf.sha: reused += 1 for row in old_by_file[key]: repo_vecs_reused.append((len(repo_records), row["row"])) repo_records.append( (sf.repo, sf.rel, sf.sha, row["start_line"], row["end_line"], row["text"], now) ) file_ends.append(len(repo_records)) continue try: # Hashed on this read, not on discovery's: a file edited during a # multi-hour build would otherwise be stored under the hash of the # version before the edit, and match it on the next run. text, sha = sf.read() except (OSError, UnicodeDecodeError): continue # changed to binary or vanished since discovery for ch in chunk_file(sf.rel, text): pending.append((len(repo_records), ch.embedding_text())) repo_records.append( (sf.repo, sf.rel, sha, ch.start_line, ch.end_line, ch.text, now) ) file_ends.append(len(repo_records))
print( f" [{repo_no}/{len(repos)}] {name:22} {len(files):5d} files, " f"{len(repo_records):6d} chunks ({reused} unchanged, {len(pending)} to embed)", file=sys.stderr, )
repo_vecs = np.zeros((len(repo_records), embed.EMBED_DIM), dtype=np.float32) if repo_vecs_reused and old_vecs is not None: dest = np.asarray([d for d, _ in repo_vecs_reused], dtype=np.int64) src = np.asarray([s for _, s in repo_vecs_reused], dtype=np.int64) repo_vecs[dest] = old_vecs[src]
# The repository joins the index only once its vectors are all present, so # a checkpoint never publishes rows whose vectors are still zero. embedded_here = 0 try: if pending: embed.start(threads=threads) started = time.time() for off in range(0, len(pending), embed.BATCH_SIZE): batch = pending[off : off + embed.BATCH_SIZE] mat = embed.embed([t for _, t in batch], is_query=False) repo_vecs[np.asarray([r for r, _ in batch], dtype=np.int64)] = mat embedded_here += len(batch) _progress(embedded_here, len(pending), started, "chunks")
since_checkpoint += len(batch) if since_checkpoint >= CHECKPOINT_EVERY: done = off + len(batch) # Every row below the next unembedded one is filled: reused # rows were written before the loop, pending rows in order. filled = ( pending[done][0] if done < len(pending) else len(repo_records) ) cut = bisect.bisect_right(file_ends, filled) if cut: hw = file_ends[cut - 1] checkpoint( f"{repo_no}/{len(repos)} repos; " f"{name} {embedded_here}/{len(pending)} chunks", repo_records[:hw], repo_vecs[:hw], ) since_checkpoint = 0 sys.stderr.write("\n") except Exception as exc: sys.stderr.write("\n") print(f" {name}: FAILED after {embedded_here} chunks: {exc}", file=sys.stderr) failed_repos.append(name) if published > len(records): # A checkpoint published the first files of this repository. The # repository is being abandoned, so those rows go with it rather # than reaching the live index at the end of the run. shadow.truncate(len(records)) published = len(records) checkpoint(f"{repo_no}/{len(repos)} repos; {name} failed") continue
records.extend(repo_records) vectors_out.append(repo_vecs) totals["reused"] += reused totals["changed"] += len(files) - reused totals["embedded"] += len(pending)
if since_checkpoint >= CHECKPOINT_EVERY or repo_no == len(repos): checkpoint(f"{repo_no}/{len(repos)} repos") since_checkpoint = 0
checkpoint("complete" if not failed_repos else f"complete; failed: {', '.join(failed_repos)}")
# The one moment search results change. Until this returns, every query has been # answered from the index that existed before the run started. live.replace_from(shadow) live.set_meta("roots", os.pathsep.join(os.path.abspath(r) for r in roots)) shadow.close() discard_shadow()
return { "repos": len(repos), "files": totals["reused"] + totals["changed"], "reused": totals["reused"], "embedded_chunks": totals["embedded"], "total_chunks": len(records), "skipped": skipped_total, "failed_repos": failed_repos, }