Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141"""Splitting source files into overlapping, line-aligned chunks."""
from __future__ import annotations
import refrom dataclasses import dataclass
# Characters per token, measured on mixed source text. Used only to size chunks;# the embedding server truncates anything that still overshoots.CHARS_PER_TOKEN = 3.6
TARGET_TOKENS = 320OVERLAP_TOKENS = 64MIN_CHUNK_CHARS = 80
TARGET_CHARS = int(TARGET_TOKENS * CHARS_PER_TOKEN)OVERLAP_CHARS = int(OVERLAP_TOKENS * CHARS_PER_TOKEN)
# A hard ceiling on chunk size, enforced after chunking regardless of how a chunk# was formed. Chunk sizing is done in characters, but the embedding server's limit# is in tokens, and the ratio between them collapses towards 1:1 on dense# punctuation, base64 payloads, and long non-prose lines. One chunk over the limit# fails the request and, before this cap existed, killed a multi-hour index.MAX_CHUNK_CHARS = 1600
# Lines that start a new top-level construct in the languages in this workspace.# A chunk boundary is nudged onto one of these when a nearby candidate exists, so# chunks tend to begin at a definition rather than mid-body.BOUNDARY = re.compile( r"^(?:\s{0,4})" r"(?:export\s+)?(?:default\s+)?(?:async\s+)?" r"(?:function\b|class\b|interface\b|type\s+\w+\s*=|enum\b|struct\b|impl\b|trait\b" r"|def\b|fn\b|pub\s+fn\b|const\s+\w+\s*=\s*(?:async\s*)?\(|let\s+\w+\s*=\s*(?:async\s*)?\(" r"|describe\(|it\(|test\(|#{1,3}\s)")
@dataclass(frozen=True)class Chunk: rel: str start_line: int # 1-based, inclusive end_line: int # 1-based, inclusive text: str # raw source text of the chunk
def embedding_text(self) -> str: """What actually gets embedded: the path, then the source.
The path carries real signal — a query like "session refresh" should match `src/auth/session.ts` even when the body never uses the word "session". """ location = self.rel.replace("/", " / ").replace("_", " ").replace("-", " ") return f"{location}\n\n{self.text}"
def _snap_to_boundary(lines: list[str], idx: int, window: int) -> int: """Move `idx` back to the nearest construct start within `window` lines.""" lo = max(0, idx - window) for probe in range(idx, lo - 1, -1): if BOUNDARY.match(lines[probe]): return probe return idx
def _enforce_cap(chunks: list[Chunk]) -> list[Chunk]: """Split any chunk over the ceiling, slicing mid-line if that is what it takes.
A single line can exceed the budget on its own — a minified fragment inside an otherwise normal file, a long JSON literal — and line-aligned splitting has no answer for it.
Each piece carries the span it actually covers, counted from the newlines before it. Handing every piece the original's span would point every hit at the whole chunk, and would also leave line-coverage checks unable to see content dropped on this path, since the spans would add up whatever the pieces held. """ out: list[Chunk] = [] for ch in chunks: if len(ch.text) <= MAX_CHUNK_CHARS: out.append(ch) continue for off in range(0, len(ch.text), MAX_CHUNK_CHARS): piece = ch.text[off : off + MAX_CHUNK_CHARS] if not piece.strip(): continue # A cut lands mid-line, so a piece shares its first line with the piece # before it and its last with the piece after. start_line = ch.start_line + ch.text.count("\n", 0, off) end_line = start_line + piece.count("\n") if piece.endswith("\n"): end_line -= 1 out.append(Chunk(ch.rel, start_line, min(end_line, ch.end_line), piece)) return out
def chunk_file(rel: str, text: str) -> list[Chunk]: lines = text.split("\n") if not any(line.strip() for line in lines): return []
# Cumulative character offsets so a line span's size is a subtraction. offsets = [0] for line in lines: offsets.append(offsets[-1] + len(line) + 1) total = offsets[-1]
if total <= TARGET_CHARS: return _enforce_cap([Chunk(rel, 1, len(lines), text)])
chunks: list[Chunk] = [] start = 0 snap_window = 12 while start < len(lines): budget = offsets[start] + TARGET_CHARS end = start while end < len(lines) and offsets[end + 1] <= budget: end += 1 if end == start: end = start + 1 # a single line longer than the budget still forms a chunk
if end < len(lines): snapped = _snap_to_boundary(lines, end, snap_window) if snapped > start: end = snapped
body = "\n".join(lines[start:end]) if len(body.strip()) >= MIN_CHUNK_CHARS: chunks.append(Chunk(rel, start + 1, end, body))
if end >= len(lines): break
# Step back by the overlap so context spanning a boundary is still findable. step_back = 0 cursor = end while cursor > start + 1 and offsets[end] - offsets[cursor] < OVERLAP_CHARS: cursor -= 1 step_back += 1 start = max(start + 1, end - step_back)
return _enforce_cap(chunks)