"""Client for the local llama.cpp embedding server, plus server lifecycle.""" from __future__ import annotations import json import os import signal import socket import subprocess import time import urllib.error import urllib.request from typing import NamedTuple import numpy as np from .paths import LLAMA_BIN, LLAMA_LIB, MODEL_PATH, RUNTIME_DIR HOST = "127.0.0.1" PORT = int(os.environ.get("CODE_INDEX_PORT", "8177")) BASE_URL = f"http://{HOST}:{PORT}" PID_FILE = os.path.join(RUNTIME_DIR, "server.pid") LOG_FILE = os.path.join(RUNTIME_DIR, "server.log") CONTEXT = 2048 # Kept below CONTEXT even at the worst character-to-token ratio, so a single input # always fits the CONTEXT tokens the sizing below allots each input. Sized in chars # because counting tokens would mean a round trip to the server for every chunk. # The densest text measured, CJK, comes to exactly one token per character, so the # margin here is 12% (eval/batch-size.md). MAX_INPUT_CHARS = 1800 # Chunks per request, and with it --ctx-size, since the whole request has to fit in # the context at once. Every token of context costs 112 KB of key/value cache under # a decoder-only model, and larger batches were not measurably faster on this # machine: 32 held 18.4 GB resident against this setting's 4.8 GB, for a difference # in indexing rate inside the run-to-run noise. See eval/batch-size.md. BATCH_SIZE = 8 class ServerError(RuntimeError): pass class ModelProfile(NamedTuple): """Everything about a model that the rest of the index has to agree with.""" dim: int pooling: str doc_prefix: str query_prefix: str # Pooling is a property of the model, not of the deployment, so it travels with # the model file rather than sitting in the server command line: llama-server # accepts any --pooling against any architecture and returns a vector of the right # shape, so a mismatch fills the entire index with meaningless vectors and raises # nothing anywhere. Task prefixes are the same trap in a quieter register — the # wrong one on either side measurably degrades every result. Keys are matched # against the GGUF filename, so quantisations and revisions need no entry each. PROFILES: dict[str, ModelProfile] = { # Decoder-only, so the whole sequence is summarised on its last token. Only # queries carry an instruction; documents are embedded bare. This wording is # what eval/model-comparison.md measured — changing it invalidates its numbers. "qwen3-embedding": ModelProfile( dim=1024, pooling="last", doc_prefix="", query_prefix=( "Instruct: Given a code search query, retrieve relevant source code " "and documentation that answer the query\nQuery: " ), ), # An encoder, trained with mean pooling and two asymmetric task prefixes. "nomic-embed-text": ModelProfile( dim=768, pooling="mean", doc_prefix="search_document: ", query_prefix="search_query: ", ), } def profile_for(model_path: str) -> ModelProfile: """The profile for a GGUF path, or an error naming what is missing. Refusing an unrecognised model is deliberate. Falling back to any default would index it without complaint and return confident nonsense, since neither wrong pooling nor a wrong prefix produces an error at any point. """ name = os.path.basename(model_path).lower() for key, profile in PROFILES.items(): if key in name: return profile raise ServerError( f"no embedding profile for {os.path.basename(model_path)}: add its " "dimensions, pooling and task prefixes to embed.PROFILES" ) MODEL_NAME = os.path.splitext(os.path.basename(MODEL_PATH))[0] PROFILE = profile_for(MODEL_PATH) EMBED_DIM = PROFILE.dim POOLING = PROFILE.pooling DOC_PREFIX = PROFILE.doc_prefix QUERY_PREFIX = PROFILE.query_prefix def is_up() -> bool: """Whether anything is listening on the port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: sock.settimeout(0.4) return sock.connect_ex((HOST, PORT)) == 0 def is_ready() -> bool: """Whether the server will actually answer an embedding request. llama-server binds its port before the model finishes loading and answers 503 until it is done, so a port check alone reports ready far too early — long enough for the first request of a session to fail. """ try: with urllib.request.urlopen(BASE_URL + "/health", timeout=2) as resp: return resp.status == 200 except (urllib.error.URLError, TimeoutError, OSError): return False def _await_ready(deadline: float, proc: subprocess.Popen | None = None) -> bool: while time.time() < deadline: if is_ready(): return True if proc is not None and proc.poll() is not None: return False time.sleep(0.3) return False def start(threads: int | None = None, wait: float = 180.0) -> None: """Ensure the server is running and has finished loading the model.""" deadline = time.time() + wait if is_ready(): return if is_up(): # Another session already started it; wait for its model load rather than # racing a second process onto the same port. if _await_ready(deadline): return if not os.path.exists(MODEL_PATH): raise ServerError(f"model not found: {MODEL_PATH}") os.makedirs(RUNTIME_DIR, exist_ok=True) threads = threads or max(1, (os.cpu_count() or 4)) env = dict(os.environ, LD_LIBRARY_PATH=LLAMA_LIB) cmd = [ os.path.join(LLAMA_BIN, "llama-server"), "--model", MODEL_PATH, "--embeddings", "--pooling", POOLING, # llama-server caches key/value state per past prompt, 8 GiB of it by # default. Indexing never repeats a prompt, so the cache fills with entries # nothing will read and holds 8.2 GB of a shared machine for the life of the # server. Search would hit it only on a repeated query, saving one ~90 ms # embedding. "--cache-ram", "0", "--ctx-size", str(CONTEXT * BATCH_SIZE), "--batch-size", str(CONTEXT * BATCH_SIZE), "--ubatch-size", str(CONTEXT), "--threads", str(threads), "--host", HOST, "--port", str(PORT), ] with open(LOG_FILE, "ab") as log: proc = subprocess.Popen( cmd, stdout=log, stderr=log, env=env, start_new_session=True ) if _await_ready(deadline, proc): # Only the process that actually bound the port owns the pid file, so # `stop` cannot kill a stranger. if proc.poll() is None: with open(PID_FILE, "w") as fh: fh.write(str(proc.pid)) return if proc.poll() is not None: # Losing a start race is normal when several sessions come up at once: one # binds the port and the rest exit. Only a genuinely absent server is an # error, and the winner may still be loading. if _await_ready(deadline): return raise ServerError(f"server exited with code {proc.returncode}; see {LOG_FILE}") proc.terminate() raise ServerError(f"server did not become ready within {wait:.0f}s; see {LOG_FILE}") def _alive(pid: int) -> bool: try: os.kill(pid, 0) except ProcessLookupError: return False except PermissionError: return True return True def _read_pid() -> int | None: """The pid recorded in the pid file, or None if there is nothing usable in it. The file is written without an intermediate rename, so a reader can catch it empty or half written. That is indistinguishable from no server at all and must not raise: `stop` is what an operator reaches for when things are already wrong. """ try: with open(PID_FILE) as fh: return int(fh.read().strip()) except (OSError, ValueError): return None def _cmdline(pid: int) -> str: """The process's argv, space-joined, or "" when it cannot be read.""" try: with open(f"/proc/{pid}/cmdline", "rb") as fh: return fh.read().decode("utf-8", "replace").replace("\0", " ").strip() except OSError: pass try: out = subprocess.run( ["ps", "-p", str(pid), "-o", "args="], capture_output=True, timeout=5, ).stdout except (OSError, subprocess.SubprocessError): return "" return out.decode("utf-8", "replace").strip() def _is_server(pid: int) -> bool: """Whether `pid` still names the embedding server. A pid file outlives the process that wrote it — a crash, a reboot, a kill from outside — and pids are reused, so the number alone is not enough to signal on: by the time anyone reads it the kernel may have handed it to something else. A pid we cannot identify is left alone rather than killed on a guess. """ return "llama-server" in _cmdline(pid) def _clear_pid_file() -> None: # Another session running `stop` concurrently may already have removed it. try: os.remove(PID_FILE) except FileNotFoundError: pass def stop(wait: float = 15.0) -> bool: """Stop the server and return once the process is actually gone. On SIGTERM llama-server closes its listening socket well before it exits, so returning as soon as the signal is sent leaves a window where the port is refused but the old process is still shutting down. Callers that restart immediately would race it, so wait for the process itself to disappear. """ if not os.path.exists(PID_FILE): return False pid = _read_pid() if pid is None or not _alive(pid) or not _is_server(pid): # Nothing here is ours to signal: the file is torn, the process is gone, or # the pid now belongs to an unrelated process. Drop the stale file. _clear_pid_file() return False try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: _clear_pid_file() return False deadline = time.time() + wait while time.time() < deadline: if not _alive(pid): break time.sleep(0.2) else: try: os.kill(pid, signal.SIGKILL) except ProcessLookupError: pass _clear_pid_file() return True def _post(path: str, payload: dict, timeout: float) -> dict: body = json.dumps(payload).encode() req = urllib.request.Request( BASE_URL + path, data=body, headers={"Content-Type": "application/json"} ) with urllib.request.urlopen(req, timeout=timeout) as resp: return json.loads(resp.read()) # The server is shared by every session on the machine, so it can disappear # mid-run — another session calling `stop`, an out-of-memory kill, a restart. A # long index would otherwise block on a socket for the full timeout and then die # with hours of work unwritten. RETRIES = 6 # Seconds to wait after a server error. A transport fault has a deterministic # signal to wait on — start() returns when /health passes — but a server that is up # and returning 5xx offers none, so this is the one place a plain delay is the only # option. It is generous because a three-hour job should ride out a restart. BACKOFF = [1, 4, 15, 30, 60] def embed(texts: list[str], is_query: bool = False, timeout: float = 180.0) -> np.ndarray: """Embed texts, returning an L2-normalised (n, EMBED_DIM) float32 matrix.""" if not texts: return np.zeros((0, EMBED_DIM), dtype=np.float32) prefix = QUERY_PREFIX if is_query else DOC_PREFIX # Last line of defence. The server rejects any single input longer than its # physical batch, and one such input fails the whole request — so an # over-length text must never reach it, whatever produced it. Chunking already # caps its output well below this; the guard exists so a future change there # degrades one result instead of killing a multi-hour index. payload = { "input": [prefix + t[:MAX_INPUT_CHARS] for t in texts], "model": MODEL_NAME, } last: Exception | None = None for attempt in range(RETRIES): try: data = _post("/v1/embeddings", payload, timeout) break except urllib.error.HTTPError as exc: # 5xx is the server failing to serve a request it accepted — loading, # restarting, out of slots — and is worth waiting out. 4xx means the # request itself is unacceptable and would be rejected again. if exc.code < 500: raise ServerError( f"embedding request failed: {exc.read()[:400]!r}" ) from exc last = exc if attempt == RETRIES - 1: raise ServerError( f"server returned {exc.code} on all {RETRIES} attempts" ) from exc # /health already passes here, so there is no state to poll: see BACKOFF. time.sleep(BACKOFF[min(attempt, len(BACKOFF) - 1)]) start() except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as exc: last = exc if attempt == RETRIES - 1: raise ServerError( f"embedding server unreachable after {RETRIES} attempts: {last!r}" ) from last # start() returns only once the port accepts connections, so this waits # on the state transition itself rather than on a guessed interval. start() rows = sorted(data["data"], key=lambda d: d["index"]) mat = np.asarray([r["embedding"] for r in rows], dtype=np.float32) if mat.ndim != 2 or mat.shape[0] != len(texts): raise ServerError(f"unexpected embedding shape {mat.shape} for {len(texts)} inputs") norms = np.linalg.norm(mat, axis=1, keepdims=True) np.divide(mat, np.maximum(norms, 1e-12), out=mat) return mat