Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257"""The embedding client, against a stub server.
The model-profile tests are the important ones. An embedding model is trained withone pooling strategy and one pair of task prefixes, and running it with anotherdegrades or destroys every result in the product without raising anything — thekind of fault a test suite has to catch because nothing else will."""
from __future__ import annotations
import osimport signalimport subprocessimport urllib.error
import numpy as npimport pytest
from code_index import embed
@pytest.fixturedef captured(monkeypatch): """Record what would be posted, and answer with fixed vectors.""" seen: list[dict] = []
def stub_post(path, payload, timeout): seen.append(payload) n = len(payload["input"]) return { "data": [ # Returned out of order on purpose: the client must sort by index. {"index": i, "embedding": [float(i + 1)] * embed.EMBED_DIM} for i in reversed(range(n)) ] }
monkeypatch.setattr(embed, "_post", stub_post) monkeypatch.setattr(embed, "start", lambda *a, **k: None) return seen
def test_documents_and_queries_use_their_own_prefixes(captured): embed.embed(["some source code"], is_query=False) embed.embed(["what does this do"], is_query=True)
# Compared in full rather than with startswith: a model whose document prefix # is empty satisfies startswith no matter which side the text came from. assert captured[0]["input"][0] == embed.DOC_PREFIX + "some source code" assert captured[1]["input"][0] == embed.QUERY_PREFIX + "what does this do" assert embed.DOC_PREFIX != embed.QUERY_PREFIX
def test_the_active_settings_come_from_the_model_being_loaded(captured): """Nothing may be pinned by hand: every one of these belongs to the model.""" profile = embed.profile_for(embed.MODEL_PATH) assert (embed.EMBED_DIM, embed.POOLING) == (profile.dim, profile.pooling) assert (embed.DOC_PREFIX, embed.QUERY_PREFIX) == ( profile.doc_prefix, profile.query_prefix, )
def test_each_model_keeps_the_pooling_it_was_trained_with(): """Qwen3-Embedding is decoder-only; nomic-embed is an encoder.""" assert embed.profile_for("/m/Qwen3-Embedding-0.6B-Q8_0.gguf").pooling == "last" assert embed.profile_for("/m/nomic-embed-text-v1.5.f16.gguf").pooling == "mean"
def test_an_unrecognised_model_is_refused_rather_than_guessed(): """Guessing would index the whole workspace into nonsense with no error.""" with pytest.raises(embed.ServerError, match="PROFILES"): embed.profile_for("/m/some-other-embedding.gguf")
def test_the_server_is_launched_with_the_model_pooling(tmp_path, monkeypatch): """The one setting llama-server accepts wrongly without complaining.""" launched: list[list[str]] = [] checks = {"n": 0}
class FakeProc: pid = 4321
def poll(self): return None
def terminate(self): pass
def fake_popen(cmd, **kwargs): launched.append(cmd) return FakeProc()
def ready_once_launched(): # False on the first call so start() launches, true afterwards so it stops # waiting — the same sequence a real server produces while it loads. checks["n"] += 1 return checks["n"] > 1
model = tmp_path / os.path.basename(embed.MODEL_PATH) model.write_bytes(b"gguf")
monkeypatch.setattr(embed, "MODEL_PATH", str(model)) monkeypatch.setattr(embed, "RUNTIME_DIR", str(tmp_path)) monkeypatch.setattr(embed, "LOG_FILE", str(tmp_path / "server.log")) monkeypatch.setattr(embed, "PID_FILE", str(tmp_path / "server.pid")) monkeypatch.setattr(embed, "is_up", lambda: False) monkeypatch.setattr(embed, "is_ready", ready_once_launched) monkeypatch.setattr(subprocess, "Popen", fake_popen)
embed.start(wait=5) cmd = launched[0] assert cmd[cmd.index("--pooling") + 1] == embed.POOLING assert cmd[cmd.index("--model") + 1] == str(model) # The context has to hold a whole request, and the prompt cache is 8 GiB of # memory that indexing can never hit (eval/batch-size.md). assert cmd[cmd.index("--ctx-size") + 1] == str(embed.CONTEXT * embed.BATCH_SIZE) assert cmd[cmd.index("--cache-ram") + 1] == "0"
def test_results_are_returned_in_request_order(captured): out = embed.embed(["a", "b", "c"], is_query=True) assert out.shape == (3, embed.EMBED_DIM) # Each row was built from its own index, so ordering is verifiable after # normalisation: every row is positive and they were distinct before it. assert np.all(out > 0)
def test_output_is_unit_length(captured): out = embed.embed(["a", "b"], is_query=True) norms = np.linalg.norm(out, axis=1) assert np.allclose(norms, 1.0, atol=1e-5)
def test_over_length_input_is_truncated_before_sending(captured): embed.embed(["x" * (embed.MAX_INPUT_CHARS * 3)], is_query=False) sent = captured[0]["input"][0] assert len(sent) == len(embed.DOC_PREFIX) + embed.MAX_INPUT_CHARS
def test_empty_input_does_not_call_the_server(captured): out = embed.embed([], is_query=False) assert out.shape == (0, embed.EMBED_DIM) assert captured == []
def _http_error(code: int) -> urllib.error.HTTPError: import io
return urllib.error.HTTPError( "http://localhost/v1/embeddings", code, "err", {}, io.BytesIO(b"{}") )
def test_server_errors_are_retried(monkeypatch): """A 5xx during a multi-hour run must not end it.""" attempts = {"n": 0}
def flaky(path, payload, timeout): attempts["n"] += 1 if attempts["n"] < 3: raise _http_error(503) return {"data": [{"index": 0, "embedding": [1.0] * embed.EMBED_DIM}]}
monkeypatch.setattr(embed, "_post", flaky) monkeypatch.setattr(embed, "start", lambda *a, **k: None) monkeypatch.setattr(embed, "BACKOFF", [0])
out = embed.embed(["x"], is_query=True) assert out.shape == (1, embed.EMBED_DIM) assert attempts["n"] == 3
def test_client_errors_are_not_retried(monkeypatch): """A rejected request would be rejected again; failing fast is correct.""" attempts = {"n": 0}
def rejecting(path, payload, timeout): attempts["n"] += 1 raise _http_error(400)
monkeypatch.setattr(embed, "_post", rejecting) monkeypatch.setattr(embed, "start", lambda *a, **k: None)
with pytest.raises(embed.ServerError): embed.embed(["x"], is_query=True) assert attempts["n"] == 1
def test_transport_failure_restarts_the_server(monkeypatch): """A vanished server is restarted rather than ending the run.""" attempts = {"n": 0} restarts = {"n": 0}
def gone(path, payload, timeout): attempts["n"] += 1 if attempts["n"] == 1: raise urllib.error.URLError("connection refused") return {"data": [{"index": 0, "embedding": [1.0] * embed.EMBED_DIM}]}
monkeypatch.setattr(embed, "_post", gone) monkeypatch.setattr(embed, "start", lambda *a, **k: restarts.__setitem__("n", restarts["n"] + 1))
out = embed.embed(["x"], is_query=True) assert out.shape == (1, embed.EMBED_DIM) assert restarts["n"] == 1, "server was not restarted after a transport failure"
@pytest.fixturedef pidfile(monkeypatch, tmp_path): """A pid file of our own, plus a record of every signal `stop` sends.""" path = tmp_path / "server.pid" monkeypatch.setattr(embed, "PID_FILE", str(path)) signalled: list[tuple[int, int]] = [] monkeypatch.setattr(embed.os, "kill", lambda pid, sig: signalled.append((pid, sig))) return path, signalled
def test_stop_with_no_pid_file_is_not_an_error(pidfile): path, signalled = pidfile assert embed.stop() is False assert signalled == []
def test_torn_pid_file_does_not_raise(pidfile): """An empty or half-written pid file is the state `stop` exists to clean up.""" path, signalled = pidfile path.write_text("")
assert embed.stop() is False assert signalled == [] assert not path.exists(), "a pid file naming nothing should not survive a stop"
def test_recycled_pid_is_not_signalled(pidfile, monkeypatch): """The pid file outlives the server, and the kernel hands the number out again.""" path, signalled = pidfile path.write_text("4321") monkeypatch.setattr(embed, "_alive", lambda pid: True) monkeypatch.setattr(embed, "_cmdline", lambda pid: "/usr/bin/postgres -D /var/lib")
assert embed.stop() is False assert signalled == [], "signalled a process that is not the embedding server" assert not path.exists()
def test_running_server_is_stopped(pidfile, monkeypatch): path, signalled = pidfile path.write_text("4321") monkeypatch.setattr(embed, "_cmdline", lambda pid: "/opt/llama/llama-server --model x") # Alive until the signal lands, so the wait loop has something to wait out. monkeypatch.setattr(embed, "_alive", lambda pid: not signalled)
assert embed.stop() is True assert signalled == [(4321, signal.SIGTERM)] assert not path.exists()