Something went wrong. Try again.
The only LLM harness that's honest about NIH syndrome being the sole reason it exists
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206#!/usr/bin/env python3"""A page is bytes, and a tool can hand back bytes that are not utf-8: the turnstill has to finish.
python3 scripts/test_encoding.py
Serves a windows-1252 page, a page with a stray invalid byte and a command thatprints one, drives them through read_url and run, then checks the transcriptcarries readable text and the turn ends. Needs the dev shell on PATH."""
import base64, json, os, socket, struct, subprocess, sys, tempfile, threading, time, urllib.requestfrom http.server import BaseHTTPRequestHandler, HTTPServer
ROOT = "/mnt/hdd0/Code/slop/hwaet"PY = sys.executableHWAET_PORT = 4785MODEL_PORT = 4786PAGE_PORT = 4787
HOME = tempfile.mkdtemp(prefix="hwaet-encoding-")ok = 0SAID = []
def good(label, extra=""): global ok ok += 1 print("ok ", label, extra)
def fail(label, detail=""): print("FAIL", label, detail, flush=True) if SAID: print("server said:\n" + "".join(SAID[-60:]), flush=True) sys.exit(1)
# ---- the pages -------------------------------------------------------------
LEGACY = "<html><head><title>caf\xe9</title></head><body><p>caf\xe9 \x97 5\x80</p></body></html>"LEGACY = LEGACY.encode("latin-1") # 0xe9 é, 0x97 em dash, 0x80 euroSTRAY = b"<html><head><title>stray</title></head><body><p>a\xffb</p></body></html>"
class Page(BaseHTTPRequestHandler): def do_GET(self): if self.path.startswith("/legacy"): body, ctype = LEGACY, "text/html; charset=windows-1252" else: body, ctype = STRAY, "text/html" self.send_response(200) self.send_header("content-type", ctype) self.send_header("content-length", str(len(body))) self.end_headers() self.wfile.write(body)
def log_message(self, *args): pass
pages = HTTPServer(("127.0.0.1", PAGE_PORT), Page)threading.Thread(target=pages.serve_forever, daemon=True).start()
json.dump({"agent_mode": "model", "base_url": f"http://127.0.0.1:{MODEL_PORT}/v1", "model": "fake", "system_prompt": "You are Hwæt.", "allow_all_tools": "on"}, open(os.path.join(HOME, "settings.json"), "w"))
SCRIPT = os.path.join(HOME, "script.json")json.dump([ {"tool": "read_url", "args": {"url": f"http://127.0.0.1:{PAGE_PORT}/legacy"}}, {"tool": "read_url", "args": {"url": f"http://127.0.0.1:{PAGE_PORT}/stray"}}, {"tool": "run", "args": {"command": "printf 'a\\xe9b'"}}, {"text": "read them"},], open(SCRIPT, "w"))
model = subprocess.Popen([PY, os.path.join(ROOT, "scripts/fake_model.py"), str(MODEL_PORT), "ping", "0", "{}"], env=dict(os.environ, FAKE_MODEL_SCRIPT=SCRIPT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)server = subprocess.Popen(["mix", "run", "--no-halt"], cwd=os.path.join(ROOT, "server"), env=dict(os.environ, HWAET_HOME=HOME, HWAET_PORT=str(HWAET_PORT), MIX_ENV="dev"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
def drain(pipe): for line in pipe: SAID.append(line) del SAID[:-400]
threading.Thread(target=drain, args=(server.stdout,), daemon=True).start()
deadline = time.time() + 90tokens_path = os.path.join(HOME, "tokens.json")while time.time() < deadline and not os.path.exists(tokens_path): if server.poll() is not None: print("server died:", "".join(SAID[-40:])) sys.exit(1) time.sleep(0.2)time.sleep(0.5)code = json.load(open(tokens_path))["code"]req = urllib.request.Request(f"http://127.0.0.1:{HWAET_PORT}/pair", data=json.dumps({"code": code}).encode(), headers={"content-type": "application/json"})token = json.load(urllib.request.urlopen(req, timeout=10))["token"]
s = socket.create_connection(("127.0.0.1", HWAET_PORT), timeout=20)key = base64.b64encode(os.urandom(16)).decode()s.sendall((f"GET /socket HTTP/1.1\r\nHost: 127.0.0.1:{HWAET_PORT}\r\nUpgrade: websocket\r\n" f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n").encode())head = b""while b"\r\n\r\n" not in head: head += s.recv(4096)head, _, buf0 = head.partition(b"\r\n\r\n")BUF = [buf0]
def send(frame): data = json.dumps(frame).encode() mask = os.urandom(4) n = len(data) if n < 126: hdr = struct.pack("!BB", 0x81, 0x80 | n) elif n < 65536: hdr = struct.pack("!BBH", 0x81, 0x80 | 126, n) else: hdr = struct.pack("!BBQ", 0x81, 0x80 | 127, n) s.sendall(hdr + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
def need(n): while len(BUF[0]) < n: chunk = s.recv(65536) if not chunk: print("socket closed; server said:\n" + "".join(SAID[-40:]), flush=True) sys.exit(1) BUF[0] += chunk out, BUF[0] = BUF[0][:n], BUF[0][n:] return out
def recv(timeout=20): s.settimeout(timeout) while True: b1, b2 = need(2) op = b1 & 0x0F n = b2 & 0x7F if n == 126: n = struct.unpack("!H", need(2))[0] elif n == 127: n = struct.unpack("!Q", need(8))[0] payload = need(n) if op == 0x8: raise EOFError("closed") if op == 0x9: continue return json.loads(payload)
def collect(until, timeout=90): got = [] deadline = time.time() + timeout while time.time() < deadline: try: f = recv(5) except socket.timeout: continue got.append(f) if until(f): return got fail("timed out waiting", json.dumps(got[-4:]))
send({"type": "hello", "id": "h1", "token": token})ready = recv()if ready.get("type") != "ready": fail("paired", ready)
send({"type": "send", "id": "e1", "conv": None, "parts": [{"type": "text", "text": "read them"}]})frames = collect(lambda f: f.get("kind") == "done")
results = [f for f in frames if f.get("kind") == "tool_result"]if len(results) != 3: fail("every tool call came back", json.dumps(results)[:600])if not all(r.get("ok") for r in results): fail("each tool call succeeded", json.dumps(results)[:600])
legacy = results[0].get("output") or ""if "café" not in legacy or "€" not in legacy: fail("a windows-1252 page reads as text", legacy[:300])good("a windows-1252 page reads as text", legacy.split("\n")[-1][:40])
stray = results[1].get("output") or ""if "a\ufffdb" not in stray: fail("a stray byte becomes the replacement character", stray[:300])good("a stray byte becomes the replacement character")
ran = results[2].get("output") or ""if "a\ufffdb" not in ran: fail("a command's bad byte is repaired too", ran[:300])good("a command's bad byte is repaired too", json.dumps(ran[:20]))
print("\n%d checks passed" % ok)server.kill()model.kill()pages.shutdown()