#!/usr/bin/env python3 """A page is bytes, and a tool can hand back bytes that are not utf-8: the turn still has to finish. python3 scripts/test_encoding.py Serves a windows-1252 page, a page with a stray invalid byte and a command that prints one, drives them through read_url and run, then checks the transcript carries readable text and the turn ends. Needs the dev shell on PATH. """ import base64, json, os, socket, struct, subprocess, sys, tempfile, threading, time, urllib.request from http.server import BaseHTTPRequestHandler, HTTPServer ROOT = "/mnt/hdd0/Code/slop/hwaet" PY = sys.executable HWAET_PORT = 4785 MODEL_PORT = 4786 PAGE_PORT = 4787 HOME = tempfile.mkdtemp(prefix="hwaet-encoding-") ok = 0 SAID = [] def good(label, extra=""): global ok ok += 1 print("ok ", label, extra) def fail(label, detail=""): print("FAIL", label, detail, flush=True) if SAID: print("server said:\n" + "".join(SAID[-60:]), flush=True) sys.exit(1) # ---- the pages ------------------------------------------------------------- LEGACY = "
caf\xe9 \x97 5\x80
" LEGACY = LEGACY.encode("latin-1") # 0xe9 é, 0x97 em dash, 0x80 euro STRAY = b"a\xffb
" class Page(BaseHTTPRequestHandler): def do_GET(self): if self.path.startswith("/legacy"): body, ctype = LEGACY, "text/html; charset=windows-1252" else: body, ctype = STRAY, "text/html" self.send_response(200) self.send_header("content-type", ctype) self.send_header("content-length", str(len(body))) self.end_headers() self.wfile.write(body) def log_message(self, *args): pass pages = HTTPServer(("127.0.0.1", PAGE_PORT), Page) threading.Thread(target=pages.serve_forever, daemon=True).start() json.dump({"agent_mode": "model", "base_url": f"http://127.0.0.1:{MODEL_PORT}/v1", "model": "fake", "system_prompt": "You are Hwæt.", "allow_all_tools": "on"}, open(os.path.join(HOME, "settings.json"), "w")) SCRIPT = os.path.join(HOME, "script.json") json.dump([ {"tool": "read_url", "args": {"url": f"http://127.0.0.1:{PAGE_PORT}/legacy"}}, {"tool": "read_url", "args": {"url": f"http://127.0.0.1:{PAGE_PORT}/stray"}}, {"tool": "run", "args": {"command": "printf 'a\\xe9b'"}}, {"text": "read them"}, ], open(SCRIPT, "w")) model = subprocess.Popen([PY, os.path.join(ROOT, "scripts/fake_model.py"), str(MODEL_PORT), "ping", "0", "{}"], env=dict(os.environ, FAKE_MODEL_SCRIPT=SCRIPT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) server = subprocess.Popen(["mix", "run", "--no-halt"], cwd=os.path.join(ROOT, "server"), env=dict(os.environ, HWAET_HOME=HOME, HWAET_PORT=str(HWAET_PORT), MIX_ENV="dev"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) def drain(pipe): for line in pipe: SAID.append(line) del SAID[:-400] threading.Thread(target=drain, args=(server.stdout,), daemon=True).start() deadline = time.time() + 90 tokens_path = os.path.join(HOME, "tokens.json") while time.time() < deadline and not os.path.exists(tokens_path): if server.poll() is not None: print("server died:", "".join(SAID[-40:])) sys.exit(1) time.sleep(0.2) time.sleep(0.5) code = json.load(open(tokens_path))["code"] req = urllib.request.Request(f"http://127.0.0.1:{HWAET_PORT}/pair", data=json.dumps({"code": code}).encode(), headers={"content-type": "application/json"}) token = json.load(urllib.request.urlopen(req, timeout=10))["token"] s = socket.create_connection(("127.0.0.1", HWAET_PORT), timeout=20) key = base64.b64encode(os.urandom(16)).decode() s.sendall((f"GET /socket HTTP/1.1\r\nHost: 127.0.0.1:{HWAET_PORT}\r\nUpgrade: websocket\r\n" f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n").encode()) head = b"" while b"\r\n\r\n" not in head: head += s.recv(4096) head, _, buf0 = head.partition(b"\r\n\r\n") BUF = [buf0] def send(frame): data = json.dumps(frame).encode() mask = os.urandom(4) n = len(data) if n < 126: hdr = struct.pack("!BB", 0x81, 0x80 | n) elif n < 65536: hdr = struct.pack("!BBH", 0x81, 0x80 | 126, n) else: hdr = struct.pack("!BBQ", 0x81, 0x80 | 127, n) s.sendall(hdr + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(data))) def need(n): while len(BUF[0]) < n: chunk = s.recv(65536) if not chunk: print("socket closed; server said:\n" + "".join(SAID[-40:]), flush=True) sys.exit(1) BUF[0] += chunk out, BUF[0] = BUF[0][:n], BUF[0][n:] return out def recv(timeout=20): s.settimeout(timeout) while True: b1, b2 = need(2) op = b1 & 0x0F n = b2 & 0x7F if n == 126: n = struct.unpack("!H", need(2))[0] elif n == 127: n = struct.unpack("!Q", need(8))[0] payload = need(n) if op == 0x8: raise EOFError("closed") if op == 0x9: continue return json.loads(payload) def collect(until, timeout=90): got = [] deadline = time.time() + timeout while time.time() < deadline: try: f = recv(5) except socket.timeout: continue got.append(f) if until(f): return got fail("timed out waiting", json.dumps(got[-4:])) send({"type": "hello", "id": "h1", "token": token}) ready = recv() if ready.get("type") != "ready": fail("paired", ready) send({"type": "send", "id": "e1", "conv": None, "parts": [{"type": "text", "text": "read them"}]}) frames = collect(lambda f: f.get("kind") == "done") results = [f for f in frames if f.get("kind") == "tool_result"] if len(results) != 3: fail("every tool call came back", json.dumps(results)[:600]) if not all(r.get("ok") for r in results): fail("each tool call succeeded", json.dumps(results)[:600]) legacy = results[0].get("output") or "" if "café" not in legacy or "€" not in legacy: fail("a windows-1252 page reads as text", legacy[:300]) good("a windows-1252 page reads as text", legacy.split("\n")[-1][:40]) stray = results[1].get("output") or "" if "a\ufffdb" not in stray: fail("a stray byte becomes the replacement character", stray[:300]) good("a stray byte becomes the replacement character") ran = results[2].get("output") or "" if "a\ufffdb" not in ran: fail("a command's bad byte is repaired too", ran[:300]) good("a command's bad byte is repaired too", json.dumps(ran[:20])) print("\n%d checks passed" % ok) server.kill() model.kill() pages.shutdown()