feat: wire browser deployment with binary tokenizer format

WASM init was hanging because std.json.parseFromSlice on the 20MB
tokenizer.json (248K vocab + 248K merges) creates millions of heap
allocations via memory.grow. Replace with a compact binary format
(.tkn) that loads via flat array reads in O(n).

- Add scripts/convert_tokenizer.py (JSON -> .tkn binary converter)
- Add Tokenizer.loadFromBinary() with TOKN magic auto-detection
- Update dist.sh to validate model files, convert tokenizer, copy
model.q4 + tokenizer.tkn into dist/
- Update app to download tokenizer.tkn + model.q4, store both in
IndexedDB, pass to init_with_tokenizer() via WASM

Co-Authored-By: Claude Opus 4.6 <[e-mail]>
Entire-Checkpoint: a3bc79e811d3

dist.sh

···
 
 ROOT="$(cd "$(dirname "$0")" && pwd)"
 DIST="$ROOT/web/dist"
+MODEL_DIR="${MODEL_DIR:-$ROOT/QwenTheBard}"
 
 echo "=== Fantasma dist ==="
 
+# Validate model files exist
+if [[ ! -f "$MODEL_DIR/model.q4" ]]; then
+  echo "ERROR: Missing $MODEL_DIR/model.q4"
+  echo "  Set MODEL_DIR= or place files in QwenTheBard/"
+  exit 1
+fi
+if [[ ! -f "$MODEL_DIR/tokenizer.json" ]]; then
+  echo "ERROR: Missing $MODEL_DIR/tokenizer.json"
+  echo "  Set MODEL_DIR= or place files in QwenTheBard/"
+  exit 1
+fi
+
+# Convert tokenizer to binary if needed
+if [[ ! -f "$MODEL_DIR/tokenizer.tkn" ]] || [[ "$MODEL_DIR/tokenizer.json" -nt "$MODEL_DIR/tokenizer.tkn" ]]; then
+  echo "[0/4] Converting tokenizer..."
+  python3 "$ROOT/scripts/convert_tokenizer.py" "$MODEL_DIR/tokenizer.json" "$MODEL_DIR/tokenizer.tkn"
+fi
+
 # Build WASM
-echo "[1/3] Building WASM..."
+echo "[1/4] Building WASM..."
 cd "$ROOT"
 zig build wasm
 echo "      fantasma.wasm: $(du -h zig-out/bin/fantasma.wasm | cut -f1)"
···
 cp zig-out/bin/fantasma.wasm "$ROOT/web/public/"
 
 # Build frontend with Vite
-echo "[2/3] Building frontend..."
+echo "[2/4] Building frontend..."
 cd "$ROOT/web"
 bun run build
 
+# Copy model assets into dist (after Vite build so they aren't hashed)
+echo "[3/4] Copying model assets..."
+cp "$MODEL_DIR/model.q4" "$DIST/"
+cp "$MODEL_DIR/tokenizer.tkn" "$DIST/"
+echo "      model.q4:       $(du -h "$DIST/model.q4" | cut -f1)"
+echo "      tokenizer.tkn:  $(du -h "$DIST/tokenizer.tkn" | cut -f1)"
+
+echo ""
 echo "      $(find "$DIST" -type f | wc -l) files -> web/dist/"
 echo ""
 ls -lh "$DIST"
···
 
 # Serve
 PORT="${1:-8080}"
-echo "[3/3] Serving at http://localhost:$PORT"
+echo "[4/4] Serving at http://localhost:$PORT"
 echo "      Ctrl+C to stop"
 echo ""
 cd "$DIST"

scripts/convert_tokenizer.py

···
+#!/usr/bin/env python3
+"""Convert HuggingFace tokenizer.json to compact binary format (.tkn).
+
+Binary format (all little-endian):
+  Header (32 bytes):
+    magic:            4 bytes  "TOKN"
+    vocab_size:       u32      max_id + 1 (total slots)
+    entry_count:      u32      populated vocab entries
+    merge_count:      u32      BPE merge rules
+    eos_id:           u32
+    im_start_id:      u32
+    im_end_id:        u32
+    string_pool_size: u32
+
+  string_pool:        [string_pool_size] u8
+
+  vocab_table:        [vocab_size] × (offset:u32, length:u32)
+                      offset=0xFFFFFFFF means empty slot
+
+  token_to_id_table:  [entry_count] × (offset:u32, length:u32, id:u32)
+                      sorted by string for binary search
+
+  merges_table:       [merge_count] × (first:u32, second:u32, result:u32, rank:u32)
+"""
+
+import json
+import struct
+import sys
+from pathlib import Path
+
+EMPTY = 0xFFFFFFFF
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(f"Usage: {sys.argv[0]} <tokenizer.json> [output.tkn]")
+        sys.exit(1)
+
+    in_path = Path(sys.argv[1])
+    out_path = Path(sys.argv[2]) if len(sys.argv) > 2 else in_path.with_suffix(".tkn")
+
+    with open(in_path) as f:
+        data = json.load(f)
+
+    model = data["model"]
+    vocab = model["vocab"]       # str -> int
+    merges = model["merges"]     # list of [str, str] or "str str"
+
+    # Special tokens
+    added = {t["content"]: t["id"] for t in data.get("added_tokens", [])}
+    eos_id = added.get("<|endoftext|>", 248046)
+    im_start_id = added.get("<|im_start|>", 248045)
+    im_end_id = added.get("<|im_end|>", 248046)
+
+    # Merge added tokens into vocab
+    all_vocab = dict(vocab)
+    for content, tid in added.items():
+        all_vocab[content] = tid
+
+    max_id = max(all_vocab.values())
+    vocab_size = max_id + 1
+    entry_count = len(all_vocab)
+
+    # Build string pool
+    pool = bytearray()
+    pool_index = {}  # text -> (offset, length)
+    for text in sorted(all_vocab.keys()):
+        offset = len(pool)
+        encoded = text.encode("utf-8")
+        pool.extend(encoded)
+        pool_index[text] = (offset, len(encoded))
+
+    # Vocab table: id -> (offset, length)
+    vocab_table = bytearray(vocab_size * 8)
+    for i in range(vocab_size):
+        struct.pack_into("<II", vocab_table, i * 8, EMPTY, 0)
+    for text, tid in all_vocab.items():
+        off, ln = pool_index[text]
+        struct.pack_into("<II", vocab_table, tid * 8, off, ln)
+
+    # Token-to-id table: sorted by text for binary search
+    sorted_entries = sorted(all_vocab.items(), key=lambda x: x[0].encode("utf-8"))
+    token_to_id_table = bytearray(entry_count * 12)
+    for i, (text, tid) in enumerate(sorted_entries):
+        off, ln = pool_index[text]
+        struct.pack_into("<III", token_to_id_table, i * 12, off, ln, tid)
+
+    # Build text->id lookup for merge resolution
+    text_to_id = {text: tid for text, tid in all_vocab.items()}
+
+    # Merges table
+    merge_entries = []
+    for rank, m in enumerate(merges):
+        if isinstance(m, list):
+            first_s, second_s = m[0], m[1]
+        else:
+            parts = m.split(" ", 1)
+            if len(parts) != 2:
+                continue
+            first_s, second_s = parts
+
+        first_id = text_to_id.get(first_s)
+        second_id = text_to_id.get(second_s)
+        result_id = text_to_id.get(first_s + second_s)
+        if first_id is None or second_id is None or result_id is None:
+            continue
+        merge_entries.append((first_id, second_id, result_id, rank))
+
+    merge_count = len(merge_entries)
+    merges_table = bytearray(merge_count * 16)
+    for i, (f, s, r, rk) in enumerate(merge_entries):
+        struct.pack_into("<IIII", merges_table, i * 16, f, s, r, rk)
+
+    # Write binary
+    header = struct.pack(
+        "<4sIIIIIII",
+        b"TOKN",
+        vocab_size,
+        entry_count,
+        merge_count,
+        eos_id,
+        im_start_id,
+        im_end_id,
+        len(pool),
+    )
+
+    with open(out_path, "wb") as f:
+        f.write(header)
+        f.write(pool)
+        f.write(vocab_table)
+        f.write(token_to_id_table)
+        f.write(merges_table)
+
+    total = len(header) + len(pool) + len(vocab_table) + len(token_to_id_table) + len(merges_table)
+    print(f"Wrote {out_path} ({total / 1048576:.1f} MB)")
+    print(f"  vocab_size={vocab_size}  entries={entry_count}  merges={merge_count}")
+    print(f"  pool={len(pool)} bytes  vocab_table={len(vocab_table)} bytes")
+
+
+if __name__ == "__main__":
+    main()

src/inference/tokenizer.zig

···
         self.allocator.free(self.string_pool);
     }
 
+    /// Load tokenizer from compact binary format (.tkn).
+    /// Much faster than JSON parsing — just flat array reads.
+    pub fn loadFromBinary(allocator: std.mem.Allocator, data: []const u8) !Tokenizer {
+        if (data.len < 32) return error.InvalidFormat;
+
+        // Parse header
+        const magic = data[0..4];
+        if (!std.mem.eql(u8, magic, "TOKN")) return error.InvalidFormat;
+
+        const vocab_size = std.mem.readInt(u32, data[4..8], .little);
+        const entry_count = std.mem.readInt(u32, data[8..12], .little);
+        const merge_count = std.mem.readInt(u32, data[12..16], .little);
+        const eos_id = std.mem.readInt(u32, data[16..20], .little);
+        const im_start_id = std.mem.readInt(u32, data[20..24], .little);
+        const im_end_id = std.mem.readInt(u32, data[24..28], .little);
+        const pool_size = std.mem.readInt(u32, data[28..32], .little);
+
+        var offset: usize = 32;
+
+        // String pool — copy so it's owned by the tokenizer
+        if (offset + pool_size > data.len) return error.InvalidFormat;
+        const string_pool = try allocator.alloc(u8, pool_size);
+        @memcpy(string_pool, data[offset .. offset + pool_size]);
+        offset += pool_size;
+
+        // Vocab table: vocab_size × (offset:u32, length:u32)
+        const vtable_size = @as(usize, vocab_size) * 8;
+        if (offset + vtable_size > data.len) return error.InvalidFormat;
+        const vtable = data[offset .. offset + vtable_size];
+        offset += vtable_size;
+
+        const vocab = try allocator.alloc([]const u8, vocab_size);
+        for (0..vocab_size) |i| {
+            const base = i * 8;
+            const str_off = std.mem.readInt(u32, vtable[base..][0..4], .little);
+            const str_len = std.mem.readInt(u32, vtable[base + 4 ..][0..4], .little);
+            if (str_off == 0xFFFFFFFF) {
+                vocab[i] = "";
+            } else {
+                vocab[i] = string_pool[str_off .. str_off + str_len];
+            }
+        }
+
+        // Token-to-id table: entry_count × (offset:u32, length:u32, id:u32)
+        const ttable_size = @as(usize, entry_count) * 12;
+        if (offset + ttable_size > data.len) return error.InvalidFormat;
+        const ttable = data[offset .. offset + ttable_size];
+        offset += ttable_size;
+
+        const token_to_id = try allocator.alloc(TokenEntry, entry_count);
+        for (0..entry_count) |i| {
+            const base = i * 12;
+            const str_off = std.mem.readInt(u32, ttable[base..][0..4], .little);
+            const str_len = std.mem.readInt(u32, ttable[base + 4 ..][0..4], .little);
+            const id = std.mem.readInt(u32, ttable[base + 8 ..][0..4], .little);
+            token_to_id[i] = .{
+                .text = string_pool[str_off .. str_off + str_len],
+                .id = id,
+            };
+        }
+
+        // Merges table: merge_count × (first:u32, second:u32, result:u32, rank:u32)
+        const mtable_size = @as(usize, merge_count) * 16;
+        if (offset + mtable_size > data.len) return error.InvalidFormat;
+        const mtable = data[offset .. offset + mtable_size];
+
+        const merges = try allocator.alloc(Merge, merge_count);
+        for (0..merge_count) |i| {
+            const base = i * 16;
+            merges[i] = .{
+                .first = std.mem.readInt(u32, mtable[base..][0..4], .little),
+                .second = std.mem.readInt(u32, mtable[base + 4 ..][0..4], .little),
+                .result = std.mem.readInt(u32, mtable[base + 8 ..][0..4], .little),
+                .rank = std.mem.readInt(u32, mtable[base + 12 ..][0..4], .little),
+            };
+        }
+
+        return Tokenizer{
+            .vocab = vocab,
+            .vocab_size = vocab_size,
+            .merges = merges,
+            .merge_count = merge_count,
+            .token_to_id = token_to_id,
+            .token_count = entry_count,
+            .eos_id = eos_id,
+            .im_start_id = im_start_id,
+            .im_end_id = im_end_id,
+            .allocator = allocator,
+            .string_pool = string_pool,
+        };
+    }
+
     /// Load tokenizer from a tokenizer.json file using std.json.
     pub fn loadFromFile(allocator: std.mem.Allocator, path: []const u8) !Tokenizer {
         const file = try std.fs.cwd().openFile(path, .{});

src/main.zig

···
 fn initFromData(weights_data: []const u8, tok_ptr: ?[*]const u8, tok_len: ?u32) bool {
     const allocator = getAllocator();
 
-    // Load tokenizer
+    // Load tokenizer (binary .tkn format for WASM, JSON for native)
     if (tok_ptr) |tp| {
         if (tok_len) |tl| {
-            g_tokenizer = inference_tokenizer.Tokenizer.loadFromJson(allocator, tp[0..tl]) catch return false;
+            const tok_data = tp[0..tl];
+            // Try binary format first (starts with "TOKN"), fall back to JSON
+            if (tok_data.len >= 4 and std.mem.eql(u8, tok_data[0..4], "TOKN")) {
+                g_tokenizer = inference_tokenizer.Tokenizer.loadFromBinary(allocator, tok_data) catch return false;
+            } else {
+                g_tokenizer = inference_tokenizer.Tokenizer.loadFromJson(allocator, tok_data) catch return false;
+            }
         }
     }
 

web/src/db.ts

···
 const DB_VERSION = 1;
 const STORE_NAME = "model";
 export const WEIGHTS_KEY = "weights";
+export const TOKENIZER_KEY = "tokenizer";
 
 function openDB(): Promise<IDBDatabase> {
   return new Promise((resolve, reject) => {

web/src/main.ts

···
 
 import { store } from "./store";
 import { loadWasm, wasmCall, loadModelWeights } from "./wasm";
-import { dbGet, dbPut, dbClear, WEIGHTS_KEY } from "./db";
+import { dbGet, dbPut, dbClear, WEIGHTS_KEY, TOKENIZER_KEY } from "./db";
 import { updateStatus, renderProfile } from "./render";
 
 const $ = (sel: string) => document.querySelector<HTMLElement>(sel)!;
···
   downloadProgress.hidden = false;
 
   try {
-    progressText.textContent = "Downloading model...";
+    // 1. Download tokenizer.tkn (compact binary, ~7MB)
+    progressText.textContent = "Downloading tokenizer...";
     progressFill.style.width = "0%";
+    const tokResponse = await fetch("/tokenizer.tkn");
+    if (!tokResponse.ok) throw new Error(`Tokenizer HTTP ${tokResponse.status}`);
+    const tokenizerBytes = await tokResponse.arrayBuffer();
+    await dbPut(TOKENIZER_KEY, tokenizerBytes);
 
-    const weightsBytes = await fetchWithProgress("/model.bin", (recv, total) => {
+    // 2. Download model weights (large, with progress)
+    progressText.textContent = "Downloading model...";
+    const weightsBytes = await fetchWithProgress("/model.q4", (recv, total) => {
       if (total > 0) {
         const pct = Math.round((recv / total) * 100);
         progressFill.style.width = pct + "%";
···
     progressFill.style.width = "100%";
 
     const { wasmInstance } = store.getState();
-    if (wasmInstance && loadModelWeights(wasmInstance, weightsBytes)) {
+    if (wasmInstance && loadModelWeights(wasmInstance, weightsBytes, tokenizerBytes)) {
       store.getState().setModelLoaded(true);
       downloadProgress.hidden = true;
       btnDownload.hidden = true;
···
     try {
       const storedWeights = await dbGet(WEIGHTS_KEY);
       if (storedWeights) {
-        if (loadModelWeights(instance, storedWeights)) {
+        const storedTokenizer = await dbGet(TOKENIZER_KEY);
+        if (loadModelWeights(instance, storedWeights, storedTokenizer)) {
           store.getState().setModelLoaded(true);
           store.getState().setStatus("model-ready");
         }

web/src/wasm.ts

···
   profile(ptr: number, len: number): number;
   neutralize(ptr: number, len: number): number;
   init(ptr: number, len: number): number;
+  init_with_tokenizer(
+    weights_ptr: number,
+    weights_len: number,
+    tok_ptr: number,
+    tok_len: number,
+  ): number;
   get_output_ptr(): number;
   get_output_len(): number;
 }
···
 export function loadModelWeights(
   instance: WebAssembly.Instance,
   weightsBytes: ArrayBuffer,
+  tokenizerBytes?: ArrayBuffer,
 ): boolean {
   const exports = instance.exports as unknown as FantasmaExports;
-  const arr = new Uint8Array(weightsBytes);
-  const ptr = exports.alloc(arr.length);
-  new Uint8Array(exports.memory.buffer, ptr, arr.length).set(arr);
-  const ok = exports.init(ptr, arr.length);
-  exports.dealloc(ptr, arr.length);
+  const wArr = new Uint8Array(weightsBytes);
+  const wPtr = exports.alloc(wArr.length);
+  new Uint8Array(exports.memory.buffer, wPtr, wArr.length).set(wArr);
+
+  let ok: number;
+  if (tokenizerBytes) {
+    const tArr = new Uint8Array(tokenizerBytes);
+    const tPtr = exports.alloc(tArr.length);
+    new Uint8Array(exports.memory.buffer, tPtr, tArr.length).set(tArr);
+    ok = exports.init_with_tokenizer(wPtr, wArr.length, tPtr, tArr.length);
+    exports.dealloc(tPtr, tArr.length);
+  } else {
+    ok = exports.init(wPtr, wArr.length);
+  }
+
+  exports.dealloc(wPtr, wArr.length);
   return !!ok;
 }