index homepage, improved opengraphs, misc other stuff

Cargo.lock

···
 
 [[package]]
 name = "const-str"
-version = "0.7.0"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4d34b8f066904ed7cfa4a6f9ee96c3214aa998cb44b69ca20bd2054f47402ed"
+checksum = "b0664d2867b4a32697dfe655557f5c3b187e9b605b38612a748e5ec99811d160"
 
 [[package]]
 name = "const_format"
···
  "base64 0.22.1",
  "bytes",
  "ciborium",
- "const-str 0.7.0",
+ "const-str 0.7.1",
  "const_format",
  "content_disposition",
  "derive_more 2.1.0",
···
 [[package]]
 name = "jacquard"
 version = "0.9.4"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "bytes",
  "getrandom 0.2.16",
···
 [[package]]
 name = "jacquard-api"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "bon",
  "bytes",
···
 [[package]]
 name = "jacquard-axum"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "axum",
  "bytes",
···
 [[package]]
 name = "jacquard-common"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "base64 0.22.1",
  "bon",
···
 [[package]]
 name = "jacquard-derive"
 version = "0.9.4"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "heck 0.5.0",
  "jacquard-lexicon",
···
 [[package]]
 name = "jacquard-identity"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "bon",
  "bytes",
···
 [[package]]
 name = "jacquard-lexicon"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "cid",
  "dashmap 6.1.0",
···
 [[package]]
 name = "jacquard-oauth"
 version = "0.9.2"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "base64 0.22.1",
  "bytes",
···
 [[package]]
 name = "jacquard-repo"
 version = "0.9.4"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "bytes",
  "cid",
···
 [[package]]
 name = "mini-moka"
 version = "0.10.99"
-source = "git+https://tangled.org/@nonbinary.computer/jacquard#d5d29a337d8f08ae52fe7bb4e31f91f8b029ff48"
+source = "git+https://tangled.org/@nonbinary.computer/jacquard#bf8f91add8747b64b1d1d74af4b358960f69d6e7"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-utils",

crates/weaver-app/Dockerfile

···
-# Build stage with cargo-chef for dependency caching
-FROM rust:1-trixie AS chef
-# Pin nightly version for reproducibility
-RUN rustup default nightly-2025-12-04 && rustup component add rust-src --toolchain nightly-2025-12-04
-RUN cargo install cargo-chef
+# Build stage
+FROM rust:1-trixie AS builder
+
 WORKDIR /app
 
-FROM chef AS planner
-COPY . .
-RUN cargo chef prepare --recipe-path recipe.json
-
-FROM chef AS builder
+# Pin nightly version for reproducibility
+RUN rustup default nightly-2025-12-04 && rustup component add rust-src llvm-tools-preview --toolchain nightly-2025-12-04
 
 # Install build dependencies
 RUN apt-get update && apt-get install -y \
     pkg-config \
     libssl-dev \
+    clang \
+    llvm \
+    binutils \
     && rm -rf /var/lib/apt/lists/*
 
 # Install wasm target and tools
···
 # Install dioxus-cli
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall dioxus-cli --root /usr/local -y --force
-
-# Cook dependencies first (cached layer)
-COPY --from=planner /app/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
 
 # Copy source code
 COPY . .
···
     --no-typescript
 
 # Bundle the app
-RUN dx bundle --release --debug-symbols false -p weaver-app
+RUN dx bundle --verbose --release --debug-symbols false -p weaver-app
 
 # Runtime stage
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
 RUN apt-get update && apt-get install -y \
     ca-certificates \

crates/weaver-app/assets/styling/theme-defaults.css

···
     font-family: "Adobe Caslon Pro";
     font-style: normal;
     font-weight: bold;
-    src: url("/assets/AdobeCaslonPro-Semibold.ttf") format("truetype");
+    src: url("/assets/AdobeCaslonPro-Bold.ttf") format("truetype");
 }
 
 @font-face {

crates/weaver-app/src/env.rs

···
 #[allow(unused)]
 pub const WEAVER_PRIVACY_POLICY_URI: &'static str = "";
 #[allow(unused)]
-pub const WEAVER_INDEXER_URL: &'static str = "http://localhost:3000";
+pub const WEAVER_INDEXER_URL: &'static str = "https://index.weaver.sh";
 #[allow(unused)]
 pub const WEAVER_INDEXER_DID: &'static str = "did:web:index.weaver.sh";
 #[allow(unused)]

crates/weaver-app/src/og/mod.rs

···
 fn get_fontdb() -> &'static fontdb::Database {
     FONTDB.get_or_init(|| {
         let mut db = fontdb::Database::new();
-        // Load IBM Plex Sans from embedded bytes
-        let font_data = include_bytes!("../../assets/fonts/IBMPlexSans-VariableFont_wdth,wght.ttf");
-        db.load_font_data(font_data.to_vec());
-        // Load IBM Plex Sans Bold (static weight for proper bold rendering)
-        let font_data = include_bytes!("../../assets/fonts/IBMPlexSans-Bold.ttf");
-        db.load_font_data(font_data.to_vec());
-        let font_data = include_bytes!("../../assets/fonts/ioskeley-mono/IoskeleyMono-Regular.ttf");
-        db.load_font_data(font_data.to_vec());
+        // Load CMU Sans Serif for headings/UI
+        db.load_font_data(
+            include_bytes!("../../assets/fonts/cmu-sans-serif/CMUSansSerif-Medium.ttf").to_vec(),
+        );
+        db.load_font_data(
+            include_bytes!("../../assets/fonts/cmu-sans-serif/CMUSansSerif-Bold.ttf").to_vec(),
+        );
+        // Load Adobe Caslon Pro for body text
+        db.load_font_data(
+            include_bytes!("../../assets/fonts/adobe-caslon/AdobeCaslonPro-Regular.ttf").to_vec(),
+        );
+        db.load_font_data(
+            include_bytes!("../../assets/fonts/adobe-caslon/AdobeCaslonPro-Bold.ttf").to_vec(),
+        );
+        // Load Ioskeley Mono for branding/handles
+        db.load_font_data(
+            include_bytes!("../../assets/fonts/ioskeley-mono/IoskeleyMono-Regular.ttf").to_vec(),
+        );
         db
     })
 }

crates/weaver-app/templates/og_hero_image.svg

···
   <!-- Bottom panel with theme colors -->
   <rect x="0" y="420" width="1200" height="210" fill="#191724"/>
 
-  <!-- Title -->
+  <!-- Title - CMU Sans Serif -->
   {% for line in title_lines %}
-  <text x="60" y="{{ 480 + loop.index0 * 56 }}" fill="#c4a7e7" font-family="IBM Plex Sans, sans-serif" font-size="52" font-weight="900">{{ line }}</text>
+  <text x="60" y="{{ 472 + loop.index0 * 56 }}" fill="#c4a7e7" font-family="CMU Sans Serif, sans-serif" font-size="52" font-weight="bold">{{ line }}</text>
   {% endfor %}
 
-  <!-- Notebook + Author row -->
-  <text x="60" y="600" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">{{ notebook_title }} · @{{ author_handle }}</text>
+  <!-- Notebook + Author row - flows after title -->
+  <text x="60" y="{{ 472 + (title_lines.len() - 1) * 56 + 52 }}" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">{{ notebook_title }} · @{{ author_handle }}</text>
 
   <!-- Weaver branding -->
-  <text x="1080" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="20">weaver.sh</text>
+  <text x="1060" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
 </svg>

crates/weaver-app/templates/og_notebook.svg

···
   <!-- Background -->
   <rect width="1200" height="630" fill="#191724"/>
 
-  <!-- Notebook title (large, wrapped) -->
+  <!-- Notebook title (large, wrapped) - CMU Sans Serif -->
   {% for line in title_lines %}
-  <text x="60" y="{{ 120 + loop.index0 * 64 }}" fill="#c4a7e7" font-family="IBM Plex Sans, sans-serif" font-size="64" font-weight="800">{{ line }}</text>
+  <text x="60" y="{{ 120 + loop.index0 * 68 }}" fill="#c4a7e7" font-family="CMU Sans Serif, sans-serif" font-size="60" font-weight="bold">{{ line }}</text>
   {% endfor %}
 
-  <!-- Author + entry count -->
-  <text x="60" y="280" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">@{{ author_handle }} · {{ entry_count }} {% if entry_count == 1 %}entry{% else %}entries{% endif %}</text>
+  <!-- Author + entry count - flows after title -->
+  <text x="60" y="{{ 120 + (title_lines.len() - 1) * 68 + 60 }}" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">@{{ author_handle }} · {{ entry_count }} {% if entry_count == 1 %}entry{% else %}entries{% endif %}</text>
 
-  <!-- Entry titles list -->
+  <!-- Entry titles list - Adobe Caslon Pro -->
   {% for entry_title in entry_titles %}
-  <text x="60" y="{{ 360 + loop.index0 * 44 }}" fill="#e0def4" font-family="IBM Plex Sans, sans-serif" font-size="28">{{ entry_title }}</text>
+  <text x="60" y="{{ 120 + (title_lines.len() - 1) * 68 + 60 + 60 + loop.index0 * 46 }}" fill="#e0def4" font-family="Adobe Caslon Pro, Georgia, serif" font-size="30">{{ entry_title }}</text>
   {% endfor %}
 
   <!-- Weaver branding -->
-  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
+  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="28">weaver.sh</text>
 </svg>

crates/weaver-app/templates/og_profile.svg

···
   <image xlink:href="{{ avatar_data.as_ref().unwrap() }}" x="940" y="100" width="200" height="200" clip-path="url(#avatar-clip)" preserveAspectRatio="xMidYMid slice"/>
   {% endif %}
 
-  <!-- Display name (large) -->
+  <!-- Display name (large) - CMU Sans Serif -->
   {% for line in display_name_lines %}
-  <text x="60" y="{{ 180 + loop.index0 * 64 }}" fill="#c4a7e7" font-family="IBM Plex Sans, sans-serif" font-size="64" font-weight="900">{{ line }}</text>
+  <text x="60" y="{{ 160 + loop.index0 * 68 }}" fill="#c4a7e7" font-family="CMU Sans Serif, sans-serif" font-size="60" font-weight="bold">{{ line }}</text>
   {% endfor %}
 
-  <!-- Handle -->
-  <text x="60" y="260" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="36">@{{ handle }}</text>
+  <!-- Handle - flows after display name -->
+  <text x="60" y="{{ 160 + (display_name_lines.len() - 1) * 68 + 56 }}" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="36">@{{ handle }}</text>
 
-  <!-- Bio snippet -->
+  <!-- Bio snippet - Adobe Caslon Pro -->
   {% for line in bio_lines %}
-  <text x="60" y="{{ 340 + loop.index0 * 44 }}" fill="#e0def4" font-family="IBM Plex Sans, sans-serif" font-size="32" font-weight="400">{{ line }}</text>
+  <text x="60" y="{{ 160 + (display_name_lines.len() - 1) * 68 + 56 + 60 + loop.index0 * 44 }}" fill="#e0def4" font-family="Adobe Caslon Pro, Georgia, serif" font-size="32">{{ line }}</text>
   {% endfor %}
 
   <!-- Notebook count -->
-  <text x="60" y="540" fill="#908caa" font-family="IBM Plex Sans, sans-serif" font-size="32" font-weight="400">{{ notebook_count }} {% if notebook_count == 1 %}notebook{% else %}notebooks{% endif %}</text>
+  <text x="60" y="540" fill="#908caa" font-family="CMU Sans Serif, sans-serif" font-size="32">{{ notebook_count }} {% if notebook_count == 1 %}notebook{% else %}notebooks{% endif %}</text>
 
   <!-- Weaver branding -->
-  <text x="60" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
+  <text x="60" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="28">weaver.sh</text>
 </svg>

crates/weaver-app/templates/og_profile_banner.svg

···
   <image xlink:href="{{ avatar_data.as_ref().unwrap() }}" x="940" y="215" width="200" height="200" clip-path="url(#avatar-clip)" preserveAspectRatio="xMidYMid slice"/>
   {% endif %}
 
-  <!-- Display name -->
+  <!-- Display name - CMU Sans Serif -->
   {% for line in display_name_lines %}
-  <text x="60" y="{{ 400 + loop.index0 * 64 }}" fill="#c4a7e7" font-family="IBM Plex Sans, sans-serif" font-size="64" font-weight="900">{{ line }}</text>
+  <text x="60" y="{{ 390 + loop.index0 * 60 }}" fill="#c4a7e7" font-family="CMU Sans Serif, sans-serif" font-size="56" font-weight="bold">{{ line }}</text>
   {% endfor %}
 
-  <!-- Handle + notebook count -->
-  <text x="60" y="450" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="36">@{{ handle }} · {{ notebook_count }} {% if notebook_count == 1 %}notebook{% else %}notebooks{% endif %}</text>
+  <!-- Handle + notebook count - flows after display name -->
+  <text x="60" y="{{ 390 + (display_name_lines.len() - 1) * 60 + 52 }}" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">@{{ handle }} · {{ notebook_count }} {% if notebook_count == 1 %}notebook{% else %}notebooks{% endif %}</text>
 
-  <!-- Bio snippet -->
+  <!-- Bio snippet - Adobe Caslon Pro -->
   {% if bio_lines.first().is_some() %}
-  <text x="60" y="500" fill="#e0def4" font-family="IBM Plex Sans, sans-serif" font-size="28" font-weight="400">{{ bio_lines.first().unwrap() }}</text>
+  <text x="60" y="{{ 390 + (display_name_lines.len() - 1) * 60 + 52 + 48 }}" fill="#e0def4" font-family="Adobe Caslon Pro, Georgia, serif" font-size="28">{{ bio_lines.first().unwrap() }}</text>
   {% endif %}
 
   <!-- Weaver branding -->
-  <text x="60" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
+  <text x="60" y="600" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="28">weaver.sh</text>
 </svg>

crates/weaver-app/templates/og_site.svg

···
   <rect width="1200" height="630" fill="#191724"/>
 
   <!-- Weaver title -->
-  <text x="60" y="280" fill="#c4a7e7" font-family="Ioskeley Mono, monospace" font-size="120" font-weight="700">Weaver</text>
+  <text x="60" y="280" fill="#c4a7e7" font-family="Ioskeley Mono, monospace" font-size="126" font-weight="bold">Weaver</text>
 
   <!-- Tagline -->
-  <text x="60" y="380" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="40">Share your words, your way.</text>
+  <text x="60" y="380" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="44">Share your words, your way.</text>
 
   <!-- Branding -->
-  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
+  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="28">weaver.sh</text>
 </svg>

crates/weaver-app/templates/og_text_only.svg

···
   <!-- Background -->
   <rect width="1200" height="630" fill="#191724"/>
 
-  <!-- Entry title (large, wrapped) -->
+  <!-- Entry title (large, wrapped) - CMU Sans Serif -->
   {% for line in title_lines %}
-  <text x="60" y="{{ 140 + loop.index0 * 56 }}" fill="#c4a7e7" font-family="IBM Plex Sans, sans-serif" font-size="64" font-weight="800">{{ line }}</text>
+  <text x="60" y="{{ 120 + loop.index0 * 68 }}" fill="#c4a7e7" font-family="CMU Sans Serif, sans-serif" font-size="60" font-weight="bold">{{ line }}</text>
   {% endfor %}
 
-  <!-- Notebook title + Author -->
-  <text x="60" y="320" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">{{ notebook_title }} · @{{ author_handle }}</text>
+  <!-- Notebook title + Author - flows after title -->
+  <text x="60" y="{{ 120 + (title_lines.len() - 1) * 68 + 60 }}" fill="#ebbcba" font-family="Ioskeley Mono, monospace" font-size="32">{{ notebook_title }} · @{{ author_handle }}</text>
 
-  <!-- Content snippet -->
+  <!-- Content snippet - Adobe Caslon Pro -->
   {% for line in content_lines %}
-  <text x="60" y="{{ 380 + loop.index0 * 36 }}" fill="#e0def4" font-family="IBM Plex Sans, sans-serif" font-size="28">{{ line }}</text>
+  <text x="60" y="{{ 120 + (title_lines.len() - 1) * 68 + 60 + 56 + loop.index0 * 40 }}" fill="#e0def4" font-family="Adobe Caslon Pro, Georgia, serif" font-size="30">{{ line }}</text>
   {% endfor %}
 
-
-
   <!-- Weaver branding -->
-  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="24">weaver.sh</text>
+  <text x="60" y="590" fill="#908caa" font-family="Ioskeley Mono, monospace" font-size="28">weaver.sh</text>
 </svg>

crates/weaver-common/src/agent.rs

···
 // Re-export jacquard for convenience
 use crate::constellation::{GetBacklinksQuery, RecordId};
 use crate::error::WeaverError;
+#[allow(unused_imports)]
+use crate::{PublishResult, W_TICKER, normalize_title_path};
 pub use jacquard;
 use jacquard::bytes::Bytes;
+#[allow(unused_imports)]
 use jacquard::client::{AgentError, AgentErrorKind, AgentSession, AgentSessionExt};
 use jacquard::error::ClientError;
 use jacquard::prelude::*;
 use jacquard::smol_str::SmolStr;
 use jacquard::types::blob::{BlobRef, MimeType};
 use jacquard::types::string::{AtUri, Did, RecordKey, Rkey};
+#[allow(unused_imports)]
 use jacquard::types::tid::Tid;
 use jacquard::types::uri::Uri;
 use jacquard::url::Url;
 use jacquard::{CowStr, IntoStatic};
 use mime_sniffer::MimeTypeSniffer;
+#[allow(unused_imports)]
 use std::path::Path;
 use weaver_api::com_atproto::repo::strong_ref::StrongRef;
 use weaver_api::sh_weaver::notebook::entry;
 use weaver_api::sh_weaver::publish::blob::Blob as PublishedBlob;
-
-use crate::{PublishResult, W_TICKER, normalize_title_path};
 
 const CONSTELLATION_URL: &str = "https://constellation.microcosm.blue";
 
···
     }
 
     /// Fetch an entry and construct EntryView
+    #[cfg(feature = "use-index")]
+    fn fetch_entry_view<'a>(
+        &self,
+        _notebook: &NotebookView<'a>,
+        entry_ref: &StrongRef<'_>,
+    ) -> impl Future<Output = Result<EntryView<'a>, WeaverError>>
+    where
+        Self: Sized,
+    {
+        async move {
+            use weaver_api::sh_weaver::notebook::get_entry::GetEntry;
+
+            let resp = self
+                .send(GetEntry::new().uri(entry_ref.uri.clone()).build())
+                .await
+                .map_err(|e| AgentError::from(ClientError::from(e)))?;
+
+            let output = resp.into_output().map_err(|e| {
+                AgentError::from(ClientError::invalid_request(format!(
+                    "Failed to get entry: {}",
+                    e
+                )))
+            })?;
+
+            Ok(output.value.into_static())
+        }
+    }
+
+    /// Fetch an entry and construct EntryView
+    #[cfg(not(feature = "use-index"))]
     fn fetch_entry_view<'a>(
         &self,
         notebook: &NotebookView<'a>,

crates/weaver-index/Cargo.toml

···
 
 # AT Protocol / Jacquard
 jacquard = { workspace = true, features = ["websocket", "zstd", "dns", "cache"] }
-jacquard-common = { workspace = true, features = ["crypto-k256"] }
+jacquard-common = { workspace = true, features = ["service-auth","crypto-ed25519"] }
 jacquard-repo = { workspace = true }
 jacquard-axum = { workspace = true }
 

crates/weaver-index/migrations/clickhouse/001_raw_records.sql

···
 -- Raw records from firehose/jetstream
 -- Core table for all AT Protocol records before denormalization
---
--- Append-only log using plain MergeTree - all versions preserved for audit/rollback.
--- Query-time deduplication via ORDER BY + LIMIT or window functions.
--- JSON column stores full record, extract fields only when needed for ORDER BY/WHERE/JOINs
 
 CREATE TABLE IF NOT EXISTS raw_records (
     -- Decomposed AT URI components (at://did/collection/rkey)
     did String,
     collection LowCardinality(String),
     rkey String,
-
-    -- Content identifier from the record (content-addressed hash)
     cid String,
-
     -- Repository revision (TID) - monotonically increasing per DID, used for ordering
     rev String,
-
-    -- Full record as native JSON (schema-flexible, queryable with record.field.subfield)
     record JSON,
-
-    -- Operation: 'create', 'update', 'delete', 'cache' (fetched on-demand)
+    -- Operation: 'create', 'update', 'delete', ('cache' - fetched on-demand)
     operation LowCardinality(String),
-
-    -- Firehose sequence number (metadata only, not for ordering - can jump on relay restart)
+    -- Firehose sequence number
     seq UInt64,
-
     -- Event timestamp from firehose
     event_time DateTime64(3),
-
-    -- When we indexed this record
+    -- When the database indexed this record
     indexed_at DateTime64(3) DEFAULT now64(3),
-
     -- Validation state: 'unchecked', 'valid', 'invalid_rev', 'invalid_gap', 'invalid_account'
-    -- Populated by async batch validation, not in hot path
     validation_state LowCardinality(String) DEFAULT 'unchecked',
-
     -- Whether this came from live firehose (true) or backfill (false)
-    -- Backfill events may not reflect current state until repo is fully synced
     is_live Bool DEFAULT true,
-
     -- Materialized AT URI for convenience
     uri String MATERIALIZED concat('at://', did, '/', collection, '/', rkey),
-
     -- Projection for fast delete lookups by (did, cid)
-    -- Delete events include CID, so we can O(1) lookup the original record
-    -- to know what to decrement (e.g., which notebook's like count)
     PROJECTION by_did_cid (
         SELECT * ORDER BY (did, cid)
     )

crates/weaver-index/migrations/clickhouse/002_identity_events.sql

···
     -- The DID this identity event is about
     did String,
 
-    -- Handle (may be empty if cleared)
+    -- Handle (may be empty)
     handle String,
 
     -- Sequence number from firehose

crates/weaver-index/migrations/clickhouse/003_account_events.sql

···
 -- Account events from firehose (#account messages)
--- Tracks account status changes: active, deactivated, deleted, suspended, takendown
 
 CREATE TABLE IF NOT EXISTS raw_account_events (
     -- The DID this account event is about

crates/weaver-index/migrations/clickhouse/005_firehose_cursor.sql

···
 -- Tracks our position in the firehose stream for resumption after restart
 
 CREATE TABLE IF NOT EXISTS firehose_cursor (
-    -- Consumer identifier (allows multiple consumers with different cursors)
     consumer_id String,
 
     -- Last successfully processed sequence number

crates/weaver-index/migrations/clickhouse/006_account_rev_state.sql

···
 -- Per-account revision state tracking
 -- Maintains latest rev/cid per DID for dedup and gap detection
---
--- AggregatingMergeTree with incremental MV from raw_records
--- Query with argMaxMerge/maxMerge to finalize aggregates
 
 CREATE TABLE IF NOT EXISTS account_rev_state (
     -- Account DID

crates/weaver-index/migrations/clickhouse/007_account_rev_state_mv.sql

···
--- Incremental MV: fires on each insert to raw_records, maintains aggregate state
--- Must be created after both account_rev_state (target) and raw_records (source) exist
+
 
 CREATE MATERIALIZED VIEW IF NOT EXISTS account_rev_state_mv TO account_rev_state AS
 SELECT

crates/weaver-index/migrations/clickhouse/010_handle_mappings_account_mv.sql

···
 -- Auto-populate freed status from account events
--- JOINs against handle_mappings to find current handle for the DID
--- If no mapping exists yet, the JOIN fails silently (can't free unknown handles)
 
 CREATE MATERIALIZED VIEW IF NOT EXISTS handle_mappings_from_account_mv TO handle_mappings AS
 SELECT

crates/weaver-index/migrations/clickhouse/011_profiles_weaver.sql

···
 -- Weaver profile source table
--- Populated by MV from raw_records, merged into profiles by refreshable MV
 
 CREATE TABLE IF NOT EXISTS profiles_weaver (
     did String,
-
-    -- Raw profile JSON
     profile String,
 
     -- Extracted fields for coalescing

crates/weaver-index/migrations/clickhouse/012_profiles_weaver_mv.sql

···
 CREATE MATERIALIZED VIEW IF NOT EXISTS profiles_weaver_mv TO profiles_weaver AS
 SELECT
     did,
-    toString(record) as profile,
+    record as profile,
     coalesce(record.displayName, '') as display_name,
     coalesce(record.description, '') as description,
     coalesce(record.avatar.ref.`$link`, '') as avatar_cid,

crates/weaver-index/migrations/clickhouse/013_profiles_bsky.sql

···
 -- Bluesky profile source table
--- Populated by MV from raw_records, merged into profiles by refreshable MV
 
 CREATE TABLE IF NOT EXISTS profiles_bsky (
     did String,
 
-    -- Raw profile JSON
     profile String,
 
     -- Extracted fields for coalescing

crates/weaver-index/migrations/clickhouse/014_profiles_bsky_mv.sql

···
 CREATE MATERIALIZED VIEW IF NOT EXISTS profiles_bsky_mv TO profiles_bsky AS
 SELECT
     did,
-    toString(record) as profile,
+    record as profile,
     coalesce(record.displayName, '') as display_name,
     coalesce(record.description, '') as description,
     coalesce(record.avatar.ref.`$link`, '') as avatar_cid,

crates/weaver-index/migrations/clickhouse/015_profiles.sql

···
 -- Unified profiles view
--- Refreshable MV that merges weaver + bsky profiles with handle resolution
--- Queries are pure reads, no merge computation needed
-
 CREATE MATERIALIZED VIEW IF NOT EXISTS profiles
 REFRESH EVERY 1 MINUTE
 ENGINE = ReplacingMergeTree(indexed_at)
···
 AS SELECT
     if(w.did != '', w.did, b.did) as did,
 
-    -- Handle from handle_mappings (empty if not resolved yet)
     coalesce(h.handle, '') as handle,
 
     -- Raw profiles per source

crates/weaver-index/migrations/clickhouse/017_notebooks.sql

···
 
     -- Soft delete (epoch = not deleted)
     deleted_at DateTime64(3) DEFAULT toDateTime64(0, 3),
-
-    -- Full record JSON for hydration
     record JSON DEFAULT '{}'
 )
 ENGINE = ReplacingMergeTree(indexed_at)

crates/weaver-index/migrations/clickhouse/019_notebook_counts.sql

···
--- Notebook engagement counts
--- Updated by MVs from likes, bookmarks, subscriptions (added later with graph tables)
--- Joined with notebooks at query time
+-- Notebook engagement counts table stub
 
 CREATE TABLE IF NOT EXISTS notebook_counts (
     did String,

crates/weaver-index/migrations/clickhouse/020_entries.sql

···
 
     -- Soft delete (epoch = not deleted)
     deleted_at DateTime64(3) DEFAULT toDateTime64(0, 3),
-
-    -- Full record JSON for hydration
     record JSON DEFAULT '{}'
 )
 ENGINE = ReplacingMergeTree(indexed_at)

crates/weaver-index/migrations/clickhouse/022_entry_counts.sql

···
--- Entry engagement counts
--- Updated by MVs from likes, bookmarks (added later with graph tables)
--- Joined with entries at query time
+-- Entry engagement counts table stub
 
 CREATE TABLE IF NOT EXISTS entry_counts (
     did String,

crates/weaver-index/migrations/clickhouse/023_drafts.sql

···
 -- Draft stub records
--- Anchors for unpublished content, enables draft discovery via queries
 
 CREATE TABLE IF NOT EXISTS drafts (
     -- Identity

crates/weaver-index/migrations/clickhouse/025_edit_nodes.sql

···
     node_type LowCardinality(String),  -- 'root' or 'diff'
 
     -- Resource being edited (extracted from doc.value)
-    -- One of these will be populated depending on doc type
     resource_type LowCardinality(String) DEFAULT '',  -- 'entry', 'notebook', 'draft'
     resource_did String DEFAULT '',
     resource_rkey String DEFAULT '',
     resource_collection LowCardinality(String) DEFAULT '',
 
-    -- For diffs: reference to root (StrongRef)
+    -- For diffs: reference to root
     root_did String DEFAULT '',
     root_rkey String DEFAULT '',
     root_cid String DEFAULT '',
 
-    -- For diffs: reference to previous node (StrongRef)
+    -- For diffs: reference to previous node
     prev_did String DEFAULT '',
     prev_rkey String DEFAULT '',
     prev_cid String DEFAULT '',
 
-    -- Whether this has inline diff data vs blob snapshot
     has_inline_diff UInt8 DEFAULT 0,
     has_snapshot UInt8 DEFAULT 0,
 

crates/weaver-index/migrations/clickhouse/026_edit_roots_mv.sql

···
         ''
     ) as resource_type,
 
-    -- Extract resource DID (parse from URI or empty)
+    -- Extract resource DID
     multiIf(
         toString(record.doc.value.entry.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.entry.uri), 'at://', ''))[1],
         toString(record.doc.value.notebook.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.notebook.uri), 'at://', ''))[1],
+        toString(record.doc.value.draftKey) != '',
+            splitByChar('/', replaceOne(toString(record.doc.value.draftKey), 'at://', ''))[1],
         ''
     ) as resource_did,
 
-    -- Extract resource rkey (parse from URI or use draftKey)
+    -- Extract resource rkey
     multiIf(
         toString(record.doc.value.entry.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.entry.uri), 'at://', ''))[3],
         toString(record.doc.value.notebook.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.notebook.uri), 'at://', ''))[3],
         toString(record.doc.value.draftKey) != '',
-            toString(record.doc.value.draftKey),
+            splitByChar('/', replaceOne(toString(record.doc.value.draftKey), 'at://', ''))[3],
         ''
     ) as resource_rkey,
 

crates/weaver-index/migrations/clickhouse/027_edit_diffs_mv.sql

···
             splitByChar('/', replaceOne(toString(record.doc.value.entry.uri), 'at://', ''))[1],
         toString(record.doc.value.notebook.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.notebook.uri), 'at://', ''))[1],
+        toString(record.doc.value.draftKey) != '',
+            splitByChar('/', replaceOne(toString(record.doc.value.draftKey), 'at://', ''))[1],
         ''
     ) as resource_did,
 
···
         toString(record.doc.value.notebook.uri) != '',
             splitByChar('/', replaceOne(toString(record.doc.value.notebook.uri), 'at://', ''))[3],
         toString(record.doc.value.draftKey) != '',
-            toString(record.doc.value.draftKey),
+            splitByChar('/', replaceOne(toString(record.doc.value.draftKey), 'at://', ''))[3],
         ''
     ) as resource_rkey,
 

crates/weaver-index/migrations/clickhouse/038_notebook_entries.sql

···
 -- Notebook entries mapping (denormalized for reverse lookup)
 -- Maps entries to the notebooks that contain them
--- Enables reverse lookup: find notebooks containing an entry
 
 CREATE TABLE IF NOT EXISTS notebook_entries (
     -- Entry being referenced

crates/weaver-index/migrations/clickhouse/039_notebook_entries_mv.sql

···
 -- Populate notebook_entries from notebooks
--- Extracts entry references from the entryList in notebook records
--- Incremental MV: triggers on INSERT to notebooks, writes to notebook_entries
 
 CREATE MATERIALIZED VIEW IF NOT EXISTS notebook_entries_mv
 TO notebook_entries
 AS
 SELECT
-    -- Parse entry URI to extract did and rkey
-    -- URI format: at://did:plc:xxx/sh.weaver.notebook.entry/rkey
-    -- assumeNotNull is safe here because WHERE filters guarantee non-null
     assumeNotNull(extract(entry_uri, 'at://([^/]+)/')) as entry_did,
     assumeNotNull(extract(entry_uri, '/sh\\.weaver\\.notebook\\.entry/([^/]+)$')) as entry_rkey,
 

crates/weaver-index/src/clickhouse/queries/notebooks.rs

···
         Ok(row)
     }
 
-    /// List entries for a notebook's author (did).
+    /// List entries for a specific notebook, ordered by position in the notebook.
     ///
-    /// Note: This is a simplified version. The full implementation would
-    /// need to join with notebook's entryList to get proper ordering.
-    /// For now, we just list entries by the same author, ordered by rkey (notebook order).
+    /// Uses notebook_entries table to get entries that belong to this notebook.
     pub async fn list_notebook_entries(
         &self,
-        did: &str,
+        notebook_did: &str,
+        notebook_rkey: &str,
         limit: u32,
-        cursor: Option<&str>,
+        cursor: Option<u32>,
     ) -> Result<Vec<EntryRow>, IndexError> {
-        // Note: rkey ordering is intentional here - it's the notebook's entry order
-        let query = if cursor.is_some() {
-            r#"
-                SELECT did, rkey, cid, uri, title, path, tags, author_dids, created_at, indexed_at, record
-                FROM (
-                    SELECT did, rkey, cid, uri, title, path, tags, author_dids, created_at, updated_at, indexed_at, record,
-                           ROW_NUMBER() OVER (PARTITION BY rkey ORDER BY updated_at DESC) as rn
-                    FROM entries FINAL
-                    WHERE did = ?
-                      AND deleted_at = toDateTime64(0, 3)
-                      AND rkey > ?
-                )
-                WHERE rn = 1
-                ORDER BY rkey ASC
-                LIMIT ?
-            "#
-        } else {
-            r#"
-                SELECT did, rkey, cid, uri, title, path, tags, author_dids, created_at, indexed_at, record
-                FROM (
-                    SELECT did, rkey, cid, uri, title, path, tags, author_dids, created_at, updated_at, indexed_at, record,
-                           ROW_NUMBER() OVER (PARTITION BY rkey ORDER BY updated_at DESC) as rn
-                    FROM entries FINAL
-                    WHERE did = ?
-                      AND deleted_at = toDateTime64(0, 3)
-                )
-                WHERE rn = 1
-                ORDER BY rkey ASC
-                LIMIT ?
-            "#
-        };
+        let query = r#"
+            SELECT
+                e.did AS did,
+                e.rkey AS rkey,
+                e.cid AS cid,
+                e.uri AS uri,
+                e.title AS title,
+                e.path AS path,
+                e.tags AS tags,
+                e.author_dids AS author_dids,
+                e.created_at AS created_at,
+                e.indexed_at AS indexed_at,
+                e.record AS record
+            FROM notebook_entries ne FINAL
+            INNER JOIN entries e ON
+                e.did = ne.entry_did
+                AND e.rkey = ne.entry_rkey
+                AND e.deleted_at = toDateTime64(0, 3)
+            WHERE ne.notebook_did = ?
+              AND ne.notebook_rkey = ?
+              AND ne.position > ?
+            ORDER BY ne.position ASC
+            LIMIT ?
+        "#;
 
-        let mut q = self.inner().query(query).bind(did);
-
-        if let Some(c) = cursor {
-            q = q.bind(c);
-        }
+        let cursor_val = cursor.unwrap_or(0);
 
-        let rows =
-            q.bind(limit)
-                .fetch_all::<EntryRow>()
-                .await
-                .map_err(|e| ClickHouseError::Query {
-                    message: "failed to list notebook entries".into(),
-                    source: e,
-                })?;
+        let rows = self
+            .inner()
+            .query(query)
+            .bind(notebook_did)
+            .bind(notebook_rkey)
+            .bind(cursor_val)
+            .bind(limit)
+            .fetch_all::<EntryRow>()
+            .await
+            .map_err(|e| ClickHouseError::Query {
+                message: "failed to list notebook entries".into(),
+                source: e,
+            })?;
 
         Ok(rows)
     }

crates/weaver-index/src/endpoints/bsky.rs

···
+//! app.bsky.* passthrough endpoints
+//!
+//! These forward requests to the Bluesky appview.
+
+use axum::{Json, extract::State};
+use jacquard::prelude::*;
+use jacquard_axum::ExtractXrpc;
+use weaver_api::app_bsky::actor::get_profile::{GetProfileOutput, GetProfileRequest};
+use weaver_api::app_bsky::feed::get_posts::{GetPostsOutput, GetPostsRequest};
+
+use crate::endpoints::repo::XrpcErrorResponse;
+use crate::server::AppState;
+
+/// Handle app.bsky.actor.getProfile (passthrough)
+pub async fn get_profile(
+    State(state): State<AppState>,
+    ExtractXrpc(args): ExtractXrpc<GetProfileRequest>,
+) -> Result<Json<GetProfileOutput<'static>>, XrpcErrorResponse> {
+    let response = state.resolver.send(args).await.map_err(|e| {
+        tracing::warn!("Appview getProfile failed: {}", e);
+        XrpcErrorResponse::internal_error("Failed to fetch profile from appview")
+    })?;
+
+    let output = response.into_output().map_err(|e| {
+        tracing::warn!("Failed to parse getProfile response: {}", e);
+        XrpcErrorResponse::internal_error("Failed to parse appview response")
+    })?;
+
+    Ok(Json(output))
+}
+
+/// Handle app.bsky.feed.getPosts (passthrough)
+pub async fn get_posts(
+    State(state): State<AppState>,
+    ExtractXrpc(args): ExtractXrpc<GetPostsRequest>,
+) -> Result<Json<GetPostsOutput<'static>>, XrpcErrorResponse> {
+    let response = state.resolver.send(args).await.map_err(|e| {
+        tracing::warn!("Appview getPosts failed: {}", e);
+        XrpcErrorResponse::internal_error("Failed to fetch posts from appview")
+    })?;
+
+    let output = response.into_output().map_err(|e| {
+        tracing::warn!("Failed to parse getPosts response: {}", e);
+        XrpcErrorResponse::internal_error("Failed to parse appview response")
+    })?;
+
+    Ok(Json(output))
+}

crates/weaver-index/src/endpoints/edit.rs

···
 use crate::clickhouse::{EditNodeRow, ProfileRow};
 use crate::endpoints::actor::{Viewer, resolve_actor};
 use crate::endpoints::collab::profile_to_view_basic;
-use crate::endpoints::resolve_uri;
 use crate::endpoints::repo::XrpcErrorResponse;
+use crate::endpoints::resolve_uri;
 use crate::server::AppState;
 
 /// Handle sh.weaver.edit.getEditHistory

crates/weaver-index/src/endpoints/identity.rs

···
+//! com.atproto.identity.* endpoint handlers
+
+use axum::{Json, extract::State};
+use jacquard::IntoStatic;
+use jacquard::types::ident::AtIdentifier;
+use jacquard_axum::ExtractXrpc;
+use weaver_api::com_atproto::identity::resolve_handle::{
+    ResolveHandleOutput, ResolveHandleRequest,
+};
+
+use crate::endpoints::actor::resolve_actor;
+use crate::endpoints::repo::XrpcErrorResponse;
+use crate::server::AppState;
+
+/// Handle com.atproto.identity.resolveHandle
+pub async fn resolve_handle(
+    State(state): State<AppState>,
+    ExtractXrpc(args): ExtractXrpc<ResolveHandleRequest>,
+) -> Result<Json<ResolveHandleOutput<'static>>, XrpcErrorResponse> {
+    let did = resolve_actor(&state, &AtIdentifier::Handle(args.handle)).await?;
+
+    Ok(Json(
+        ResolveHandleOutput {
+            did: did.into_static(),
+            extra_data: None,
+        }
+        .into_static(),
+    ))
+}

crates/weaver-index/src/endpoints/mod.rs

···
 use self::repo::XrpcErrorResponse;
 
 pub mod actor;
+pub mod bsky;
 pub mod collab;
 pub mod edit;
+pub mod identity;
 pub mod notebook;
 pub mod repo;
 

crates/weaver-index/src/endpoints/notebook.rs

···
     let did_str = did.as_str();
     let name = args.name.as_ref();
 
-    // Fetch notebook and entries in parallel - both just need the DID
     let limit = args.entry_limit.unwrap_or(50).clamp(1, 100) as u32;
-    let cursor = args.entry_cursor.as_deref();
+    let cursor: Option<u32> = args
+        .entry_cursor
+        .as_deref()
+        .and_then(|c| c.parse().ok());
 
-    let (notebook_result, entries_result) = tokio::try_join!(
-        async {
-            state
-                .clickhouse
-                .resolve_notebook(did_str, name)
-                .await
-                .map_err(|e| {
-                    tracing::error!("Failed to resolve notebook: {}", e);
-                    XrpcErrorResponse::internal_error("Database query failed")
-                })
-        },
-        async {
-            state
-                .clickhouse
-                .list_notebook_entries(did_str, limit + 1, cursor)
-                .await
-                .map_err(|e| {
-                    tracing::error!("Failed to list entries: {}", e);
-                    XrpcErrorResponse::internal_error("Database query failed")
-                })
-        }
-    )?;
+    // Fetch notebook first to get its rkey
+    let notebook_row = state
+        .clickhouse
+        .resolve_notebook(did_str, name)
+        .await
+        .map_err(|e| {
+            tracing::error!("Failed to resolve notebook: {}", e);
+            XrpcErrorResponse::internal_error("Database query failed")
+        })?
+        .ok_or_else(|| XrpcErrorResponse::not_found("Notebook not found"))?;
 
-    let notebook_row =
-        notebook_result.ok_or_else(|| XrpcErrorResponse::not_found("Notebook not found"))?;
-    let entry_rows = entries_result;
+    // Now fetch entries using notebook's rkey
+    let entry_rows = state
+        .clickhouse
+        .list_notebook_entries(did_str, &notebook_row.rkey, limit + 1, cursor)
+        .await
+        .map_err(|e| {
+            tracing::error!("Failed to list entries: {}", e);
+            XrpcErrorResponse::internal_error("Database query failed")
+        })?;
 
     // Fetch notebook contributors (evidence-based)
     let notebook_contributors = state
···
         entries.push(book_entry);
     }
 
-    // Build cursor for pagination
+    // Build cursor for pagination (position-based)
     let next_cursor = if has_more {
-        entry_rows.last().map(|e| e.rkey.to_string().into())
+        // Position = cursor offset + number of entries returned
+        let last_position = cursor.unwrap_or(0) + entry_rows.len() as u32;
+        Some(last_position.to_string().into())
     } else {
         None
     };

crates/weaver-index/src/landing.html

···
+<!doctype html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1" />
+        <title>Weaver Index</title>
+        <style>
+            @font-face {
+                font-family: "Ioskeley Mono";
+                font-style: normal;
+                font-weight: normal;
+                src: url("/assets/IoskeleyMono-Regular.woff2") format("woff2");
+            }
+            :root {
+                --color-base: #faf4ed;
+                --color-surface: #fffaf3;
+                --color-text: #1f1d2e;
+                --color-muted: #635e74;
+                --color-subtle: #4a4560;
+                --color-primary: #907aa9;
+                --color-secondary: #56949f;
+                --color-link: #d7827e;
+                --color-border: #908caa;
+            }
+            @media (prefers-color-scheme: dark) {
+                :root {
+                    --color-base: #191724;
+                    --color-surface: #1f1d2e;
+                    --color-text: #e0def4;
+                    --color-muted: #6e6a86;
+                    --color-subtle: #908caa;
+                    --color-primary: #c4a7e7;
+                    --color-secondary: #3e8fb0;
+                    --color-link: #ebbcba;
+                    --color-border: #403d52;
+                }
+            }
+            * {
+                box-sizing: border-box;
+                margin: 0;
+                padding: 0;
+            }
+            body {
+                font-family: "Ioskeley Mono", "IBM Plex Mono", "Berkeley Mono", Consolas, monospace;
+                font-size: 14px;
+                line-height: 1.6;
+                color: var(--color-text);
+                background: var(--color-base);
+                max-width: 700px;
+                margin: 0 auto;
+                padding: 3rem 1.5rem;
+            }
+            pre {
+                font-size: 0.7rem;
+                line-height: 1.1;
+                color: var(--color-primary);
+                margin-bottom: 1.5rem;
+            }
+            h1 {
+                font-size: 1.25rem;
+                font-weight: 600;
+                color: var(--color-secondary);
+                margin-bottom: 0.25rem;
+            }
+            .subtitle {
+                color: var(--color-muted);
+                margin-bottom: 1.5rem;
+            }
+            p {
+                margin-bottom: 1rem;
+            }
+            a {
+                color: var(--color-link);
+                text-decoration: none;
+            }
+            a:hover {
+                text-decoration: underline;
+            }
+            code {
+                background: var(--color-surface);
+                padding: 0.125rem 0.375rem;
+                border-radius: 3px;
+                border: 1px solid var(--color-border);
+            }
+            ul {
+                list-style: none;
+                margin-top: 1.5rem;
+                padding-top: 1rem;
+                border-top: 1px solid var(--color-border);
+            }
+            li {
+                margin-bottom: 0.5rem;
+            }
+            li::before {
+                content: "→ ";
+                color: var(--color-muted);
+            }
+        </style>
+    </head>
+    <body>
+        <pre>
+
+    .dP'    db    `Yb                .dP'
+    dP'    db    db  88              dP'
+                88
+    `Yb d888b    'Yb     'Yb     88 d88b d88b     'Yb   `Yb    dP'
+    88P    88    88      88     88P   8Y   8b     88     Yb  dP
+    88     8P    88      88     88    8P   88     88      YbdP
+    88   .dP    .8P     .8P     88  .dP' .dP'    .8P      .8P
+    .88888888b.                  8888888888888b.         dP'  b
+                                        Y.  ,P
+                                            `""'</pre
+        >
+        <h1>Weaver Index</h1>
+        <p class="subtitle">AT Protocol Record Index</p>
+        <p>This is an AT Protocol record index serving the Weaver writing platform.</p>
+        <p>Most API endpoints are available under <code>/xrpc/</code>.</p>
+        <ul>
+            <li>Web App (alpha): <a href="https://alpha.weaver.sh">alpha.weaver.sh</a></li>
+            <li>
+                Source Code:
+                <a href="https://tangled.org/@nonbinary.computer/weaver">tangled.org/@nonbinary.computer/weaver</a>
+            </li>
+            <li><a href="https://atproto.com">AT Protocol</a></li>
+        </ul>
+    </body>
+</html>

crates/weaver-index/src/server.rs

···
 use std::net::SocketAddr;
 use std::sync::Arc;
 
-use axum::{Json, Router, extract::State, http::StatusCode, response::IntoResponse, routing::get};
+use axum::{
+    Json, Router,
+    extract::State,
+    http::{StatusCode, header},
+    response::{Html, IntoResponse},
+    routing::get,
+};
 use jacquard::api::com_atproto::repo::{
     get_record::GetRecordRequest, list_records::ListRecordsRequest,
 };
···
 use tower_http::cors::CorsLayer;
 use tower_http::trace::TraceLayer;
 use tracing::info;
+use weaver_api::app_bsky::actor::get_profile::GetProfileRequest as BskyGetProfileRequest;
+use weaver_api::app_bsky::feed::get_posts::GetPostsRequest as BskyGetPostsRequest;
+use weaver_api::com_atproto::identity::resolve_handle::ResolveHandleRequest;
 use weaver_api::sh_weaver::actor::{
     get_actor_entries::GetActorEntriesRequest, get_actor_notebooks::GetActorNotebooksRequest,
     get_profile::GetProfileRequest,
···
 
 use crate::clickhouse::Client;
 use crate::config::ShardConfig;
-use crate::endpoints::{actor, collab, edit, notebook, repo};
+use crate::endpoints::{actor, bsky, collab, edit, identity, notebook, repo};
 use crate::error::{IndexError, ServerError};
 use crate::sqlite::ShardRouter;
 
···
         Self {
             clickhouse: Arc::new(clickhouse),
             shards: Arc::new(ShardRouter::new(shard_config.base_path)),
-            resolver: UnauthenticatedSession::new_slingshot(),
+            resolver: UnauthenticatedSession::new_public(),
             service_did,
         }
     }
···
 /// Build the axum router with all XRPC endpoints
 pub fn router(state: AppState, did_doc: DidDocument<'static>) -> Router {
     Router::new()
-        // did:web document
+        .route("/", get(landing))
+        .route(
+            "/assets/IoskeleyMono-Regular.woff2",
+            get(font_ioskeley_regular),
+        )
+        .route("/assets/IoskeleyMono-Bold.woff2", get(font_ioskeley_bold))
+        .route(
+            "/assets/IoskeleyMono-Italic.woff2",
+            get(font_ioskeley_italic),
+        )
         .route("/xrpc/_health", get(health))
         .route("/metrics", get(metrics))
+        // com.atproto.identity.* endpoints
+        .merge(ResolveHandleRequest::into_router(identity::resolve_handle))
         // com.atproto.repo.* endpoints (record cache)
         .merge(GetRecordRequest::into_router(repo::get_record))
         .merge(ListRecordsRequest::into_router(repo::list_records))
+        // app.bsky.* passthrough endpoints
+        .merge(BskyGetProfileRequest::into_router(bsky::get_profile))
+        .merge(BskyGetPostsRequest::into_router(bsky::get_posts))
         // sh.weaver.actor.* endpoints
         .merge(GetProfileRequest::into_router(actor::get_profile))
         .merge(GetActorNotebooksRequest::into_router(
···
 /// Prometheus metrics endpoint
 async fn metrics() -> String {
     telemetry::render()
+}
+
+// Embedded font files
+const IOSKELEY_MONO_REGULAR: &[u8] =
+    include_bytes!("../../weaver-app/assets/fonts/ioskeley-mono/IoskeleyMono-Regular.woff2");
+const IOSKELEY_MONO_BOLD: &[u8] =
+    include_bytes!("../../weaver-app/assets/fonts/ioskeley-mono/IoskeleyMono-Bold.woff2");
+const IOSKELEY_MONO_ITALIC: &[u8] =
+    include_bytes!("../../weaver-app/assets/fonts/ioskeley-mono/IoskeleyMono-Italic.woff2");
+
+/// Serve the Ioskeley Mono Regular font
+async fn font_ioskeley_regular() -> impl IntoResponse {
+    (
+        [(header::CONTENT_TYPE, "font/woff2")],
+        IOSKELEY_MONO_REGULAR,
+    )
+}
+/// Serve the Ioskeley Mono Regular font
+async fn font_ioskeley_bold() -> impl IntoResponse {
+    ([(header::CONTENT_TYPE, "font/woff2")], IOSKELEY_MONO_BOLD)
+}
+
+/// Serve the Ioskeley Mono Regular font
+async fn font_ioskeley_italic() -> impl IntoResponse {
+    ([(header::CONTENT_TYPE, "font/woff2")], IOSKELEY_MONO_ITALIC)
+}
+
+const LANDING_HTML: &str = include_str!("./landing.html");
+
+/// Landing page
+async fn landing() -> Html<&'static str> {
+    Html(LANDING_HTML)
 }
 
 /// Health check response

crates/weaver-index/src/service_identity.rs

···
     /// Encode the public key as a multikey string
     fn encode_public_key(signing_key: &SigningKey) -> String {
         let verifying_key = signing_key.verifying_key();
-        let point = verifying_key.to_encoded_point(true); // compressed
-        let bytes = point.as_bytes();
+        let bytes = verifying_key.to_sec1_bytes();
         // 0xE7 is the multicodec for secp256k1-pub
-        multikey(0xE7, bytes)
+        multikey(0xE7, bytes.as_ref())
     }
 
     /// Get the public key multibase string
···
             "@context": [
                 "https://www.w3.org/ns/did/v1",
                 "https://w3id.org/security/multikey/v1",
-                "https://w3id.org/security/suites/secp256k1-2019/v1"
             ],
             "id": did_str,
             "verificationMethod": [{
···
             "@context": [
                 "https://www.w3.org/ns/did/v1",
                 "https://w3id.org/security/multikey/v1",
-                "https://w3id.org/security/suites/secp256k1-2019/v1"
             ],
             "id": did_str,
             "verificationMethod": [{

crates/weaver-index/src/sqlite.rs

···
     }
 
     pub fn last_accessed(&self) -> Instant {
-        self.last_accessed
-            .lock()
-            .map(|t| *t)
-            .unwrap_or_else(|_| Instant::now())
+        self.last_accessed.lock().map(|t| *t).expect("poisoned")
     }
 
     /// Execute a read operation on the shard

docker-compose.yml

···
       TAP_BIND: ":2480"
       TAP_DISABLE_ACKS: "false"
       TAP_LOG_LEVEL: info
-      TAP_SIGNAL_COLLECTION: sh.tangled.actor.profile
-      TAP_COLLECTION_FILTERS: "sh.weaver.*,app.bsky.actor.profile,sh.tangled.*,pub.leaflet.*"
+      TAP_OUTBOX_PARALLELISM: 5
+      #TAP_FULL_NETWORK: true
+      TAP_SIGNAL_COLLECTION: place.stream.chat.profile
+      TAP_COLLECTION_FILTERS: "sh.weaver.*,app.bsky.actor.profile,sh.tangled.*,pub.leaflet.*,net.anisota.*,place.stream.*"
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:2480/health"]
       interval: 20s
···
     restart: unless-stopped
 
   # Weaver indexer - consumes from tap
-  indexer:
-    container_name: weaver-indexer
+  index:
+    container_name: weaver-index
     image: ${REGISTRY_HOST:-localhost}:5000/weaver-index:latest
     ports:
       - "3000:3000"
+    command: ["run"]
+    volumes:
+      - index_data:/app/data
     environment:
       RUST_LOG: info,weaver_index=debug,hyper_util::client::legacy::pool=info
       CLICKHOUSE_URL: ${CLICKHOUSE_URL}
···
       TAP_URL: ws://tap:2480/channel
       TAP_SEND_ACKS: "true"
       FIREHOSE_RELAY_URL: wss://bsky.network
-      INDEXER_COLLECTIONS: "sh.weaver.*,app.bsky.actor.profile,sh.tangled.*,pub.leaflet.*"
+      INDEXER_COLLECTIONS: "sh.weaver.*,app.bsky.actor.profile,sh.tangled.*,pub.leaflet.*,net.anisota.*,place.stream.*"
     depends_on:
       tap:
         condition: service_healthy
     healthcheck:
-      test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/xrpc/_health"]
+      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:3000/xrpc/_health"]
       interval: 20s
       timeout: 5s
       retries: 3
···
       PORT: 8080
       IP: 0.0.0.0
       RUST_LOG: info
-    depends_on:
-      indexer:
-        condition: service_healthy
     healthcheck:
       test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/"]
       interval: 20s
···
 volumes:
   registry_data:
   tap_data:
+  index_data:

weaver_notes/.obsidian/workspace.json

···
             "state": {
               "type": "markdown",
               "state": {
-                "file": "Why I rewrote pdsls in Rust (tm).md",
+                "file": "Writing the AppView Last.md",
                 "mode": "source",
                 "source": false
               },
               "icon": "lucide-file",
-              "title": "Why I rewrote pdsls in Rust (tm)"
-            }
-          },
-          {
-            "id": "4b584137400d2323",
-            "type": "leaf",
-            "state": {
-              "type": "release-notes",
-              "state": {
-                "currentVersion": "1.10.6"
-              },
-              "icon": "lucide-book-up",
-              "title": "Release Notes 1.10.6"
+              "title": "Writing the AppView Last"
             }
           }
         ],
-        "currentTab": 3
+        "currentTab": 2
       }
     ],
     "direction": "vertical"
···
       "bases:Create new base": false
     }
   },
-  "active": "4b584137400d2323",
+  "active": "6029beecc3d03bce",
   "lastOpenFiles": [
+    "diff_record.png",
+    "Why I rewrote pdsls in Rust (tm).md",
+    "Writing the AppView Last.md",
     "light_mode_excerpt.png",
     "notebook_entry_preview.png",
     "xkcd_345_excerpt.png",
     "bug notes.md",
-    "Why I rewrote pdsls in Rust (tm).md",
     "meta.png",
     "Pasted image 20251114125028.png",
     "invalid_record.png",
···
     "json_editor_with_errors.png",
     "pretty_editor.png",
     "Arch.md",
-    "Weaver - Long-form writing.md",
-    "weaver_photo_med.jpg"
+    "Weaver - Long-form writing.md"
   ]
 }

weaver_notes/Writing the AppView Last.md

···
+If you've been to this site before, you maybe noticed it loaded a fair bit more quickly this time. That's not really because the web server creating this HTML got a whole lot better. It did require some refactoring, but it was mostly in the vein of taking some code and adding new code that did the same thing gated behind a cargo feature. This did, however, have the side effect of, in the final binary, replacing functions that are literally hundreds of lines, that in turn call functions that may also be hundreds of lines, making several cascading network requests, with functions that look like this, which make by and large a single network request and return exactly what is required.
+
+```rust
+#[cfg(feature = "use-index")]
+fn fetch_entry_view(
+	&self,
+	entry_ref: &StrongRef<'_>,
+) -> impl Future<Output = Result<EntryView<'static>, WeaverError>>
+where
+	Self: Sized,
+{
+	async move {
+		use weaver_api::sh_weaver::notebook::get_entry::GetEntry;
+
+		let resp = self
+			.send(GetEntry::new().uri(entry_ref.uri.clone()).build())
+			.await
+			.map_err(|e| AgentError::from(ClientError::from(e)))?;
+
+		let output = resp.into_output().map_err(|e| {
+			AgentError::xrpc(e.into))
+		})?;
+
+		Ok(output.value.into_static())
+	}
+}
+```
+
+Of course the reason is that I finally got round to building the Weaver AppView. I'm going to be calling mine the Index, because Weaver is about writing and I think "AppView" as a term kind of sucks and "index" is much more elegant, on top of being a good descriptor of what the big backend service now powering Weaver does. ![[at://did:plc:ragtjsm2j2vknwkz3zp4oxrd/app.bsky.feed.post/3lyucxfxq622w]]
+For the uninitiated, because I expect at least some people reading this aren't big into AT Protocol development, an AppView is an instance of the kind of big backend service that Bluesky PBLLC runs which powers essentially every Bluesky client, with a few notable exceptions, such as [Red Dwarf](https://reddwarf.app/), and (partially, eventually more completely) [Blacksky](https://blacksky.community/). It listens to the [Firehose](https://bsky.network/) [event stream](https://atproto.com/specs/event-stream) from the main Bluesky Relay and analyzes the data which comes through that pertains to Bluesky, producing your timeline feeds, figuring out who follows you, who you block and who blocks you (and filtering them out of your view of the app), how many people liked your last post, and so on. Because the records in your PDS (and those of all the other people on Bluesky) need context and relationship and so on to give them meaning, and then that context can be passed along to you without your app having to go collect it all. ![[at://did:plc:uu5axsmbm2or2dngy4gwchec/app.bsky.feed.post/3lsc2tzfsys2f]]
+It's a very normal backend with some weird constraints because of the protocol, and in it's practice the thing that separates the day-to-day Bluesky experience from the Mastodon experience the most. It's also by far the most centralising force in the network, because it also does moderation, and because it's quite expensive to run. A full index of all Bluesky activity takes a lot of storage (futur's Zeppelin experiment detailed above took about 16 terabytes of storage using PostgreSQL for the database and cost $200/month to run), and then it takes that much more computing power to calculate all the relationships between the data on the fly as new events come in and then serve personalized versions to everyone that uses it.
+
+It's not the only AppView out there, most atproto apps have something like this. Tangled, Streamplace, Leaflet, and so on all have substantial backends. Some (like Tangled) actually combine the front end you interact with and the AppView into a single service. But in general these are big, complicated persistent services you have to backfill from existing data to bootstrap, and they really strongly shape your app, whether they're literally part of the same executable or hosted on the same server or not. And when I started building Weaver in earnest, not only did I still have a few big unanswered questions about how I wanted Weaver to work, how it needed to work, I also didn't want to fundamentally tie it to some big server, create this centralising force. I wanted it to be possible for someone else to run it without being dependent on me personally, ideally possible even if all they had access to was a static site host like GitHub Pages or a browser runtime platform like Cloudflare Workers, so long as someone somewhere was running a couple of generic services. I wanted to be able to distribute the fullstack server version as basically just an executable in a directory of files with no other dependencies, which could easily be run in any container hosting environment with zero persistent storage required. Hell, you could technically serve it as a blob or series of blobs from your PDS with the right entry point if I did my job right.
+
+I succeeded. 
+
+Well, I don't know if you can serve `weaver-app` purely via  `com.atproto.sync.getBlob` request, but it doesn't need much.
+## Constellation
+![[at://did:plc:ttdrpj45ibqunmfhdsb4zdwq/app.bsky.feed.post/3m6pckslkt222]] Ana's leaflet does a good job of explaining more or less how Weaver worked up until now. It used direct requests to personal data servers (mostly mine) as well as many calls to [Constellation](https://constellation.microcosm.blue/) and [Slingshot](https://slingshot.microcosm.blue/), and some even to [UFOs](https://ufos.microcosm.blue/), plus a couple of judicious calls to the Bluesky AppView for profiles and post embeds. ![[at://did:plc:hdhoaan3xa3jiuq4fg4mefid/app.bsky.feed.post/3m5jzclsvpc2c]]
+The three things linked above are generic services that provide back-links, a record cache, and a running feed of the most recent instances of all lexicons on the network, respectively. That's more than enough to build an app with, though it's not always easy. For some things it can be pretty straightforward. Constellation can tell you what notebooks an entry is in. It can tell you which edit history records are related to this notebook entry. For single-layer relationships it's straightforward. However you then have to also fetch the records individually, because it doesn't provide you the records, just the URIs you need to find them. Slingshot doesn't currently have an endpoint that will batch fetch a list of URIs for you. And the PDS only has endpoints like [`com.atproto.repo.listRecords`](https://docs.bsky.app/docs/api/com-atproto-repo-list-records), which gives you a paginated list of all records of a specific type, but doesn't let you narrow that down easily, so you have to page through until you find what you wanted. 
+
+This wouldn't be too bad if I was fine with almost everything after the hostname in my web URLs being gobbledegook record keys, but I wanted people to be able to link within a notebook like they normally would if they were linking within an Obsidian Vault, by name or by path, something human-readable. So some queries became the good old N+1 requests, because I had to list a lot of records and fetch them until I could find the one that matched. Or worse still, particularly once I introduce collaboration and draft syncing to the editor. Loading a draft of an entry with a lot of edit history could take 100 or more requests, to check permissions, find all the edit records, figure out which ones mattered, publish the collaboration session record, check for collaborators, and so on. It was pretty slow going, particularly when one could not pre-fetch and cache and generate everything server-side on a real CPU rather than in a browser after downloading a nice chunk of WebAssembly code. My profile page [alpha.weaver.sh/nonbinary.computer](https://alpha.weaver.sh/nonbinary.computer) often took quite some time to load due to a frustrating quirk of Dioxus, the Rust web framework I've used for the front-end, which prevented server-side rendering from waiting until everything important had been fetched to render the complete page on that specific route, forcing me to load it client-side.
+
+Some stuff is just complicated to graph out, to find and pull all the relevant data together in order, and some connections aren't the kinds of things you can graph generically. For example, in order to work without any sort of service that has access to indefinite authenticated sessions of more than one person at once, Weaver handles collaborative writing and publishing by having each collaborator write to their own repository and publish there, and then, when the published version is requested, figuring out which version of an entry or notebook is most up-to-date, and displaying that one. It matches by record key across more than one repository, determined at request time by the state of multiple other records in those users' repositories.
+
+# Shape of Data
+All of that being said, this was still the correct route, particularly for me. Because not only does this provide a powerful fallback mode, built-in protection against me going AWOL, it was critical in the design process of the index. My friend Ollie, when talking about database and API design, always says that, regardless of the specific technology you use, you need to structure your data based on how you need to query into it. Whatever interface you put in front of it, be it GraphQL, SQL, gRPC, XRPC, server functions, AJAX, literally any way that you can have the part of your app that people interact with pull the specific data they want from where it's stored, how well that performs, how many cycles your server or client spends collecting it, sorting it, or waiting on it, how much memory it takes, how much bandwidth it takes, depends on how that data is shaped, and you, when you are designing your app and all the services that go into it, get to choose that shape. 
+
+Bluesky developers have said that hydrating blocks, mutes, and labels and applying the appropriate ones to the feed content based on the preferences of the user takes quite a bit of compute at scale, and that even the seemingly simple [Following feed](https://jazco.dev/2025/02/19/imperfection/), which is mostly a reverse-chronological feed of posts by people you follow explicitly (plus a few simple rules), is remarkably resource-intensive to produce for them. The extremely clever [string interning](https://jazco.dev/2025/09/26/interning/) and [bitmap tricks](https://jazco.dev/2024/04/20/roaring-bitmaps/) implemented by a brilliant engineer during their time at Bluesky are all oriented toward figuring out the most efficient way to structure the data to make the desired query emerge naturally from it. ![Roaring Bitmaps Diagram from the Original Publication at https://arxiv.org/pdf/1709.07821](https://jazco.dev/public/images/2025-09-26/roaring_bitmaps_diagram.png)
+
+It's intuitive that this matters a lot when you use something like RocksDB, or FoundationDB, or Redis, which are fundamentally key-value stores. What your key contains there determines almost everything about how easy it is to find and manipulate the values you want. Fig and I have had some struggles getting a backup of their Constellation service running in real-time and keeping up with Jetstream on my home server, because the only storage on said home server with enough free space for Constellation's full index is a ZFS pool that's primarily hard-drive based, and the way the Constellation RocksDB backend storage is structured makes processing delete events extremely expensive on a hard drive where seek times are nontrivial. On a Pi 4 with an SSD, it runs just fine. ![[at://did:plc:44ybard66vv44zksje25o7dz/app.bsky.feed.post/3m7e3hnyh5c2u]]
+But it's a problem for every database. Custom feed builder service [graze.social](https://graze.social/) ran into difficulties with Postgres early on in their development, as they rapidly gained popularity. They ended up using the same database I did, Clickhouse, for many of the same reasons. ![[at://did:plc:i6y3jdklpvkjvynvsrnqfdoq/app.bsky.feed.post/3m7ecmqcwys23]]
+And while thankfully I don't think that a platform oriented around long-form written content will ever have the kinds of following timeline graph write amplification problems Bluesky has dealt with, even if it becomes successful beyond my wildest dreams, there are definitely going to be areas where latency matters a ton and the workload is very write-heavy, like real-time collaboration, particularly if a large number of people work on a document simultaneously, even while the vast majority of requests will primarily be reading data out.
+
+One reason why the edit records for Weaver have three link fields (and may get more!), even though it may seem a bit redundant, is precisely because those links make it easy to graph the relationships between them, to trace a tree of edits backward to the root, while also allowing direct access and a direct relationship to the root snapshot and the thing it's associated with.
+
+In contrast, notebook entry records lack links to other parts of the notebook in and of themselves because calculating them would be challenging, and updating one entry would require not just updating the entry itself and notebook it's in, but also neighbouring entries in said notebook. With the shape of collaborative publishing in Weaver, that would result in up to 4 writes to the PDS when you publish an entry, in addition to any blob uploads. And trying to link the other way in edit history (root to edit head) is similarly challenging.
+
+I anticipated some of these. but others emerged only because I ran into them while building the web app. I've had to manually fix up records more than once because I made breaking changes to my lexicons after discovering I really wanted X piece of metadata or cross-linkage. If I'd built the index first or alongside—particularly if the index remained a separate service from the web app as I intended it to, to keep the web app simple—it would likely have constrained my choices and potentially cut off certain solutions, due to the time it takes to dump the database and re-run backfill even at a very small scale. Building a big chunk of the front end first told me exactly what the index needed to provide easy access to.
+# ClickHAUS
+So what does Weaver's index look like? Well it starts with either the firehose or the new Tap sync tool. The index ingests from either over a WebSocket connection, does a bit of processing (less is required when ingesting from Tap, and that's currently what I've deployed) and then dumps them in the Clickhouse database. I chose it as the primary index database on recommendation from a friend, and after doing a lot of reading. It fits atproto data well, as Graze found. Because it isolates concurrent inserts and selects so that you can just dump data in, while it cleans things up asynchronously after, it does wonderfully when you have a single major input point or a set of them to dump into that fans out, which you can then transform and then read from. 
+
+I will not claim that the tables you can find in the weaver repository are especially **good** database design overall, but they work, and we'll see how they scale. This is one of three main input tables. One for record writes, one for identity events, and one for account events.
+```SQL
+CREATE TABLE IF NOT EXISTS raw_records (
+    did String,
+    collection LowCardinality(String),
+    rkey String,
+    cid String,
+    -- Repository revision (TID)
+    rev String,
+    record JSON,
+    -- Operation: 'create', 'update', 'delete', 'cache' (fetched on-demand)
+    operation LowCardinality(String),
+    -- Firehose sequence number
+    seq UInt64,
+    -- Event timestamp from firehose
+    event_time DateTime64(3),
+    -- When the database indexed this record
+    indexed_at DateTime64(3) DEFAULT now64(3),
+    -- Validation state: 'unchecked', 'valid', 'invalid_rev', 'invalid_gap', 'invalid_account'
+    validation_state LowCardinality(String) DEFAULT 'unchecked',
+    -- Whether this came from live firehose (true) or backfill (false)
+    is_live Bool DEFAULT true,
+    -- Materialized AT URI for convenience
+    uri String MATERIALIZED concat('at://', did, '/', collection, '/', rkey),
+    -- Projection for fast delete lookups by (did, cid)
+    PROJECTION by_did_cid (
+        SELECT * ORDER BY (did, cid)
+    )
+)
+ENGINE = MergeTree()
+ORDER BY (collection, did, rkey, event_time, indexed_at);
+```
+From here we fan out into a cascading series of materialized views and other specialised tables. These break out the different record types, calculate metadata, and pull critical fields out of the record JSON for easier querying. Clickhouse's wild-ass compression means we're not too badly off replicating data on disk this way. Seriously, their JSON type ends up being the same size as a CBOR BLOB on disk in my testing, though it *does* have some quirks, as I discovered when I read back Datetime fields and got...not the format I put in. Thankfully there's a config setting for that. ![Clickhouse animation showing parallel inserts into a source table and a transformation query into a materialized view](https://clickhouse.com/docs/assets/images/incremental_materialized_view-1158726e31b08dc9808d96671239467f.gif)We also build out the list of who contributed to a published entry and determine the canonical record for it, so that fetching a fully hydrated entry with all contributor profiles only takes a couple of `SELECT` queries that themselves avoid performing extensive table scans due to reasonable choices of `ORDER BY` fields in the denormalized tables they query. And then I can do quirky things like power a profile fetch endpoint that will provide either a Weaver or a Bluesky profile, while also unifying fields so that we can easily get at the critical stuff in common. This is a relatively expensive calculation, but people thankfully don't edit their profiles that often, and this is why we don't keep the stats in the same table. 
+
+However, this is ***also*** why Clickhouse will not be the only database used in the index.
+
+# Why is it always SQLite?
+When it comes to things like real-time collaboration sessions with almost keystroke-level cursor tracking and rapid per-user writeback/readback, where latency matters and we can't wait around for the merge cycle to produce the right state, *don't* work well in Clickhouse. But they sure do in SQLite!
+
+If there's one thing the AT Protocol developer community loves more than base32-encoded timestamps it's SQLite. In fairness, we're in good company, the whole world loves SQLite. It's a good fucking embedded database and very hard to beat for write or read performance so long as you're not trying to hit it massively concurrently. Of course, that concurrency limitation does end up mattering as you scale. And here we take a cue from the Typescript PDS implementation and discover the magic of buying, well, a lot more than two of them, and of using the filesystem like a hierarchical key-value store.
+
+<iframe width="560" height="315" src="https://www.youtube.com/embed/CZs-YcmxyUw?si=bd3GmSxMVQGdqHAR" title="YouTube video player" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
+
+This part of the data backend is still *very* much a work-in-progress and isn't used yet in the deployed version, but I did want to discuss the architecture. Unlike the PDS, we don't divide primarily by DID, instead we shard by resource, designated by collection and record key.
+
+```rust
+pub struct ShardKey {
+    pub collection: SmolStr,
+    pub rkey: SmolStr,
+}
+
+impl ShardKey {
+...
+    /// Directory path: {base}/{hash(collection,rkey)[0..2]}/{rkey}/
+    fn dir_path(&self, base: &Path) -> PathBuf {
+        base.join(self.hash_prefix()).join(self.rkey.as_str())
+    }
+...
+}
+/// A single SQLite shard for a resource
+pub struct SqliteShard {
+    conn: Mutex<Connection>,
+    path: PathBuf,
+    last_accessed: Mutex<Instant>,
+}
+/// Routes resources to their SQLite shards
+pub struct ShardRouter {
+    base_path: PathBuf,
+    shards: DashMap<ShardKey, std::sync::Arc<SqliteShard>>,
+}
+```
+
+The hash of the shard key plus the record key gives us the directory where we put the database file for this resource. Ultimately this may be moved out of the main index off onto something more comparable to the Tangled knot server or Streamplace nodes, depending on what constraints we run into if things go exceptionally well, but for now it lives as part of the index. In there we can tee off raw events from the incoming firehose and then transform them into the correct forms in memory, optionally persisted to disk, alongside Clickhouse and probably, for the specific things we want it for with a local scope, faster. 
+
+And direct communication, either by using something like oatproxy to swap the auth relationships around a bit (currently the index is accessed via service proxying through the PDS when authenticated) or via an iroh channel from the client, gets stuff there without having to wait for the relay to pick it up and fan it out to us, which then means that users can read their own writes very effectively. The handler hits the relevant SQLite shard if present and Clickhouse in parallel, merging the data to provide the most up-to-date form. For real-time collaboration this is critical. The current `iroh-gossip` implementation works well and requires only a generic iroh relay, but it runs into the problem every gossip protocol runs into the more concurrent users you have.
+
+The exact method of authentication of that side-channel is by far the largest remaining unanswered question about Weaver right now, aside from "Will anyone (else) use it?" 
+
+If people have ideas, I'm all ears.
+
+I hope you found this interesting. I enjoyed writing it out.