better indexing on the server

src/client/components/feed-import-podcasts.tsx

···
+import {useSignal} from '@preact/signals'
+import {useCallback} from 'preact/hooks'
+
+import {useSkypod} from '#client/skypod/context'
+import {useRealmIdentity} from '#realm/client/context-identity'
+
+// Popular podcasts
+const PODCAST_FEEDS = [
+  {url: 'https://feeds.simplecast.com/54nAGcIl', title: 'The Daily (NYT)'},
+  {url: 'https://feeds.megaphone.fm/WWO3519750118', title: 'Stuff You Should Know'},
+  {
+    url: 'https://www.omnycontent.com/d/playlist/e73c998e-6e60-432f-8610-ae210140c5b1/A91018A4-EA4F-4130-BF55-AE270180FF7C/44710E1C-10BB-48D0-9524-AE270180FF93/podcast.rss',
+    title: 'This American Life',
+  },
+  {url: 'https://feeds.npr.org/510318/podcast.xml', title: 'Up First (NPR)'},
+  {url: 'https://feeds.megaphone.fm/darknetdiaries', title: 'Darknet Diaries'},
+  {url: 'https://changelog.com/podcast/feed', title: 'The Changelog'},
+  {url: 'https://feeds.simplecast.com/wgl4xEgL', title: 'Reply All'},
+  {url: 'https://feeds.megaphone.fm/ADL9840290619', title: 'The Ezra Klein Show'},
+]
+
+export const FeedImportPodcasts: preact.FunctionComponent = () => {
+  const {identity} = useRealmIdentity()
+  const store = useSkypod()
+
+  const importing$ = useSignal(false)
+  const imported$ = useSignal(0)
+  const errors$ = useSignal<string[]>([])
+
+  const importAll = useCallback(() => {
+    importing$.value = true
+    imported$.value = 0
+    errors$.value = []
+
+    const go = async () => {
+      for (const feed of PODCAST_FEEDS) {
+        try {
+          const action = store.action('feed:add', {
+            url: feed.url,
+            lock: {by: identity.identid},
+            private: false,
+          })
+
+          await store.dispatch(action)
+          imported$.value++
+        } catch (ex: unknown) {
+          const msg = ex instanceof Error ? ex.message : String(ex)
+          errors$.value = [...errors$.value, `${feed.title}: ${msg}`]
+          console.error(`Failed to import ${feed.title}:`, ex)
+        }
+      }
+
+      importing$.value = false
+    }
+
+    go().catch((exc: unknown) => {
+      console.error('error loading podcast feeds', exc)
+    })
+  }, [store, identity.identid, importing$, imported$, errors$])
+
+  return (
+    <div className="feed-import-podcasts">
+      <button type="button" onClick={importAll} disabled={importing$.value}>
+        {importing$.value
+          ? `Importing... (${imported$.value}/${PODCAST_FEEDS.length})`
+          : 'Import Podcast Feeds'}
+      </button>
+
+      {errors$.value.length > 0 && (
+        <details>
+          <summary>Errors ({errors$.value.length})</summary>
+          <ul>
+            {errors$.value.map((err, i) => (
+              <li key={i}>{err}</li>
+            ))}
+          </ul>
+        </details>
+      )}
+    </div>
+  )
+}

src/client/components/feed-import-tech.tsx

···
+import {useSignal} from '@preact/signals'
+import {useCallback} from 'preact/hooks'
+
+import {useSkypod} from '#client/skypod/context'
+import {useRealmIdentity} from '#realm/client/context-identity'
+
+// Popular tech/programming feeds
+const TECH_FEEDS = [
+  {url: 'https://hnrss.org/frontpage', title: 'Hacker News - Front Page'},
+  {url: 'https://hnrss.org/newest', title: 'Hacker News - Newest'},
+  {url: 'https://lobste.rs/rss', title: 'Lobsters'},
+  {url: 'https://www.theverge.com/rss/index.xml', title: 'The Verge'},
+  {url: 'https://techcrunch.com/feed/', title: 'TechCrunch'},
+  {url: 'https://arstechnica.com/feed/', title: 'Ars Technica'},
+  {url: 'https://www.wired.com/feed/rss', title: 'Wired'},
+  {url: 'https://stackoverflow.blog/feed/', title: 'Stack Overflow Blog'},
+  {url: 'https://github.blog/feed/', title: 'GitHub Blog'},
+  {url: 'https://blog.cloudflare.com/rss/', title: 'Cloudflare Blog'},
+]
+
+export const FeedImportTech: preact.FunctionComponent = () => {
+  const {identity} = useRealmIdentity()
+  const store = useSkypod()
+
+  const importing$ = useSignal(false)
+  const imported$ = useSignal(0)
+  const errors$ = useSignal<string[]>([])
+
+  const importAll = useCallback(() => {
+    importing$.value = true
+    imported$.value = 0
+    errors$.value = []
+
+    const go = async () => {
+      for (const feed of TECH_FEEDS) {
+        try {
+          const action = store.action('feed:add', {
+            url: feed.url,
+            lock: {by: identity.identid},
+            private: false,
+          })
+
+          await store.dispatch(action)
+          imported$.value++
+        } catch (ex: unknown) {
+          const msg = ex instanceof Error ? ex.message : String(ex)
+          errors$.value = [...errors$.value, `${feed.title}: ${msg}`]
+          console.error(`Failed to import ${feed.title}:`, ex)
+        }
+      }
+
+      importing$.value = false
+    }
+
+    go().catch((exc: unknown) => {
+      console.error('error loading tech feeds', exc)
+    })
+  }, [store, identity.identid, importing$, imported$, errors$])
+
+  return (
+    <div className="feed-import-tech">
+      <button type="button" onClick={importAll} disabled={importing$.value}>
+        {importing$.value
+          ? `Importing... (${imported$.value}/${TECH_FEEDS.length})`
+          : 'Import Tech Feeds'}
+      </button>
+
+      {errors$.value.length > 0 && (
+        <details>
+          <summary>Errors ({errors$.value.length})</summary>
+          <ul>
+            {errors$.value.map((err, i) => (
+              <li key={i}>{err}</li>
+            ))}
+          </ul>
+        </details>
+      )}
+    </div>
+  )
+}

src/client/page-app.tsx

···
 
 import {DebugNuke} from './components/debug-nuke'
 import {FeedImportNYTimes} from './components/feed-import-nytimes'
+import {FeedImportPodcasts} from './components/feed-import-podcasts'
+import {FeedImportTech} from './components/feed-import-tech'
 import {Messenger} from './components/messenger'
 import {PeerList} from './components/peer-list'
 
···
             <RealmConnectionManager />
             <PeerList />
             <FeedImportNYTimes />
+            <FeedImportTech />
+            <FeedImportPodcasts />
             <Messenger />
             <DebugNuke />
           </SkypodProvider>

src/realm/client/service-connection-sync.ts

···
 
   async buildSyncDelta(clocks: PeerClocks): Promise<StoredAction[]> {
     const states = await this.buildSyncState()
-    const initial = this.#db.actions.where('[actor+clock]').equals([Dexie.minKey, Dexie.minKey])
-    // this is the root for the reduction; dexie doesn't have an "additive identity" for `or` - table.toCollection is (all) not (none)
+    const [first, ...known] = Object.keys(states) as IdentID[]
+    if (!first) return []
+
+    const initial = this.#db.actions.where('[actor+clock]').between(
+      [first, clocks[first] ?? Dexie.minKey],
+      [first, states[first]],
+      clocks[first] == null, // include lower only if they don't have it
+      true, // always include the most recent
+    )
 
-    const known = Object.keys(states) as IdentID[]
     const actions = known.reduce((memo, actor) => {
       const clock = clocks[actor] ?? null
-      return memo.or('[actor+clock]').between(
-        [actor, clock ?? Dexie.minKey],
-        [actor, states[actor]],
-        clock == null, // include lower only if they don't have it
-        true, // always include the most recent
-      )
+      return memo
+        .or('[actor+clock]')
+        .between([actor, clock ?? Dexie.minKey], [actor, states[actor]], clock == null, true)
     }, initial)
 
     // already in clock order

src/realm/protocol/logical-clock.ts

···
   // hlc format: 'lc.seconds.counter.identid'
 
   static #hlc = Symbol('hlc')
-  static readonly pattern = `lc:\\d+:\\d+:${IdentBrand.pattern}`
+  static readonly pattern = `lc:\\d+:\\d{6}:${IdentBrand.pattern}`
   static readonly regexp = new RegExp(`^${this.pattern}`)
   static readonly schema = z.string().regex(this.regexp).brand(this.#hlc)
 
···
       this.#seconds = seconds
     }
 
-    const hlcstr = `lc:${this.#seconds.toFixed(0)}:${this.#counter}:${this.#identid}`
+    const counter = String(this.#counter).padStart(6, '0')
+    const hlcstr = `lc:${this.#seconds.toFixed(0)}:${counter}:${this.#identid}`
     return this.tick(LogicalClock.parse(hlcstr))
   }
 

src/realm/server/state-storage.ts

···
 
 type RealmMetadata = z.infer<typeof realmMetadataSchema>
 
+const CLOCK_KEY_PREFIX = 'clock:'
+const ACTION_KEY_PREFIX = 'action:'
+
 export class RealmStorage {
   private static async storagedir() {
     const dir = process.env.REALM_STORAGE_DIR || './data/realms'
···
       try {
         const {identid: actor} = LogicalClock.extract(action.clk)
         const storedAction: StoredAction = {actor, clock: action.clk, action}
-        await this.#db.put(action.clk, JSON.stringify(storedAction))
+
+        // store action with compound key [actor:clock]
+        const actionKey = `${ACTION_KEY_PREFIX}${actor}:${action.clk}`
+        await this.#db.put(actionKey, JSON.stringify(storedAction))
+
+        // update latest clock for this actor
+        const clockKey = `${CLOCK_KEY_PREFIX}${actor}`
+        const existingClock = await this.#getClock(actor)
+        if (!existingClock || LogicalClock.compare(action.clk, existingClock) > 0) {
+          await this.#db.put(clockKey, action.clk)
+        }
+
         console.debug(`Stored action: ${action.clk} from ${actor}`)
       } catch (err) {
         console.error('Failed to store action:', action, err)
···
     const states: Record<IdentID, LCTimestamp | null> = {}
 
     try {
-      // Iterate through all actions (leveldb stores by clock which is sortable)
-      for await (const value of this.#db.values({reverse: true})) {
-        const stored = storedActionSchema.parse(JSON.parse(value))
-        if (!states[stored.actor]) {
-          states[stored.actor] = stored.clock
-        }
+      // iterate _just_ over the "latest clock" keys
+      for await (const [key, value] of this.#db.iterator({
+        gte: CLOCK_KEY_PREFIX,
+        lte: `${CLOCK_KEY_PREFIX}\xFF`,
+      })) {
+        const actor = IdentBrand.parse(key.slice(CLOCK_KEY_PREFIX.length))
+        states[actor] = LogicalClock.parse(value)
       }
     } catch (err) {
       console.error('Error building sync state:', err)
···
 
   async buildSyncDelta(clocks: PeerClocks): Promise<StoredAction[]> {
     const results: StoredAction[] = []
+
     try {
-      for await (const value of this.#db.values()) {
-        const stored = storedActionSchema.parse(JSON.parse(value))
-        const knownClock = clocks[stored.actor]
+      const state = await this.buildSyncState()
+      for (const [key, ourClock] of Object.entries(state)) {
+        if (!ourClock) continue // we don't have anything for this clock?
+
+        const actor = key as IdentID
+        const theirClock = clocks[actor] ?? null
+
+        const gte = theirClock
+          ? `${ACTION_KEY_PREFIX}${actor}:${theirClock}\x01` // exclude theirClock (they have it)
+          : `${ACTION_KEY_PREFIX}${actor}:` // include all if they have nothing
+
+        const lte = `${ACTION_KEY_PREFIX}${actor}:${ourClock}`
 
-        // include if we don't know about this actor or our clock is newer
-        if (!knownClock || LogicalClock.compare(stored.clock, knownClock) > 0) {
+        for await (const value of this.#db.values({gte, lte})) {
+          const stored = storedActionSchema.parse(JSON.parse(value))
           results.push(stored)
         }
       }
···
       console.error('Error building sync delta:', err)
     }
 
-    // results are already sorted by clock (leveldb key order)
-    return results
+    return results.sort((a, b) => LogicalClock.compare(a.clock, b.clock))
+  }
+
+  async #getClock(actor: IdentID): Promise<LCTimestamp | null> {
+    try {
+      const clockKey = `${CLOCK_KEY_PREFIX}${actor}`
+      return (await this.#db.get(clockKey)) as LCTimestamp
+    } catch (err) {
+      // Key doesn't exist
+      return null
+    }
   }
 
   async close(): Promise<void> {