server: index old appview records

this doesnt include ingesting their blobs since that'll happen at request time

server/src/backfill/index.ts

···
 import * as schema from "../db/schema.ts";
 import { db as db_type } from "../utils.ts";
 import { Client, simpleFetchHandler } from "@atcute/client";
+import oldRecords from "./old-records.ts";
 
 const db: db_type = drizzle<typeof schema>(Deno.env.get("DB_FILE_NAME")!);
 
···
   res = p_res;
 });
 
+// clear old records
+// accounts could have deleted their sites or anything
+// so we should just nuke them
+await db.delete(routes);
+
 const backfillClient = new Client({
   handler: simpleFetchHandler({
     service:
···
   }),
 });
 
-let repos: `did:${string}:${string}`[] = [];
-let cursor: string | undefined;
-while (true) {
-  const { data, ok } = await backfillClient.get(
-    "com.atproto.sync.listReposByCollection",
-    {
-      params: {
-        collection: "dev.atcities.route",
-        cursor,
-      },
-    }
-  );
-
-  if (!ok) {
-    throw data.error + "\n" + data.message;
-  }
-
-  repos = [...repos, ...data.repos.map((x) => x.did)];
-  cursor = data.cursor;
-  if (!cursor) break;
-}
+oldRecords(backfillClient, db);
 
 res(db);

server/src/backfill/old-records.ts

···
+import { Client, simpleFetchHandler } from "@atcute/client";
+import { db, getPds, isDid, rkeyToUrl } from "../utils.ts";
+import { is } from "@atcute/lexicons";
+import { DevAtcitiesRoute } from "../lexicons/index.ts";
+import { routes } from "../db/schema.ts";
+
+async function index(did: `did:${"web" | "plc"}:${string}`, db: db) {
+  const pds = await getPds(did);
+  if (!pds) return console.error(did, "could not be resolved to a pds.");
+  const pdsClient = new Client({
+    handler: simpleFetchHandler({ service: pds }),
+  });
+  let cursor: string | undefined;
+  while (true) {
+    const { data, ok } = await pdsClient.get("com.atproto.repo.listRecords", {
+      params: {
+        repo: did,
+        collection: "dev.atcities.route",
+        cursor,
+      },
+    });
+
+    if (!ok) {
+      console.error("Got error:", data.error, data.message);
+      break;
+    }
+
+    for (const record of data.records)
+      if (is(DevAtcitiesRoute.mainSchema, record.value)) {
+        if (record.value.page.$type === "dev.atcities.route#blob") {
+          const url = rkeyToUrl(record.uri.split("/")[4]);
+          if (!url) continue;
+          const data: typeof routes.$inferInsert = {
+            did: did,
+            url_route: url,
+            blob_cid:
+              "ref" in record.value.page.blob
+                ? record.value.page.blob.ref.$link
+                : record.value.page.blob.cid,
+            mime: record.value.page.blob.mimeType,
+          };
+          db.insert(routes).values(data);
+        }
+      } else console.warn("Invalid record:", record.uri);
+
+    cursor = data.cursor;
+    if (!cursor) break;
+  }
+}
+
+export default async function (backfillClient: Client, db: db) {
+  let repos: `did:${string}:${string}`[] = [];
+  let cursor: string | undefined;
+  while (true) {
+    const { data, ok } = await backfillClient.get(
+      "com.atproto.sync.listReposByCollection",
+      {
+        params: {
+          collection: "dev.atcities.route",
+          cursor,
+        },
+      }
+    );
+
+    if (!ok) {
+      throw data.error + "\n" + data.message;
+    }
+
+    repos = [...repos, ...data.repos.map((x) => x.did)];
+    cursor = data.cursor;
+    if (!cursor) break;
+  }
+
+  for (const i in repos) {
+    const did = repos[i];
+    console.log(`indexing ${Number(i) + 1}/${repos.length} ${did}`);
+    if (isDid(did)) index(did, db);
+  }
+}

server/src/routes/user.ts

···
   req: Request,
   user:
     | { handle: `${string}.${string}` }
-    | { did: `did:plc:${string}` | `did:web:${string}` },
+    | { did: `did:plc:${string}` | `did:web:${string}` }
 ): Promise<Response> {
   // if handle: resolve did
   let did: `did:${"plc" | "web"}:${string}`;
···
       return new Response(
         `${ascii}
         
-This handle
+This handle could not be resolved.
 `,
         {
           status: 500,
           statusText: "Internal Server Error",
-        },
+        }
       );
     }
   } else did = user.did;
 
   // look up in db
-  const db_res = (
-    await db
-      .select()
-      .from(routes)
-      .where(
-        and(
-          eq(routes.did, did),
-          eq(routes.url_route, new URL(req.url).pathname),
-        ),
-      )
-  ).at(0) ??
+  const db_res =
+    (
+      await db
+        .select()
+        .from(routes)
+        .where(
+          and(
+            eq(routes.did, did),
+            eq(routes.url_route, new URL(req.url).pathname)
+          )
+        )
+    ).at(0) ??
     (
       await db
         .select()
···
   }
   try {
     const file = await Deno.readFile(
-      `./blobs/${db_res.did}/${db_res.blob_cid}`,
+      `./blobs/${db_res.did}/${db_res.blob_cid}`
     );
     return new Response(file, {
       headers: {

server/src/utils.ts

···
 import { LibSQLDatabase } from "drizzle-orm/libsql";
 import { Client } from "@libsql/client";
 import * as schema from "./db/schema.ts";
+import {
+  CompositeDidDocumentResolver,
+  PlcDidDocumentResolver,
+  WebDidDocumentResolver,
+} from "@atcute/identity-resolver";
 
 export type db = LibSQLDatabase<typeof schema> & {
   $client: Client;
···
     head.append("Set-Cookie", `${key}=; Max-Age=-1`);
   }
   return head;
+}
+
+const docResolver = new CompositeDidDocumentResolver({
+  methods: {
+    plc: new PlcDidDocumentResolver(),
+    web: new WebDidDocumentResolver(),
+  },
+});
+
+export async function getPds(
+  did: `did:${"plc" | "web"}:${string}`
+): Promise<string | undefined> {
+  try {
+    const doc = await docResolver.resolve(did);
+    const pds = doc.service?.filter(
+      (x) =>
+        x.id.endsWith("#atproto_pds") && x.type === "AtprotoPersonalDataServer"
+    )[0].serviceEndpoint;
+    return typeof pds === "string" ? pds : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function isDid(did: unknown): did is `did:${"plc" | "web"}:${string}` {
+  return (
+    typeof did === "string" &&
+    (did.startsWith("did:web:") || did.startsWith("did:plc:"))
+  );
 }
 
 /**