slices / packages / cli / src / commands / lexicon / pull.ts

import { parseArgs } from "@std/cli/parse-args";
import { join, dirname } from "@std/path";
import { ensureDir } from "@std/fs";
import type { AtProtoClient } from "../../generated_client.ts";
import { ConfigManager } from "../../auth/config.ts";
import { createAuthenticatedClient } from "../../utils/client.ts";
import { logger } from "../../utils/logger.ts";
import { SlicesConfigLoader, mergeConfig } from "../../utils/config.ts";

function showPullHelp() {
  console.log(`
slices lexicon pull - Pull lexicon files from your slice

USAGE:
    slices lexicon pull [OPTIONS]

OPTIONS:
    --path <PATH>          Directory to save lexicon files (default: ./lexicons or from slices.json)
    --slice <SLICE_URI>    Source slice URI (required, or from slices.json)
    --nsid <PATTERN>      Filter lexicons by NSID pattern (supports wildcards with *)
    --api-url <URL>       Slices API base URL (default: https://api.slices.network or from slices.json)
    -h, --help            Show this help message

EXAMPLES:
    slices lexicon pull --slice at://did:plc:example/slice
    slices lexicon pull --path ./my-lexicons --slice at://did:plc:example/slice
    slices lexicon pull --nsid "app.bsky.*" --slice at://did:plc:example/slice
    slices lexicon pull --nsid "app.bsky.actor.*" --slice at://did:plc:example/slice
    slices lexicon pull  # Uses config from slices.json

NOTE:
    When using wildcards (*), wrap the pattern in quotes to prevent shell expansion
`);
}

interface PullStats {
  fetched: number;
  written: number;
  failed: number;
  errors: Array<{ nsid: string; error: string }>;
}

function nsidToPath(nsid: string, basePath: string): string {
  const parts = nsid.split(".");
  const dirParts = parts.slice(0, -1);
  const fileName = parts[parts.length - 1] + ".json";

  return join(basePath, ...dirParts, fileName);
}

function matchesNsidPattern(nsid: string, pattern: string): boolean {
  const regexPattern = pattern.replace(/\./g, "\\.").replace(/\*/g, ".*");
  const regex = new RegExp(`^${regexPattern}$`);
  return regex.test(nsid);
}

async function pullLexicons(
  sliceUri: string,
  lexiconPath: string,
  client: AtProtoClient,
  nsidPattern?: string
): Promise<PullStats> {
  const stats: PullStats = {
    fetched: 0,
    written: 0,
    failed: 0,
    errors: [],
  };

  try {
    const response = await client.network.slices.lexicon.getRecords({
      where: { slice: { eq: sliceUri } },
      limit: 100,
    });

    stats.fetched = response.records.length;

    for (const record of response.records) {
      try {
        const nsid = record.value.nsid;

        if (nsidPattern && !matchesNsidPattern(nsid, nsidPattern)) {
          continue;
        }
        const definitions = JSON.parse(record.value.definitions);

        const lexiconDoc = {
          lexicon: 1,
          id: nsid,
          defs: definitions,
        };

        const filePath = nsidToPath(nsid, lexiconPath);

        await ensureDir(dirname(filePath));

        await Deno.writeTextFile(
          filePath,
          JSON.stringify(lexiconDoc, null, 2) + "\n"
        );

        logger.info(`Wrote: ${filePath}`);
        stats.written++;
      } catch (error) {
        const err = error as Error;
        stats.failed++;
        stats.errors.push({
          nsid: record.value.nsid,
          error: err.message,
        });
      }
    }
  } catch (error) {
    const err = error as Error;
    logger.error(`Failed to fetch lexicons: ${err.message}`);
    throw error;
  }

  return stats;
}

export async function pullCommand(
  commandArgs: unknown[],
  _globalArgs: Record<string, unknown>
): Promise<void> {
  const args = parseArgs(commandArgs as string[], {
    boolean: ["help"],
    string: ["path", "slice", "api-url", "nsid"],
    alias: {
      h: "help",
    },
  });

  if (args.help) {
    showPullHelp();
    return;
  }

  const configLoader = new SlicesConfigLoader();
  const slicesConfig = await configLoader.load();
  const mergedConfig = mergeConfig(slicesConfig, args);

  if (!mergedConfig.slice) {
    logger.error("--slice is required");
    if (!slicesConfig.slice) {
      logger.info(
        "Tip: Create a slices.json file with your slice URI to avoid passing --slice every time"
      );
    }
    console.log("\nRun 'slices lexicon pull --help' for usage information.");
    Deno.exit(1);
  }

  const lexiconPath = mergedConfig.lexiconPath!;
  const sliceUri = mergedConfig.slice!;
  const apiUrl = mergedConfig.apiUrl!;
  const nsidPattern = args.nsid as string | undefined;

  const config = new ConfigManager();
  await config.load();

  if (!config.isAuthenticated()) {
    logger.error("Not authenticated. Run 'slices login' first.");
    Deno.exit(1);
  }

  const client = await createAuthenticatedClient(sliceUri, apiUrl);

  const pullStats = await pullLexicons(
    sliceUri,
    lexiconPath,
    client,
    nsidPattern
  );

  if (pullStats.failed > 0) {
    logger.warn(`${pullStats.failed} lexicons failed to write`);
    for (const error of pullStats.errors) {
      logger.error(`${error.nsid}: ${error.error}`);
    }
  }

  if (pullStats.written > 0) {
    const filterMsg = nsidPattern ? ` matching '${nsidPattern}'` : "";
    logger.success(
      `Pulled ${pullStats.written} lexicons${filterMsg} to ${lexiconPath}`
    );
  } else {
    const filterMsg = nsidPattern ? ` matching '${nsidPattern}'` : "";
    logger.info(`No lexicons found${filterMsg}`);
  }
}