Configurable backfill

.env.example

···
 #   psql -U postgres -c "CREATE DATABASE atproto;"
 DATABASE_URL=postgresql://postgres:password@localhost:5432/atproto
 
+# Redis connection string for caching and metrics
+# Format: redis://host:port
+# Default: redis://localhost:6379
+REDIS_URL=redis://localhost:6379
+
 # ============================================
 # REQUIRED: Security
 # ============================================
···
 # Default: wss://bsky.network
 RELAY_URL=wss://bsky.network
 
-# Enable historical backfill (DANGEROUS - will consume massive resources)
-# WARNING: Do NOT enable this in production without proper resource planning
-# Backfill will attempt to download and process ALL historical data from the network
-# This can take days/weeks and requires significant disk space and memory
-# Default: false
-ENABLE_BACKFILL=false
+# Historical backfill configuration (in days)
+# 0 = disabled (no backfill)
+# >0 = backfill X days of historical data
+# WARNING: Resource-intensive! Each day of backfill can take hours and significant disk space
+# Recommended: Start with 1-7 days for testing
+# Default: 0 (disabled)
+BACKFILL_DAYS=0
+
+# Data retention configuration (in days)
+# 0 = keep all data forever
+# >0 = automatically prune posts/likes/reposts older than X days (runs daily)
+# Note: User profiles and follows are never pruned (preserves social graph)
+# Recommended: 30-90 days for production to manage disk usage
+# Default: 0 (keep forever)
+DATA_RETENTION_DAYS=0
 
 # ============================================
 # OPTIONAL: AppView Identity
···
 # Node environment
 # Options: development, production
 NODE_ENV=production
+
+# Database connection pool size
+# Higher values support more concurrent connections
+# Default: 32 (docker-compose uses 100 for production)
+DB_POOL_SIZE=32
+
+# Maximum concurrent event processing operations
+# Balances throughput vs memory usage
+# Default: 80
+MAX_CONCURRENT_OPS=80
+
+# ============================================
+# OPTIONAL: Dashboard Authentication
+# ============================================
+# Password for dashboard access (leave blank for no authentication)
+# Recommended: Set a strong password in production
+DASHBOARD_PASSWORD=

Dockerfile

···
 ENV REDIS_URL=redis://localhost:6379
 ENV PORT=5000
 ENV APPVIEW_DID=did:web:appview.local
-ENV ENABLE_BACKFILL=false
+ENV BACKFILL_DAYS=0
+ENV DATA_RETENTION_DAYS=0
 ENV DB_POOL_SIZE=32
 ENV MAX_CONCURRENT_OPS=80
 ENV NODE_OPTIONS="--max-old-space-size=2048"

README.md

···
 
 ### Prerequisites
 - PostgreSQL database
+- Redis (for caching and metrics)
 - Node.js 20+
 - (Optional) Domain for `did:web` identifier
 
···
 
 1. **Clone and Install**
 ```bash
-git clone <your-repo>
-cd at-protocol-appview
+git clone <your-repo> PublicAppView
+cd PublicAppView
 npm install
 ```
 
···
 
 ### Building the Docker Image
 
-1. **Create a Dockerfile**
-```dockerfile
-FROM node:20-alpine AS builder
+The Dockerfile is already included in the repository. It uses a multi-stage build with:
+- Node.js 20-slim base image
+- PM2 cluster mode for multi-worker deployment
+- Automatic database migrations on startup
+- Health checks and production optimizations
 
-WORKDIR /app
-
-# Copy package files
-COPY package*.json ./
-
-# Install all dependencies (including devDependencies for build)
-RUN npm ci
-
-# Copy source code
-COPY . .
-
-# Build the application
-RUN npm run build
-
-# Production stage
-FROM node:20-alpine
-
-WORKDIR /app
-
-# Copy package files
-COPY package*.json ./
-
-# Install only production dependencies
-RUN npm ci --production
-
-# Copy built application from builder
-COPY --from=builder /app/dist ./dist
-COPY --from=builder /app/client/dist ./client/dist
-
-# Expose port
-EXPOSE 5000
-
-# Start the application
-CMD ["npm", "start"]
-```
-
-2. **Build the Image**
+**Build the Image**
 ```bash
-docker build -t at-protocol-appview .
+# Build all services (use --no-cache for clean build)
+sudo docker-compose build --no-cache
+
+# Or build without cache flag
+sudo docker-compose build
 ```
 
 ### Running with Docker
 
-**Basic Run**
+**Basic Run (requires external PostgreSQL and Redis)**
 ```bash
 docker run -d \
-  --name appview \
+  --name at-protocol-appview \
   -p 5000:5000 \
   -e DATABASE_URL="postgresql://user:pass@host:5432/dbname" \
+  -e REDIS_URL="redis://host:6379" \
   -e SESSION_SECRET="your-secure-secret" \
   -e NODE_ENV="production" \
   at-protocol-appview
 ```
 
-**With Docker Compose**
-
-Create `docker-compose.yml`:
-```yaml
-version: '3.8'
+**With Docker Compose (Recommended)**
 
-services:
-  appview:
-    build: .
-    ports:
-      - "5000:5000"
-    environment:
-      DATABASE_URL: postgresql://user:pass@postgres:5432/appview
-      SESSION_SECRET: ${SESSION_SECRET}
-      NODE_ENV: production
-      APPVIEW_DID: did:web:your-domain.com
-    restart: unless-stopped
-    depends_on:
-      - postgres
-    healthcheck:
-      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:5000/health"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-
-  postgres:
-    image: postgres:16-alpine
-    environment:
-      POSTGRES_DB: appview
-      POSTGRES_USER: user
-      POSTGRES_PASSWORD: pass
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    restart: unless-stopped
-
-volumes:
-  postgres_data:
-```
+A complete `docker-compose.yml` is included in the repository with:
+- **Redis** service (in-memory caching and metrics)
+- **PostgreSQL** service (database with production tuning)
+- **App** service (AppView with all dependencies)
 
-Then run:
+Start all services:
 ```bash
 docker-compose up -d
 ```
 
-### Monitoring Docker Container
+The docker-compose setup includes:
+- PostgreSQL 14 with 5000 max connections and 20GB shared buffers
+- Redis 7 with 8GB memory and LRU eviction
+- Health checks for all services
+- Automatic dependency ordering
+- Volume persistence for database
 
-**View Container Status**
+### Monitoring Docker Services
+
+**View Service Status**
 ```bash
-# Check if container is running
-docker ps | grep appview
+# Check all services
+docker-compose ps
 
-# View detailed container info
-docker inspect appview
+# View detailed service info
+docker-compose logs
 
 # Check resource usage
-docker stats appview
+docker stats
 ```
 
 **View Logs**
 ```bash
-# Follow logs in real-time
-docker logs -f appview
+# Follow all logs in real-time
+docker-compose logs -f
+
+# Follow specific service logs
+docker-compose logs -f app
+docker-compose logs -f db
+docker-compose logs -f redis
 
 # View last 100 lines
-docker logs --tail 100 appview
+docker-compose logs --tail 100 app
 
-# View logs with timestamps
-docker logs -t appview
+# Or use container names directly
+docker logs -f publicappview-app-1
+docker logs --tail 100 publicappview-db-1
 ```
 
 **Health Checks**
 ```bash
-# Check health status
-docker inspect --format='{{.State.Health.Status}}' appview
+# Check service health
+docker-compose ps
 
 # Manual health check
 curl http://localhost:5000/health
 curl http://localhost:5000/ready
 
-# View health check logs
-docker inspect --format='{{json .State.Health}}' appview | jq
+# View health status
+docker inspect --format='{{.State.Health.Status}}' publicappview-app-1
 ```
 
 **Performance Monitoring**
 ```bash
-# Real-time stats (CPU, memory, network)
-docker stats appview --no-stream
+# Real-time stats (all services)
+docker stats
 
-# Export metrics
-docker stats appview --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.NetIO}}"
+# Specific containers
+docker stats publicappview-app-1 publicappview-db-1 publicappview-redis-1
 ```
 
-**Container Management**
+**Service Management**
 ```bash
-# Stop container
-docker stop appview
+# Stop all services
+docker-compose stop
 
-# Start container
-docker start appview
+# Start all services
+docker-compose start
+
+# Restart all services
+docker-compose restart
+
+# Restart specific service
+docker-compose restart app
 
-# Restart container
-docker restart appview
+# Remove all services
+docker-compose down
 
-# Remove container
-docker rm -f appview
+# Remove with volumes (WARNING: deletes data)
+docker-compose down -v
 ```
 
 **Entering the Container**
 ```bash
-# Open shell in running container
-docker exec -it appview sh
+# Open shell in app container
+docker-compose exec app sh
 
 # Run commands in container
-docker exec appview npm run db:push
+docker-compose exec app npm run db:push
+
+# Or use container name directly (note: Docker Compose uses directory name as prefix)
+docker exec -it publicappview-app-1 sh
 ```
 
+**Note:** Docker Compose creates container names using the pattern `{directory-name}-{service-name}-{number}`. Since this repo is cloned to `PublicAppView`, the actual container names are:
+- `publicappview-app-1` (main AppView service)
+- `publicappview-db-1` (PostgreSQL database)
+- `publicappview-redis-1` (Redis cache)
+
 ## Environment Variables
 
 ### Required
 - `DATABASE_URL`: PostgreSQL connection string
+- `REDIS_URL`: Redis connection string (default: `redis://localhost:6379`)
 - `SESSION_SECRET`: JWT secret (generate with `openssl rand -base64 32`)
 
 ### Optional
···
 - `APPVIEW_DID`: DID for this AppView instance (default: `did:web:appview.local`)
 - `PORT`: Server port (default: `5000`)
 - `NODE_ENV`: Environment mode (`development` or `production`)
-- `ENABLE_BACKFILL`: Enable historical backfill (**NOT recommended**, default: `false`)
+- `BACKFILL_DAYS`: Historical backfill in days (0=disabled, >0=backfill X days, default: `0`)
+- `DATA_RETENTION_DAYS`: Auto-prune old data (0=keep forever, >0=prune after X days, default: `0`)
+- `DASHBOARD_PASSWORD`: Optional password for dashboard authentication
+- `DB_POOL_SIZE`: Database connection pool size (default: `32`)
+- `MAX_CONCURRENT_OPS`: Max concurrent event processing (default: `80`)
 
 ## Production Deployment
 

docker-compose.yml

···
       - SESSION_SECRET=${SESSION_SECRET:-change-this-to-a-random-secret-in-production}
       - DASHBOARD_PASSWORD=${DASHBOARD_PASSWORD:-}
       - APPVIEW_DID=${APPVIEW_DID:-did:web:appview.local}
-      - ENABLE_BACKFILL=${ENABLE_BACKFILL:-false}
+      - BACKFILL_DAYS=${BACKFILL_DAYS:-0}
+      - DATA_RETENTION_DAYS=${DATA_RETENTION_DAYS:-0}
       - DB_POOL_SIZE=100
       - MAX_CONCURRENT_OPS=80
       - PORT=5000

server/services/backfill.ts

···
   private readonly BATCH_SIZE = 100; // Process in batches for memory efficiency
   private readonly PROGRESS_SAVE_INTERVAL = 1000; // Save progress every 1000 events
   private readonly MAX_EVENTS_PER_RUN = 100000; // Limit for safety
+  private readonly backfillDays: number;
+  private cutoffDate: Date | null = null;
   
   constructor(
     private relayUrl: string = process.env.RELAY_URL || "wss://bsky.network"
-  ) {}
+  ) {
+    // 0 or not set = backfill disabled, >0 = backfill X days of historical data
+    const backfillDaysRaw = parseInt(process.env.BACKFILL_DAYS || "0");
+    this.backfillDays = !isNaN(backfillDaysRaw) && backfillDaysRaw >= 0 ? backfillDaysRaw : 0;
+    
+    if (process.env.BACKFILL_DAYS && isNaN(backfillDaysRaw)) {
+      console.warn(`[BACKFILL] Invalid BACKFILL_DAYS value "${process.env.BACKFILL_DAYS}" - using default (0)`);
+    }
+  }
 
   async start(startCursor?: string): Promise<void> {
     if (this.isRunning) {
       throw new Error("Backfill is already running");
     }
 
-    if (process.env.ENABLE_BACKFILL !== "true") {
-      console.warn("[BACKFILL] Backfill is disabled. Set ENABLE_BACKFILL=true to enable.");
+    if (this.backfillDays === 0) {
+      console.log("[BACKFILL] Backfill is disabled (BACKFILL_DAYS=0)");
       return;
     }
 
-    console.log("[BACKFILL] Starting historical backfill...");
-    logCollector.info("Starting historical backfill", { startCursor });
+    // Calculate cutoff date - only backfill data from the last X days
+    this.cutoffDate = new Date();
+    this.cutoffDate.setDate(this.cutoffDate.getDate() - this.backfillDays);
+
+    console.log(`[BACKFILL] Starting historical backfill for last ${this.backfillDays} days (since ${this.cutoffDate.toISOString()})...`);
+    logCollector.info("Starting historical backfill", { 
+      startCursor, 
+      backfillDays: this.backfillDays,
+      cutoffDate: this.cutoffDate.toISOString()
+    });
 
     this.isRunning = true;
     this.progress = {
···
             this.progress.currentCursor = String((commit as any).seq);
           }
 
+          // Check if any record is older than cutoff date (stop backfilling if so)
+          if (this.cutoffDate) {
+            for (const op of commit.ops) {
+              if (op.action !== 'delete' && 'record' in op && op.record) {
+                const record = op.record as any;
+                if (record.createdAt) {
+                  const recordDate = new Date(record.createdAt);
+                  if (recordDate < this.cutoffDate) {
+                    console.log(`[BACKFILL] Reached cutoff date (${this.cutoffDate.toISOString()}). Stopping backfill.`);
+                    logCollector.info("Backfill reached cutoff date", { 
+                      recordDate: recordDate.toISOString(),
+                      cutoffDate: this.cutoffDate.toISOString(),
+                      eventsProcessed: this.progress.eventsProcessed
+                    });
+                    await this.stop();
+                    resolve();
+                    return;
+                  }
+                }
+              }
+            }
+          }
+
           const event = {
             repo: commit.repo,
             ops: commit.ops.map((op) => {
···
     
     if (this.client) {
       try {
-        this.client.removeAllListeners();
-        // @ts-ignore
-        if (typeof this.client.close === 'function') {
-          this.client.close();
+        if (typeof (this.client as any).removeAllListeners === 'function') {
+          (this.client as any).removeAllListeners();
+        }
+        if (typeof (this.client as any).close === 'function') {
+          (this.client as any).close();
         }
       } catch (error) {
         console.error("[BACKFILL] Error closing client:", error);

server/services/data-pruning.ts

···
 
   constructor() {
     // 0 = keep forever, >0 = prune after X days
-    this.retentionDays = parseInt(process.env.DATA_RETENTION_DAYS || "0");
+    const retentionDaysRaw = parseInt(process.env.DATA_RETENTION_DAYS || "0");
+    this.retentionDays = !isNaN(retentionDaysRaw) && retentionDaysRaw >= 0 ? retentionDaysRaw : 0;
+    
+    if (process.env.DATA_RETENTION_DAYS && isNaN(retentionDaysRaw)) {
+      console.warn(`[DATA_PRUNING] Invalid DATA_RETENTION_DAYS value "${process.env.DATA_RETENTION_DAYS}" - using default (0)`);
+    }
     
     if (this.retentionDays > 0) {
       console.log(`[DATA_PRUNING] Enabled - will prune content older than ${this.retentionDays} days`);