update

.gitignore

···
 *.db-*
 atscanner
 .DS_Store
-plc_cache\.tmp/*
+plc_cache\.tmp/*
+plc_bundles*

config.yaml

···
 plc:
   directory_url: "https://plc.directory"
   scan_interval: "6h"
-  cache_dir: "./plc_cache"    # Cache directory
-  use_cache: true             # Enable caching
+  bundle_dir: "./plc_bundles"
+  use_cache: true
 
 pds:
   scan_interval: "15m"

internal/api/handlers.go

···
 	lastBundle := bundles[len(bundles)-1]
 
 	// Compute file path
-	filePath := filepath.Join(s.plcCacheDir, fmt.Sprintf("%06d.jsonl.zst", lastBundle.BundleNumber))
+	filePath := filepath.Join(s.plcBundleDir, fmt.Sprintf("%06d.jsonl.zst", lastBundle.BundleNumber))
 
 	operations, err := s.loadBundleOperations(filePath)
 	if err != nil {
···
 
 	for _, bundle := range bundles {
 		// Compute file path
-		filePath := filepath.Join(s.plcCacheDir, fmt.Sprintf("%06d.jsonl.zst", bundle.BundleNumber))
+		filePath := filepath.Join(s.plcBundleDir, fmt.Sprintf("%06d.jsonl.zst", bundle.BundleNumber))
 
 		operations, err := s.loadBundleOperations(filePath)
 		if err != nil {
···
 	lastBundle, _ := s.db.GetLastBundleNumber(ctx)
 
 	for bundleNum := startBundle; bundleNum <= lastBundle && len(allOps) < count; bundleNum++ {
-		bundlePath := filepath.Join(s.plcCacheDir, fmt.Sprintf("%06d.jsonl.zst", bundleNum))
+		bundlePath := filepath.Join(s.plcBundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNum))
 
 		ops, err := s.loadBundleOperations(bundlePath)
 		if err != nil {

internal/api/server.go

···
 )
 
 type Server struct {
-	router      *mux.Router
-	server      *http.Server
-	db          storage.Database
-	plcClient   *plc.Client
-	plcCacheDir string // NEW: Store cache dir
+	router       *mux.Router
+	server       *http.Server
+	db           storage.Database
+	plcClient    *plc.Client
+	plcBundleDir string // NEW: Store cache dir
 }
 
 func NewServer(db storage.Database, apiCfg config.APIConfig, plcCfg config.PLCConfig) *Server {
 	s := &Server{
-		router:      mux.NewRouter(),
-		db:          db,
-		plcClient:   plc.NewClient(plcCfg.DirectoryURL),
-		plcCacheDir: plcCfg.CacheDir, // NEW
+		router:       mux.NewRouter(),
+		db:           db,
+		plcClient:    plc.NewClient(plcCfg.DirectoryURL),
+		plcBundleDir: plcCfg.BundleDir, // NEW
 	}
 
 	s.setupRoutes()

internal/config/config.go

···
 	DirectoryURL string        `yaml:"directory_url"`
 	ScanInterval time.Duration `yaml:"scan_interval"`
 	BatchSize    int           `yaml:"batch_size"`
-	CacheDir     string        `yaml:"cache_dir"` // NEW: Cache directory
-	UseCache     bool          `yaml:"use_cache"` // NEW: Enable/disable cache
+	BundleDir    string        `yaml:"bundles_dir"` // NEW: Cache directory
+	UseCache     bool          `yaml:"use_cache"`   // NEW: Enable/disable cache
 }
 
 type PDSConfig struct {
···
 	if cfg.PLC.BatchSize == 0 {
 		cfg.PLC.BatchSize = 1000
 	}
-	if cfg.PLC.CacheDir == "" {
-		cfg.PLC.CacheDir = "./plc_cache"
+	if cfg.PLC.BundleDir == "" {
+		cfg.PLC.BundleDir = "./plc_bundles"
 	}
 	if cfg.PDS.ScanInterval == 0 {
 		cfg.PDS.ScanInterval = 15 * time.Minute

internal/plc/scanner.go

···
 }
 
 func NewScanner(db storage.Database, cfg config.PLCConfig) *Scanner {
-	bundleManager, err := NewBundleManager(cfg.CacheDir, cfg.UseCache, db)
+	bundleManager, err := NewBundleManager(cfg.BundleDir, cfg.UseCache, db)
 	if err != nil {
 		log.Error("Warning: failed to initialize bundle manager: %v", err)
 		bundleManager = &BundleManager{enabled: false}

internal/storage/types.go

···
 }
 
 // GetFilePath returns the computed file path for this bundle
-func (b *PLCBundle) GetFilePath(cacheDir string) string {
-	return filepath.Join(cacheDir, fmt.Sprintf("%06d.jsonl.zst", b.BundleNumber))
+func (b *PLCBundle) GetFilePath(bundleDir string) string {
+	return filepath.Join(bundleDir, fmt.Sprintf("%06d.jsonl.zst", b.BundleNumber))
 }
 
 // OperationCount() returns 1000 (all bundles have exactly 1000 operations)

utils/verify-export.sh

···
 #!/bin/bash
-# verify-export.sh - Verify local PLC export endpoint against plc.directory
+# verify-export.sh - Verify local PLC export endpoint against multiple remotes
 # Usage: ./verify-export.sh [after_timestamp] [count]
 
 AFTER="${1:-}"
-COUNT="${2:-50}"
+COUNT="${2:-1000}"
 LOCAL_URL="http://localhost:8080/api/v1/plc/export"
-REMOTE_URL="https://plc.directory/export"
+
+# Remote URLs (one per line: "name|url")
+REMOTES=(
+    "plc.directory|https://plc.directory/export"
+    "plc.wtf|https://plc.wtf/export"
+)
 
 echo "=== PLC Export Verification ==="
 echo "Count: $COUNT"
···
     PARAMS="${PARAMS}&after=${AFTER}"
 fi
 
+# Fetch from local
 echo "Fetching from local API..."
 echo "curl -s \"${LOCAL_URL}?${PARAMS}\""
 LOCAL_DATA=$(curl -s "${LOCAL_URL}?${PARAMS}")
···
 echo "  Hash: $LOCAL_HASH"
 echo ""
 
-echo "Fetching from plc.directory..."
-REMOTE_DATA=$(curl -s "${REMOTE_URL}?${PARAMS}")
-REMOTE_COUNT=$(echo "$REMOTE_DATA" | wc -l | tr -d ' ')
-REMOTE_HASH=$(echo "$REMOTE_DATA" | shasum -a 256 | cut -d' ' -f1)
+# Arrays to store remote data
+REMOTE_NAMES=()
+REMOTE_URLS=()
+REMOTE_DATA_ARR=()
+REMOTE_COUNT_ARR=()
+REMOTE_HASH_ARR=()
 
-echo "  Operations: $REMOTE_COUNT"
-echo "  Hash: $REMOTE_HASH"
-echo ""
+# Fetch from all remotes
+for i in "${!REMOTES[@]}"; do
+    IFS='|' read -r name url <<< "${REMOTES[$i]}"
+    
+    echo "Fetching from ${name}..."
+    echo "curl -s \"${url}?${PARAMS}\""
+    
+    data=$(curl -s "${url}?${PARAMS}")
+    count=$(echo "$data" | wc -l | tr -d ' ')
+    hash=$(echo "$data" | shasum -a 256 | cut -d' ' -f1)
+    
+    REMOTE_NAMES+=("$name")
+    REMOTE_URLS+=("$url")
+    REMOTE_DATA_ARR+=("$data")
+    REMOTE_COUNT_ARR+=("$count")
+    REMOTE_HASH_ARR+=("$hash")
+    
+    echo "  Operations: $count"
+    echo "  Hash: $hash"
+    echo ""
+done
 
 # Compare
 echo "=== COMPARISON ==="
-if [ "$LOCAL_HASH" = "$REMOTE_HASH" ]; then
-    echo "✅ MATCH! Hashes are identical"
+
+# Check local vs each remote
+ALL_MATCH=true
+MATCHES=()
+for i in "${!REMOTE_NAMES[@]}"; do
+    name="${REMOTE_NAMES[$i]}"
+    hash="${REMOTE_HASH_ARR[$i]}"
+    
+    if [ "$LOCAL_HASH" = "$hash" ]; then
+        echo "Local vs ${name}: ✅ MATCH"
+        MATCHES+=("true")
+    else
+        echo "Local vs ${name}: ❌ MISMATCH"
+        MATCHES+=("false")
+        ALL_MATCH=false
+    fi
+done
+
+echo ""
+
+# Check remotes against each other
+REMOTES_MATCH=true
+if [ ${#REMOTE_NAMES[@]} -gt 1 ]; then
+    for ((i=0; i<${#REMOTE_NAMES[@]}-1; i++)); do
+        for ((j=i+1; j<${#REMOTE_NAMES[@]}; j++)); do
+            name1="${REMOTE_NAMES[$i]}"
+            name2="${REMOTE_NAMES[$j]}"
+            hash1="${REMOTE_HASH_ARR[$i]}"
+            hash2="${REMOTE_HASH_ARR[$j]}"
+            
+            if [ "$hash1" = "$hash2" ]; then
+                echo "${name1} vs ${name2}: ✅ MATCH"
+            else
+                echo "${name1} vs ${name2}: ❌ MISMATCH"
+                REMOTES_MATCH=false
+            fi
+        done
+    done
     echo ""
-    echo "Local and remote exports are in sync! 🎯"
+fi
+
+if [ "$ALL_MATCH" = "true" ]; then
+    echo "🎉 ALL MATCH! All endpoints are in perfect sync! 🎯"
     exit 0
 else
-    echo "❌ MISMATCH! Hashes differ"
+    echo "❌ DISCREPANCIES DETECTED"
     echo ""
     
-    # Show counts
-    if [ "$LOCAL_COUNT" != "$REMOTE_COUNT" ]; then
-        echo "⚠️  Operation count differs:"
-        echo "   Local:  $LOCAL_COUNT operations"
-        echo "   Remote: $REMOTE_COUNT operations"
-        echo "   Diff:   $((REMOTE_COUNT - LOCAL_COUNT))"
-        echo ""
-    fi
+    # Show counts comparison
+    echo "=== OPERATION COUNTS ==="
+    echo "Local: $LOCAL_COUNT operations"
+    for i in "${!REMOTE_NAMES[@]}"; do
+        name="${REMOTE_NAMES[$i]}"
+        count="${REMOTE_COUNT_ARR[$i]}"
+        diff=$((count - LOCAL_COUNT))
+        echo "${name}: ${count} operations (diff: ${diff})"
+    done
+    echo ""
     
-    # Sample first and last operations
-    echo "First operation (local):"
+    # Sample operations from each source
+    echo "=== FIRST OPERATIONS ==="
+    echo "Local:"
     echo "$LOCAL_DATA" | head -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
     echo ""
     
-    echo "First operation (remote):"
-    echo "$REMOTE_DATA" | head -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
-    echo ""
+    for i in "${!REMOTE_NAMES[@]}"; do
+        name="${REMOTE_NAMES[$i]}"
+        data="${REMOTE_DATA_ARR[$i]}"
+        echo "${name}:"
+        echo "$data" | head -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
+        echo ""
+    done
     
-    echo "Last operation (local):"
+    echo "=== LAST OPERATIONS ==="
+    echo "Local:"
     echo "$LOCAL_DATA" | tail -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
     echo ""
     
-    echo "Last operation (remote):"
-    echo "$REMOTE_DATA" | tail -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
-    echo ""
+    for i in "${!REMOTE_NAMES[@]}"; do
+        name="${REMOTE_NAMES[$i]}"
+        data="${REMOTE_DATA_ARR[$i]}"
+        echo "${name}:"
+        echo "$data" | tail -1 | jq -r '[.did, .cid, .createdAt] | @tsv' 2>/dev/null || echo "(parse error)"
+        echo ""
+    done
     
-    # Find first difference
-    echo "Finding first difference..."
-    diff <(echo "$LOCAL_DATA" | jq -r '.cid' 2>/dev/null | head -20) \
-         <(echo "$REMOTE_DATA" | jq -r '.cid' 2>/dev/null | head -20) || true
+    # Find first differences for mismatches
+    for i in "${!REMOTE_NAMES[@]}"; do
+        if [ "${MATCHES[$i]}" = "false" ]; then
+            name="${REMOTE_NAMES[$i]}"
+            data="${REMOTE_DATA_ARR[$i]}"
+            echo "=== FIRST DIFFERENCES (Local vs ${name}) ==="
+            diff <(echo "$LOCAL_DATA" | jq -r '.cid' 2>/dev/null | head -20) \
+                 <(echo "$data" | jq -r '.cid' 2>/dev/null | head -20) || true
+            echo ""
+        fi
+    done
+    
+    # Check differences between remotes
+    if [ "$REMOTES_MATCH" = "false" ] && [ ${#REMOTE_NAMES[@]} -gt 1 ]; then
+        for ((i=0; i<${#REMOTE_NAMES[@]}-1; i++)); do
+            for ((j=i+1; j<${#REMOTE_NAMES[@]}; j++)); do
+                name1="${REMOTE_NAMES[$i]}"
+                name2="${REMOTE_NAMES[$j]}"
+                hash1="${REMOTE_HASH_ARR[$i]}"
+                hash2="${REMOTE_HASH_ARR[$j]}"
+                
+                if [ "$hash1" != "$hash2" ]; then
+                    data1="${REMOTE_DATA_ARR[$i]}"
+                    data2="${REMOTE_DATA_ARR[$j]}"
+                    echo "=== FIRST DIFFERENCES (${name1} vs ${name2}) ==="
+                    diff <(echo "$data1" | jq -r '.cid' 2>/dev/null | head -20) \
+                         <(echo "$data2" | jq -r '.cid' 2>/dev/null | head -20) || true
+                    echo ""
+                fi
+            done
+        done
+    fi
     
     exit 1
 fi