A Transparent and Verifiable Way to Sync the AT Protocol's PLC Directory

misc updates

Changed files
+194 -80
bundle
cmd
plcbundle
commands
server
+8 -8
bundle/manager.go
··· 587 587 } 588 588 589 589 if !quiet { 590 - msg := fmt.Sprintf("→ Bundle %06d | %s | time: %s (%d reqs)", 590 + msg := fmt.Sprintf("→ Bundle %06d | %s | fetch: %s (%d reqs)", 591 591 bundle.BundleNumber, 592 592 bundle.Hash[0:7], 593 593 stats.TotalDuration.Round(time.Millisecond), ··· 1401 1401 // ResolveDID resolves a DID to its current document with detailed timing metrics 1402 1402 func (m *Manager) ResolveDID(ctx context.Context, did string) (*ResolveDIDResult, error) { 1403 1403 if err := plcclient.ValidateDIDFormat(did); err != nil { 1404 - // Track error 1405 1404 atomic.AddInt64(&m.resolverStats.errors, 1) 1406 1405 return nil, err 1407 1406 } ··· 1409 1408 result := &ResolveDIDResult{} 1410 1409 totalStart := time.Now() 1411 1410 1412 - // STEP 1: Check mempool first (most recent data) - OPTIMIZED 1411 + // STEP 1: Check mempool first 1413 1412 mempoolStart := time.Now() 1414 - 1415 1413 var latestMempoolOp *plcclient.PLCOperation 1416 1414 if m.mempool != nil { 1417 1415 latestMempoolOp = m.mempool.FindLatestDIDOperation(did) ··· 1427 1425 } 1428 1426 1429 1427 result.Document = doc 1428 + result.LatestOperation = latestMempoolOp // NEW: Include the operation 1430 1429 result.Source = "mempool" 1431 1430 result.TotalTime = time.Since(totalStart) 1432 1431 1433 - // Record stats 1434 1432 m.recordResolverTiming(result, nil) 1435 - 1436 1433 return result, nil 1437 1434 } 1438 1435 ··· 1493 1490 } 1494 1491 1495 1492 result.Document = doc 1493 + result.LatestOperation = op // NEW: Include the operation 1496 1494 result.Source = "bundle" 1497 1495 result.TotalTime = time.Since(totalStart) 1498 1496 1499 - // Record stats 1500 1497 m.recordResolverTiming(result, nil) 1501 - 1502 1498 return result, nil 1503 1499 } 1504 1500 ··· 1842 1838 m.resolverStats.recentTimes = make([]resolverTiming, m.resolverStats.recentSize) 1843 1839 m.resolverStats.recentIdx = 0 1844 1840 } 1841 + 1842 + func (m *Manager) SetQuiet(quiet bool) { 1843 + m.config.Quiet = quiet 1844 + }
+10 -9
bundle/types.go
··· 197 197 198 198 // ResolveDIDResult contains DID resolution with timing metrics 199 199 type ResolveDIDResult struct { 200 - Document *plcclient.DIDDocument 201 - MempoolTime time.Duration 202 - IndexTime time.Duration 203 - LoadOpTime time.Duration 204 - TotalTime time.Duration 205 - ResolvedHandle string 206 - Source string // "mempool" or "bundle" 207 - BundleNumber int // if from bundle 208 - Position int // if from bundle 200 + Document *plcclient.DIDDocument 201 + LatestOperation *plcclient.PLCOperation 202 + MempoolTime time.Duration 203 + IndexTime time.Duration 204 + LoadOpTime time.Duration 205 + TotalTime time.Duration 206 + ResolvedHandle string 207 + Source string // "mempool" or "bundle" 208 + BundleNumber int // if from bundle 209 + Position int // if from bundle 209 210 } 210 211 211 212 type resolverTiming struct {
+1
cmd/plcbundle/commands/common.go
··· 50 50 ScanDirectoryParallel(workers int, progressCallback func(current, total int, bytesProcessed int64)) (*bundle.DirectoryScanResult, error) 51 51 LoadBundleForDIDIndex(ctx context.Context, bundleNumber int) (*didindex.BundleData, error) 52 52 ResolveHandleOrDID(ctx context.Context, input string) (string, time.Duration, error) 53 + SetQuiet(quiet bool) 53 54 } 54 55 55 56 // PLCOperationWithLocation wraps operation with location info
+1
cmd/plcbundle/commands/server.go
··· 131 131 if syncMode { 132 132 go runServerSyncLoop(ctx, mgr, syncInterval, maxBundles, verbose) 133 133 } 134 + mgr.SetQuiet(true) 134 135 135 136 // Create and start HTTP server 136 137 serverConfig := &server.Config{
+81 -63
server/handlers.go
··· 74 74 sb.WriteString("immutable, cryptographically-chained bundles of 10,000 operations.\n\n") 75 75 sb.WriteString("More info: https://tangled.org/@atscan.net/plcbundle\n\n") 76 76 77 + origin := s.manager.GetPLCOrigin() 78 + 77 79 if bundleCount > 0 { 78 80 sb.WriteString("Bundles\n") 79 81 sb.WriteString("━━━━━━━\n") 82 + sb.WriteString(fmt.Sprintf(" Origin: %s\n", origin)) 80 83 sb.WriteString(fmt.Sprintf(" Bundle count: %d\n", bundleCount)) 81 84 82 85 firstBundle := stats["first_bundle"].(int) ··· 111 114 mempoolStats := s.manager.GetMempoolStats() 112 115 count := mempoolStats["count"].(int) 113 116 targetBundle := mempoolStats["target_bundle"].(int) 114 - canCreate := mempoolStats["can_create_bundle"].(bool) 115 117 116 - sb.WriteString("\nMempool Stats\n") 117 - sb.WriteString("━━━━━━━━━━━━━\n") 118 + sb.WriteString("\nMempool\n") 119 + sb.WriteString("━━━━━━━\n") 118 120 sb.WriteString(fmt.Sprintf(" Target bundle: %d\n", targetBundle)) 119 121 sb.WriteString(fmt.Sprintf(" Operations: %d / %d\n", count, types.BUNDLE_SIZE)) 120 - sb.WriteString(fmt.Sprintf(" Can create bundle: %v\n", canCreate)) 121 122 122 123 if count > 0 { 123 124 progress := float64(count) / float64(types.BUNDLE_SIZE) * 100 ··· 142 143 } 143 144 } 144 145 145 - if didStats := s.manager.GetDIDIndexStats(); didStats["exists"].(bool) { 146 - sb.WriteString("\nDID Index\n") 147 - sb.WriteString("━━━━━━━━━\n") 146 + if s.config.EnableResolver { 147 + 148 + sb.WriteString("\nResolver\n") 149 + sb.WriteString("━━━━━━━━\n") 148 150 sb.WriteString(" Status: enabled\n") 149 151 150 - indexedDIDs := didStats["indexed_dids"].(int64) 151 - mempoolDIDs := didStats["mempool_dids"].(int64) 152 - totalDIDs := didStats["total_dids"].(int64) 152 + if didStats := s.manager.GetDIDIndexStats(); didStats["exists"].(bool) { 153 + indexedDIDs := didStats["indexed_dids"].(int64) 154 + mempoolDIDs := didStats["mempool_dids"].(int64) 155 + totalDIDs := didStats["total_dids"].(int64) 153 156 154 - if mempoolDIDs > 0 { 155 - sb.WriteString(fmt.Sprintf(" Total DIDs: %s (%s indexed + %s mempool)\n", 156 - formatNumber(int(totalDIDs)), 157 - formatNumber(int(indexedDIDs)), 158 - formatNumber(int(mempoolDIDs)))) 159 - } else { 160 - sb.WriteString(fmt.Sprintf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))) 157 + if mempoolDIDs > 0 { 158 + sb.WriteString(fmt.Sprintf(" Total DIDs: %s (%s indexed + %s mempool)\n", 159 + formatNumber(int(totalDIDs)), 160 + formatNumber(int(indexedDIDs)), 161 + formatNumber(int(mempoolDIDs)))) 162 + } else { 163 + sb.WriteString(fmt.Sprintf(" Total DIDs: %s\n", formatNumber(int(totalDIDs)))) 164 + } 161 165 } 162 - 163 - sb.WriteString(fmt.Sprintf(" Cached shards: %d / %d\n", 164 - didStats["cached_shards"], didStats["cache_limit"])) 165 166 sb.WriteString("\n") 166 167 } 167 168 168 169 sb.WriteString("Server Stats\n") 169 170 sb.WriteString("━━━━━━━━━━━━\n") 170 - sb.WriteString(fmt.Sprintf(" Version: %s\n", s.config.Version)) 171 - if origin := s.manager.GetPLCOrigin(); origin != "" { 172 - sb.WriteString(fmt.Sprintf(" Origin: %s\n", origin)) 173 - } 174 - sb.WriteString(fmt.Sprintf(" Sync mode: %v\n", s.config.SyncMode)) 175 - sb.WriteString(fmt.Sprintf(" WebSocket: %v\n", s.config.EnableWebSocket)) 176 - sb.WriteString(fmt.Sprintf(" Resolver: %v\n", s.config.EnableResolver)) 177 - sb.WriteString(fmt.Sprintf(" Uptime: %s\n", time.Since(s.startTime).Round(time.Second))) 171 + sb.WriteString(fmt.Sprintf(" Version: %s\n", s.config.Version)) 172 + sb.WriteString(fmt.Sprintf(" Sync mode: %v\n", s.config.SyncMode)) 173 + sb.WriteString(fmt.Sprintf(" WebSocket: %v\n", s.config.EnableWebSocket)) 174 + sb.WriteString(fmt.Sprintf(" Handle Resolver: %v\n", s.manager.GetHandleResolver().GetBaseURL())) 175 + sb.WriteString(fmt.Sprintf(" Uptime: %s\n", time.Since(s.startTime).Round(time.Second))) 178 176 179 177 sb.WriteString("\n\nAPI Endpoints\n") 180 178 sb.WriteString("━━━━━━━━━━━━━\n") ··· 193 191 sb.WriteString(" GET /:did DID Document (W3C format)\n") 194 192 sb.WriteString(" GET /:did/data PLC State (raw format)\n") 195 193 sb.WriteString(" GET /:did/log/audit Operation history\n") 196 - 197 - didStats := s.manager.GetDIDIndexStats() 198 - if didStats["exists"].(bool) { 199 - sb.WriteString(fmt.Sprintf("\n Index: %s DIDs indexed\n", 200 - formatNumber(int(didStats["total_dids"].(int64))))) 201 - } else { 202 - sb.WriteString("\n ⚠️ Index: not built (will use slow scan)\n") 203 - } 204 - sb.WriteString("\n") 205 194 } 206 195 207 196 if s.config.EnableWebSocket { ··· 684 673 path := strings.TrimPrefix(r.URL.Path, "/") 685 674 686 675 parts := strings.SplitN(path, "/", 2) 687 - did := parts[0] 676 + input := parts[0] // Could be DID or handle 688 677 689 - if !strings.HasPrefix(did, "did:plc:") { 690 - sendJSON(w, 404, map[string]string{"error": "not found"}) 691 - return 692 - } 678 + // Accept both DIDs and handles 679 + // DIDs: did:plc:*, did:web:* 680 + // Handles: tree.fail, ngerakines.me, etc. 693 681 694 682 if len(parts) == 1 { 695 - s.handleDIDDocument(did)(w, r) 683 + s.handleDIDDocument(input)(w, r) 696 684 } else if parts[1] == "data" { 697 - s.handleDIDData(did)(w, r) 685 + s.handleDIDData(input)(w, r) 698 686 } else if parts[1] == "log/audit" { 699 - s.handleDIDAuditLog(did)(w, r) 687 + s.handleDIDAuditLog(input)(w, r) 700 688 } else { 701 689 sendJSON(w, 404, map[string]string{"error": "not found"}) 702 690 } 703 691 } 704 692 705 - func (s *Server) handleDIDDocument(did string) http.HandlerFunc { 693 + func (s *Server) handleDIDDocument(input string) http.HandlerFunc { 706 694 return func(w http.ResponseWriter, r *http.Request) { 707 - // OPTIONS already handled by middleware, but extra safety check 708 695 if r.Method == "OPTIONS" { 709 696 return 710 697 } 711 698 712 - // Track only actual GET requests 699 + // Resolve handle to DID 700 + did, handleResolveTime, err := s.manager.ResolveHandleOrDID(r.Context(), input) 701 + if err != nil { 702 + if strings.Contains(err.Error(), "appears to be a handle") { 703 + sendJSON(w, 400, map[string]string{ 704 + "error": "Handle resolver not configured", 705 + "hint": "Start server with --handle-resolver flag", 706 + }) 707 + } else { 708 + sendJSON(w, 400, map[string]string{"error": err.Error()}) 709 + } 710 + return 711 + } 712 + 713 + resolvedHandle := "" 714 + if handleResolveTime > 0 { 715 + resolvedHandle = input 716 + } 717 + 718 + // Single call gets both document AND operation metadata 713 719 result, err := s.manager.ResolveDID(r.Context(), did) 714 720 if err != nil { 715 721 if strings.Contains(err.Error(), "deactivated") { ··· 722 728 return 723 729 } 724 730 725 - // Add timing headers in MILLISECONDS 726 - w.Header().Set("X-Resolution-Time-Ms", fmt.Sprintf("%.3f", float64(result.TotalTime.Microseconds())/1000.0)) 727 - w.Header().Set("X-Resolution-Source", result.Source) 728 - w.Header().Set("X-Mempool-Time-Ms", fmt.Sprintf("%.3f", float64(result.MempoolTime.Microseconds())/1000.0)) 731 + // Early ETag check - operation is already in result.LatestOperation 732 + if result.LatestOperation != nil { 733 + etag := fmt.Sprintf(`"%s"`, result.LatestOperation.CID) 729 734 730 - if result.Source == "bundle" { 731 - w.Header().Set("X-Bundle-Number", fmt.Sprintf("%d", result.BundleNumber)) 732 - w.Header().Set("X-Bundle-Position", fmt.Sprintf("%d", result.Position)) 733 - w.Header().Set("X-Index-Time-Ms", fmt.Sprintf("%.3f", float64(result.IndexTime.Microseconds())/1000.0)) 734 - w.Header().Set("X-Load-Time-Ms", fmt.Sprintf("%.3f", float64(result.LoadOpTime.Microseconds())/1000.0)) 735 + if match := r.Header.Get("If-None-Match"); match != "" { 736 + // Strip quotes if present 737 + matchClean := strings.Trim(match, `"`) 738 + if matchClean == result.LatestOperation.CID { 739 + // Set minimal headers for 304 response 740 + w.Header().Set("ETag", etag) 741 + w.Header().Set("Cache-Control", "public, max-age=300") 742 + w.WriteHeader(http.StatusNotModified) 743 + return 744 + } 745 + } 735 746 } 747 + 748 + // Set all headers (now with result.LatestOperation available) 749 + setDIDDocumentHeaders(w, r, did, resolvedHandle, result, handleResolveTime) 736 750 737 751 w.Header().Set("Content-Type", "application/did+ld+json") 738 752 sendJSON(w, 200, result.Document) 739 753 } 740 754 } 741 755 742 - func (s *Server) handleDIDData(did string) http.HandlerFunc { 756 + func (s *Server) handleDIDData(input string) http.HandlerFunc { 743 757 return func(w http.ResponseWriter, r *http.Request) { 744 - if err := plcclient.ValidateDIDFormat(did); err != nil { 745 - sendJSON(w, 400, map[string]string{"error": "Invalid DID format"}) 758 + // Resolve handle to DID 759 + did, _, err := s.manager.ResolveHandleOrDID(r.Context(), input) 760 + if err != nil { 761 + sendJSON(w, 400, map[string]string{"error": err.Error()}) 746 762 return 747 763 } 748 764 ··· 771 787 } 772 788 } 773 789 774 - func (s *Server) handleDIDAuditLog(did string) http.HandlerFunc { 790 + func (s *Server) handleDIDAuditLog(input string) http.HandlerFunc { 775 791 return func(w http.ResponseWriter, r *http.Request) { 776 - if err := plcclient.ValidateDIDFormat(did); err != nil { 777 - sendJSON(w, 400, map[string]string{"error": "Invalid DID format"}) 792 + // Resolve handle to DID 793 + did, _, err := s.manager.ResolveHandleOrDID(r.Context(), input) 794 + if err != nil { 795 + sendJSON(w, 400, map[string]string{"error": err.Error()}) 778 796 return 779 797 } 780 798
+93
server/helpers.go
··· 3 3 import ( 4 4 "fmt" 5 5 "net/http" 6 + "time" 7 + 8 + "tangled.org/atscan.net/plcbundle/bundle" 6 9 ) 7 10 8 11 // getScheme determines the HTTP scheme ··· 56 59 } 57 60 return string(result) 58 61 } 62 + 63 + func setDIDDocumentHeaders( 64 + w http.ResponseWriter, 65 + _ *http.Request, 66 + did string, 67 + resolvedHandle string, 68 + result *bundle.ResolveDIDResult, 69 + handleResolveTime time.Duration, 70 + ) { 71 + // === Identity === 72 + w.Header().Set("X-DID", did) 73 + 74 + if resolvedHandle != "" { 75 + w.Header().Set("X-Handle-Resolved", resolvedHandle) 76 + w.Header().Set("X-Handle-Resolution-Time-Ms", 77 + fmt.Sprintf("%.3f", float64(handleResolveTime.Microseconds())/1000.0)) 78 + w.Header().Set("X-Request-Type", "handle") 79 + } else { 80 + w.Header().Set("X-Request-Type", "did") 81 + } 82 + 83 + // === Resolution Source & Location === 84 + w.Header().Set("X-Resolution-Source", result.Source) 85 + 86 + if result.Source == "bundle" { 87 + w.Header().Set("X-Bundle-Number", fmt.Sprintf("%d", result.BundleNumber)) 88 + w.Header().Set("X-Bundle-Position", fmt.Sprintf("%d", result.Position)) 89 + globalPos := (result.BundleNumber * 10000) + result.Position 90 + w.Header().Set("X-Global-Position", fmt.Sprintf("%d", globalPos)) 91 + w.Header().Set("X-Pointer", fmt.Sprintf("%d:%d", result.BundleNumber, result.Position)) 92 + } else { 93 + w.Header().Set("X-Mempool", "true") 94 + } 95 + 96 + // === Operation Metadata (from result.LatestOperation) === 97 + if result.LatestOperation != nil { 98 + op := result.LatestOperation 99 + 100 + w.Header().Set("X-Operation-CID", op.CID) 101 + w.Header().Set("X-Operation-Created", op.CreatedAt.Format(time.RFC3339)) 102 + 103 + opAge := time.Since(op.CreatedAt) 104 + w.Header().Set("X-Operation-Age-Seconds", fmt.Sprintf("%d", int(opAge.Seconds()))) 105 + 106 + if len(op.RawJSON) > 0 { 107 + w.Header().Set("X-Operation-Size", fmt.Sprintf("%d", len(op.RawJSON))) 108 + } 109 + 110 + // Nullification status 111 + if op.IsNullified() { 112 + w.Header().Set("X-Operation-Nullified", "true") 113 + if nullCID := op.GetNullifyingCID(); nullCID != "" { 114 + w.Header().Set("X-Operation-Nullified-By", nullCID) 115 + } 116 + } else { 117 + w.Header().Set("X-Operation-Nullified", "false") 118 + } 119 + 120 + // Standard HTTP headers 121 + w.Header().Set("Last-Modified", op.CreatedAt.UTC().Format(http.TimeFormat)) 122 + w.Header().Set("ETag", fmt.Sprintf(`"%s"`, op.CID)) 123 + } 124 + 125 + // === Performance Metrics === 126 + totalTime := handleResolveTime + result.TotalTime 127 + w.Header().Set("X-Resolution-Time-Ms", 128 + fmt.Sprintf("%.3f", float64(totalTime.Microseconds())/1000.0)) 129 + w.Header().Set("X-Mempool-Time-Ms", 130 + fmt.Sprintf("%.3f", float64(result.MempoolTime.Microseconds())/1000.0)) 131 + 132 + if result.Source == "bundle" { 133 + w.Header().Set("X-Index-Time-Ms", 134 + fmt.Sprintf("%.3f", float64(result.IndexTime.Microseconds())/1000.0)) 135 + w.Header().Set("X-Load-Time-Ms", 136 + fmt.Sprintf("%.3f", float64(result.LoadOpTime.Microseconds())/1000.0)) 137 + } 138 + 139 + // === Caching Strategy === 140 + if result.Source == "bundle" { 141 + // Bundled data: cache 5min, stale-while-revalidate 10min 142 + w.Header().Set("Cache-Control", 143 + "public, max-age=300, stale-while-revalidate=600, stale-if-error=3600") 144 + } else { 145 + // Mempool data: cache 1min, stale-while-revalidate 5min 146 + w.Header().Set("Cache-Control", 147 + "public, max-age=60, stale-while-revalidate=300, stale-if-error=600") 148 + } 149 + 150 + w.Header().Set("Vary", "Accept, If-None-Match") 151 + }