update fetcher

bundle/manager.go

···
 
 	// Update DID index if enabled (ONLY when bundle is created)
 	if m.didIndex != nil && m.didIndex.Exists() {
+		indexUpdateStart := time.Now()
+
 		if err := m.updateDIDIndexForBundle(ctx, bundle); err != nil {
 			m.logger.Printf("Warning: failed to update DID index: %v", err)
+		} else {
+			indexUpdateDuration := time.Since(indexUpdateStart)
+
+			if !quiet {
+				m.logger.Printf("  [DID Index] Updated in %s", indexUpdateDuration)
+			}
 		}
 	}
 
···
 		}
 	}
 
-	// ✨ Use mempool's last time if available
+	// Use mempool's last time if available
 	if m.mempool.Count() > 0 {
 		mempoolLastTime := m.mempool.GetLastTime()
 		if mempoolLastTime != "" {
···
 		m.logger.Printf("Preparing bundle %06d (mempool: %d ops)...", nextBundleNum, m.mempool.Count())
 	}
 
-	// ✨ NEW: Loop until we have enough OR catch up to latest
-	maxAttempts := 50 // Safety limit
+	// Track total fetches across all attempts
+	totalFetches := 0
+	maxAttempts := 50
 	attempt := 0
 	caughtUp := false
+	attemptStart := time.Now()
 
 	for m.mempool.Count() < types.BUNDLE_SIZE && attempt < maxAttempts {
 		attempt++
-
 		needed := types.BUNDLE_SIZE - m.mempool.Count()
 
 		if !quiet && attempt > 1 {
 			m.logger.Printf("  Attempt %d: Need %d more ops...", attempt, needed)
 		}
 
-		newOps, err := m.syncer.FetchToMempool(
+		newOps, fetchCount, err := m.syncer.FetchToMempool(
 			ctx,
 			afterTime,
 			prevBoundaryCIDs,
 			needed,
 			quiet,
 			m.mempool.Count(),
+			totalFetches, // Pass current total
 		)
 
+		// Update total fetch counter
+		totalFetches += fetchCount
+
+		// Check if we got an incomplete batch
+		gotIncompleteBatch := len(newOps) > 0 && len(newOps) < needed && err == nil
+
 		// Add operations if we got any
 		if len(newOps) > 0 {
 			added, addErr := m.mempool.Add(newOps)
···
 			}
 
 			// Update cursor for next fetch
-			if len(newOps) > 0 {
-				afterTime = newOps[len(newOps)-1].CreatedAt.Format(time.RFC3339Nano)
-			}
+			afterTime = newOps[len(newOps)-1].CreatedAt.Format(time.RFC3339Nano)
 		}
 
-		// Check if we caught up (got incomplete batch)
-		if err != nil || len(newOps) == 0 {
+		// Stop if caught up or error
+		if err != nil || len(newOps) == 0 || gotIncompleteBatch {
 			caughtUp = true
-			if !quiet {
+			if !quiet && totalFetches > 0 {
 				m.logger.Printf("  Caught up to latest PLC data")
 			}
 			break
 		}
 
-		// If we got a full batch but still need more, continue looping
-		if len(newOps) > 0 && m.mempool.Count() < types.BUNDLE_SIZE {
-			if !quiet {
-				m.logger.Printf("  Got full batch, fetching more...")
-			}
-			continue
-		}
-
 		// If we have enough, break
 		if m.mempool.Count() >= types.BUNDLE_SIZE {
 			break
···
 	// Save mempool state
 	m.mempool.Save()
 
+	totalDuration := time.Since(attemptStart)
+
 	// Check if we have enough for a bundle
 	if m.mempool.Count() < types.BUNDLE_SIZE {
 		if caughtUp {
···
 	// Create bundle structure
 	syncBundle := internalsync.CreateBundle(nextBundleNum, operations, afterTime, prevBundleHash, m.operations)
 
-	// Convert to bundle.Bundle
 	bundle := &Bundle{
 		BundleNumber: syncBundle.BundleNumber,
 		StartTime:    syncBundle.StartTime,
···
 	}
 
 	if !quiet {
-		m.logger.Printf("✓ Bundle %06d ready (%d ops, %d DIDs)",
-			bundle.BundleNumber, len(bundle.Operations), bundle.DIDCount)
+		avgPerFetch := float64(types.BUNDLE_SIZE) / float64(totalFetches)
+		m.logger.Printf("✓ Bundle %06d ready (%d ops, %d DIDs) - %d fetches in %s (avg %.0f ops/fetch)",
+			bundle.BundleNumber, len(bundle.Operations), bundle.DIDCount,
+			totalFetches, totalDuration.Round(time.Millisecond), avgPerFetch)
 	}
 
 	return bundle, nil

internal/didindex/builder.go

···
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
+	"sync"
+	"time"
 )
 
 // newShardBuilder creates a new shard builder
···
 	return int64(len(builder.entries)), nil
 }
 
-// UpdateIndexForBundle adds operations from a new bundle (incremental + ATOMIC)
+// UpdateIndexForBundle adds operations from a new bundle (incremental + ATOMIC + PARALLEL)
 func (dim *Manager) UpdateIndexForBundle(ctx context.Context, bundle *BundleData) error {
 	dim.indexMu.Lock()
 	defer dim.indexMu.Unlock()
 
-	// Group operations by shard
+	totalStart := time.Now()
+
+	// STEP 1: Group operations by shard
+	groupStart := time.Now()
 	shardOps := make(map[uint8]map[string][]OpLocation)
 
 	for pos, op := range bundle.Operations {
···
 		})
 	}
 
-	// PHASE 1: Write ALL shards to .tmp files FIRST
+	groupDuration := time.Since(groupStart)
+	dim.logger.Printf("  [DID Index] Grouped operations into %d shards in %s",
+		len(shardOps), groupDuration)
+
+	// STEP 2: Write ALL shards to .tmp files FIRST (PARALLEL)
+	writeStart := time.Now()
+
 	tmpShards := make(map[uint8]string)
+	var tmpShardsMu sync.Mutex
 	var deltaCount int64
+	var deltaCountMu sync.Mutex
 
+	// Error handling
+	errChan := make(chan error, len(shardOps))
+
+	// Worker pool
+	workers := runtime.NumCPU()
+	if workers > len(shardOps) {
+		workers = len(shardOps)
+	}
+	if workers < 1 {
+		workers = 1
+	}
+
+	semaphore := make(chan struct{}, workers)
+	var wg sync.WaitGroup
+
+	dim.logger.Printf("  [DID Index] Updating %d shards in parallel (%d workers)...",
+		len(shardOps), workers)
+
+	// Process each shard in parallel
 	for shardNum, newOps := range shardOps {
-		tmpPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx.tmp", shardNum))
+		wg.Add(1)
+		go func(sNum uint8, ops map[string][]OpLocation) {
+			defer wg.Done()
 
-		addedCount, err := dim.updateShardToTemp(shardNum, newOps, tmpPath)
-		if err != nil {
-			dim.cleanupTempShards(tmpShards)
-			return fmt.Errorf("failed to prepare shard %02x: %w", shardNum, err)
-		}
+			// Acquire semaphore (limit concurrency)
+			semaphore <- struct{}{}
+			defer func() { <-semaphore }()
 
-		tmpShards[shardNum] = tmpPath
-		deltaCount += addedCount
+			shardStart := time.Now()
+			tmpPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx.tmp", sNum))
+
+			addedCount, err := dim.updateShardToTemp(sNum, ops, tmpPath)
+			if err != nil {
+				errChan <- fmt.Errorf("shard %02x: %w", sNum, err)
+				return
+			}
+
+			shardDuration := time.Since(shardStart)
+
+			// Update shared state
+			tmpShardsMu.Lock()
+			tmpShards[sNum] = tmpPath
+			tmpShardsMu.Unlock()
+
+			deltaCountMu.Lock()
+			deltaCount += addedCount
+			deltaCountMu.Unlock()
+
+			// Debug log for each shard
+			if dim.verbose {
+				dim.logger.Printf("    Shard %02x: +%d DIDs in %s (%d ops)",
+					sNum, addedCount, shardDuration, len(ops))
+			}
+		}(shardNum, newOps)
 	}
 
-	// PHASE 2: Atomically commit ALL shards
+	// Wait for all workers
+	wg.Wait()
+	close(errChan)
+
+	writeDuration := time.Since(writeStart)
+	dim.logger.Printf("  [DID Index] Wrote %d temp files in %s (%.1f shards/sec)",
+		len(tmpShards), writeDuration, float64(len(tmpShards))/writeDuration.Seconds())
+
+	// Check for errors
+	if err := <-errChan; err != nil {
+		dim.cleanupTempShards(tmpShards)
+		return err
+	}
+
+	// STEP 3: Atomically commit ALL shards
+	commitStart := time.Now()
+
 	for shardNum, tmpPath := range tmpShards {
 		finalPath := filepath.Join(dim.shardDir, fmt.Sprintf("%02x.idx", shardNum))
 
···
 		dim.invalidateShard(shardNum)
 	}
 
-	// PHASE 3: Update config
+	commitDuration := time.Since(commitStart)
+
+	// STEP 4: Update config
+	configStart := time.Now()
+
 	dim.config.TotalDIDs += deltaCount
 	dim.config.LastBundle = bundle.BundleNumber
 
-	return dim.saveIndexConfig()
+	if err := dim.saveIndexConfig(); err != nil {
+		return fmt.Errorf("failed to save config: %w", err)
+	}
+
+	configDuration := time.Since(configStart)
+	totalDuration := time.Since(totalStart)
+
+	// Summary log
+	dim.logger.Printf("  [DID Index] ✓ Bundle %06d indexed: +%d DIDs, %d shards updated in %s",
+		bundle.BundleNumber, deltaCount, len(tmpShards), totalDuration)
+
+	if dim.verbose {
+		dim.logger.Printf("    Breakdown: group=%s write=%s commit=%s config=%s",
+			groupDuration, writeDuration, commitDuration, configDuration)
+		dim.logger.Printf("    Throughput: %.0f ops/sec",
+			float64(len(bundle.Operations))/totalDuration.Seconds())
+	}
+
+	return nil
 }
 
 // updateShardToTemp updates a shard and writes to temp file

internal/sync/fetcher.go

···
 	target int,
 	quiet bool,
 	currentMempoolCount int,
-) ([]plcclient.PLCOperation, error) {
+	totalFetchesSoFar int,
+) ([]plcclient.PLCOperation, int, error) {
 
 	seenCIDs := make(map[string]bool)
 
···
 	currentAfter := afterTime
 	maxFetches := 20
 	var allNewOps []plcclient.PLCOperation
+	fetchesMade := 0
 
 	for fetchNum := 0; fetchNum < maxFetches; fetchNum++ {
+		fetchesMade++
 		remaining := target - len(allNewOps)
 		if remaining <= 0 {
 			break
 		}
 
-		batchSize := 1000
-		if remaining < 500 {
+		// ✨ SMART BATCH SIZING
+		var batchSize int
+		switch {
+		case remaining <= 50:
+			batchSize = 50 // Fetch exactly what we need (with small buffer)
+		case remaining <= 100:
+			batchSize = 100
+		case remaining <= 500:
 			batchSize = 200
+		default:
+			batchSize = 1000
 		}
+
+		fetchStart := time.Now()
 
 		if !quiet {
-			f.logger.Printf("  Fetch #%d: requesting %d operations (after: %s)",
-				fetchNum+1, batchSize, currentAfter[:19])
+			f.logger.Printf("  Fetch #%d: requesting %d operations (need %d, after: %s)",
+				totalFetchesSoFar+fetchesMade, batchSize, remaining, currentAfter[:19])
 		}
 
 		batch, err := f.plcClient.Export(ctx, plcclient.ExportOptions{
···
 			After: currentAfter,
 		})
 
+		fetchDuration := time.Since(fetchStart)
+
 		if err != nil {
-			return allNewOps, fmt.Errorf("export failed: %w", err)
+			return allNewOps, fetchesMade, fmt.Errorf("export failed: %w", err)
 		}
 
 		if len(batch) == 0 {
 			if !quiet {
-				f.logger.Printf("  No more operations available from PLC")
+				f.logger.Printf("  No more operations available from PLC (in %s)", fetchDuration)
 			}
-			// Return what we have (might be incomplete)
-			return allNewOps, nil
+			return allNewOps, fetchesMade, nil
 		}
 
 		// Deduplicate
···
 			}
 		}
 
-		if !quiet && len(batch) > 0 {
-			deduped := len(batch) - (len(allNewOps) - beforeDedup)
+		uniqueAdded := len(allNewOps) - beforeDedup
+		deduped := len(batch) - uniqueAdded
+
+		// ✨ DETAILED METRICS
+		if !quiet {
+			opsPerSec := float64(len(batch)) / fetchDuration.Seconds()
+
 			if deduped > 0 {
-				f.logger.Printf("  Received %d ops (%d duplicates filtered)", len(batch), deduped)
+				f.logger.Printf("  Received %d ops (%d unique, %d dupes) in %s (%.0f ops/sec)",
+					len(batch), uniqueAdded, deduped, fetchDuration, opsPerSec)
+			} else {
+				f.logger.Printf("  Received %d ops in %s (%.0f ops/sec)",
+					len(batch), fetchDuration, opsPerSec)
 			}
 		}
 
···
 			currentAfter = batch[len(batch)-1].CreatedAt.Format(time.RFC3339Nano)
 		}
 
-		// ✨ KEY: Stop if we got incomplete batch (caught up!)
+		// Stop if we got incomplete batch (caught up!)
 		if len(batch) < batchSize {
 			if !quiet {
 				f.logger.Printf("  Received incomplete batch (%d/%d) → caught up to latest",
 					len(batch), batchSize)
 			}
-			return allNewOps, nil
+			return allNewOps, fetchesMade, nil
 		}
 
 		// If we have enough, stop
 		if len(allNewOps) >= target {
+			if !quiet {
+				f.logger.Printf("  ✓ Target reached (%d/%d unique ops collected)",
+					len(allNewOps), target)
+			}
 			break
 		}
 	}
 
-	return allNewOps, nil
+	return allNewOps, fetchesMade, nil
 }