better did batch resolve

bundle/manager.go

···
 	return m.operations.LoadOperationAtPosition(path, position)
 }
 
+// LoadOperations loads multiple operations from a bundle efficiently
+func (m *Manager) LoadOperations(ctx context.Context, bundleNumber int, positions []int) (map[int]*plcclient.PLCOperation, error) {
+	// Build file path
+	path := filepath.Join(m.config.BundleDir, fmt.Sprintf("%06d.jsonl.zst", bundleNumber))
+	if !m.operations.FileExists(path) {
+		return nil, fmt.Errorf("bundle file not found: %s", path)
+	}
+
+	return m.operations.LoadOperationsAtPositions(path, positions)
+}
+
 // filterBundleFiles filters out files starting with . or _
 func filterBundleFiles(files []string) []string {
 	filtered := make([]string, 0, len(files))

cmd/plcbundle/commands/common.go

···
 	GetDIDOperationsFromMempool(did string) ([]plcclient.PLCOperation, error)
 	GetLatestDIDOperation(ctx context.Context, did string) (*plcclient.PLCOperation, error)
 	LoadOperation(ctx context.Context, bundleNum, position int) (*plcclient.PLCOperation, error)
+	LoadOperations(ctx context.Context, bundleNumber int, positions []int) (map[int]*plcclient.PLCOperation, error)
 	CloneFromRemote(ctx context.Context, opts internalsync.CloneOptions) (*internalsync.CloneResult, error)
 	ResolveDID(ctx context.Context, did string) (*bundle.ResolveDIDResult, error)
 	RunSyncOnce(ctx context.Context, config *internalsync.SyncLoopConfig, verbose bool) (int, error)

cmd/plcbundle/commands/did.go

···
 	"github.com/goccy/go-json"
 	"github.com/spf13/cobra"
 	"tangled.org/atscan.net/plcbundle/cmd/plcbundle/ui"
+	"tangled.org/atscan.net/plcbundle/internal/didindex"
 	"tangled.org/atscan.net/plcbundle/internal/plcclient"
 )
 
···
 }
 
 func batchResolve(mgr BundleManager, dids []string, output *os.File, workers int) error {
-	progress := ui.NewProgressBar(len(dids))
 	ctx := context.Background()
+	overallStart := time.Now()
+
+	// =================================================================
+	// PHASE 1: Batch lookup locations
+	// =================================================================
+	fmt.Fprintf(os.Stderr, "Phase 1/2: Looking up %d DID locations...\n", len(dids))
+	phase1Start := time.Now()
 
-	type resolveResult struct {
-		did string
-		doc *plcclient.DIDDocument
-		err error
+	type didLocation struct {
+		did         string
+		bundle      int
+		position    int
+		fromMempool bool
+		mempoolOp   *plcclient.PLCOperation
+		err         error
 	}
 
-	// Now use it
-	jobs := make(chan string, len(dids))
-	results := make(chan resolveResult, len(dids))
+	locations := make([]didLocation, len(dids))
+
+	// Mempool check
+	for i, did := range dids {
+		if mempoolOp := findLatestInMempool(mgr, did); mempoolOp != nil {
+			locations[i] = didLocation{did: did, fromMempool: true, mempoolOp: mempoolOp}
+		}
+	}
+
+	// Batch index lookup
+	needsLookup := make([]string, 0)
+	lookupMap := make(map[string]int)
+	for i, did := range dids {
+		if !locations[i].fromMempool {
+			needsLookup = append(needsLookup, did)
+			lookupMap[did] = i
+		}
+	}
 
-	// Start worker pool
-	var wg sync.WaitGroup
-	for w := 0; w < workers; w++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for did := range jobs {
-				result, err := mgr.ResolveDID(ctx, did)
-				if err != nil {
-					results <- resolveResult{did: did, err: err}
-				} else {
-					results <- resolveResult{did: did, doc: result.Document}
+	if len(needsLookup) > 0 {
+		batchResults, _ := mgr.GetDIDIndex().BatchGetDIDLocations(needsLookup)
+		for did, locs := range batchResults {
+			var latest *didindex.OpLocation
+			for i := range locs {
+				if !locs[i].Nullified() && (latest == nil || locs[i].IsAfter(*latest)) {
+					latest = &locs[i]
 				}
 			}
-		}()
+
+			idx := lookupMap[did]
+			if latest != nil {
+				locations[idx] = didLocation{did: did, bundle: latest.BundleInt(), position: latest.PositionInt()}
+			} else {
+				locations[idx] = didLocation{did: did, err: fmt.Errorf("not found")}
+			}
+		}
+
+		for _, did := range needsLookup {
+			if idx := lookupMap[did]; locations[idx].bundle == 0 && locations[idx].err == nil {
+				locations[idx] = didLocation{did: did, err: fmt.Errorf("not found")}
+			}
+		}
 	}
 
-	// Send jobs
-	for _, did := range dids {
-		jobs <- did
+	phase1Duration := time.Since(phase1Start)
+	fmt.Fprintf(os.Stderr, "  ✓ %s\n\n", phase1Duration.Round(time.Millisecond))
+
+	// =================================================================
+	// PHASE 2: Group by bundle, load ops, resolve (MERGED, parallel)
+	// =================================================================
+	fmt.Fprintf(os.Stderr, "Phase 2/2: Loading and resolving (%d workers)...\n", workers)
+
+	// Group DIDs by bundle
+	type bundleGroup struct {
+		bundleNum int
+		dids      []int // indices into locations array
+	}
+
+	bundleMap := make(map[int][]int)
+	mempoolDIDs := make([]int, 0)
+	errorDIDs := make([]int, 0)
+
+	for i, loc := range locations {
+		if loc.err != nil {
+			errorDIDs = append(errorDIDs, i)
+		} else if loc.fromMempool {
+			mempoolDIDs = append(mempoolDIDs, i)
+		} else {
+			bundleMap[loc.bundle] = append(bundleMap[loc.bundle], i)
+		}
 	}
-	close(jobs)
 
-	// Collect results in background
-	go func() {
-		wg.Wait()
-		close(results)
-	}()
+	bundles := make([]bundleGroup, 0, len(bundleMap))
+	for bn, didIndices := range bundleMap {
+		bundles = append(bundles, bundleGroup{bundleNum: bn, dids: didIndices})
+	}
 
-	// Process results
+	fmt.Fprintf(os.Stderr, "  %d bundles, %d mempool, %d errors\n",
+		len(bundles), len(mempoolDIDs), len(errorDIDs))
+
+	// Setup output
 	writer := bufio.NewWriterSize(output, 512*1024)
 	defer writer.Flush()
 
-	resolved := 0
-	failed := 0
-	processed := 0
+	var (
+		resolved  int
+		failed    int
+		processed int
+		mu        sync.Mutex // Single lock for all counters
+	)
+
+	progress := ui.NewProgressBar(len(dids))
+
+	writeResult := func(doc *plcclient.DIDDocument, err error) {
+		mu.Lock()
+		defer mu.Unlock()
 
-	for res := range results {
 		processed++
+		progress.Set(processed)
 
-		if res.err != nil {
+		if err != nil {
 			failed++
-			if failed < 10 {
-				fmt.Fprintf(os.Stderr, "Failed to resolve %s: %v\n", res.did, res.err)
-			}
 		} else {
 			resolved++
-			data, _ := json.Marshal(res.doc)
+			data, _ := json.Marshal(doc)
 			writer.Write(data)
 			writer.WriteByte('\n')
-
-			if processed%100 == 0 {
+			if resolved%100 == 0 {
 				writer.Flush()
 			}
 		}
+	}
 
-		progress.Set(processed)
+	// Process mempool DIDs (already have ops)
+	for _, idx := range mempoolDIDs {
+		loc := locations[idx]
+		doc, err := plcclient.ResolveDIDDocument(loc.did, []plcclient.PLCOperation{*loc.mempoolOp})
+		writeResult(doc, err)
+	}
+
+	// Process errors
+	for range errorDIDs {
+		writeResult(nil, fmt.Errorf("not found"))
 	}
 
+	// ✨ Process bundles in parallel - LoadOperations once per bundle
+	bundleJobs := make(chan bundleGroup, len(bundles))
+	var wg sync.WaitGroup
+
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for job := range bundleJobs {
+				// Collect all positions needed from this bundle
+				positions := make([]int, len(job.dids))
+				for i, didIdx := range job.dids {
+					positions[i] = locations[didIdx].position
+				}
+
+				// ✨ Load operations once for this bundle
+				ops, err := mgr.LoadOperations(ctx, job.bundleNum, positions)
+
+				if err != nil {
+					// All DIDs from this bundle fail
+					for range job.dids {
+						writeResult(nil, err)
+					}
+					continue
+				}
+
+				// Resolve each DID using loaded operations
+				for i, didIdx := range job.dids {
+					loc := locations[didIdx]
+
+					if op, ok := ops[positions[i]]; ok {
+						doc, resolveErr := plcclient.ResolveDIDDocument(loc.did, []plcclient.PLCOperation{*op})
+						writeResult(doc, resolveErr)
+					} else {
+						writeResult(nil, fmt.Errorf("operation not loaded"))
+					}
+				}
+			}
+		}()
+	}
+
+	// Send bundle jobs
+	for _, bg := range bundles {
+		bundleJobs <- bg
+	}
+	close(bundleJobs)
+
+	wg.Wait()
 	writer.Flush()
 	progress.Finish()
+
+	totalDuration := time.Since(overallStart)
 
 	fmt.Fprintf(os.Stderr, "\n✓ Batch resolve complete\n")
-	fmt.Fprintf(os.Stderr, "  DIDs input: %d\n", len(dids))
-	fmt.Fprintf(os.Stderr, "  Resolved:   %d\n", resolved)
+	fmt.Fprintf(os.Stderr, "  Resolved:    %d/%d\n", resolved, len(dids))
 	if failed > 0 {
-		fmt.Fprintf(os.Stderr, "  Failed:     %d\n", failed)
+		fmt.Fprintf(os.Stderr, "  Failed:      %d\n", failed)
+	}
+	fmt.Fprintf(os.Stderr, "  Total:       %s (%.1f DIDs/sec)\n",
+		totalDuration.Round(time.Millisecond),
+		float64(resolved)/totalDuration.Seconds())
+
+	return nil
+}
+
+// Helper function to find latest non-nullified op in mempool
+func findLatestInMempool(mgr BundleManager, did string) *plcclient.PLCOperation {
+	ops, err := mgr.GetDIDOperationsFromMempool(did)
+	if err != nil || len(ops) == 0 {
+		return nil
+	}
+
+	// Search backwards from most recent
+	for i := len(ops) - 1; i >= 0; i-- {
+		if !ops[i].IsNullified() {
+			return &ops[i]
+		}
 	}
 
 	return nil

internal/didindex/lookup.go

···
 	"context"
 	"fmt"
 	"sort"
+	"sync"
 
 	"tangled.org/atscan.net/plcbundle/internal/plcclient"
 )
···
 	// Load ONLY the specific operation (efficient!)
 	return provider.LoadOperation(ctx, latestLoc.BundleInt(), latestLoc.PositionInt())
 }
+
+// BatchGetDIDLocations retrieves locations for multiple DIDs efficiently
+// Returns map[did][]OpLocation - only locations, no operation loading
+func (dim *Manager) BatchGetDIDLocations(dids []string) (map[string][]OpLocation, error) {
+	if !dim.Exists() {
+		return nil, fmt.Errorf("DID index not available")
+	}
+
+	// Group DIDs by shard to minimize shard loads
+	type shardQuery struct {
+		shardNum    uint8
+		identifiers []string
+		didMap      map[string]string // identifier -> original DID
+	}
+
+	shardQueries := make(map[uint8]*shardQuery)
+
+	for _, did := range dids {
+		identifier, err := extractDIDIdentifier(did)
+		if err != nil {
+			continue
+		}
+
+		shardNum := dim.calculateShard(identifier)
+
+		if shardQueries[shardNum] == nil {
+			shardQueries[shardNum] = &shardQuery{
+				shardNum:    shardNum,
+				identifiers: make([]string, 0),
+				didMap:      make(map[string]string),
+			}
+		}
+
+		sq := shardQueries[shardNum]
+		sq.identifiers = append(sq.identifiers, identifier)
+		sq.didMap[identifier] = did
+	}
+
+	if dim.verbose {
+		dim.logger.Printf("DEBUG: Batch lookup: %d DIDs across %d shards", len(dids), len(shardQueries))
+	}
+
+	// Process each shard (load once, search multiple times)
+	results := make(map[string][]OpLocation)
+	var mu sync.Mutex
+
+	var wg sync.WaitGroup
+	for _, sq := range shardQueries {
+		wg.Add(1)
+		go func(query *shardQuery) {
+			defer wg.Done()
+
+			// Load shard once
+			shard, err := dim.loadShard(query.shardNum)
+			if err != nil {
+				if dim.verbose {
+					dim.logger.Printf("DEBUG: Failed to load shard %02x: %v", query.shardNum, err)
+				}
+				return
+			}
+			defer dim.releaseShard(shard)
+
+			if shard.data == nil {
+				return
+			}
+
+			// Search for all identifiers in this shard
+			for _, identifier := range query.identifiers {
+				locations := dim.searchShard(shard, identifier)
+				if len(locations) > 0 {
+					originalDID := query.didMap[identifier]
+					mu.Lock()
+					results[originalDID] = locations
+					mu.Unlock()
+				}
+			}
+		}(sq)
+	}
+
+	wg.Wait()
+
+	return results, nil
+}

internal/storage/storage.go

···
 
 	return operations[startIdx:]
 }
+
+// LoadOperationsAtPositions loads multiple operations from a bundle in one pass
+func (op *Operations) LoadOperationsAtPositions(path string, positions []int) (map[int]*plcclient.PLCOperation, error) {
+	if len(positions) == 0 {
+		return make(map[int]*plcclient.PLCOperation), nil
+	}
+
+	// Create position set for fast lookup
+	posSet := make(map[int]bool)
+	maxPos := 0
+	for _, pos := range positions {
+		if pos < 0 {
+			continue
+		}
+		posSet[pos] = true
+		if pos > maxPos {
+			maxPos = pos
+		}
+	}
+
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close()
+
+	reader := gozstd.NewReader(file)
+	defer reader.Close()
+
+	bufPtr := scannerBufPool.Get().(*[]byte)
+	defer scannerBufPool.Put(bufPtr)
+
+	scanner := bufio.NewScanner(reader)
+	scanner.Buffer(*bufPtr, 512*1024)
+
+	results := make(map[int]*plcclient.PLCOperation)
+	lineNum := 0
+
+	for scanner.Scan() {
+		// Early exit if we found everything
+		if len(results) == len(posSet) {
+			break
+		}
+
+		// Only parse if this position is requested
+		if posSet[lineNum] {
+			line := scanner.Bytes()
+			var operation plcclient.PLCOperation
+			if err := json.UnmarshalNoEscape(line, &operation); err != nil {
+				return nil, fmt.Errorf("failed to parse operation at position %d: %w", lineNum, err)
+			}
+
+			operation.RawJSON = make([]byte, len(line))
+			copy(operation.RawJSON, line)
+			results[lineNum] = &operation
+		}
+
+		lineNum++
+
+		// Early exit if we passed the max position we need
+		if lineNum > maxPos {
+			break
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("scanner error: %w", err)
+	}
+
+	return results, nil
+}