update

internal/api/handlers.go

···
 	r.w.Header().Set("X-Bundle-Start-Time", bundle.StartTime.Format(time.RFC3339Nano))
 	r.w.Header().Set("X-Bundle-End-Time", bundle.EndTime.Format(time.RFC3339Nano))
 	r.w.Header().Set("X-Bundle-Operation-Count", fmt.Sprintf("%d", plc.BUNDLE_SIZE))
-	r.w.Header().Set("X-Bundle-DID-Count", fmt.Sprintf("%d", len(bundle.DIDs)))
+	r.w.Header().Set("X-Bundle-DID-Count", fmt.Sprintf("%d", bundle.DIDCount))
 }
 
 // ===== REQUEST HELPERS =====
···
 		"start_time":        bundle.StartTime,
 		"end_time":          bundle.EndTime,
 		"operation_count":   plc.BUNDLE_SIZE,
-		"did_count":         len(bundle.DIDs),
+		"did_count":         bundle.DIDCount, // Use DIDCount instead of len(DIDs)
 		"hash":              bundle.Hash,
 		"compressed_hash":   bundle.CompressedHash,
 		"compressed_size":   bundle.CompressedSize,
···
 		"progress_percent":          float64(mempoolCount) / float64(plc.BUNDLE_SIZE) * 100,
 		"operations_needed":         operationsNeeded,
 		"did_count":                 uniqueDIDCount,
-		"start_time":                firstOp.CreatedAt, // This is FIXED once first op exists
-		"current_end_time":          lastOp.CreatedAt,  // This will change as more ops arrive
+		"start_time":                firstOp.CreatedAt,
+		"current_end_time":          lastOp.CreatedAt,
 		"uncompressed_size":         uncompressedSize,
 		"estimated_compressed_size": estimatedCompressedSize,
 		"compression_ratio":         float64(uncompressedSize) / float64(estimatedCompressedSize),
···
 		result["current_rate_per_second"] = currentRate
 	}
 
-	// Get actual mempool operations if requested
+	// Get actual mempool operations if requested (for DIDs list)
 	if r.URL.Query().Get("include_dids") == "true" {
 		ops, err := s.db.GetMempoolOperations(ctx, plc.BUNDLE_SIZE)
 		if err == nil {
···
 		return
 	}
 
+	// Query DIDs from dids table instead
+	dids, err := s.db.GetDIDsForBundle(r.Context(), bundleNum)
+	if err != nil {
+		resp.error(fmt.Sprintf("failed to get DIDs: %v", err), http.StatusInternalServerError)
+		return
+	}
+
 	resp.json(map[string]interface{}{
 		"plc_bundle_number": bundle.BundleNumber,
-		"did_count":         len(bundle.DIDs),
-		"dids":              bundle.DIDs,
+		"did_count":         bundle.DIDCount,
+		"dids":              dids,
 	})
 }
 

internal/plc/bundle.go

···
 
 func NewBundleManager(dir string, enabled bool, db storage.Database, indexDIDs bool) (*BundleManager, error) {
 	if !enabled {
+		log.Verbose("BundleManager disabled (enabled=false)")
 		return &BundleManager{enabled: false}, nil
 	}
 
···
 		return nil, err
 	}
 
+	log.Verbose("BundleManager initialized: enabled=%v, indexDIDs=%v, dir=%s", enabled, indexDIDs, dir)
+
 	return &BundleManager{
 		dir:       dir,
 		enabled:   enabled,
 		encoder:   encoder,
 		decoder:   decoder,
 		db:        db,
-		indexDIDs: indexDIDs, // NEW
+		indexDIDs: indexDIDs,
 	}, nil
 }
 
···
 // ===== BUNDLE INDEXING =====
 
 func (bm *BundleManager) indexBundle(ctx context.Context, bundleNum int, bf *bundleFile, cursor string) error {
+	log.Verbose("indexBundle called for bundle %06d: indexDIDs=%v", bundleNum, bm.indexDIDs)
+
 	prevHash := ""
 	if bundleNum > 1 {
 		if prev, err := bm.db.GetBundleByNumber(ctx, bundleNum-1); err == nil {
···
 	}
 
 	dids := bm.extractUniqueDIDs(bf.operations)
+	log.Verbose("Extracted %d unique DIDs from bundle %06d", len(dids), bundleNum)
+
 	compressedFileSize := bm.getFileSize(bf.path)
 
 	// Calculate uncompressed size
 	uncompressedSize := int64(0)
 	for _, op := range bf.operations {
-		uncompressedSize += int64(len(op.RawJSON)) + 1 // +1 for newline
+		uncompressedSize += int64(len(op.RawJSON)) + 1
 	}
 
 	// Get time range from operations
···
 		BundleNumber:     bundleNum,
 		StartTime:        firstSeenAt,
 		EndTime:          lastSeenAt,
-		DIDs:             dids,
+		DIDCount:         len(dids),
 		Hash:             bf.uncompressedHash,
 		CompressedHash:   bf.compressedHash,
 		CompressedSize:   compressedFileSize,
···
 		CreatedAt:        time.Now().UTC(),
 	}
 
+	log.Verbose("About to create bundle %06d in database (DIDCount=%d)", bundleNum, bundle.DIDCount)
+
 	// Create bundle first
 	if err := bm.db.CreateBundle(ctx, bundle); err != nil {
+		log.Error("Failed to create bundle %06d in database: %v", bundleNum, err)
 		return err
 	}
 
-	// NEW: Only index DIDs if enabled
+	log.Verbose("Bundle %06d created successfully in database", bundleNum)
+
+	// Index DIDs if enabled
 	if bm.indexDIDs {
 		start := time.Now()
+		log.Verbose("Starting DID indexing for bundle %06d: %d unique DIDs", bundleNum, len(dids))
 
-		// Extract handle and PDS for each DID using centralized helper
+		// Extract handle and PDS for each DID
 		didInfoMap := ExtractDIDInfoMap(bf.operations)
+		log.Verbose("Extracted info for %d DIDs from operations", len(didInfoMap))
 
-		if err := bm.db.AddBundleDIDs(ctx, bundleNum, dids); err != nil {
-			log.Error("Failed to index DIDs for bundle %06d: %v", bundleNum, err)
-			// Don't return error - bundle is already created
-		} else {
-			// Update handle and PDS for each DID
-			for did, info := range didInfoMap {
-				// Validate handle length before saving
-				validHandle := ValidateHandle(info.Handle)
+		successCount := 0
+		errorCount := 0
+		invalidHandleCount := 0
 
-				if err := bm.db.UpsertDID(ctx, did, bundleNum, validHandle, info.PDS); err != nil {
-					log.Error("Failed to update DID %s metadata: %v", did, err)
-				}
+		// Upsert each DID with handle, pds, and bundle number
+		for did, info := range didInfoMap {
+			validHandle := ValidateHandle(info.Handle)
+			if info.Handle != "" && validHandle == "" {
+				//log.Verbose("Bundle %06d: Skipping invalid handle for DID %s (length: %d)", bundleNum, did, len(info.Handle))
+				invalidHandleCount++
 			}
 
-			elapsed := time.Since(start)
-			log.Verbose("✓ Indexed %d unique DIDs for bundle %06d in %v", len(dids), bundleNum, elapsed)
+			if err := bm.db.UpsertDID(ctx, did, bundleNum, validHandle, info.PDS); err != nil {
+				log.Error("Failed to index DID %s for bundle %06d: %v", did, bundleNum, err)
+				errorCount++
+			} else {
+				successCount++
+			}
 		}
+
+		elapsed := time.Since(start)
+		log.Info("✓ Indexed bundle %06d: %d DIDs succeeded, %d errors, %d invalid handles in %v",
+			bundleNum, successCount, errorCount, invalidHandleCount, elapsed)
 	} else {
 		log.Verbose("⊘ Skipped DID indexing for bundle %06d (disabled in config)", bundleNum)
 	}

internal/plc/scanner.go

···
 }
 
 func NewScanner(db storage.Database, cfg config.PLCConfig) *Scanner {
-	bundleManager, err := NewBundleManager(cfg.BundleDir, cfg.UseCache, db, cfg.IndexDIDs) // NEW: pass IndexDIDs
+	log.Verbose("NewScanner: IndexDIDs config = %v", cfg.IndexDIDs)
+
+	bundleManager, err := NewBundleManager(cfg.BundleDir, cfg.UseCache, db, cfg.IndexDIDs)
 	if err != nil {
 		log.Error("Warning: failed to initialize bundle manager: %v", err)
 		bundleManager = &BundleManager{enabled: false}

internal/storage/db.go

···
 	GetBundleByNumber(ctx context.Context, bundleNumber int) (*PLCBundle, error)
 	GetBundles(ctx context.Context, limit int) ([]*PLCBundle, error)
 	GetBundlesForDID(ctx context.Context, did string) ([]*PLCBundle, error)
+	GetDIDsForBundle(ctx context.Context, bundleNum int) ([]string, error)
 	GetBundleStats(ctx context.Context) (count, compressedSize, uncompressedSize, lastBundle int64, err error)
 	GetLastBundleNumber(ctx context.Context) (int, error)
 	GetBundleForTimestamp(ctx context.Context, afterTime time.Time) (int, error)

internal/storage/postgres.go

···
         records_processed BIGINT DEFAULT 0
     );
 
-    CREATE TABLE IF NOT EXISTS plc_bundles (
-        bundle_number INTEGER PRIMARY KEY,
-        start_time TIMESTAMP NOT NULL,
-        end_time TIMESTAMP NOT NULL,
-        dids JSONB NOT NULL,
-        hash TEXT NOT NULL,
-        compressed_hash TEXT NOT NULL,
-        compressed_size BIGINT NOT NULL,
-        uncompressed_size BIGINT NOT NULL,
-        cumulative_compressed_size BIGINT NOT NULL,
-        cumulative_uncompressed_size BIGINT NOT NULL,
-        cursor TEXT,
-        prev_bundle_hash TEXT,
-        compressed BOOLEAN DEFAULT true,
-        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-    );
+	CREATE TABLE IF NOT EXISTS plc_bundles (
+		bundle_number INTEGER PRIMARY KEY,
+		start_time TIMESTAMP NOT NULL,
+		end_time TIMESTAMP NOT NULL,
+		did_count INTEGER NOT NULL DEFAULT 0,
+		hash TEXT NOT NULL,
+		compressed_hash TEXT NOT NULL,
+		compressed_size BIGINT NOT NULL,
+		uncompressed_size BIGINT NOT NULL,
+		cumulative_compressed_size BIGINT NOT NULL,
+		cumulative_uncompressed_size BIGINT NOT NULL,
+		cursor TEXT,
+		prev_bundle_hash TEXT,
+		compressed BOOLEAN DEFAULT true,
+		created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+	);
 
-    CREATE INDEX IF NOT EXISTS idx_plc_bundles_time ON plc_bundles(start_time, end_time);
-    CREATE INDEX IF NOT EXISTS idx_plc_bundles_hash ON plc_bundles(hash);
-    CREATE INDEX IF NOT EXISTS idx_plc_bundles_prev ON plc_bundles(prev_bundle_hash);
-    CREATE INDEX IF NOT EXISTS idx_plc_bundles_number_desc ON plc_bundles(bundle_number DESC);
-    CREATE INDEX IF NOT EXISTS idx_plc_bundles_dids ON plc_bundles USING gin(dids);
+	CREATE INDEX IF NOT EXISTS idx_plc_bundles_time ON plc_bundles(start_time, end_time);
+	CREATE INDEX IF NOT EXISTS idx_plc_bundles_hash ON plc_bundles(hash);
+	CREATE INDEX IF NOT EXISTS idx_plc_bundles_prev ON plc_bundles(prev_bundle_hash);
+	CREATE INDEX IF NOT EXISTS idx_plc_bundles_number_desc ON plc_bundles(bundle_number DESC);
 
     CREATE TABLE IF NOT EXISTS plc_mempool (
         id BIGSERIAL PRIMARY KEY,
···
 // ===== BUNDLE OPERATIONS =====
 
 func (p *PostgresDB) CreateBundle(ctx context.Context, bundle *PLCBundle) error {
-	didsJSON, err := json.Marshal(bundle.DIDs)
-	if err != nil {
-		return err
-	}
-
 	// Calculate cumulative sizes from previous bundle
 	if bundle.BundleNumber > 1 {
 		prevBundle, err := p.GetBundleByNumber(ctx, bundle.BundleNumber-1)
···
 
 	query := `
         INSERT INTO plc_bundles (
-            bundle_number, start_time, end_time, dids, 
+            bundle_number, start_time, end_time, did_count, 
             hash, compressed_hash, compressed_size, uncompressed_size, 
             cumulative_compressed_size, cumulative_uncompressed_size,
             cursor, prev_bundle_hash, compressed
···
         ON CONFLICT(bundle_number) DO UPDATE SET
             start_time = EXCLUDED.start_time,
             end_time = EXCLUDED.end_time,
-            dids = EXCLUDED.dids,
+            did_count = EXCLUDED.did_count,
             hash = EXCLUDED.hash,
             compressed_hash = EXCLUDED.compressed_hash,
             compressed_size = EXCLUDED.compressed_size,
···
             prev_bundle_hash = EXCLUDED.prev_bundle_hash,
             compressed = EXCLUDED.compressed
     `
-	_, err = p.db.ExecContext(ctx, query,
+	_, err := p.db.ExecContext(ctx, query,
 		bundle.BundleNumber, bundle.StartTime, bundle.EndTime,
-		didsJSON, bundle.Hash, bundle.CompressedHash,
+		bundle.DIDCount, bundle.Hash, bundle.CompressedHash,
 		bundle.CompressedSize, bundle.UncompressedSize,
 		bundle.CumulativeCompressedSize, bundle.CumulativeUncompressedSize,
 		bundle.Cursor, bundle.PrevBundleHash, bundle.Compressed,
···
 
 func (p *PostgresDB) GetBundleByNumber(ctx context.Context, bundleNumber int) (*PLCBundle, error) {
 	query := `
-        SELECT bundle_number, start_time, end_time, dids, hash, compressed_hash, 
+        SELECT bundle_number, start_time, end_time, did_count, hash, compressed_hash, 
                compressed_size, uncompressed_size, cumulative_compressed_size, 
                cumulative_uncompressed_size, cursor, prev_bundle_hash, compressed, created_at
         FROM plc_bundles
···
     `
 
 	var bundle PLCBundle
-	var didsJSON []byte
 	var prevHash sql.NullString
 	var cursor sql.NullString
 
 	err := p.db.QueryRowContext(ctx, query, bundleNumber).Scan(
 		&bundle.BundleNumber, &bundle.StartTime, &bundle.EndTime,
-		&didsJSON, &bundle.Hash, &bundle.CompressedHash,
+		&bundle.DIDCount, &bundle.Hash, &bundle.CompressedHash,
 		&bundle.CompressedSize, &bundle.UncompressedSize,
 		&bundle.CumulativeCompressedSize, &bundle.CumulativeUncompressedSize,
 		&cursor, &prevHash, &bundle.Compressed, &bundle.CreatedAt,
···
 		bundle.Cursor = cursor.String
 	}
 
-	json.Unmarshal(didsJSON, &bundle.DIDs)
 	return &bundle, nil
 }
 
 func (p *PostgresDB) GetBundles(ctx context.Context, limit int) ([]*PLCBundle, error) {
 	query := `
-        SELECT bundle_number, start_time, end_time, dids, hash, compressed_hash, 
+        SELECT bundle_number, start_time, end_time, did_count, hash, compressed_hash, 
                compressed_size, uncompressed_size, cumulative_compressed_size, 
                cumulative_uncompressed_size, cursor, prev_bundle_hash, compressed, created_at
         FROM plc_bundles
···
 }
 
 func (p *PostgresDB) GetBundlesForDID(ctx context.Context, did string) ([]*PLCBundle, error) {
-	query := `
-        SELECT bundle_number, start_time, end_time, dids, hash, compressed_hash, 
+	// Get bundle numbers from dids table
+	var bundleNumbersJSON []byte
+	err := p.db.QueryRowContext(ctx, `
+		SELECT bundle_numbers FROM dids WHERE did = $1
+	`, did).Scan(&bundleNumbersJSON)
+
+	if err == sql.ErrNoRows {
+		return []*PLCBundle{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	var bundleNumbers []int
+	if err := json.Unmarshal(bundleNumbersJSON, &bundleNumbers); err != nil {
+		return nil, err
+	}
+
+	if len(bundleNumbers) == 0 {
+		return []*PLCBundle{}, nil
+	}
+
+	// Build query with IN clause
+	placeholders := make([]string, len(bundleNumbers))
+	args := make([]interface{}, len(bundleNumbers))
+	for i, num := range bundleNumbers {
+		placeholders[i] = fmt.Sprintf("$%d", i+1)
+		args[i] = num
+	}
+
+	query := fmt.Sprintf(`
+        SELECT bundle_number, start_time, end_time, did_count, hash, compressed_hash, 
                compressed_size, uncompressed_size, cumulative_compressed_size, 
                cumulative_uncompressed_size, cursor, prev_bundle_hash, compressed, created_at
         FROM plc_bundles
-        WHERE dids ? $1
+        WHERE bundle_number IN (%s)
         ORDER BY bundle_number ASC
-    `
+    `, strings.Join(placeholders, ","))
 
-	rows, err := p.db.QueryContext(ctx, query, did)
+	rows, err := p.db.QueryContext(ctx, query, args...)
 	if err != nil {
 		return nil, err
 	}
···
 	return p.scanBundles(rows)
 }
 
+func (p *PostgresDB) GetDIDsForBundle(ctx context.Context, bundleNum int) ([]string, error) {
+	query := `
+		SELECT did 
+		FROM dids 
+		WHERE bundle_numbers @> $1::jsonb
+		ORDER BY did
+	`
+
+	rows, err := p.db.QueryContext(ctx, query, fmt.Sprintf("[%d]", bundleNum))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var dids []string
+	for rows.Next() {
+		var did string
+		if err := rows.Scan(&did); err != nil {
+			return nil, err
+		}
+		dids = append(dids, did)
+	}
+
+	return dids, rows.Err()
+}
+
 func (p *PostgresDB) scanBundles(rows *sql.Rows) ([]*PLCBundle, error) {
 	var bundles []*PLCBundle
 
 	for rows.Next() {
 		var bundle PLCBundle
-		var didsJSON []byte
 		var prevHash sql.NullString
 		var cursor sql.NullString
 
···
 			&bundle.BundleNumber,
 			&bundle.StartTime,
 			&bundle.EndTime,
-			&didsJSON,
+			&bundle.DIDCount,
 			&bundle.Hash,
 			&bundle.CompressedHash,
 			&bundle.CompressedSize,
···
 			bundle.Cursor = cursor.String
 		}
 
-		json.Unmarshal(didsJSON, &bundle.DIDs)
 		bundles = append(bundles, &bundle)
 	}
 

internal/storage/types.go

···
 	StartTime                  time.Time
 	EndTime                    time.Time
 	BoundaryCIDs               []string
-	DIDs                       []string
+	DIDCount                   int // Changed from DIDs []string
 	Hash                       string
 	CompressedHash             string
 	CompressedSize             int64