update

internal/api/handlers.go

···
 		"hash":              bundle.Hash,
 		"compressed_hash":   bundle.CompressedHash,
 		"compressed_size":   bundle.CompressedSize,
+		"uncompressed_size": bundle.UncompressedSize,
+		"compression_ratio": float64(bundle.UncompressedSize) / float64(bundle.CompressedSize),
+		"cursor":            bundle.Cursor,
 		"prev_bundle_hash":  bundle.PrevBundleHash,
 		"created_at":        bundle.CreatedAt,
 	}
···
 		return
 	}
 
+	uniqueDIDCount, err := s.db.GetMempoolUniqueDIDCount(ctx)
+	if err != nil {
+		resp.error(err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	uncompressedSize, err := s.db.GetMempoolUncompressedSize(ctx)
+	if err != nil {
+		resp.error(err.Error(), http.StatusInternalServerError)
+		return
+	}
+
 	result := map[string]interface{}{
-		"operation_count":   count,
-		"can_create_bundle": count >= plc.BUNDLE_SIZE,
+		"operation_count":      count,
+		"unique_did_count":     uniqueDIDCount,
+		"uncompressed_size":    uncompressedSize,
+		"uncompressed_size_mb": float64(uncompressedSize) / 1024 / 1024,
+		"can_create_bundle":    count >= plc.BUNDLE_SIZE,
 	}
 
 	if count > 0 {

internal/plc/bundle.go

···
 	if _, err := bm.db.GetBundleByNumber(ctx, bundleNum); err != nil {
 		bf.compressedHash = bm.hashFile(bf.path)
 		bf.uncompressedHash = bm.hash(bm.serializeJSONL(bf.operations))
-		bm.indexBundle(ctx, bundleNum, bf)
+		cursor := "" // Unknown for existing files
+		bm.indexBundle(ctx, bundleNum, bf, cursor)
 	}
 
 	return bf.operations, true, nil
···
 
 	ops, isComplete := fetcher.fetchUntilComplete(ctx, BUNDLE_SIZE)
 
-	log.Info("  Collected %d unique operations after %d fetches (complete=%v)",
-		len(ops), fetcher.fetchCount, isComplete)
-
 	if isComplete {
 		bf.operations = ops
 		if err := bm.save(bf); err != nil {
 			log.Error("Warning: failed to save bundle: %v", err)
 		} else {
-			bm.indexBundle(ctx, bundleNum, bf)
+			cursor := afterTime // Store the cursor used
+			bm.indexBundle(ctx, bundleNum, bf, cursor)
 			log.Info("✓ Bundle %06d saved [%d ops, hash: %s...]",
 				bundleNum, len(ops), bf.uncompressedHash[:16])
 		}
···
 
 // ===== BUNDLE INDEXING =====
 
-func (bm *BundleManager) indexBundle(ctx context.Context, bundleNum int, bf *bundleFile) error {
+func (bm *BundleManager) indexBundle(ctx context.Context, bundleNum int, bf *bundleFile, cursor string) error {
 	prevHash := ""
 	if bundleNum > 1 {
 		if prev, err := bm.db.GetBundleByNumber(ctx, bundleNum-1); err == nil {
···
 	}
 
 	dids := bm.extractUniqueDIDs(bf.operations)
-	fileSize := bm.getFileSize(bf.path)
+	compressedFileSize := bm.getFileSize(bf.path)
+
+	// Calculate uncompressed size
+	uncompressedSize := int64(0)
+	for _, op := range bf.operations {
+		uncompressedSize += int64(len(op.RawJSON)) + 1 // +1 for newline
+	}
 
 	bundle := &storage.PLCBundle{
-		BundleNumber:   bundleNum,
-		StartTime:      bf.operations[0].CreatedAt,
-		EndTime:        bf.operations[len(bf.operations)-1].CreatedAt,
-		DIDs:           dids,
-		Hash:           bf.uncompressedHash,
-		CompressedHash: bf.compressedHash,
-		CompressedSize: fileSize,
-		PrevBundleHash: prevHash,
-		Compressed:     true,
-		CreatedAt:      time.Now(),
+		BundleNumber:     bundleNum,
+		StartTime:        bf.operations[0].CreatedAt,
+		EndTime:          bf.operations[len(bf.operations)-1].CreatedAt,
+		DIDs:             dids,
+		Hash:             bf.uncompressedHash,
+		CompressedHash:   bf.compressedHash,
+		CompressedSize:   compressedFileSize,
+		UncompressedSize: uncompressedSize,
+		Cursor:           cursor,
+		PrevBundleHash:   prevHash,
+		Compressed:       true,
+		CreatedAt:        time.Now(),
 	}
 
 	return bm.db.CreateBundle(ctx, bundle)
···
 
 // ===== MEMPOOL BUNDLE CREATION =====
 
-func (bm *BundleManager) CreateBundleFromMempool(ctx context.Context, operations []PLCOperation) (int, error) {
+func (bm *BundleManager) CreateBundleFromMempool(ctx context.Context, operations []PLCOperation, cursor string) (int, error) {
 	if !bm.enabled {
 		return 0, fmt.Errorf("bundle manager disabled")
 	}
···
 		return 0, err
 	}
 
-	if err := bm.indexBundle(ctx, bundleNum, bf); err != nil {
+	if err := bm.indexBundle(ctx, bundleNum, bf, cursor); err != nil {
 		return 0, err
 	}
 

internal/plc/scanner.go

···
 
 	// Handle existing mempool first
 	if hasMempool, _ := s.hasSufficientMempool(ctx); hasMempool {
-		return s.handleMempoolOnly(ctx, metrics, cursor)
+		return s.handleMempoolOnly(ctx, metrics)
 	}
 
 	// Process bundles until incomplete or error
···
 	return count > 0, nil
 }
 
-func (s *Scanner) handleMempoolOnly(ctx context.Context, m *ScanMetrics, cursor *storage.ScanCursor) error {
+func (s *Scanner) handleMempoolOnly(ctx context.Context, m *ScanMetrics) error {
 	count, _ := s.db.GetMempoolCount(ctx)
 	log.Info("→ Mempool has %d operations, continuing to fill it before fetching new bundles", count)
 
···
 
 		log.Info("→ Creating bundle from mempool (%d operations available)...", count)
 
-		bundleNum, ops, err := s.createBundleFromMempool(ctx)
+		// Updated to receive 4 values instead of 3
+		bundleNum, ops, cursor, err := s.createBundleFromMempool(ctx)
 		if err != nil {
 			return err
 		}
···
 		newEndpointsFound := sumCounts(m.endpointCounts) - countsBefore
 
 		m.totalProcessed += int64(len(ops))
-		m.newEndpoints += newEndpointsFound // NEW: Track new endpoints
+		m.newEndpoints += newEndpointsFound
 		m.currentBundle = bundleNum
 
 		if err := s.updateCursorForBundle(ctx, bundleNum, m.totalProcessed); err != nil {
 			log.Error("Warning: failed to update cursor: %v", err)
 		}
 
-		log.Info("✓ Created bundle %06d from mempool", bundleNum)
+		log.Info("✓ Created bundle %06d from mempool (cursor: %s)", bundleNum, cursor)
 	}
 }
 
-func (s *Scanner) createBundleFromMempool(ctx context.Context) (int, []PLCOperation, error) {
+func (s *Scanner) createBundleFromMempool(ctx context.Context) (int, []PLCOperation, string, error) {
 	mempoolOps, err := s.db.GetMempoolOperations(ctx, BUNDLE_SIZE)
 	if err != nil {
-		return 0, nil, err
+		return 0, nil, "", err
 	}
 
 	ops, ids := s.deduplicateMempool(mempoolOps)
 	if len(ops) < BUNDLE_SIZE {
-		return 0, nil, fmt.Errorf("only got %d unique operations from mempool, need %d", len(ops), BUNDLE_SIZE)
+		return 0, nil, "", fmt.Errorf("only got %d unique operations from mempool, need %d", len(ops), BUNDLE_SIZE)
 	}
 
-	bundleNum, err := s.bundleManager.CreateBundleFromMempool(ctx, ops)
+	// Determine cursor from last bundle
+	cursor := ""
+	lastBundle, err := s.db.GetLastBundleNumber(ctx)
+	if err == nil && lastBundle > 0 {
+		if bundle, err := s.db.GetBundleByNumber(ctx, lastBundle); err == nil {
+			cursor = bundle.EndTime.Format(time.RFC3339Nano)
+		}
+	}
+
+	bundleNum, err := s.bundleManager.CreateBundleFromMempool(ctx, ops, cursor)
 	if err != nil {
-		return 0, nil, err
+		return 0, nil, "", err
 	}
 
 	if err := s.db.DeleteFromMempool(ctx, ids[:len(ops)]); err != nil {
-		return 0, nil, err
+		return 0, nil, "", err
 	}
 
-	return bundleNum, ops, nil
+	return bundleNum, ops, cursor, nil
 }
 
 func (s *Scanner) deduplicateMempool(mempoolOps []storage.MempoolOperation) ([]PLCOperation, []int64) {

internal/storage/db.go

···
 	DeleteFromMempool(ctx context.Context, ids []int64) error
 	GetFirstMempoolOperation(ctx context.Context) (*MempoolOperation, error)
 	GetLastMempoolOperation(ctx context.Context) (*MempoolOperation, error)
+	GetMempoolUniqueDIDCount(ctx context.Context) (int, error)
+	GetMempoolUncompressedSize(ctx context.Context) (int64, error)
 
 	// Metrics
 	StorePLCMetrics(ctx context.Context, metrics *PLCMetrics) error

internal/storage/sqlite.go

···
         start_time TIMESTAMP NOT NULL,
         end_time TIMESTAMP NOT NULL,
         dids TEXT NOT NULL,
-        hash TEXT NOT NULL,                -- SHA256 of uncompressed JSONL
-        compressed_hash TEXT NOT NULL,     -- SHA256 of compressed file
-        compressed_size INTEGER NOT NULL,  -- Size of compressed file
-        prev_bundle_hash TEXT,             -- Chain link
+        hash TEXT NOT NULL,
+        compressed_hash TEXT NOT NULL,
+        compressed_size INTEGER NOT NULL,
+        uncompressed_size INTEGER NOT NULL,     -- NEW
+        cursor TEXT,                             -- NEW
+        prev_bundle_hash TEXT,
         compressed BOOLEAN DEFAULT 1,
         created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
     );
···
 // GetBundleByNumber
 func (s *SQLiteDB) GetBundleByNumber(ctx context.Context, bundleNumber int) (*PLCBundle, error) {
 	query := `
-        SELECT bundle_number, start_time, end_time, dids, hash, compressed_hash, compressed_size, prev_bundle_hash, compressed, created_at
+        SELECT bundle_number, start_time, end_time, dids, hash, compressed_hash, 
+               compressed_size, uncompressed_size, cursor, prev_bundle_hash, compressed, created_at
         FROM plc_bundles
         WHERE bundle_number = ?
     `
···
 	var bundle PLCBundle
 	var didsJSON string
 	var prevHash sql.NullString
+	var cursor sql.NullString
 
 	err := s.db.QueryRowContext(ctx, query, bundleNumber).Scan(
 		&bundle.BundleNumber, &bundle.StartTime, &bundle.EndTime,
 		&didsJSON, &bundle.Hash, &bundle.CompressedHash,
-		&bundle.CompressedSize, &prevHash, &bundle.Compressed, &bundle.CreatedAt,
+		&bundle.CompressedSize, &bundle.UncompressedSize, &cursor,
+		&prevHash, &bundle.Compressed, &bundle.CreatedAt,
 	)
 	if err != nil {
 		return nil, err
···
 
 	if prevHash.Valid {
 		bundle.PrevBundleHash = prevHash.String
+	}
+	if cursor.Valid {
+		bundle.Cursor = cursor.String
 	}
 
 	json.Unmarshal([]byte(didsJSON), &bundle.DIDs)
···
 	query := `
         INSERT INTO plc_bundles (
             bundle_number, start_time, end_time, dids, 
-            hash, compressed_hash, compressed_size, prev_bundle_hash, compressed
+            hash, compressed_hash, compressed_size, uncompressed_size, cursor, prev_bundle_hash, compressed
         )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ON CONFLICT(bundle_number) DO UPDATE SET
             start_time = excluded.start_time,
             end_time = excluded.end_time,
···
             hash = excluded.hash,
             compressed_hash = excluded.compressed_hash,
             compressed_size = excluded.compressed_size,
+            uncompressed_size = excluded.uncompressed_size,
+            cursor = excluded.cursor,
             prev_bundle_hash = excluded.prev_bundle_hash,
             compressed = excluded.compressed
     `
 	_, err = s.db.ExecContext(ctx, query,
 		bundle.BundleNumber, bundle.StartTime, bundle.EndTime,
 		string(didsJSON), bundle.Hash, bundle.CompressedHash,
-		bundle.CompressedSize, bundle.PrevBundleHash, bundle.Compressed,
+		bundle.CompressedSize, bundle.UncompressedSize, bundle.Cursor,
+		bundle.PrevBundleHash, bundle.Compressed,
 	)
 
 	return err
 }
 
+// GetMempoolUniqueDIDCount returns the number of unique DIDs in mempool
+func (s *SQLiteDB) GetMempoolUniqueDIDCount(ctx context.Context) (int, error) {
+	query := "SELECT COUNT(DISTINCT did) FROM plc_mempool"
+	var count int
+	err := s.db.QueryRowContext(ctx, query).Scan(&count)
+	return count, err
+}
+
+// GetMempoolUncompressedSize returns total uncompressed size of all operations
+func (s *SQLiteDB) GetMempoolUncompressedSize(ctx context.Context) (int64, error) {
+	query := "SELECT COALESCE(SUM(LENGTH(operation)), 0) FROM plc_mempool"
+	var size int64
+	err := s.db.QueryRowContext(ctx, query).Scan(&size)
+	return size, err
+}
+
 // GetBundles
 func (s *SQLiteDB) GetBundles(ctx context.Context, limit int) ([]*PLCBundle, error) {
 	query := `
···
 	return &bundle, nil
 }
 
-// scanBundles - Make sure it reads 10 columns
 func (s *SQLiteDB) scanBundles(rows *sql.Rows) ([]*PLCBundle, error) {
 	var bundles []*PLCBundle
 
···
 		var bundle PLCBundle
 		var didsJSON string
 		var prevHash sql.NullString
+		var cursor sql.NullString
 
-		// Scan: bundle_number, start_time, end_time, dids, hash, compressed_hash, compressed_size, prev_bundle_hash, compressed, created_at
 		if err := rows.Scan(
 			&bundle.BundleNumber,
 			&bundle.StartTime,
 			&bundle.EndTime,
 			&didsJSON,
-			&bundle.Hash,           // Uncompressed hash
-			&bundle.CompressedHash, // Compressed hash
-			&bundle.CompressedSize, // Compressed size
+			&bundle.Hash,
+			&bundle.CompressedHash,
+			&bundle.CompressedSize,
+			&bundle.UncompressedSize,
+			&cursor,
 			&prevHash,
 			&bundle.Compressed,
 			&bundle.CreatedAt,
···
 
 		if prevHash.Valid {
 			bundle.PrevBundleHash = prevHash.String
+		}
+		if cursor.Valid {
+			bundle.Cursor = cursor.String
 		}
 
 		json.Unmarshal([]byte(didsJSON), &bundle.DIDs)

internal/storage/types.go

···
 
 // PLCBundle represents a cached bundle of PLC operations
 type PLCBundle struct {
-	BundleNumber   int
-	StartTime      time.Time
-	EndTime        time.Time
-	BoundaryCIDs   []string
-	DIDs           []string
-	Hash           string
-	CompressedHash string
-	CompressedSize int64
-	PrevBundleHash string
-	Compressed     bool
-	CreatedAt      time.Time
+	BundleNumber     int
+	StartTime        time.Time
+	EndTime          time.Time
+	BoundaryCIDs     []string
+	DIDs             []string
+	Hash             string
+	CompressedHash   string
+	CompressedSize   int64
+	UncompressedSize int64  // NEW: uncompressed size
+	Cursor           string // NEW: PLC cursor used to create this bundle
+	PrevBundleHash   string
+	Compressed       bool
+	CreatedAt        time.Time
 }
 
 // GetFilePath returns the computed file path for this bundle