add flags for backfill workers/consumers

cmd/monarch/backfill.go

···
 
 import "github.com/bluesky-social/indigo/backfill"
 
-func NewBackfillService(store backfill.Store, h *HandlerService) *backfill.Backfiller {
+func NewBackfillService(store backfill.Store, h *HandlerService, workers int, consumers int) *backfill.Backfiller {
 	opts := &backfill.BackfillOptions{
-		ParallelBackfills:     1,
-		ParallelRecordCreates: 1,
+		ParallelBackfills:     workers,
+		ParallelRecordCreates: consumers,
 		NSIDFilter:            "",
-		SyncRequestsPerSecond: 2,
+		SyncRequestsPerSecond: 10,
 		RelayHost:             "https://bsky.network",
 	}
 

cmd/monarch/main.go

···
 	"log/slog"
 	"os"
 	"os/signal"
+	"runtime"
 	"syscall"
 	"time"
 
···
 	}
 }
 
-func (app *App) Start(ctx context.Context) error {
+func (app *App) Start(ctx context.Context, cctx *cli.Context) error {
 	slog.Info("starting up")
 
 	app.cursor = NewCursorService(app.state)
···
 
 	app.handler = NewHandlerService(app.content)
 
-	app.backfill = NewBackfillService(backfill.NewGormstore(app.state), app.handler)
+	workers := cctx.Int("backfill-workers")
+	consumers := cctx.Int("backfill-consumers")
+	app.backfill = NewBackfillService(backfill.NewGormstore(app.state), app.handler, workers, consumers)
 	go app.backfill.Start()
 
 	app.census = NewCensusService(app.cursor, app.backfill)
···
 		},
 		&cli.IntFlag{
 			Name:  "max-db-connections",
-			Value: 1,
+			Value: runtime.NumCPU(),
+		},
+		&cli.IntFlag{
+			Name:  "backfill-workers",
+			Value: 50,
+		},
+		&cli.IntFlag{
+			Name:  "backfill-consumers",
+			Value: 100,
 		},
 	}
 
···
 		}
 
 		app := NewApp(statedb, contentdb)
-		if err := app.Start(ctx); err != nil {
+		if err := app.Start(ctx, cctx); err != nil {
 			slog.Error("failed to start backfiller", "err", err)
 		}