an app.bsky.* indexer

drop locks

Changed files
+5 -33
cmd
+4 -20
cmd/monarch/census.go
··· 14 14 ) 15 15 16 16 type CensusService struct { 17 - cursor *CursorService 18 - backfill *backfill.Backfiller 19 - 17 + cursor *CursorService 18 + backfill *backfill.Backfiller 20 19 seenHosts map[string]bool 21 - seenLk sync.Mutex 22 - 23 - storeLk sync.Mutex 24 20 } 25 21 26 22 type jobMaker interface { ··· 70 66 71 67 for _, host := range res.Hosts { 72 68 // don't reprocess hosts already handled 73 - cs.seenLk.Lock() 74 - _, ok := cs.seenHosts[host.Hostname] 75 - cs.seenLk.Unlock() 76 - if ok { 69 + seen := cs.seenHosts[host.Hostname] 70 + if seen { 77 71 slog.Info("already processed host, skipping", "host", host) 78 72 continue 79 73 } ··· 114 108 return 115 109 } 116 110 117 - cs.storeLk.Lock() 118 111 hcur, err := cs.cursor.GetHostCursor(host) 119 112 if err != nil { 120 113 slog.Error("error fetching host cursor", "err", err) 121 114 } 122 - cs.storeLk.Unlock() 123 115 124 116 var added int 125 117 curs := hcur.Cursor ··· 138 130 continue 139 131 } 140 132 141 - cs.storeLk.Lock() 142 133 for _, repo := range res.Repos { 143 134 _, err := jmstore.GetOrCreateJob(ctx, repo.Did, backfill.StateEnqueued) 144 135 if err != nil { ··· 147 138 added += 1 148 139 } 149 140 } 150 - cs.storeLk.Unlock() 151 141 152 142 if res.Cursor != nil && *res.Cursor != "" { 153 143 curs = *res.Cursor 154 - cs.storeLk.Lock() 155 144 if err := cs.cursor.SetHostCursor(host, curs); err != nil { 156 145 slog.Error("error updating cursor for host", "err", err) 157 146 } 158 - cs.storeLk.Unlock() 159 147 } else { 160 148 break 161 149 } 162 150 } 163 151 164 152 slog.Info("finished listing repos", "host", host) 165 - 166 - cs.seenLk.Lock() 167 - defer cs.seenLk.Unlock() 168 - 169 153 cs.seenHosts[host] = true 170 154 } 171 155
+1 -4
cmd/monarch/cursors.go
··· 3 3 import ( 4 4 "context" 5 5 "log/slog" 6 - "sync" 7 6 "time" 8 7 9 8 "gorm.io/gorm" 10 9 ) 11 10 12 11 type CursorService struct { 13 - store *gorm.DB 14 - 15 - firehoseLk sync.Mutex 12 + store *gorm.DB 16 13 firehoseSeq int64 17 14 } 18 15
-9
cmd/monarch/firehose.go
··· 35 35 } 36 36 37 37 func (cs *CursorService) GetFirehoseCursor() (int64, error) { 38 - cs.firehoseLk.Lock() 39 - defer cs.firehoseLk.Unlock() 40 - 41 38 var fcur firehoseCursor 42 39 if err := cs.store.Where(firehoseCursor{ 43 40 Key: "firehose", ··· 51 48 } 52 49 53 50 func (cs *CursorService) SetFirehoseCursor(seq int64) { 54 - cs.firehoseLk.Lock() 55 - defer cs.firehoseLk.Unlock() 56 - 57 51 cs.firehoseSeq = seq 58 52 } 59 53 60 54 func (cs *CursorService) PersistFirehoseCursor() error { 61 - cs.firehoseLk.Lock() 62 - defer cs.firehoseLk.Unlock() 63 - 64 55 if err := cs.store.Model(&firehoseCursor{}).Where(firehoseCursor{Key: "firehose"}).Update("val", cs.firehoseSeq).Error; err != nil { 65 56 return fmt.Errorf("error persisting firehose seq: %w", err) 66 57 }