+4
-20
cmd/monarch/census.go
+4
-20
cmd/monarch/census.go
···
14
14
)
15
15
16
16
type CensusService struct {
17
-
cursor *CursorService
18
-
backfill *backfill.Backfiller
19
-
17
+
cursor *CursorService
18
+
backfill *backfill.Backfiller
20
19
seenHosts map[string]bool
21
-
seenLk sync.Mutex
22
-
23
-
storeLk sync.Mutex
24
20
}
25
21
26
22
type jobMaker interface {
···
70
66
71
67
for _, host := range res.Hosts {
72
68
// don't reprocess hosts already handled
73
-
cs.seenLk.Lock()
74
-
_, ok := cs.seenHosts[host.Hostname]
75
-
cs.seenLk.Unlock()
76
-
if ok {
69
+
seen := cs.seenHosts[host.Hostname]
70
+
if seen {
77
71
slog.Info("already processed host, skipping", "host", host)
78
72
continue
79
73
}
···
114
108
return
115
109
}
116
110
117
-
cs.storeLk.Lock()
118
111
hcur, err := cs.cursor.GetHostCursor(host)
119
112
if err != nil {
120
113
slog.Error("error fetching host cursor", "err", err)
121
114
}
122
-
cs.storeLk.Unlock()
123
115
124
116
var added int
125
117
curs := hcur.Cursor
···
138
130
continue
139
131
}
140
132
141
-
cs.storeLk.Lock()
142
133
for _, repo := range res.Repos {
143
134
_, err := jmstore.GetOrCreateJob(ctx, repo.Did, backfill.StateEnqueued)
144
135
if err != nil {
···
147
138
added += 1
148
139
}
149
140
}
150
-
cs.storeLk.Unlock()
151
141
152
142
if res.Cursor != nil && *res.Cursor != "" {
153
143
curs = *res.Cursor
154
-
cs.storeLk.Lock()
155
144
if err := cs.cursor.SetHostCursor(host, curs); err != nil {
156
145
slog.Error("error updating cursor for host", "err", err)
157
146
}
158
-
cs.storeLk.Unlock()
159
147
} else {
160
148
break
161
149
}
162
150
}
163
151
164
152
slog.Info("finished listing repos", "host", host)
165
-
166
-
cs.seenLk.Lock()
167
-
defer cs.seenLk.Unlock()
168
-
169
153
cs.seenHosts[host] = true
170
154
}
171
155
+1
-4
cmd/monarch/cursors.go
+1
-4
cmd/monarch/cursors.go
-9
cmd/monarch/firehose.go
-9
cmd/monarch/firehose.go
···
35
35
}
36
36
37
37
func (cs *CursorService) GetFirehoseCursor() (int64, error) {
38
-
cs.firehoseLk.Lock()
39
-
defer cs.firehoseLk.Unlock()
40
-
41
38
var fcur firehoseCursor
42
39
if err := cs.store.Where(firehoseCursor{
43
40
Key: "firehose",
···
51
48
}
52
49
53
50
func (cs *CursorService) SetFirehoseCursor(seq int64) {
54
-
cs.firehoseLk.Lock()
55
-
defer cs.firehoseLk.Unlock()
56
-
57
51
cs.firehoseSeq = seq
58
52
}
59
53
60
54
func (cs *CursorService) PersistFirehoseCursor() error {
61
-
cs.firehoseLk.Lock()
62
-
defer cs.firehoseLk.Unlock()
63
-
64
55
if err := cs.store.Model(&firehoseCursor{}).Where(firehoseCursor{Key: "firehose"}).Update("val", cs.firehoseSeq).Error; err != nil {
65
56
return fmt.Errorf("error persisting firehose seq: %w", err)
66
57
}