oppi.li
fork of indigo with slightly nicer lexgen
Requeue dequeued jobs on restart, add a command to re-enqueue failed jobs
+705
+666
cmd/netsync/dash.json
+666
cmd/netsync/dash.json
···
1
+
{
2
+
"__inputs": [
3
+
{
4
+
"name": "DS_PROMETHEUS",
5
+
"label": "Prometheus",
6
+
"description": "",
7
+
"type": "datasource",
8
+
"pluginId": "prometheus",
9
+
"pluginName": "Prometheus"
10
+
}
11
+
],
12
+
"__elements": {},
13
+
"__requires": [
14
+
{
15
+
"type": "grafana",
16
+
"id": "grafana",
17
+
"name": "Grafana",
18
+
"version": "9.4.3"
19
+
},
20
+
{
21
+
"type": "datasource",
22
+
"id": "prometheus",
23
+
"name": "Prometheus",
24
+
"version": "1.0.0"
25
+
},
26
+
{
27
+
"type": "panel",
28
+
"id": "stat",
29
+
"name": "Stat",
30
+
"version": ""
31
+
},
32
+
{
33
+
"type": "panel",
34
+
"id": "timeseries",
35
+
"name": "Time series",
36
+
"version": ""
37
+
}
38
+
],
39
+
"annotations": {
40
+
"list": [
41
+
{
42
+
"builtIn": 1,
43
+
"datasource": {
44
+
"type": "grafana",
45
+
"uid": "-- Grafana --"
46
+
},
47
+
"enable": true,
48
+
"hide": true,
49
+
"iconColor": "rgba(0, 211, 255, 1)",
50
+
"name": "Annotations & Alerts",
51
+
"target": {
52
+
"limit": 100,
53
+
"matchAny": false,
54
+
"tags": [],
55
+
"type": "dashboard"
56
+
},
57
+
"type": "dashboard"
58
+
}
59
+
]
60
+
},
61
+
"editable": true,
62
+
"fiscalYearStartMonth": 0,
63
+
"graphTooltip": 0,
64
+
"id": null,
65
+
"links": [],
66
+
"liveNow": false,
67
+
"panels": [
68
+
{
69
+
"datasource": {
70
+
"type": "prometheus",
71
+
"uid": "${DS_PROMETHEUS}"
72
+
},
73
+
"fieldConfig": {
74
+
"defaults": {
75
+
"color": {
76
+
"mode": "palette-classic"
77
+
},
78
+
"custom": {
79
+
"axisCenteredZero": false,
80
+
"axisColorMode": "series",
81
+
"axisLabel": "",
82
+
"axisPlacement": "auto",
83
+
"barAlignment": 0,
84
+
"drawStyle": "line",
85
+
"fillOpacity": 0,
86
+
"gradientMode": "none",
87
+
"hideFrom": {
88
+
"legend": false,
89
+
"tooltip": false,
90
+
"viz": false
91
+
},
92
+
"lineInterpolation": "linear",
93
+
"lineWidth": 1,
94
+
"pointSize": 5,
95
+
"scaleDistribution": {
96
+
"type": "linear"
97
+
},
98
+
"showPoints": "auto",
99
+
"spanNulls": false,
100
+
"stacking": {
101
+
"group": "A",
102
+
"mode": "none"
103
+
},
104
+
"thresholdsStyle": {
105
+
"mode": "off"
106
+
}
107
+
},
108
+
"decimals": 0,
109
+
"mappings": [],
110
+
"thresholds": {
111
+
"mode": "absolute",
112
+
"steps": [
113
+
{
114
+
"color": "green",
115
+
"value": null
116
+
},
117
+
{
118
+
"color": "red",
119
+
"value": 80
120
+
}
121
+
]
122
+
},
123
+
"unit": "locale"
124
+
},
125
+
"overrides": [
126
+
{
127
+
"matcher": {
128
+
"id": "byName",
129
+
"options": "Jobs Finished"
130
+
},
131
+
"properties": [
132
+
{
133
+
"id": "custom.axisPlacement",
134
+
"value": "right"
135
+
}
136
+
]
137
+
}
138
+
]
139
+
},
140
+
"gridPos": {
141
+
"h": 9,
142
+
"w": 9,
143
+
"x": 0,
144
+
"y": 0
145
+
},
146
+
"id": 7,
147
+
"options": {
148
+
"legend": {
149
+
"calcs": [],
150
+
"displayMode": "list",
151
+
"placement": "bottom",
152
+
"showLegend": true
153
+
},
154
+
"tooltip": {
155
+
"mode": "single",
156
+
"sort": "none"
157
+
}
158
+
},
159
+
"targets": [
160
+
{
161
+
"datasource": {
162
+
"type": "prometheus",
163
+
"uid": "${DS_PROMETHEUS}"
164
+
},
165
+
"editorMode": "code",
166
+
"expr": "netsync_enqueued_jobs{}",
167
+
"legendFormat": "Jobs Enqueued",
168
+
"range": true,
169
+
"refId": "A"
170
+
},
171
+
{
172
+
"datasource": {
173
+
"type": "prometheus",
174
+
"uid": "${DS_PROMETHEUS}"
175
+
},
176
+
"editorMode": "code",
177
+
"expr": "netsync_finished_jobs{}",
178
+
"hide": false,
179
+
"legendFormat": "Jobs Finished",
180
+
"range": true,
181
+
"refId": "B"
182
+
}
183
+
],
184
+
"title": "Job Backlog",
185
+
"type": "timeseries"
186
+
},
187
+
{
188
+
"datasource": {
189
+
"type": "prometheus",
190
+
"uid": "${DS_PROMETHEUS}"
191
+
},
192
+
"fieldConfig": {
193
+
"defaults": {
194
+
"color": {
195
+
"mode": "thresholds"
196
+
},
197
+
"mappings": [],
198
+
"thresholds": {
199
+
"mode": "absolute",
200
+
"steps": [
201
+
{
202
+
"color": "green",
203
+
"value": null
204
+
}
205
+
]
206
+
},
207
+
"unit": "dtdhms"
208
+
},
209
+
"overrides": []
210
+
},
211
+
"gridPos": {
212
+
"h": 4,
213
+
"w": 3,
214
+
"x": 9,
215
+
"y": 0
216
+
},
217
+
"id": 8,
218
+
"options": {
219
+
"colorMode": "value",
220
+
"graphMode": "area",
221
+
"justifyMode": "auto",
222
+
"orientation": "auto",
223
+
"reduceOptions": {
224
+
"calcs": [
225
+
"lastNotNull"
226
+
],
227
+
"fields": "",
228
+
"values": false
229
+
},
230
+
"textMode": "auto"
231
+
},
232
+
"pluginVersion": "9.4.3",
233
+
"targets": [
234
+
{
235
+
"datasource": {
236
+
"type": "prometheus",
237
+
"uid": "${DS_PROMETHEUS}"
238
+
},
239
+
"editorMode": "code",
240
+
"exemplar": false,
241
+
"expr": "(netsync_enqueued_jobs / rate(netsync_finished_jobs{}[$__rate_interval]))",
242
+
"instant": false,
243
+
"legendFormat": "Seconds Until Completion",
244
+
"range": true,
245
+
"refId": "A"
246
+
}
247
+
],
248
+
"title": "Time until Completion",
249
+
"type": "stat"
250
+
},
251
+
{
252
+
"datasource": {
253
+
"type": "prometheus",
254
+
"uid": "${DS_PROMETHEUS}"
255
+
},
256
+
"fieldConfig": {
257
+
"defaults": {
258
+
"color": {
259
+
"mode": "palette-classic"
260
+
},
261
+
"custom": {
262
+
"axisCenteredZero": false,
263
+
"axisColorMode": "text",
264
+
"axisLabel": "",
265
+
"axisPlacement": "auto",
266
+
"barAlignment": 0,
267
+
"drawStyle": "line",
268
+
"fillOpacity": 0,
269
+
"gradientMode": "none",
270
+
"hideFrom": {
271
+
"legend": false,
272
+
"tooltip": false,
273
+
"viz": false
274
+
},
275
+
"lineInterpolation": "linear",
276
+
"lineWidth": 1,
277
+
"pointSize": 5,
278
+
"scaleDistribution": {
279
+
"type": "linear"
280
+
},
281
+
"showPoints": "auto",
282
+
"spanNulls": false,
283
+
"stacking": {
284
+
"group": "A",
285
+
"mode": "none"
286
+
},
287
+
"thresholdsStyle": {
288
+
"mode": "off"
289
+
}
290
+
},
291
+
"mappings": [],
292
+
"thresholds": {
293
+
"mode": "absolute",
294
+
"steps": [
295
+
{
296
+
"color": "green",
297
+
"value": null
298
+
},
299
+
{
300
+
"color": "red",
301
+
"value": 80
302
+
}
303
+
]
304
+
},
305
+
"unit": "bytes"
306
+
},
307
+
"overrides": []
308
+
},
309
+
"gridPos": {
310
+
"h": 9,
311
+
"w": 12,
312
+
"x": 12,
313
+
"y": 0
314
+
},
315
+
"id": 3,
316
+
"options": {
317
+
"legend": {
318
+
"calcs": [],
319
+
"displayMode": "list",
320
+
"placement": "bottom",
321
+
"showLegend": true
322
+
},
323
+
"tooltip": {
324
+
"mode": "single",
325
+
"sort": "none"
326
+
}
327
+
},
328
+
"targets": [
329
+
{
330
+
"datasource": {
331
+
"type": "prometheus",
332
+
"uid": "${DS_PROMETHEUS}"
333
+
},
334
+
"editorMode": "code",
335
+
"expr": "netsync_bytes_processed{}",
336
+
"legendFormat": "Bytes Read",
337
+
"range": true,
338
+
"refId": "A"
339
+
}
340
+
],
341
+
"title": "Backfilled Bytes",
342
+
"type": "timeseries"
343
+
},
344
+
{
345
+
"datasource": {
346
+
"type": "prometheus",
347
+
"uid": "${DS_PROMETHEUS}"
348
+
},
349
+
"fieldConfig": {
350
+
"defaults": {
351
+
"color": {
352
+
"mode": "palette-classic"
353
+
},
354
+
"custom": {
355
+
"axisCenteredZero": false,
356
+
"axisColorMode": "text",
357
+
"axisLabel": "",
358
+
"axisPlacement": "auto",
359
+
"barAlignment": 0,
360
+
"drawStyle": "line",
361
+
"fillOpacity": 0,
362
+
"gradientMode": "none",
363
+
"hideFrom": {
364
+
"legend": false,
365
+
"tooltip": false,
366
+
"viz": false
367
+
},
368
+
"lineInterpolation": "linear",
369
+
"lineWidth": 1,
370
+
"pointSize": 5,
371
+
"scaleDistribution": {
372
+
"type": "linear"
373
+
},
374
+
"showPoints": "auto",
375
+
"spanNulls": false,
376
+
"stacking": {
377
+
"group": "A",
378
+
"mode": "none"
379
+
},
380
+
"thresholdsStyle": {
381
+
"mode": "off"
382
+
}
383
+
},
384
+
"mappings": [],
385
+
"thresholds": {
386
+
"mode": "absolute",
387
+
"steps": [
388
+
{
389
+
"color": "green",
390
+
"value": null
391
+
}
392
+
]
393
+
},
394
+
"unit": "cps"
395
+
},
396
+
"overrides": [
397
+
{
398
+
"matcher": {
399
+
"id": "byFrameRefID",
400
+
"options": "B"
401
+
},
402
+
"properties": [
403
+
{
404
+
"id": "color",
405
+
"value": {
406
+
"fixedColor": "red",
407
+
"mode": "fixed"
408
+
}
409
+
}
410
+
]
411
+
}
412
+
]
413
+
},
414
+
"gridPos": {
415
+
"h": 9,
416
+
"w": 12,
417
+
"x": 0,
418
+
"y": 9
419
+
},
420
+
"id": 2,
421
+
"options": {
422
+
"legend": {
423
+
"calcs": [],
424
+
"displayMode": "list",
425
+
"placement": "bottom",
426
+
"showLegend": true
427
+
},
428
+
"tooltip": {
429
+
"mode": "single",
430
+
"sort": "none"
431
+
}
432
+
},
433
+
"targets": [
434
+
{
435
+
"datasource": {
436
+
"type": "prometheus",
437
+
"uid": "${DS_PROMETHEUS}"
438
+
},
439
+
"editorMode": "code",
440
+
"expr": "rate(netsync_repo_clone_duration_seconds_count{status=\"success\"}[$__rate_interval])",
441
+
"legendFormat": "{{status}}",
442
+
"range": true,
443
+
"refId": "A"
444
+
},
445
+
{
446
+
"datasource": {
447
+
"type": "prometheus",
448
+
"uid": "${DS_PROMETHEUS}"
449
+
},
450
+
"editorMode": "code",
451
+
"expr": "rate(netsync_repo_clone_duration_seconds_count{status!=\"success\"}[$__rate_interval])",
452
+
"hide": false,
453
+
"legendFormat": "{{status}}",
454
+
"range": true,
455
+
"refId": "B"
456
+
}
457
+
],
458
+
"title": "Backfilled Repos",
459
+
"type": "timeseries"
460
+
},
461
+
{
462
+
"datasource": {
463
+
"type": "prometheus",
464
+
"uid": "${DS_PROMETHEUS}"
465
+
},
466
+
"fieldConfig": {
467
+
"defaults": {
468
+
"color": {
469
+
"mode": "palette-classic"
470
+
},
471
+
"custom": {
472
+
"axisCenteredZero": false,
473
+
"axisColorMode": "text",
474
+
"axisLabel": "",
475
+
"axisPlacement": "auto",
476
+
"barAlignment": 0,
477
+
"drawStyle": "line",
478
+
"fillOpacity": 0,
479
+
"gradientMode": "none",
480
+
"hideFrom": {
481
+
"legend": false,
482
+
"tooltip": false,
483
+
"viz": false
484
+
},
485
+
"lineInterpolation": "linear",
486
+
"lineWidth": 1,
487
+
"pointSize": 5,
488
+
"scaleDistribution": {
489
+
"type": "linear"
490
+
},
491
+
"showPoints": "auto",
492
+
"spanNulls": false,
493
+
"stacking": {
494
+
"group": "A",
495
+
"mode": "none"
496
+
},
497
+
"thresholdsStyle": {
498
+
"mode": "off"
499
+
}
500
+
},
501
+
"mappings": [],
502
+
"thresholds": {
503
+
"mode": "absolute",
504
+
"steps": [
505
+
{
506
+
"color": "green",
507
+
"value": null
508
+
},
509
+
{
510
+
"color": "red",
511
+
"value": 80
512
+
}
513
+
]
514
+
},
515
+
"unit": "s"
516
+
},
517
+
"overrides": []
518
+
},
519
+
"gridPos": {
520
+
"h": 9,
521
+
"w": 6,
522
+
"x": 12,
523
+
"y": 9
524
+
},
525
+
"id": 5,
526
+
"options": {
527
+
"legend": {
528
+
"calcs": [],
529
+
"displayMode": "list",
530
+
"placement": "bottom",
531
+
"showLegend": true
532
+
},
533
+
"tooltip": {
534
+
"mode": "single",
535
+
"sort": "none"
536
+
}
537
+
},
538
+
"targets": [
539
+
{
540
+
"datasource": {
541
+
"type": "prometheus",
542
+
"uid": "${DS_PROMETHEUS}"
543
+
},
544
+
"editorMode": "code",
545
+
"expr": "rate(process_cpu_seconds_total{job=\"netsync\"}[$__rate_interval])",
546
+
"legendFormat": "CPU Usage per Second",
547
+
"range": true,
548
+
"refId": "A"
549
+
}
550
+
],
551
+
"title": "CPU Usage",
552
+
"type": "timeseries"
553
+
},
554
+
{
555
+
"datasource": {
556
+
"type": "prometheus",
557
+
"uid": "${DS_PROMETHEUS}"
558
+
},
559
+
"fieldConfig": {
560
+
"defaults": {
561
+
"color": {
562
+
"mode": "palette-classic"
563
+
},
564
+
"custom": {
565
+
"axisCenteredZero": false,
566
+
"axisColorMode": "text",
567
+
"axisLabel": "",
568
+
"axisPlacement": "auto",
569
+
"barAlignment": 0,
570
+
"drawStyle": "line",
571
+
"fillOpacity": 0,
572
+
"gradientMode": "none",
573
+
"hideFrom": {
574
+
"legend": false,
575
+
"tooltip": false,
576
+
"viz": false
577
+
},
578
+
"lineInterpolation": "linear",
579
+
"lineWidth": 1,
580
+
"pointSize": 5,
581
+
"scaleDistribution": {
582
+
"type": "linear"
583
+
},
584
+
"showPoints": "auto",
585
+
"spanNulls": false,
586
+
"stacking": {
587
+
"group": "A",
588
+
"mode": "none"
589
+
},
590
+
"thresholdsStyle": {
591
+
"mode": "off"
592
+
}
593
+
},
594
+
"mappings": [],
595
+
"thresholds": {
596
+
"mode": "absolute",
597
+
"steps": [
598
+
{
599
+
"color": "green",
600
+
"value": null
601
+
},
602
+
{
603
+
"color": "red",
604
+
"value": 80
605
+
}
606
+
]
607
+
},
608
+
"unit": "bytes"
609
+
},
610
+
"overrides": []
611
+
},
612
+
"gridPos": {
613
+
"h": 9,
614
+
"w": 6,
615
+
"x": 18,
616
+
"y": 9
617
+
},
618
+
"id": 6,
619
+
"options": {
620
+
"legend": {
621
+
"calcs": [],
622
+
"displayMode": "list",
623
+
"placement": "bottom",
624
+
"showLegend": true
625
+
},
626
+
"tooltip": {
627
+
"mode": "single",
628
+
"sort": "none"
629
+
}
630
+
},
631
+
"targets": [
632
+
{
633
+
"datasource": {
634
+
"type": "prometheus",
635
+
"uid": "${DS_PROMETHEUS}"
636
+
},
637
+
"editorMode": "code",
638
+
"expr": "process_resident_memory_bytes{job=\"netsync\"}",
639
+
"legendFormat": "Bytes Used",
640
+
"range": true,
641
+
"refId": "A"
642
+
}
643
+
],
644
+
"title": "Memory Usage",
645
+
"type": "timeseries"
646
+
}
647
+
],
648
+
"refresh": "10s",
649
+
"revision": 1,
650
+
"schemaVersion": 38,
651
+
"style": "dark",
652
+
"tags": [],
653
+
"templating": {
654
+
"list": []
655
+
},
656
+
"time": {
657
+
"from": "now-1h",
658
+
"to": "now"
659
+
},
660
+
"timepicker": {},
661
+
"timezone": "",
662
+
"title": "Netsync",
663
+
"uid": "n5sI_QRSz",
664
+
"version": 13,
665
+
"weekStart": ""
666
+
}
+39
cmd/netsync/main.go
+39
cmd/netsync/main.go
···
9
9
"net/http"
10
10
"os"
11
11
"os/signal"
12
+
"strings"
12
13
"sync"
13
14
"syscall"
14
15
"time"
···
81
82
Usage: "header value to send with checkout request",
82
83
Value: "",
83
84
EnvVars: []string{"MAGIC_HEADER_VAL"},
85
+
},
86
+
}
87
+
88
+
app.Commands = []*cli.Command{
89
+
{
90
+
Name: "retry",
91
+
Usage: "requeue failed repos",
92
+
Action: func(cctx *cli.Context) error {
93
+
state := &NetsyncState{
94
+
StatePath: cctx.String("state-file"),
95
+
}
96
+
97
+
err := state.Resume()
98
+
if err != nil {
99
+
return err
100
+
}
101
+
102
+
// Look through finished repos for failed ones
103
+
for _, repoState := range state.FinishedRepos {
104
+
// Don't retry repos that failed due to a 400 (they've been deleted)
105
+
if strings.HasPrefix(repoState.State, "failed") && repoState.State != "failed (status: 400)" {
106
+
state.EnqueuedRepos[repoState.Repo] = &RepoState{
107
+
Repo: repoState.Repo,
108
+
State: "enqueued",
109
+
}
110
+
}
111
+
}
112
+
113
+
// Save state
114
+
return state.Save()
115
+
},
84
116
},
85
117
}
86
118
···
257
289
err = state.Resume()
258
290
if state.EnqueuedRepos == nil {
259
291
state.EnqueuedRepos = make(map[string]*RepoState)
292
+
} else {
293
+
// Reset any dequeued repos
294
+
for _, repoState := range state.EnqueuedRepos {
295
+
if repoState.State == "dequeued" {
296
+
repoState.State = "enqueued"
297
+
}
298
+
}
260
299
}
261
300
262
301
if state.FinishedRepos == nil {