why.bsky.team
+19
LICENSE-MIT
+19
LICENSE-MIT
···
1
+
MIT License
2
+
3
+
Permission is hereby granted, free of charge, to any person obtaining a copy
4
+
of this software and associated documentation files (the "Software"), to deal
5
+
in the Software without restriction, including without limitation the rights
6
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+
copies of the Software, and to permit persons to whom the Software is
8
+
furnished to do so, subject to the following conditions:
9
+
10
+
The above copyright notice and this permission notice shall be included in all
11
+
copies or substantial portions of the Software.
12
+
13
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+
SOFTWARE.
+49
README.md
+49
README.md
···
190
190
This takes a while on first load since its building everything.
191
191
After that, load the localhost url it gives you and it _should_ work.
192
192
193
+
## Selective Backfill
194
+
195
+
If you'd like to backfill a particular repo, just hit the following endpoint:
196
+
197
+
```
198
+
curl http://localhost:4444/rescan/<DID OR HANDLE>
199
+
200
+
```
201
+
202
+
It will take a minute but it should pull all records from that user.
203
+
204
+
## Upstream Firehose Configuration
205
+
206
+
Konbini supports both standard firehose endpoints as well as jetstream. If
207
+
bandwidth and CPU usage is a concern, and you trust the jetstream endpoint,
208
+
then it may be worth trying that out.
209
+
210
+
The configuration file is formatted as follows:
211
+
212
+
```json
213
+
{
214
+
"backends": [
215
+
{
216
+
"type": "jetstream",
217
+
"host": "jetstream1.us-west.bsky.network"
218
+
}
219
+
]
220
+
}
221
+
```
222
+
223
+
The default (implicit) configuration file looks like this:
224
+
225
+
```json
226
+
{
227
+
"backends": [
228
+
{
229
+
"type": "firehose",
230
+
"host": "bsky.network"
231
+
}
232
+
]
233
+
}
234
+
```
235
+
236
+
Note that this is an array of backends, you can specify multiple upstreams, and
237
+
konbini will read from all of them. The main intended purpose of this is to be
238
+
able to subscribe directly to PDSs. PDSs currently only support the full
239
+
firehose endpoint, not jetstream, so be sure to specify a type of "firehose"
240
+
for individual PDS endpoints.
241
+
193
242
## License
194
243
195
244
MIT (whyrusleeping)
+536
backend/backend.go
+536
backend/backend.go
···
1
+
package backend
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
"strings"
8
+
"sync"
9
+
"time"
10
+
11
+
"github.com/bluesky-social/indigo/api/atproto"
12
+
"github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/bluesky-social/indigo/atproto/identity"
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/bluesky-social/indigo/util"
16
+
"github.com/bluesky-social/indigo/xrpc"
17
+
lru "github.com/hashicorp/golang-lru/v2"
18
+
"github.com/jackc/pgx/v5"
19
+
"github.com/jackc/pgx/v5/pgconn"
20
+
"github.com/jackc/pgx/v5/pgxpool"
21
+
. "github.com/whyrusleeping/konbini/models"
22
+
"github.com/whyrusleeping/market/models"
23
+
"gorm.io/gorm"
24
+
"gorm.io/gorm/clause"
25
+
"gorm.io/gorm/logger"
26
+
)
27
+
28
+
// PostgresBackend handles database operations
29
+
type PostgresBackend struct {
30
+
db *gorm.DB
31
+
pgx *pgxpool.Pool
32
+
33
+
dir identity.Directory
34
+
35
+
client *xrpc.Client
36
+
37
+
mydid string
38
+
myrepo *models.Repo
39
+
40
+
relevantDids map[string]bool
41
+
rdLk sync.Mutex
42
+
43
+
revCache *lru.TwoQueueCache[uint, string]
44
+
45
+
repoCache *lru.TwoQueueCache[string, *Repo]
46
+
reposLk sync.Mutex
47
+
48
+
didByIDCache *lru.TwoQueueCache[uint, string]
49
+
50
+
postInfoCache *lru.TwoQueueCache[string, cachedPostInfo]
51
+
52
+
missingRecords chan MissingRecord
53
+
}
54
+
55
+
type cachedPostInfo struct {
56
+
ID uint
57
+
Author uint
58
+
}
59
+
60
+
// NewPostgresBackend creates a new PostgresBackend
61
+
func NewPostgresBackend(mydid string, db *gorm.DB, pgx *pgxpool.Pool, client *xrpc.Client, dir identity.Directory) (*PostgresBackend, error) {
62
+
rc, _ := lru.New2Q[string, *Repo](1_000_000)
63
+
pc, _ := lru.New2Q[string, cachedPostInfo](1_000_000)
64
+
revc, _ := lru.New2Q[uint, string](1_000_000)
65
+
dbic, _ := lru.New2Q[uint, string](1_000_000)
66
+
67
+
b := &PostgresBackend{
68
+
client: client,
69
+
mydid: mydid,
70
+
db: db,
71
+
pgx: pgx,
72
+
relevantDids: make(map[string]bool),
73
+
repoCache: rc,
74
+
postInfoCache: pc,
75
+
revCache: revc,
76
+
didByIDCache: dbic,
77
+
dir: dir,
78
+
79
+
missingRecords: make(chan MissingRecord, 1000),
80
+
}
81
+
82
+
r, err := b.GetOrCreateRepo(context.TODO(), mydid)
83
+
if err != nil {
84
+
return nil, err
85
+
}
86
+
87
+
b.myrepo = r
88
+
89
+
go b.missingRecordFetcher()
90
+
return b, nil
91
+
}
92
+
93
+
// TrackMissingRecord implements the RecordTracker interface
94
+
func (b *PostgresBackend) TrackMissingRecord(identifier string, wait bool) {
95
+
mr := MissingRecord{
96
+
Type: mrTypeFromIdent(identifier),
97
+
Identifier: identifier,
98
+
Wait: wait,
99
+
}
100
+
101
+
b.addMissingRecord(context.TODO(), mr)
102
+
}
103
+
104
+
func mrTypeFromIdent(ident string) MissingRecordType {
105
+
if strings.HasPrefix(ident, "did:") {
106
+
return MissingRecordTypeProfile
107
+
}
108
+
109
+
puri, _ := syntax.ParseATURI(ident)
110
+
switch puri.Collection().String() {
111
+
case "app.bsky.feed.post":
112
+
return MissingRecordTypePost
113
+
case "app.bsky.feed.generator":
114
+
return MissingRecordTypeFeedGenerator
115
+
default:
116
+
return MissingRecordTypeUnknown
117
+
}
118
+
119
+
}
120
+
121
+
// DidToID converts a DID to a database ID
122
+
func (b *PostgresBackend) DidToID(ctx context.Context, did string) (uint, error) {
123
+
r, err := b.GetOrCreateRepo(ctx, did)
124
+
if err != nil {
125
+
return 0, err
126
+
}
127
+
return r.ID, nil
128
+
}
129
+
130
+
func (b *PostgresBackend) GetOrCreateRepo(ctx context.Context, did string) (*Repo, error) {
131
+
r, ok := b.repoCache.Get(did)
132
+
if !ok {
133
+
b.reposLk.Lock()
134
+
135
+
r, ok = b.repoCache.Get(did)
136
+
if !ok {
137
+
r = &Repo{}
138
+
r.Did = did
139
+
b.repoCache.Add(did, r)
140
+
}
141
+
142
+
b.reposLk.Unlock()
143
+
}
144
+
145
+
r.Lk.Lock()
146
+
defer r.Lk.Unlock()
147
+
if r.Setup {
148
+
return r, nil
149
+
}
150
+
151
+
row := b.pgx.QueryRow(ctx, "SELECT id, created_at, did FROM repos WHERE did = $1", did)
152
+
153
+
err := row.Scan(&r.ID, &r.CreatedAt, &r.Did)
154
+
if err == nil {
155
+
// found it!
156
+
r.Setup = true
157
+
return r, nil
158
+
}
159
+
160
+
if err != pgx.ErrNoRows {
161
+
return nil, err
162
+
}
163
+
164
+
r.Did = did
165
+
if err := b.db.Create(r).Error; err != nil {
166
+
return nil, err
167
+
}
168
+
169
+
r.Setup = true
170
+
171
+
return r, nil
172
+
}
173
+
174
+
func (b *PostgresBackend) GetOrCreateList(ctx context.Context, uri string) (*List, error) {
175
+
puri, err := util.ParseAtUri(uri)
176
+
if err != nil {
177
+
return nil, err
178
+
}
179
+
180
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
181
+
if err != nil {
182
+
return nil, err
183
+
}
184
+
185
+
// TODO: needs upsert treatment when we actually find the list
186
+
var list List
187
+
if err := b.db.FirstOrCreate(&list, map[string]any{
188
+
"author": r.ID,
189
+
"rkey": puri.Rkey,
190
+
}).Error; err != nil {
191
+
return nil, err
192
+
}
193
+
return &list, nil
194
+
}
195
+
196
+
func (b *PostgresBackend) postIDForUri(ctx context.Context, uri string) (uint, error) {
197
+
// getPostByUri implicitly fills the cache
198
+
p, err := b.postInfoForUri(ctx, uri)
199
+
if err != nil {
200
+
return 0, err
201
+
}
202
+
203
+
return p.ID, nil
204
+
}
205
+
206
+
func (b *PostgresBackend) postInfoForUri(ctx context.Context, uri string) (cachedPostInfo, error) {
207
+
v, ok := b.postInfoCache.Get(uri)
208
+
if ok {
209
+
return v, nil
210
+
}
211
+
212
+
// getPostByUri implicitly fills the cache
213
+
p, err := b.getOrCreatePostBare(ctx, uri)
214
+
if err != nil {
215
+
return cachedPostInfo{}, err
216
+
}
217
+
218
+
return cachedPostInfo{ID: p.ID, Author: p.Author}, nil
219
+
}
220
+
221
+
func (b *PostgresBackend) tryLoadPostInfo(ctx context.Context, uid uint, rkey string) (*Post, error) {
222
+
var p Post
223
+
q := "SELECT id, author FROM posts WHERE author = $1 AND rkey = $2"
224
+
if err := b.pgx.QueryRow(ctx, q, uid, rkey).Scan(&p.ID, &p.Author); err != nil {
225
+
if errors.Is(err, pgx.ErrNoRows) {
226
+
return nil, nil
227
+
}
228
+
return nil, err
229
+
}
230
+
231
+
return &p, nil
232
+
}
233
+
234
+
func (b *PostgresBackend) getOrCreatePostBare(ctx context.Context, uri string) (*Post, error) {
235
+
puri, err := util.ParseAtUri(uri)
236
+
if err != nil {
237
+
return nil, err
238
+
}
239
+
240
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
241
+
if err != nil {
242
+
return nil, err
243
+
}
244
+
245
+
post, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
246
+
if err != nil {
247
+
return nil, err
248
+
}
249
+
250
+
if post == nil {
251
+
post = &Post{
252
+
Rkey: puri.Rkey,
253
+
Author: r.ID,
254
+
NotFound: true,
255
+
}
256
+
257
+
err := b.pgx.QueryRow(ctx, "INSERT INTO posts (rkey, author, not_found) VALUES ($1, $2, $3) RETURNING id", puri.Rkey, r.ID, true).Scan(&post.ID)
258
+
if err != nil {
259
+
pgErr, ok := err.(*pgconn.PgError)
260
+
if !ok || pgErr.Code != "23505" {
261
+
return nil, err
262
+
}
263
+
264
+
out, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
265
+
if err != nil {
266
+
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
267
+
}
268
+
if out == nil {
269
+
return nil, fmt.Errorf("postgres is lying to us: %d %s", r.ID, puri.Rkey)
270
+
}
271
+
272
+
post = out
273
+
}
274
+
275
+
}
276
+
277
+
b.postInfoCache.Add(uri, cachedPostInfo{
278
+
ID: post.ID,
279
+
Author: post.Author,
280
+
})
281
+
282
+
return post, nil
283
+
}
284
+
285
+
func (b *PostgresBackend) GetPostByUri(ctx context.Context, uri string, fields string) (*Post, error) {
286
+
puri, err := util.ParseAtUri(uri)
287
+
if err != nil {
288
+
return nil, err
289
+
}
290
+
291
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
292
+
if err != nil {
293
+
return nil, err
294
+
}
295
+
296
+
q := "SELECT " + fields + " FROM posts WHERE author = ? AND rkey = ?"
297
+
298
+
var post Post
299
+
if err := b.db.Raw(q, r.ID, puri.Rkey).Scan(&post).Error; err != nil {
300
+
return nil, err
301
+
}
302
+
303
+
if post.ID == 0 {
304
+
post.Rkey = puri.Rkey
305
+
post.Author = r.ID
306
+
post.NotFound = true
307
+
308
+
if err := b.db.Session(&gorm.Session{
309
+
Logger: logger.Default.LogMode(logger.Silent),
310
+
}).Create(&post).Error; err != nil {
311
+
if !errors.Is(err, gorm.ErrDuplicatedKey) {
312
+
return nil, err
313
+
}
314
+
if err := b.db.Find(&post, "author = ? AND rkey = ?", r.ID, puri.Rkey).Error; err != nil {
315
+
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
316
+
}
317
+
}
318
+
319
+
}
320
+
321
+
b.postInfoCache.Add(uri, cachedPostInfo{
322
+
ID: post.ID,
323
+
Author: post.Author,
324
+
})
325
+
326
+
return &post, nil
327
+
}
328
+
329
+
func (b *PostgresBackend) revForRepo(rr *Repo) (string, error) {
330
+
lrev, ok := b.revCache.Get(rr.ID)
331
+
if ok {
332
+
return lrev, nil
333
+
}
334
+
335
+
var rev string
336
+
if err := b.pgx.QueryRow(context.TODO(), "SELECT COALESCE(rev, '') FROM sync_infos WHERE repo = $1", rr.ID).Scan(&rev); err != nil {
337
+
if errors.Is(err, pgx.ErrNoRows) {
338
+
return "", nil
339
+
}
340
+
return "", err
341
+
}
342
+
343
+
if rev != "" {
344
+
b.revCache.Add(rr.ID, rev)
345
+
}
346
+
return rev, nil
347
+
}
348
+
349
+
func (b *PostgresBackend) AddRelevantDid(did string) {
350
+
b.rdLk.Lock()
351
+
defer b.rdLk.Unlock()
352
+
b.relevantDids[did] = true
353
+
}
354
+
355
+
func (b *PostgresBackend) DidIsRelevant(did string) bool {
356
+
b.rdLk.Lock()
357
+
defer b.rdLk.Unlock()
358
+
return b.relevantDids[did]
359
+
}
360
+
361
+
func (b *PostgresBackend) anyRelevantIdents(idents ...string) bool {
362
+
for _, id := range idents {
363
+
if strings.HasPrefix(id, "did:") {
364
+
if b.DidIsRelevant(id) {
365
+
return true
366
+
}
367
+
} else if strings.HasPrefix(id, "at://") {
368
+
puri, err := syntax.ParseATURI(id)
369
+
if err != nil {
370
+
continue
371
+
}
372
+
373
+
if b.DidIsRelevant(puri.Authority().String()) {
374
+
return true
375
+
}
376
+
}
377
+
}
378
+
379
+
return false
380
+
}
381
+
382
+
func (b *PostgresBackend) GetRelevantDids() []string {
383
+
b.rdLk.Lock()
384
+
var out []string
385
+
for k := range b.relevantDids {
386
+
out = append(out, k)
387
+
}
388
+
b.rdLk.Unlock()
389
+
return out
390
+
}
391
+
392
+
func (b *PostgresBackend) GetRepoByID(ctx context.Context, id uint) (*models.Repo, error) {
393
+
var r models.Repo
394
+
if err := b.db.Find(&r, "id = ?", id).Error; err != nil {
395
+
return nil, err
396
+
}
397
+
398
+
return &r, nil
399
+
}
400
+
401
+
func (b *PostgresBackend) DidFromID(ctx context.Context, uid uint) (string, error) {
402
+
val, ok := b.didByIDCache.Get(uid)
403
+
if ok {
404
+
return val, nil
405
+
}
406
+
407
+
r, err := b.GetRepoByID(ctx, uid)
408
+
if err != nil {
409
+
return "", err
410
+
}
411
+
412
+
b.didByIDCache.Add(uid, r.Did)
413
+
return r.Did, nil
414
+
}
415
+
416
+
func (b *PostgresBackend) checkPostExists(ctx context.Context, repo *Repo, rkey string) (bool, error) {
417
+
var id uint
418
+
var notfound bool
419
+
if err := b.pgx.QueryRow(ctx, "SELECT id, not_found FROM posts WHERE author = $1 AND rkey = $2", repo.ID, rkey).Scan(&id, ¬found); err != nil {
420
+
if errors.Is(err, pgx.ErrNoRows) {
421
+
return false, nil
422
+
}
423
+
return false, err
424
+
}
425
+
426
+
if id != 0 && !notfound {
427
+
return true, nil
428
+
}
429
+
430
+
return false, nil
431
+
}
432
+
433
+
func (b *PostgresBackend) LoadRelevantDids() error {
434
+
ctx := context.TODO()
435
+
436
+
if err := b.ensureFollowsScraped(ctx, b.mydid); err != nil {
437
+
return fmt.Errorf("failed to scrape follows: %w", err)
438
+
}
439
+
440
+
r, err := b.GetOrCreateRepo(ctx, b.mydid)
441
+
if err != nil {
442
+
return err
443
+
}
444
+
445
+
var dids []string
446
+
if err := b.db.Raw("select did from follows left join repos on follows.subject = repos.id where follows.author = ?", r.ID).Scan(&dids).Error; err != nil {
447
+
return err
448
+
}
449
+
450
+
b.relevantDids[b.mydid] = true
451
+
for _, d := range dids {
452
+
fmt.Println("adding did: ", d)
453
+
b.relevantDids[d] = true
454
+
}
455
+
456
+
return nil
457
+
}
458
+
459
+
type SyncInfo struct {
460
+
Repo uint `gorm:"index"`
461
+
FollowsSynced bool
462
+
Rev string
463
+
}
464
+
465
+
func (b *PostgresBackend) ensureFollowsScraped(ctx context.Context, user string) error {
466
+
r, err := b.GetOrCreateRepo(ctx, user)
467
+
if err != nil {
468
+
return err
469
+
}
470
+
471
+
var si SyncInfo
472
+
if err := b.db.Find(&si, "repo = ?", r.ID).Error; err != nil {
473
+
return err
474
+
}
475
+
476
+
// not found
477
+
if si.Repo == 0 {
478
+
if err := b.db.Create(&SyncInfo{
479
+
Repo: r.ID,
480
+
}).Error; err != nil {
481
+
return err
482
+
}
483
+
}
484
+
485
+
if si.FollowsSynced {
486
+
return nil
487
+
}
488
+
489
+
var follows []Follow
490
+
var cursor string
491
+
for {
492
+
resp, err := atproto.RepoListRecords(ctx, b.client, "app.bsky.graph.follow", cursor, 100, b.mydid, false)
493
+
if err != nil {
494
+
return err
495
+
}
496
+
497
+
for _, rec := range resp.Records {
498
+
if fol, ok := rec.Value.Val.(*bsky.GraphFollow); ok {
499
+
fr, err := b.GetOrCreateRepo(ctx, fol.Subject)
500
+
if err != nil {
501
+
return err
502
+
}
503
+
504
+
puri, err := syntax.ParseATURI(rec.Uri)
505
+
if err != nil {
506
+
return err
507
+
}
508
+
509
+
follows = append(follows, Follow{
510
+
Created: time.Now(),
511
+
Indexed: time.Now(),
512
+
Rkey: puri.RecordKey().String(),
513
+
Author: r.ID,
514
+
Subject: fr.ID,
515
+
})
516
+
}
517
+
}
518
+
519
+
if resp.Cursor == nil || len(resp.Records) == 0 {
520
+
break
521
+
}
522
+
cursor = *resp.Cursor
523
+
}
524
+
525
+
if err := b.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(follows, 200).Error; err != nil {
526
+
return err
527
+
}
528
+
529
+
if err := b.db.Model(SyncInfo{}).Where("repo = ?", r.ID).Update("follows_synced", true).Error; err != nil {
530
+
return err
531
+
}
532
+
533
+
fmt.Println("Got follows: ", len(follows))
534
+
535
+
return nil
536
+
}
+1208
backend/events.go
+1208
backend/events.go
···
1
+
package backend
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"encoding/json"
7
+
"fmt"
8
+
"log/slog"
9
+
"strings"
10
+
"time"
11
+
12
+
"github.com/bluesky-social/indigo/api/atproto"
13
+
"github.com/bluesky-social/indigo/api/bsky"
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
lexutil "github.com/bluesky-social/indigo/lex/util"
16
+
"github.com/bluesky-social/indigo/repo"
17
+
jsmodels "github.com/bluesky-social/jetstream/pkg/models"
18
+
"github.com/ipfs/go-cid"
19
+
"github.com/jackc/pgx/v5/pgconn"
20
+
"github.com/prometheus/client_golang/prometheus"
21
+
"github.com/prometheus/client_golang/prometheus/promauto"
22
+
23
+
. "github.com/whyrusleeping/konbini/models"
24
+
)
25
+
26
+
var handleOpHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
27
+
Name: "handle_op_duration",
28
+
Help: "A histogram of op handling durations",
29
+
Buckets: prometheus.ExponentialBuckets(1, 2, 15),
30
+
}, []string{"op", "collection"})
31
+
32
+
func (b *PostgresBackend) HandleEvent(ctx context.Context, evt *atproto.SyncSubscribeRepos_Commit) error {
33
+
r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks))
34
+
if err != nil {
35
+
return fmt.Errorf("failed to read event repo: %w", err)
36
+
}
37
+
38
+
for _, op := range evt.Ops {
39
+
switch op.Action {
40
+
case "create":
41
+
c, rec, err := r.GetRecordBytes(ctx, op.Path)
42
+
if err != nil {
43
+
return err
44
+
}
45
+
if err := b.HandleCreate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
46
+
return fmt.Errorf("create record failed: %w", err)
47
+
}
48
+
case "update":
49
+
c, rec, err := r.GetRecordBytes(ctx, op.Path)
50
+
if err != nil {
51
+
return err
52
+
}
53
+
if err := b.HandleUpdate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
54
+
return fmt.Errorf("update record failed: %w", err)
55
+
}
56
+
case "delete":
57
+
if err := b.HandleDelete(ctx, evt.Repo, evt.Rev, op.Path); err != nil {
58
+
return fmt.Errorf("delete record failed: %w", err)
59
+
}
60
+
}
61
+
}
62
+
63
+
// TODO: sync with the Since field to make sure we don't miss events we care about
64
+
/*
65
+
if err := bf.Store.UpdateRev(ctx, evt.Repo, evt.Rev); err != nil {
66
+
return fmt.Errorf("failed to update rev: %w", err)
67
+
}
68
+
*/
69
+
70
+
return nil
71
+
}
72
+
73
+
func cborBytesFromEvent(evt *jsmodels.Event) ([]byte, error) {
74
+
val, err := lexutil.NewFromType(evt.Commit.Collection)
75
+
if err != nil {
76
+
return nil, fmt.Errorf("failed to load event record type: %w", err)
77
+
}
78
+
79
+
if err := json.Unmarshal(evt.Commit.Record, val); err != nil {
80
+
return nil, err
81
+
}
82
+
83
+
cval, ok := val.(lexutil.CBOR)
84
+
if !ok {
85
+
return nil, fmt.Errorf("decoded type was not cbor marshalable")
86
+
}
87
+
88
+
buf := new(bytes.Buffer)
89
+
if err := cval.MarshalCBOR(buf); err != nil {
90
+
return nil, fmt.Errorf("failed to marshal event to cbor: %w", err)
91
+
}
92
+
93
+
rec := buf.Bytes()
94
+
return rec, nil
95
+
}
96
+
97
+
func (b *PostgresBackend) HandleEventJetstream(ctx context.Context, evt *jsmodels.Event) error {
98
+
99
+
path := evt.Commit.Collection + "/" + evt.Commit.RKey
100
+
switch evt.Commit.Operation {
101
+
case jsmodels.CommitOperationCreate:
102
+
rec, err := cborBytesFromEvent(evt)
103
+
if err != nil {
104
+
return err
105
+
}
106
+
107
+
c, err := cid.Decode(evt.Commit.CID)
108
+
if err != nil {
109
+
return err
110
+
}
111
+
112
+
if err := b.HandleCreate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
113
+
return fmt.Errorf("create record failed: %w", err)
114
+
}
115
+
case jsmodels.CommitOperationUpdate:
116
+
rec, err := cborBytesFromEvent(evt)
117
+
if err != nil {
118
+
return err
119
+
}
120
+
121
+
c, err := cid.Decode(evt.Commit.CID)
122
+
if err != nil {
123
+
return err
124
+
}
125
+
126
+
if err := b.HandleUpdate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
127
+
return fmt.Errorf("update record failed: %w", err)
128
+
}
129
+
case jsmodels.CommitOperationDelete:
130
+
if err := b.HandleDelete(ctx, evt.Did, evt.Commit.Rev, path); err != nil {
131
+
return fmt.Errorf("delete record failed: %w", err)
132
+
}
133
+
}
134
+
135
+
return nil
136
+
}
137
+
138
+
func (b *PostgresBackend) HandleCreate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
139
+
start := time.Now()
140
+
141
+
rr, err := b.GetOrCreateRepo(ctx, repo)
142
+
if err != nil {
143
+
return fmt.Errorf("get user failed: %w", err)
144
+
}
145
+
146
+
lrev, err := b.revForRepo(rr)
147
+
if err != nil {
148
+
return err
149
+
}
150
+
if lrev != "" {
151
+
if rev < lrev {
152
+
slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
153
+
return nil
154
+
}
155
+
}
156
+
157
+
parts := strings.Split(path, "/")
158
+
if len(parts) != 2 {
159
+
return fmt.Errorf("invalid path in HandleCreate: %q", path)
160
+
}
161
+
col := parts[0]
162
+
rkey := parts[1]
163
+
164
+
defer func() {
165
+
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
166
+
}()
167
+
168
+
if rkey == "" {
169
+
fmt.Printf("messed up path: %q\n", rkey)
170
+
}
171
+
172
+
switch col {
173
+
case "app.bsky.feed.post":
174
+
if err := b.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
175
+
return err
176
+
}
177
+
case "app.bsky.feed.like":
178
+
if err := b.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
179
+
return err
180
+
}
181
+
case "app.bsky.feed.repost":
182
+
if err := b.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
183
+
return err
184
+
}
185
+
case "app.bsky.graph.follow":
186
+
if err := b.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
187
+
return err
188
+
}
189
+
case "app.bsky.graph.block":
190
+
if err := b.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
191
+
return err
192
+
}
193
+
case "app.bsky.graph.list":
194
+
if err := b.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
195
+
return err
196
+
}
197
+
case "app.bsky.graph.listitem":
198
+
if err := b.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
199
+
return err
200
+
}
201
+
case "app.bsky.graph.listblock":
202
+
if err := b.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
203
+
return err
204
+
}
205
+
case "app.bsky.actor.profile":
206
+
if err := b.HandleCreateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
207
+
return err
208
+
}
209
+
case "app.bsky.feed.generator":
210
+
if err := b.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
211
+
return err
212
+
}
213
+
case "app.bsky.feed.threadgate":
214
+
if err := b.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
215
+
return err
216
+
}
217
+
case "chat.bsky.actor.declaration":
218
+
if err := b.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
219
+
return err
220
+
}
221
+
case "app.bsky.feed.postgate":
222
+
if err := b.HandleCreatePostGate(ctx, rr, rkey, *rec, *cid); err != nil {
223
+
return err
224
+
}
225
+
case "app.bsky.graph.starterpack":
226
+
if err := b.HandleCreateStarterPack(ctx, rr, rkey, *rec, *cid); err != nil {
227
+
return err
228
+
}
229
+
default:
230
+
slog.Debug("unrecognized record type", "repo", repo, "path", path, "rev", rev)
231
+
}
232
+
233
+
b.revCache.Add(rr.ID, rev)
234
+
return nil
235
+
}
236
+
237
+
func (b *PostgresBackend) HandleCreatePost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
238
+
exists, err := b.checkPostExists(ctx, repo, rkey)
239
+
if err != nil {
240
+
return err
241
+
}
242
+
243
+
// still technically a race condition if two creates for the same post happen concurrently... probably fine
244
+
if exists {
245
+
return nil
246
+
}
247
+
248
+
var rec bsky.FeedPost
249
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
250
+
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
251
+
slog.Warn("skipping post with malformed data", "uri", uri, "error", err)
252
+
return nil // Skip this post rather than failing the entire event
253
+
}
254
+
255
+
reldids := []string{repo.Did}
256
+
// care about a post if its in a thread of a user we are interested in
257
+
if rec.Reply != nil && rec.Reply.Parent != nil && rec.Reply.Root != nil {
258
+
reldids = append(reldids, rec.Reply.Parent.Uri, rec.Reply.Root.Uri)
259
+
}
260
+
// TODO: maybe also care if its mentioning a user we care about or quoting a user we care about?
261
+
if !b.anyRelevantIdents(reldids...) {
262
+
return nil
263
+
}
264
+
265
+
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
266
+
slog.Warn("adding post", "uri", uri)
267
+
268
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
269
+
if err != nil {
270
+
return fmt.Errorf("invalid timestamp: %w", err)
271
+
}
272
+
273
+
p := Post{
274
+
Created: created.Time(),
275
+
Indexed: time.Now(),
276
+
Author: repo.ID,
277
+
Rkey: rkey,
278
+
Raw: recb,
279
+
Cid: cc.String(),
280
+
}
281
+
282
+
if rec.Reply != nil && rec.Reply.Parent != nil {
283
+
if rec.Reply.Root == nil {
284
+
return fmt.Errorf("post reply had nil root")
285
+
}
286
+
287
+
pinfo, err := b.postInfoForUri(ctx, rec.Reply.Parent.Uri)
288
+
if err != nil {
289
+
return fmt.Errorf("getting reply parent: %w", err)
290
+
}
291
+
292
+
p.ReplyTo = pinfo.ID
293
+
p.ReplyToUsr = pinfo.Author
294
+
295
+
thread, err := b.postIDForUri(ctx, rec.Reply.Root.Uri)
296
+
if err != nil {
297
+
return fmt.Errorf("getting thread root: %w", err)
298
+
}
299
+
300
+
p.InThread = thread
301
+
302
+
r, err := b.GetOrCreateRepo(ctx, b.mydid)
303
+
if err != nil {
304
+
return err
305
+
}
306
+
307
+
if p.ReplyToUsr == r.ID {
308
+
if err := b.AddNotification(ctx, r.ID, p.Author, uri, cc, NotifKindReply); err != nil {
309
+
slog.Warn("failed to create notification", "uri", uri, "error", err)
310
+
}
311
+
}
312
+
}
313
+
314
+
if rec.Embed != nil {
315
+
var rpref string
316
+
if rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {
317
+
rpref = rec.Embed.EmbedRecord.Record.Uri
318
+
}
319
+
if rec.Embed.EmbedRecordWithMedia != nil &&
320
+
rec.Embed.EmbedRecordWithMedia.Record != nil &&
321
+
rec.Embed.EmbedRecordWithMedia.Record.Record != nil {
322
+
rpref = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri
323
+
}
324
+
325
+
if rpref != "" && strings.Contains(rpref, "app.bsky.feed.post") {
326
+
rp, err := b.postIDForUri(ctx, rpref)
327
+
if err != nil {
328
+
return fmt.Errorf("getting quote subject: %w", err)
329
+
}
330
+
331
+
p.Reposting = rp
332
+
}
333
+
}
334
+
335
+
if err := b.doPostCreate(ctx, &p); err != nil {
336
+
return err
337
+
}
338
+
339
+
// Check for mentions and create notifications
340
+
if rec.Facets != nil {
341
+
for _, facet := range rec.Facets {
342
+
for _, feature := range facet.Features {
343
+
if feature.RichtextFacet_Mention != nil {
344
+
mentionDid := feature.RichtextFacet_Mention.Did
345
+
// This is a mention
346
+
mentionedRepo, err := b.GetOrCreateRepo(ctx, mentionDid)
347
+
if err != nil {
348
+
slog.Warn("failed to get repo for mention", "did", mentionDid, "error", err)
349
+
continue
350
+
}
351
+
352
+
// Create notification if the mentioned user is the current user
353
+
if mentionedRepo.ID == b.myrepo.ID {
354
+
if err := b.AddNotification(ctx, b.myrepo.ID, p.Author, uri, cc, NotifKindMention); err != nil {
355
+
slog.Warn("failed to create mention notification", "uri", uri, "error", err)
356
+
}
357
+
}
358
+
}
359
+
}
360
+
}
361
+
}
362
+
363
+
b.postInfoCache.Add(uri, cachedPostInfo{
364
+
ID: p.ID,
365
+
Author: p.Author,
366
+
})
367
+
368
+
return nil
369
+
}
370
+
371
+
func (b *PostgresBackend) doPostCreate(ctx context.Context, p *Post) error {
372
+
/*
373
+
if err := b.db.Clauses(clause.OnConflict{
374
+
Columns: []clause.Column{{Name: "author"}, {Name: "rkey"}},
375
+
DoUpdates: clause.AssignmentColumns([]string{"cid", "not_found", "raw", "created", "indexed"}),
376
+
}).Create(p).Error; err != nil {
377
+
return err
378
+
}
379
+
*/
380
+
381
+
query := `
382
+
INSERT INTO posts (author, rkey, cid, not_found, raw, created, indexed, reposting, reply_to, reply_to_usr, in_thread)
383
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
384
+
ON CONFLICT (author, rkey)
385
+
DO UPDATE SET
386
+
cid = $3,
387
+
not_found = $4,
388
+
raw = $5,
389
+
created = $6,
390
+
indexed = $7,
391
+
reposting = $8,
392
+
reply_to = $9,
393
+
reply_to_usr = $10,
394
+
in_thread = $11
395
+
RETURNING id
396
+
`
397
+
398
+
// Execute the query with parameters from the Post struct
399
+
if err := b.pgx.QueryRow(
400
+
ctx,
401
+
query,
402
+
p.Author,
403
+
p.Rkey,
404
+
p.Cid,
405
+
p.NotFound,
406
+
p.Raw,
407
+
p.Created,
408
+
p.Indexed,
409
+
p.Reposting,
410
+
p.ReplyTo,
411
+
p.ReplyToUsr,
412
+
p.InThread,
413
+
).Scan(&p.ID); err != nil {
414
+
return err
415
+
}
416
+
417
+
return nil
418
+
}
419
+
420
+
func (b *PostgresBackend) HandleCreateLike(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
421
+
var rec bsky.FeedLike
422
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
423
+
return err
424
+
}
425
+
426
+
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
427
+
return nil
428
+
}
429
+
430
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
431
+
if err != nil {
432
+
return fmt.Errorf("invalid timestamp: %w", err)
433
+
}
434
+
435
+
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
436
+
if err != nil {
437
+
return fmt.Errorf("getting like subject: %w", err)
438
+
}
439
+
440
+
if _, err := b.pgx.Exec(ctx, `INSERT INTO "likes" ("created","indexed","author","rkey","subject","cid") VALUES ($1, $2, $3, $4, $5, $6)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID, cc.String()); err != nil {
441
+
pgErr, ok := err.(*pgconn.PgError)
442
+
if ok && pgErr.Code == "23505" {
443
+
return nil
444
+
}
445
+
return err
446
+
}
447
+
448
+
// Create notification if the liked post belongs to the current user
449
+
if pinfo.Author == b.myrepo.ID {
450
+
uri := fmt.Sprintf("at://%s/app.bsky.feed.like/%s", repo.Did, rkey)
451
+
if err := b.AddNotification(ctx, b.myrepo.ID, repo.ID, uri, cc, NotifKindLike); err != nil {
452
+
slog.Warn("failed to create like notification", "uri", uri, "error", err)
453
+
}
454
+
}
455
+
456
+
return nil
457
+
}
458
+
459
+
func (b *PostgresBackend) HandleCreateRepost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
460
+
var rec bsky.FeedRepost
461
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
462
+
return err
463
+
}
464
+
465
+
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
466
+
return nil
467
+
}
468
+
469
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
470
+
if err != nil {
471
+
return fmt.Errorf("invalid timestamp: %w", err)
472
+
}
473
+
474
+
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
475
+
if err != nil {
476
+
return fmt.Errorf("getting repost subject: %w", err)
477
+
}
478
+
479
+
if _, err := b.pgx.Exec(ctx, `INSERT INTO "reposts" ("created","indexed","author","rkey","subject") VALUES ($1, $2, $3, $4, $5)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID); err != nil {
480
+
pgErr, ok := err.(*pgconn.PgError)
481
+
if ok && pgErr.Code == "23505" {
482
+
return nil
483
+
}
484
+
return err
485
+
}
486
+
487
+
// Create notification if the reposted post belongs to the current user
488
+
if pinfo.Author == b.myrepo.ID {
489
+
uri := fmt.Sprintf("at://%s/app.bsky.feed.repost/%s", repo.Did, rkey)
490
+
if err := b.AddNotification(ctx, b.myrepo.ID, repo.ID, uri, cc, NotifKindRepost); err != nil {
491
+
slog.Warn("failed to create repost notification", "uri", uri, "error", err)
492
+
}
493
+
}
494
+
495
+
return nil
496
+
}
497
+
498
+
func (b *PostgresBackend) HandleCreateFollow(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
499
+
var rec bsky.GraphFollow
500
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
501
+
return err
502
+
}
503
+
504
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
505
+
return nil
506
+
}
507
+
508
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
509
+
if err != nil {
510
+
return fmt.Errorf("invalid timestamp: %w", err)
511
+
}
512
+
513
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
514
+
if err != nil {
515
+
return err
516
+
}
517
+
518
+
if _, err := b.pgx.Exec(ctx, "INSERT INTO follows (created, indexed, author, rkey, subject) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", created.Time(), time.Now(), repo.ID, rkey, subj.ID); err != nil {
519
+
return err
520
+
}
521
+
522
+
return nil
523
+
}
524
+
525
+
func (b *PostgresBackend) HandleCreateBlock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
526
+
var rec bsky.GraphBlock
527
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
528
+
return err
529
+
}
530
+
531
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
532
+
return nil
533
+
}
534
+
535
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
536
+
if err != nil {
537
+
return fmt.Errorf("invalid timestamp: %w", err)
538
+
}
539
+
540
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
541
+
if err != nil {
542
+
return err
543
+
}
544
+
545
+
if err := b.db.Create(&Block{
546
+
Created: created.Time(),
547
+
Indexed: time.Now(),
548
+
Author: repo.ID,
549
+
Rkey: rkey,
550
+
Subject: subj.ID,
551
+
}).Error; err != nil {
552
+
return err
553
+
}
554
+
555
+
return nil
556
+
}
557
+
558
+
func (b *PostgresBackend) HandleCreateList(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
559
+
var rec bsky.GraphList
560
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
561
+
return err
562
+
}
563
+
564
+
if !b.anyRelevantIdents(repo.Did) {
565
+
return nil
566
+
}
567
+
568
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
569
+
if err != nil {
570
+
return fmt.Errorf("invalid timestamp: %w", err)
571
+
}
572
+
573
+
if err := b.db.Create(&List{
574
+
Created: created.Time(),
575
+
Indexed: time.Now(),
576
+
Author: repo.ID,
577
+
Rkey: rkey,
578
+
Raw: recb,
579
+
}).Error; err != nil {
580
+
return err
581
+
}
582
+
583
+
return nil
584
+
}
585
+
586
+
func (b *PostgresBackend) HandleCreateListitem(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
587
+
var rec bsky.GraphListitem
588
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
589
+
return err
590
+
}
591
+
if !b.anyRelevantIdents(repo.Did) {
592
+
return nil
593
+
}
594
+
595
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
596
+
if err != nil {
597
+
return fmt.Errorf("invalid timestamp: %w", err)
598
+
}
599
+
600
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
601
+
if err != nil {
602
+
return err
603
+
}
604
+
605
+
list, err := b.GetOrCreateList(ctx, rec.List)
606
+
if err != nil {
607
+
return err
608
+
}
609
+
610
+
if err := b.db.Create(&ListItem{
611
+
Created: created.Time(),
612
+
Indexed: time.Now(),
613
+
Author: repo.ID,
614
+
Rkey: rkey,
615
+
Subject: subj.ID,
616
+
List: list.ID,
617
+
}).Error; err != nil {
618
+
return err
619
+
}
620
+
621
+
return nil
622
+
}
623
+
624
+
func (b *PostgresBackend) HandleCreateListblock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
625
+
var rec bsky.GraphListblock
626
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
627
+
return err
628
+
}
629
+
630
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
631
+
return nil
632
+
}
633
+
634
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
635
+
if err != nil {
636
+
return fmt.Errorf("invalid timestamp: %w", err)
637
+
}
638
+
639
+
list, err := b.GetOrCreateList(ctx, rec.Subject)
640
+
if err != nil {
641
+
return err
642
+
}
643
+
644
+
if err := b.db.Create(&ListBlock{
645
+
Created: created.Time(),
646
+
Indexed: time.Now(),
647
+
Author: repo.ID,
648
+
Rkey: rkey,
649
+
List: list.ID,
650
+
}).Error; err != nil {
651
+
return err
652
+
}
653
+
654
+
return nil
655
+
}
656
+
657
+
func (b *PostgresBackend) HandleCreateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
658
+
if !b.anyRelevantIdents(repo.Did) {
659
+
return nil
660
+
}
661
+
662
+
if err := b.db.Create(&Profile{
663
+
//Created: created.Time(),
664
+
Indexed: time.Now(),
665
+
Repo: repo.ID,
666
+
Raw: recb,
667
+
Rev: rev,
668
+
}).Error; err != nil {
669
+
return err
670
+
}
671
+
672
+
return nil
673
+
}
674
+
675
+
func (b *PostgresBackend) HandleUpdateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
676
+
if !b.anyRelevantIdents(repo.Did) {
677
+
return nil
678
+
}
679
+
680
+
if err := b.db.Create(&Profile{
681
+
Indexed: time.Now(),
682
+
Repo: repo.ID,
683
+
Raw: recb,
684
+
Rev: rev,
685
+
}).Error; err != nil {
686
+
return err
687
+
}
688
+
689
+
return nil
690
+
}
691
+
692
+
func (b *PostgresBackend) HandleCreateFeedGenerator(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
693
+
if !b.anyRelevantIdents(repo.Did) {
694
+
return nil
695
+
}
696
+
697
+
var rec bsky.FeedGenerator
698
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
699
+
return err
700
+
}
701
+
702
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
703
+
if err != nil {
704
+
return fmt.Errorf("invalid timestamp: %w", err)
705
+
}
706
+
707
+
if err := b.db.Create(&FeedGenerator{
708
+
Created: created.Time(),
709
+
Indexed: time.Now(),
710
+
Author: repo.ID,
711
+
Rkey: rkey,
712
+
Did: rec.Did,
713
+
Raw: recb,
714
+
}).Error; err != nil {
715
+
return err
716
+
}
717
+
718
+
return nil
719
+
}
720
+
721
+
func (b *PostgresBackend) HandleCreateThreadgate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
722
+
if !b.anyRelevantIdents(repo.Did) {
723
+
return nil
724
+
}
725
+
var rec bsky.FeedThreadgate
726
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
727
+
return err
728
+
}
729
+
730
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
731
+
if err != nil {
732
+
return fmt.Errorf("invalid timestamp: %w", err)
733
+
}
734
+
735
+
pid, err := b.postIDForUri(ctx, rec.Post)
736
+
if err != nil {
737
+
return err
738
+
}
739
+
740
+
if err := b.db.Create(&ThreadGate{
741
+
Created: created.Time(),
742
+
Indexed: time.Now(),
743
+
Author: repo.ID,
744
+
Rkey: rkey,
745
+
Post: pid,
746
+
}).Error; err != nil {
747
+
return err
748
+
}
749
+
750
+
return nil
751
+
}
752
+
753
+
func (b *PostgresBackend) HandleCreateChatDeclaration(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
754
+
// TODO: maybe track these?
755
+
return nil
756
+
}
757
+
758
+
func (b *PostgresBackend) HandleCreatePostGate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
759
+
if !b.anyRelevantIdents(repo.Did) {
760
+
return nil
761
+
}
762
+
var rec bsky.FeedPostgate
763
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
764
+
return err
765
+
}
766
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
767
+
if err != nil {
768
+
return fmt.Errorf("invalid timestamp: %w", err)
769
+
}
770
+
771
+
refPost, err := b.postInfoForUri(ctx, rec.Post)
772
+
if err != nil {
773
+
return err
774
+
}
775
+
776
+
if err := b.db.Create(&PostGate{
777
+
Created: created.Time(),
778
+
Indexed: time.Now(),
779
+
Author: repo.ID,
780
+
Rkey: rkey,
781
+
Subject: refPost.ID,
782
+
Raw: recb,
783
+
}).Error; err != nil {
784
+
return err
785
+
}
786
+
787
+
return nil
788
+
}
789
+
790
+
func (b *PostgresBackend) HandleCreateStarterPack(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
791
+
if !b.anyRelevantIdents(repo.Did) {
792
+
return nil
793
+
}
794
+
var rec bsky.GraphStarterpack
795
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
796
+
return err
797
+
}
798
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
799
+
if err != nil {
800
+
return fmt.Errorf("invalid timestamp: %w", err)
801
+
}
802
+
803
+
list, err := b.GetOrCreateList(ctx, rec.List)
804
+
if err != nil {
805
+
return err
806
+
}
807
+
808
+
if err := b.db.Create(&StarterPack{
809
+
Created: created.Time(),
810
+
Indexed: time.Now(),
811
+
Author: repo.ID,
812
+
Rkey: rkey,
813
+
Raw: recb,
814
+
List: list.ID,
815
+
}).Error; err != nil {
816
+
return err
817
+
}
818
+
819
+
return nil
820
+
}
821
+
822
+
func (b *PostgresBackend) HandleUpdate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
823
+
start := time.Now()
824
+
825
+
rr, err := b.GetOrCreateRepo(ctx, repo)
826
+
if err != nil {
827
+
return fmt.Errorf("get user failed: %w", err)
828
+
}
829
+
830
+
lrev, err := b.revForRepo(rr)
831
+
if err != nil {
832
+
return err
833
+
}
834
+
if lrev != "" {
835
+
if rev < lrev {
836
+
//slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
837
+
return nil
838
+
}
839
+
}
840
+
841
+
parts := strings.Split(path, "/")
842
+
if len(parts) != 2 {
843
+
return fmt.Errorf("invalid path in HandleCreate: %q", path)
844
+
}
845
+
col := parts[0]
846
+
rkey := parts[1]
847
+
848
+
defer func() {
849
+
handleOpHist.WithLabelValues("update", col).Observe(float64(time.Since(start).Milliseconds()))
850
+
}()
851
+
852
+
if rkey == "" {
853
+
fmt.Printf("messed up path: %q\n", rkey)
854
+
}
855
+
856
+
switch col {
857
+
/*
858
+
case "app.bsky.feed.post":
859
+
if err := s.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
860
+
return err
861
+
}
862
+
case "app.bsky.feed.like":
863
+
if err := s.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
864
+
return err
865
+
}
866
+
case "app.bsky.feed.repost":
867
+
if err := s.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
868
+
return err
869
+
}
870
+
case "app.bsky.graph.follow":
871
+
if err := s.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
872
+
return err
873
+
}
874
+
case "app.bsky.graph.block":
875
+
if err := s.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
876
+
return err
877
+
}
878
+
case "app.bsky.graph.list":
879
+
if err := s.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
880
+
return err
881
+
}
882
+
case "app.bsky.graph.listitem":
883
+
if err := s.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
884
+
return err
885
+
}
886
+
case "app.bsky.graph.listblock":
887
+
if err := s.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
888
+
return err
889
+
}
890
+
*/
891
+
case "app.bsky.actor.profile":
892
+
if err := b.HandleUpdateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
893
+
return err
894
+
}
895
+
/*
896
+
case "app.bsky.feed.generator":
897
+
if err := s.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
898
+
return err
899
+
}
900
+
case "app.bsky.feed.threadgate":
901
+
if err := s.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
902
+
return err
903
+
}
904
+
case "chat.bsky.actor.declaration":
905
+
if err := s.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
906
+
return err
907
+
}
908
+
*/
909
+
default:
910
+
slog.Debug("unrecognized record type in update", "repo", repo, "path", path, "rev", rev)
911
+
}
912
+
913
+
return nil
914
+
}
915
+
916
+
func (b *PostgresBackend) HandleDelete(ctx context.Context, repo string, rev string, path string) error {
917
+
start := time.Now()
918
+
919
+
rr, err := b.GetOrCreateRepo(ctx, repo)
920
+
if err != nil {
921
+
return fmt.Errorf("get user failed: %w", err)
922
+
}
923
+
924
+
lrev, ok := b.revCache.Get(rr.ID)
925
+
if ok {
926
+
if rev < lrev {
927
+
//slog.Info("skipping old rev delete", "did", rr.Did, "rev", rev, "oldrev", lrev)
928
+
return nil
929
+
}
930
+
}
931
+
932
+
parts := strings.Split(path, "/")
933
+
if len(parts) != 2 {
934
+
return fmt.Errorf("invalid path in HandleDelete: %q", path)
935
+
}
936
+
col := parts[0]
937
+
rkey := parts[1]
938
+
939
+
defer func() {
940
+
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
941
+
}()
942
+
943
+
switch col {
944
+
case "app.bsky.feed.post":
945
+
if err := b.HandleDeletePost(ctx, rr, rkey); err != nil {
946
+
return err
947
+
}
948
+
case "app.bsky.feed.like":
949
+
if err := b.HandleDeleteLike(ctx, rr, rkey); err != nil {
950
+
return err
951
+
}
952
+
case "app.bsky.feed.repost":
953
+
if err := b.HandleDeleteRepost(ctx, rr, rkey); err != nil {
954
+
return err
955
+
}
956
+
case "app.bsky.graph.follow":
957
+
if err := b.HandleDeleteFollow(ctx, rr, rkey); err != nil {
958
+
return err
959
+
}
960
+
case "app.bsky.graph.block":
961
+
if err := b.HandleDeleteBlock(ctx, rr, rkey); err != nil {
962
+
return err
963
+
}
964
+
case "app.bsky.graph.list":
965
+
if err := b.HandleDeleteList(ctx, rr, rkey); err != nil {
966
+
return err
967
+
}
968
+
case "app.bsky.graph.listitem":
969
+
if err := b.HandleDeleteListitem(ctx, rr, rkey); err != nil {
970
+
return err
971
+
}
972
+
case "app.bsky.graph.listblock":
973
+
if err := b.HandleDeleteListblock(ctx, rr, rkey); err != nil {
974
+
return err
975
+
}
976
+
case "app.bsky.actor.profile":
977
+
if err := b.HandleDeleteProfile(ctx, rr, rkey); err != nil {
978
+
return err
979
+
}
980
+
case "app.bsky.feed.generator":
981
+
if err := b.HandleDeleteFeedGenerator(ctx, rr, rkey); err != nil {
982
+
return err
983
+
}
984
+
case "app.bsky.feed.threadgate":
985
+
if err := b.HandleDeleteThreadgate(ctx, rr, rkey); err != nil {
986
+
return err
987
+
}
988
+
default:
989
+
slog.Warn("delete unrecognized record type", "repo", repo, "path", path, "rev", rev)
990
+
}
991
+
992
+
b.revCache.Add(rr.ID, rev)
993
+
return nil
994
+
}
995
+
996
+
func (b *PostgresBackend) HandleDeletePost(ctx context.Context, repo *Repo, rkey string) error {
997
+
var p Post
998
+
if err := b.db.Find(&p, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
999
+
return err
1000
+
}
1001
+
1002
+
if p.ID == 0 {
1003
+
//slog.Warn("delete of unknown post record", "repo", repo.Did, "rkey", rkey)
1004
+
return nil
1005
+
}
1006
+
1007
+
if err := b.db.Delete(&Post{}, p.ID).Error; err != nil {
1008
+
return err
1009
+
}
1010
+
1011
+
return nil
1012
+
}
1013
+
1014
+
func (b *PostgresBackend) HandleDeleteLike(ctx context.Context, repo *Repo, rkey string) error {
1015
+
var like Like
1016
+
if err := b.db.Find(&like, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1017
+
return err
1018
+
}
1019
+
1020
+
if like.ID == 0 {
1021
+
//slog.Warn("delete of missing like", "repo", repo.Did, "rkey", rkey)
1022
+
return nil
1023
+
}
1024
+
1025
+
if err := b.db.Exec("DELETE FROM likes WHERE id = ?", like.ID).Error; err != nil {
1026
+
return err
1027
+
}
1028
+
1029
+
return nil
1030
+
}
1031
+
1032
+
func (b *PostgresBackend) HandleDeleteRepost(ctx context.Context, repo *Repo, rkey string) error {
1033
+
var repost Repost
1034
+
if err := b.db.Find(&repost, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1035
+
return err
1036
+
}
1037
+
1038
+
if repost.ID == 0 {
1039
+
//return fmt.Errorf("delete of missing repost: %s %s", repo.Did, rkey)
1040
+
return nil
1041
+
}
1042
+
1043
+
if err := b.db.Exec("DELETE FROM reposts WHERE id = ?", repost.ID).Error; err != nil {
1044
+
return err
1045
+
}
1046
+
1047
+
return nil
1048
+
}
1049
+
1050
+
func (b *PostgresBackend) HandleDeleteFollow(ctx context.Context, repo *Repo, rkey string) error {
1051
+
var follow Follow
1052
+
if err := b.db.Find(&follow, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1053
+
return err
1054
+
}
1055
+
1056
+
if follow.ID == 0 {
1057
+
//slog.Warn("delete of missing follow", "repo", repo.Did, "rkey", rkey)
1058
+
return nil
1059
+
}
1060
+
1061
+
if err := b.db.Exec("DELETE FROM follows WHERE id = ?", follow.ID).Error; err != nil {
1062
+
return err
1063
+
}
1064
+
1065
+
return nil
1066
+
}
1067
+
1068
+
func (b *PostgresBackend) HandleDeleteBlock(ctx context.Context, repo *Repo, rkey string) error {
1069
+
var block Block
1070
+
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1071
+
return err
1072
+
}
1073
+
1074
+
if block.ID == 0 {
1075
+
//slog.Warn("delete of missing block", "repo", repo.Did, "rkey", rkey)
1076
+
return nil
1077
+
}
1078
+
1079
+
if err := b.db.Exec("DELETE FROM blocks WHERE id = ?", block.ID).Error; err != nil {
1080
+
return err
1081
+
}
1082
+
1083
+
return nil
1084
+
}
1085
+
1086
+
func (b *PostgresBackend) HandleDeleteList(ctx context.Context, repo *Repo, rkey string) error {
1087
+
var list List
1088
+
if err := b.db.Find(&list, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1089
+
return err
1090
+
}
1091
+
1092
+
if list.ID == 0 {
1093
+
return nil
1094
+
//return fmt.Errorf("delete of missing list: %s %s", repo.Did, rkey)
1095
+
}
1096
+
1097
+
if err := b.db.Exec("DELETE FROM lists WHERE id = ?", list.ID).Error; err != nil {
1098
+
return err
1099
+
}
1100
+
1101
+
return nil
1102
+
}
1103
+
1104
+
func (b *PostgresBackend) HandleDeleteListitem(ctx context.Context, repo *Repo, rkey string) error {
1105
+
var item ListItem
1106
+
if err := b.db.Find(&item, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1107
+
return err
1108
+
}
1109
+
1110
+
if item.ID == 0 {
1111
+
return nil
1112
+
//return fmt.Errorf("delete of missing listitem: %s %s", repo.Did, rkey)
1113
+
}
1114
+
1115
+
if err := b.db.Exec("DELETE FROM list_items WHERE id = ?", item.ID).Error; err != nil {
1116
+
return err
1117
+
}
1118
+
1119
+
return nil
1120
+
}
1121
+
1122
+
func (b *PostgresBackend) HandleDeleteListblock(ctx context.Context, repo *Repo, rkey string) error {
1123
+
var block ListBlock
1124
+
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1125
+
return err
1126
+
}
1127
+
1128
+
if block.ID == 0 {
1129
+
return nil
1130
+
//return fmt.Errorf("delete of missing listblock: %s %s", repo.Did, rkey)
1131
+
}
1132
+
1133
+
if err := b.db.Exec("DELETE FROM list_blocks WHERE id = ?", block.ID).Error; err != nil {
1134
+
return err
1135
+
}
1136
+
1137
+
return nil
1138
+
}
1139
+
1140
+
func (b *PostgresBackend) HandleDeleteFeedGenerator(ctx context.Context, repo *Repo, rkey string) error {
1141
+
var feedgen FeedGenerator
1142
+
if err := b.db.Find(&feedgen, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1143
+
return err
1144
+
}
1145
+
1146
+
if feedgen.ID == 0 {
1147
+
return nil
1148
+
//return fmt.Errorf("delete of missing feedgen: %s %s", repo.Did, rkey)
1149
+
}
1150
+
1151
+
if err := b.db.Exec("DELETE FROM feed_generators WHERE id = ?", feedgen.ID).Error; err != nil {
1152
+
return err
1153
+
}
1154
+
1155
+
return nil
1156
+
}
1157
+
1158
+
func (b *PostgresBackend) HandleDeleteThreadgate(ctx context.Context, repo *Repo, rkey string) error {
1159
+
var threadgate ThreadGate
1160
+
if err := b.db.Find(&threadgate, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1161
+
return err
1162
+
}
1163
+
1164
+
if threadgate.ID == 0 {
1165
+
return nil
1166
+
//return fmt.Errorf("delete of missing threadgate: %s %s", repo.Did, rkey)
1167
+
}
1168
+
1169
+
if err := b.db.Exec("DELETE FROM thread_gates WHERE id = ?", threadgate.ID).Error; err != nil {
1170
+
return err
1171
+
}
1172
+
1173
+
return nil
1174
+
}
1175
+
1176
+
func (b *PostgresBackend) HandleDeleteProfile(ctx context.Context, repo *Repo, rkey string) error {
1177
+
var profile Profile
1178
+
if err := b.db.Find(&profile, "repo = ?", repo.ID).Error; err != nil {
1179
+
return err
1180
+
}
1181
+
1182
+
if profile.ID == 0 {
1183
+
return nil
1184
+
}
1185
+
1186
+
if err := b.db.Exec("DELETE FROM profiles WHERE id = ?", profile.ID).Error; err != nil {
1187
+
return err
1188
+
}
1189
+
1190
+
return nil
1191
+
}
1192
+
1193
+
const (
1194
+
NotifKindReply = "reply"
1195
+
NotifKindLike = "like"
1196
+
NotifKindMention = "mention"
1197
+
NotifKindRepost = "repost"
1198
+
)
1199
+
1200
+
func (b *PostgresBackend) AddNotification(ctx context.Context, forUser, author uint, recordUri string, recordCid cid.Cid, kind string) error {
1201
+
return b.db.Create(&Notification{
1202
+
For: forUser,
1203
+
Author: author,
1204
+
Source: recordUri,
1205
+
SourceCid: recordCid.String(),
1206
+
Kind: kind,
1207
+
}).Error
1208
+
}
+12
backend/interface.go
+12
backend/interface.go
···
1
+
package backend
2
+
3
+
// RecordTracker is an interface for tracking missing records that need to be fetched
4
+
type RecordTracker interface {
5
+
// TrackMissingRecord queues a missing record for fetching
6
+
// identifier can be:
7
+
// - A DID (e.g., "did:plc:...") for actors/profiles
8
+
// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.post/...") for posts
9
+
// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.generator/...") for feed generators
10
+
// wait: if true, blocks until the record is fetched
11
+
TrackMissingRecord(identifier string, wait bool)
12
+
}
+211
backend/missing.go
+211
backend/missing.go
···
1
+
package backend
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"log/slog"
8
+
9
+
"github.com/bluesky-social/indigo/api/atproto"
10
+
"github.com/bluesky-social/indigo/api/bsky"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
xrpclib "github.com/bluesky-social/indigo/xrpc"
13
+
"github.com/ipfs/go-cid"
14
+
)
15
+
16
+
type MissingRecordType string
17
+
18
+
const (
19
+
MissingRecordTypeProfile MissingRecordType = "profile"
20
+
MissingRecordTypePost MissingRecordType = "post"
21
+
MissingRecordTypeFeedGenerator MissingRecordType = "feedgenerator"
22
+
MissingRecordTypeUnknown MissingRecordType = "unknown"
23
+
)
24
+
25
+
type MissingRecord struct {
26
+
Type MissingRecordType
27
+
Identifier string // DID for profiles, AT-URI for posts/feedgens
28
+
Wait bool
29
+
30
+
waitch chan struct{}
31
+
}
32
+
33
+
func (b *PostgresBackend) addMissingRecord(ctx context.Context, rec MissingRecord) {
34
+
if rec.Wait {
35
+
rec.waitch = make(chan struct{})
36
+
}
37
+
38
+
select {
39
+
case b.missingRecords <- rec:
40
+
case <-ctx.Done():
41
+
}
42
+
43
+
if rec.Wait {
44
+
select {
45
+
case <-rec.waitch:
46
+
case <-ctx.Done():
47
+
}
48
+
}
49
+
}
50
+
51
+
func (b *PostgresBackend) missingRecordFetcher() {
52
+
for rec := range b.missingRecords {
53
+
var err error
54
+
switch rec.Type {
55
+
case MissingRecordTypeProfile:
56
+
err = b.fetchMissingProfile(context.TODO(), rec.Identifier)
57
+
case MissingRecordTypePost:
58
+
err = b.fetchMissingPost(context.TODO(), rec.Identifier)
59
+
case MissingRecordTypeFeedGenerator:
60
+
err = b.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
61
+
default:
62
+
slog.Error("unknown missing record type", "type", rec.Type)
63
+
continue
64
+
}
65
+
66
+
if err != nil {
67
+
slog.Warn("failed to fetch missing record", "type", rec.Type, "identifier", rec.Identifier, "error", err)
68
+
}
69
+
70
+
if rec.Wait {
71
+
close(rec.waitch)
72
+
}
73
+
}
74
+
}
75
+
76
+
func (b *PostgresBackend) fetchMissingProfile(ctx context.Context, did string) error {
77
+
b.AddRelevantDid(did)
78
+
79
+
repo, err := b.GetOrCreateRepo(ctx, did)
80
+
if err != nil {
81
+
return err
82
+
}
83
+
84
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
85
+
if err != nil {
86
+
return err
87
+
}
88
+
89
+
c := &xrpclib.Client{
90
+
Host: resp.PDSEndpoint(),
91
+
}
92
+
93
+
rec, err := atproto.RepoGetRecord(ctx, c, "", "app.bsky.actor.profile", did, "self")
94
+
if err != nil {
95
+
return err
96
+
}
97
+
98
+
prof, ok := rec.Value.Val.(*bsky.ActorProfile)
99
+
if !ok {
100
+
return fmt.Errorf("record we got back wasnt a profile somehow")
101
+
}
102
+
103
+
buf := new(bytes.Buffer)
104
+
if err := prof.MarshalCBOR(buf); err != nil {
105
+
return err
106
+
}
107
+
108
+
cc, err := cid.Decode(*rec.Cid)
109
+
if err != nil {
110
+
return err
111
+
}
112
+
113
+
return b.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
114
+
}
115
+
116
+
func (b *PostgresBackend) fetchMissingPost(ctx context.Context, uri string) error {
117
+
puri, err := syntax.ParseATURI(uri)
118
+
if err != nil {
119
+
return fmt.Errorf("invalid AT URI: %s", uri)
120
+
}
121
+
122
+
did := puri.Authority().String()
123
+
collection := puri.Collection().String()
124
+
rkey := puri.RecordKey().String()
125
+
126
+
b.AddRelevantDid(did)
127
+
128
+
repo, err := b.GetOrCreateRepo(ctx, did)
129
+
if err != nil {
130
+
return err
131
+
}
132
+
133
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
134
+
if err != nil {
135
+
return err
136
+
}
137
+
138
+
c := &xrpclib.Client{
139
+
Host: resp.PDSEndpoint(),
140
+
}
141
+
142
+
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
143
+
if err != nil {
144
+
return err
145
+
}
146
+
147
+
post, ok := rec.Value.Val.(*bsky.FeedPost)
148
+
if !ok {
149
+
return fmt.Errorf("record we got back wasn't a post somehow")
150
+
}
151
+
152
+
buf := new(bytes.Buffer)
153
+
if err := post.MarshalCBOR(buf); err != nil {
154
+
return err
155
+
}
156
+
157
+
cc, err := cid.Decode(*rec.Cid)
158
+
if err != nil {
159
+
return err
160
+
}
161
+
162
+
return b.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
163
+
}
164
+
165
+
func (b *PostgresBackend) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
166
+
puri, err := syntax.ParseATURI(uri)
167
+
if err != nil {
168
+
return fmt.Errorf("invalid AT URI: %s", uri)
169
+
}
170
+
171
+
did := puri.Authority().String()
172
+
collection := puri.Collection().String()
173
+
rkey := puri.RecordKey().String()
174
+
b.AddRelevantDid(did)
175
+
176
+
repo, err := b.GetOrCreateRepo(ctx, did)
177
+
if err != nil {
178
+
return err
179
+
}
180
+
181
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
182
+
if err != nil {
183
+
return err
184
+
}
185
+
186
+
c := &xrpclib.Client{
187
+
Host: resp.PDSEndpoint(),
188
+
}
189
+
190
+
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
191
+
if err != nil {
192
+
return err
193
+
}
194
+
195
+
feedGen, ok := rec.Value.Val.(*bsky.FeedGenerator)
196
+
if !ok {
197
+
return fmt.Errorf("record we got back wasn't a feed generator somehow")
198
+
}
199
+
200
+
buf := new(bytes.Buffer)
201
+
if err := feedGen.MarshalCBOR(buf); err != nil {
202
+
return err
203
+
}
204
+
205
+
cc, err := cid.Decode(*rec.Cid)
206
+
if err != nil {
207
+
return err
208
+
}
209
+
210
+
return b.HandleCreateFeedGenerator(ctx, repo, rkey, buf.Bytes(), cc)
211
+
}
-1128
events.go
-1128
events.go
···
1
-
package main
2
-
3
-
import (
4
-
"bytes"
5
-
"context"
6
-
"fmt"
7
-
"log/slog"
8
-
"strings"
9
-
"sync"
10
-
"time"
11
-
12
-
"github.com/bluesky-social/indigo/api/atproto"
13
-
"github.com/bluesky-social/indigo/api/bsky"
14
-
"github.com/bluesky-social/indigo/atproto/syntax"
15
-
"github.com/bluesky-social/indigo/repo"
16
-
lru "github.com/hashicorp/golang-lru/v2"
17
-
"github.com/ipfs/go-cid"
18
-
"github.com/jackc/pgx/v5/pgconn"
19
-
"github.com/jackc/pgx/v5/pgxpool"
20
-
"gorm.io/gorm"
21
-
22
-
. "github.com/whyrusleeping/konbini/models"
23
-
)
24
-
25
-
type PostgresBackend struct {
26
-
db *gorm.DB
27
-
pgx *pgxpool.Pool
28
-
s *Server
29
-
30
-
relevantDids map[string]bool
31
-
rdLk sync.Mutex
32
-
33
-
revCache *lru.TwoQueueCache[uint, string]
34
-
35
-
repoCache *lru.TwoQueueCache[string, *Repo]
36
-
reposLk sync.Mutex
37
-
38
-
postInfoCache *lru.TwoQueueCache[string, cachedPostInfo]
39
-
}
40
-
41
-
func (b *PostgresBackend) HandleEvent(ctx context.Context, evt *atproto.SyncSubscribeRepos_Commit) error {
42
-
r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks))
43
-
if err != nil {
44
-
return fmt.Errorf("failed to read event repo: %w", err)
45
-
}
46
-
47
-
for _, op := range evt.Ops {
48
-
switch op.Action {
49
-
case "create":
50
-
c, rec, err := r.GetRecordBytes(ctx, op.Path)
51
-
if err != nil {
52
-
return err
53
-
}
54
-
if err := b.HandleCreate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
55
-
return fmt.Errorf("create record failed: %w", err)
56
-
}
57
-
case "update":
58
-
c, rec, err := r.GetRecordBytes(ctx, op.Path)
59
-
if err != nil {
60
-
return err
61
-
}
62
-
if err := b.HandleUpdate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
63
-
return fmt.Errorf("update record failed: %w", err)
64
-
}
65
-
case "delete":
66
-
if err := b.HandleDelete(ctx, evt.Repo, evt.Rev, op.Path); err != nil {
67
-
return fmt.Errorf("delete record failed: %w", err)
68
-
}
69
-
}
70
-
}
71
-
72
-
// TODO: sync with the Since field to make sure we don't miss events we care about
73
-
/*
74
-
if err := bf.Store.UpdateRev(ctx, evt.Repo, evt.Rev); err != nil {
75
-
return fmt.Errorf("failed to update rev: %w", err)
76
-
}
77
-
*/
78
-
79
-
return nil
80
-
}
81
-
82
-
func (b *PostgresBackend) HandleCreate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
83
-
start := time.Now()
84
-
85
-
rr, err := b.getOrCreateRepo(ctx, repo)
86
-
if err != nil {
87
-
return fmt.Errorf("get user failed: %w", err)
88
-
}
89
-
90
-
lrev, err := b.revForRepo(rr)
91
-
if err != nil {
92
-
return err
93
-
}
94
-
if lrev != "" {
95
-
if rev < lrev {
96
-
slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
97
-
return nil
98
-
}
99
-
}
100
-
101
-
parts := strings.Split(path, "/")
102
-
if len(parts) != 2 {
103
-
return fmt.Errorf("invalid path in HandleCreate: %q", path)
104
-
}
105
-
col := parts[0]
106
-
rkey := parts[1]
107
-
108
-
defer func() {
109
-
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
110
-
}()
111
-
112
-
if rkey == "" {
113
-
fmt.Printf("messed up path: %q\n", rkey)
114
-
}
115
-
116
-
switch col {
117
-
case "app.bsky.feed.post":
118
-
if err := b.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
119
-
return err
120
-
}
121
-
case "app.bsky.feed.like":
122
-
if err := b.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
123
-
return err
124
-
}
125
-
case "app.bsky.feed.repost":
126
-
if err := b.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
127
-
return err
128
-
}
129
-
case "app.bsky.graph.follow":
130
-
if err := b.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
131
-
return err
132
-
}
133
-
case "app.bsky.graph.block":
134
-
if err := b.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
135
-
return err
136
-
}
137
-
case "app.bsky.graph.list":
138
-
if err := b.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
139
-
return err
140
-
}
141
-
case "app.bsky.graph.listitem":
142
-
if err := b.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
143
-
return err
144
-
}
145
-
case "app.bsky.graph.listblock":
146
-
if err := b.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
147
-
return err
148
-
}
149
-
case "app.bsky.actor.profile":
150
-
if err := b.HandleCreateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
151
-
return err
152
-
}
153
-
case "app.bsky.feed.generator":
154
-
if err := b.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
155
-
return err
156
-
}
157
-
case "app.bsky.feed.threadgate":
158
-
if err := b.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
159
-
return err
160
-
}
161
-
case "chat.bsky.actor.declaration":
162
-
if err := b.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
163
-
return err
164
-
}
165
-
case "app.bsky.feed.postgate":
166
-
if err := b.HandleCreatePostGate(ctx, rr, rkey, *rec, *cid); err != nil {
167
-
return err
168
-
}
169
-
case "app.bsky.graph.starterpack":
170
-
if err := b.HandleCreateStarterPack(ctx, rr, rkey, *rec, *cid); err != nil {
171
-
return err
172
-
}
173
-
default:
174
-
slog.Debug("unrecognized record type", "repo", repo, "path", path, "rev", rev)
175
-
}
176
-
177
-
b.revCache.Add(rr.ID, rev)
178
-
return nil
179
-
}
180
-
181
-
func (b *PostgresBackend) HandleCreatePost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
182
-
exists, err := b.checkPostExists(ctx, repo, rkey)
183
-
if err != nil {
184
-
return err
185
-
}
186
-
187
-
// still technically a race condition if two creates for the same post happen concurrently... probably fine
188
-
if exists {
189
-
return nil
190
-
}
191
-
192
-
var rec bsky.FeedPost
193
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
194
-
return err
195
-
}
196
-
197
-
reldids := []string{repo.Did}
198
-
// care about a post if its in a thread of a user we are interested in
199
-
if rec.Reply != nil && rec.Reply.Parent != nil && rec.Reply.Root != nil {
200
-
reldids = append(reldids, rec.Reply.Parent.Uri, rec.Reply.Root.Uri)
201
-
}
202
-
// TODO: maybe also care if its mentioning a user we care about or quoting a user we care about?
203
-
if !b.anyRelevantIdents(reldids...) {
204
-
return nil
205
-
}
206
-
207
-
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
208
-
slog.Warn("adding post", "uri", uri)
209
-
210
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
211
-
if err != nil {
212
-
return fmt.Errorf("invalid timestamp: %w", err)
213
-
}
214
-
215
-
p := Post{
216
-
Created: created.Time(),
217
-
Indexed: time.Now(),
218
-
Author: repo.ID,
219
-
Rkey: rkey,
220
-
Raw: recb,
221
-
Cid: cc.String(),
222
-
}
223
-
224
-
if rec.Reply != nil && rec.Reply.Parent != nil {
225
-
if rec.Reply.Root == nil {
226
-
return fmt.Errorf("post reply had nil root")
227
-
}
228
-
229
-
pinfo, err := b.postInfoForUri(ctx, rec.Reply.Parent.Uri)
230
-
if err != nil {
231
-
return fmt.Errorf("getting reply parent: %w", err)
232
-
}
233
-
234
-
p.ReplyTo = pinfo.ID
235
-
p.ReplyToUsr = pinfo.Author
236
-
237
-
thread, err := b.postIDForUri(ctx, rec.Reply.Root.Uri)
238
-
if err != nil {
239
-
return fmt.Errorf("getting thread root: %w", err)
240
-
}
241
-
242
-
p.InThread = thread
243
-
244
-
if p.ReplyToUsr == b.s.myrepo.ID {
245
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, p.Author, uri, cc, NotifKindReply); err != nil {
246
-
slog.Warn("failed to create notification", "uri", uri, "error", err)
247
-
}
248
-
}
249
-
}
250
-
251
-
if rec.Embed != nil {
252
-
var rpref string
253
-
if rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {
254
-
rpref = rec.Embed.EmbedRecord.Record.Uri
255
-
}
256
-
if rec.Embed.EmbedRecordWithMedia != nil &&
257
-
rec.Embed.EmbedRecordWithMedia.Record != nil &&
258
-
rec.Embed.EmbedRecordWithMedia.Record.Record != nil {
259
-
rpref = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri
260
-
}
261
-
262
-
if rpref != "" && strings.Contains(rpref, "app.bsky.feed.post") {
263
-
rp, err := b.postIDForUri(ctx, rpref)
264
-
if err != nil {
265
-
return fmt.Errorf("getting quote subject: %w", err)
266
-
}
267
-
268
-
p.Reposting = rp
269
-
}
270
-
}
271
-
272
-
if err := b.doPostCreate(ctx, &p); err != nil {
273
-
return err
274
-
}
275
-
276
-
// Check for mentions and create notifications
277
-
if rec.Facets != nil {
278
-
for _, facet := range rec.Facets {
279
-
for _, feature := range facet.Features {
280
-
if feature.RichtextFacet_Mention != nil {
281
-
mentionDid := feature.RichtextFacet_Mention.Did
282
-
// This is a mention
283
-
mentionedRepo, err := b.getOrCreateRepo(ctx, mentionDid)
284
-
if err != nil {
285
-
slog.Warn("failed to get repo for mention", "did", mentionDid, "error", err)
286
-
continue
287
-
}
288
-
289
-
// Create notification if the mentioned user is the current user
290
-
if mentionedRepo.ID == b.s.myrepo.ID {
291
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, p.Author, uri, cc, NotifKindMention); err != nil {
292
-
slog.Warn("failed to create mention notification", "uri", uri, "error", err)
293
-
}
294
-
}
295
-
}
296
-
}
297
-
}
298
-
}
299
-
300
-
b.postInfoCache.Add(uri, cachedPostInfo{
301
-
ID: p.ID,
302
-
Author: p.Author,
303
-
})
304
-
305
-
return nil
306
-
}
307
-
308
-
func (b *PostgresBackend) doPostCreate(ctx context.Context, p *Post) error {
309
-
/*
310
-
if err := b.db.Clauses(clause.OnConflict{
311
-
Columns: []clause.Column{{Name: "author"}, {Name: "rkey"}},
312
-
DoUpdates: clause.AssignmentColumns([]string{"cid", "not_found", "raw", "created", "indexed"}),
313
-
}).Create(p).Error; err != nil {
314
-
return err
315
-
}
316
-
*/
317
-
318
-
query := `
319
-
INSERT INTO posts (author, rkey, cid, not_found, raw, created, indexed, reposting, reply_to, reply_to_usr, in_thread)
320
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
321
-
ON CONFLICT (author, rkey)
322
-
DO UPDATE SET
323
-
cid = $3,
324
-
not_found = $4,
325
-
raw = $5,
326
-
created = $6,
327
-
indexed = $7,
328
-
reposting = $8,
329
-
reply_to = $9,
330
-
reply_to_usr = $10,
331
-
in_thread = $11
332
-
RETURNING id
333
-
`
334
-
335
-
// Execute the query with parameters from the Post struct
336
-
if err := b.pgx.QueryRow(
337
-
ctx,
338
-
query,
339
-
p.Author,
340
-
p.Rkey,
341
-
p.Cid,
342
-
p.NotFound,
343
-
p.Raw,
344
-
p.Created,
345
-
p.Indexed,
346
-
p.Reposting,
347
-
p.ReplyTo,
348
-
p.ReplyToUsr,
349
-
p.InThread,
350
-
).Scan(&p.ID); err != nil {
351
-
return err
352
-
}
353
-
354
-
return nil
355
-
}
356
-
357
-
func (b *PostgresBackend) HandleCreateLike(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
358
-
var rec bsky.FeedLike
359
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
360
-
return err
361
-
}
362
-
363
-
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
364
-
return nil
365
-
}
366
-
367
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
368
-
if err != nil {
369
-
return fmt.Errorf("invalid timestamp: %w", err)
370
-
}
371
-
372
-
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
373
-
if err != nil {
374
-
return fmt.Errorf("getting like subject: %w", err)
375
-
}
376
-
377
-
if _, err := b.pgx.Exec(ctx, `INSERT INTO "likes" ("created","indexed","author","rkey","subject","cid") VALUES ($1, $2, $3, $4, $5, $6)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID, cc.String()); err != nil {
378
-
pgErr, ok := err.(*pgconn.PgError)
379
-
if ok && pgErr.Code == "23505" {
380
-
return nil
381
-
}
382
-
return err
383
-
}
384
-
385
-
// Create notification if the liked post belongs to the current user
386
-
if pinfo.Author == b.s.myrepo.ID {
387
-
uri := fmt.Sprintf("at://%s/app.bsky.feed.like/%s", repo.Did, rkey)
388
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, repo.ID, uri, cc, NotifKindLike); err != nil {
389
-
slog.Warn("failed to create like notification", "uri", uri, "error", err)
390
-
}
391
-
}
392
-
393
-
return nil
394
-
}
395
-
396
-
func (b *PostgresBackend) HandleCreateRepost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
397
-
var rec bsky.FeedRepost
398
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
399
-
return err
400
-
}
401
-
402
-
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
403
-
return nil
404
-
}
405
-
406
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
407
-
if err != nil {
408
-
return fmt.Errorf("invalid timestamp: %w", err)
409
-
}
410
-
411
-
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
412
-
if err != nil {
413
-
return fmt.Errorf("getting repost subject: %w", err)
414
-
}
415
-
416
-
if _, err := b.pgx.Exec(ctx, `INSERT INTO "reposts" ("created","indexed","author","rkey","subject") VALUES ($1, $2, $3, $4, $5)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID); err != nil {
417
-
pgErr, ok := err.(*pgconn.PgError)
418
-
if ok && pgErr.Code == "23505" {
419
-
return nil
420
-
}
421
-
return err
422
-
}
423
-
424
-
// Create notification if the reposted post belongs to the current user
425
-
if pinfo.Author == b.s.myrepo.ID {
426
-
uri := fmt.Sprintf("at://%s/app.bsky.feed.repost/%s", repo.Did, rkey)
427
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, repo.ID, uri, cc, NotifKindRepost); err != nil {
428
-
slog.Warn("failed to create repost notification", "uri", uri, "error", err)
429
-
}
430
-
}
431
-
432
-
return nil
433
-
}
434
-
435
-
func (b *PostgresBackend) HandleCreateFollow(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
436
-
var rec bsky.GraphFollow
437
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
438
-
return err
439
-
}
440
-
441
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
442
-
return nil
443
-
}
444
-
445
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
446
-
if err != nil {
447
-
return fmt.Errorf("invalid timestamp: %w", err)
448
-
}
449
-
450
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
451
-
if err != nil {
452
-
return err
453
-
}
454
-
455
-
if _, err := b.pgx.Exec(ctx, "INSERT INTO follows (created, indexed, author, rkey, subject) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", created.Time(), time.Now(), repo.ID, rkey, subj.ID); err != nil {
456
-
return err
457
-
}
458
-
459
-
return nil
460
-
}
461
-
462
-
func (b *PostgresBackend) HandleCreateBlock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
463
-
var rec bsky.GraphBlock
464
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
465
-
return err
466
-
}
467
-
468
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
469
-
return nil
470
-
}
471
-
472
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
473
-
if err != nil {
474
-
return fmt.Errorf("invalid timestamp: %w", err)
475
-
}
476
-
477
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
478
-
if err != nil {
479
-
return err
480
-
}
481
-
482
-
if err := b.db.Create(&Block{
483
-
Created: created.Time(),
484
-
Indexed: time.Now(),
485
-
Author: repo.ID,
486
-
Rkey: rkey,
487
-
Subject: subj.ID,
488
-
}).Error; err != nil {
489
-
return err
490
-
}
491
-
492
-
return nil
493
-
}
494
-
495
-
func (b *PostgresBackend) HandleCreateList(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
496
-
var rec bsky.GraphList
497
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
498
-
return err
499
-
}
500
-
501
-
if !b.anyRelevantIdents(repo.Did) {
502
-
return nil
503
-
}
504
-
505
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
506
-
if err != nil {
507
-
return fmt.Errorf("invalid timestamp: %w", err)
508
-
}
509
-
510
-
if err := b.db.Create(&List{
511
-
Created: created.Time(),
512
-
Indexed: time.Now(),
513
-
Author: repo.ID,
514
-
Rkey: rkey,
515
-
Raw: recb,
516
-
}).Error; err != nil {
517
-
return err
518
-
}
519
-
520
-
return nil
521
-
}
522
-
523
-
func (b *PostgresBackend) HandleCreateListitem(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
524
-
var rec bsky.GraphListitem
525
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
526
-
return err
527
-
}
528
-
if !b.anyRelevantIdents(repo.Did) {
529
-
return nil
530
-
}
531
-
532
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
533
-
if err != nil {
534
-
return fmt.Errorf("invalid timestamp: %w", err)
535
-
}
536
-
537
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
538
-
if err != nil {
539
-
return err
540
-
}
541
-
542
-
list, err := b.getOrCreateList(ctx, rec.List)
543
-
if err != nil {
544
-
return err
545
-
}
546
-
547
-
if err := b.db.Create(&ListItem{
548
-
Created: created.Time(),
549
-
Indexed: time.Now(),
550
-
Author: repo.ID,
551
-
Rkey: rkey,
552
-
Subject: subj.ID,
553
-
List: list.ID,
554
-
}).Error; err != nil {
555
-
return err
556
-
}
557
-
558
-
return nil
559
-
}
560
-
561
-
func (b *PostgresBackend) HandleCreateListblock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
562
-
var rec bsky.GraphListblock
563
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
564
-
return err
565
-
}
566
-
567
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
568
-
return nil
569
-
}
570
-
571
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
572
-
if err != nil {
573
-
return fmt.Errorf("invalid timestamp: %w", err)
574
-
}
575
-
576
-
list, err := b.getOrCreateList(ctx, rec.Subject)
577
-
if err != nil {
578
-
return err
579
-
}
580
-
581
-
if err := b.db.Create(&ListBlock{
582
-
Created: created.Time(),
583
-
Indexed: time.Now(),
584
-
Author: repo.ID,
585
-
Rkey: rkey,
586
-
List: list.ID,
587
-
}).Error; err != nil {
588
-
return err
589
-
}
590
-
591
-
return nil
592
-
}
593
-
594
-
func (b *PostgresBackend) HandleCreateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
595
-
if !b.anyRelevantIdents(repo.Did) {
596
-
return nil
597
-
}
598
-
599
-
if err := b.db.Create(&Profile{
600
-
//Created: created.Time(),
601
-
Indexed: time.Now(),
602
-
Repo: repo.ID,
603
-
Raw: recb,
604
-
Rev: rev,
605
-
}).Error; err != nil {
606
-
return err
607
-
}
608
-
609
-
return nil
610
-
}
611
-
612
-
func (b *PostgresBackend) HandleUpdateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
613
-
if !b.anyRelevantIdents(repo.Did) {
614
-
return nil
615
-
}
616
-
617
-
if err := b.db.Create(&Profile{
618
-
Indexed: time.Now(),
619
-
Repo: repo.ID,
620
-
Raw: recb,
621
-
Rev: rev,
622
-
}).Error; err != nil {
623
-
return err
624
-
}
625
-
626
-
return nil
627
-
}
628
-
629
-
func (b *PostgresBackend) HandleCreateFeedGenerator(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
630
-
if !b.anyRelevantIdents(repo.Did) {
631
-
return nil
632
-
}
633
-
634
-
var rec bsky.FeedGenerator
635
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
636
-
return err
637
-
}
638
-
639
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
640
-
if err != nil {
641
-
return fmt.Errorf("invalid timestamp: %w", err)
642
-
}
643
-
644
-
if err := b.db.Create(&FeedGenerator{
645
-
Created: created.Time(),
646
-
Indexed: time.Now(),
647
-
Author: repo.ID,
648
-
Rkey: rkey,
649
-
Did: rec.Did,
650
-
Raw: recb,
651
-
}).Error; err != nil {
652
-
return err
653
-
}
654
-
655
-
return nil
656
-
}
657
-
658
-
func (b *PostgresBackend) HandleCreateThreadgate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
659
-
if !b.anyRelevantIdents(repo.Did) {
660
-
return nil
661
-
}
662
-
var rec bsky.FeedThreadgate
663
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
664
-
return err
665
-
}
666
-
667
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
668
-
if err != nil {
669
-
return fmt.Errorf("invalid timestamp: %w", err)
670
-
}
671
-
672
-
pid, err := b.postIDForUri(ctx, rec.Post)
673
-
if err != nil {
674
-
return err
675
-
}
676
-
677
-
if err := b.db.Create(&ThreadGate{
678
-
Created: created.Time(),
679
-
Indexed: time.Now(),
680
-
Author: repo.ID,
681
-
Rkey: rkey,
682
-
Post: pid,
683
-
}).Error; err != nil {
684
-
return err
685
-
}
686
-
687
-
return nil
688
-
}
689
-
690
-
func (b *PostgresBackend) HandleCreateChatDeclaration(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
691
-
// TODO: maybe track these?
692
-
return nil
693
-
}
694
-
695
-
func (b *PostgresBackend) HandleCreatePostGate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
696
-
if !b.anyRelevantIdents(repo.Did) {
697
-
return nil
698
-
}
699
-
var rec bsky.FeedPostgate
700
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
701
-
return err
702
-
}
703
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
704
-
if err != nil {
705
-
return fmt.Errorf("invalid timestamp: %w", err)
706
-
}
707
-
708
-
refPost, err := b.postInfoForUri(ctx, rec.Post)
709
-
if err != nil {
710
-
return err
711
-
}
712
-
713
-
if err := b.db.Create(&PostGate{
714
-
Created: created.Time(),
715
-
Indexed: time.Now(),
716
-
Author: repo.ID,
717
-
Rkey: rkey,
718
-
Subject: refPost.ID,
719
-
Raw: recb,
720
-
}).Error; err != nil {
721
-
return err
722
-
}
723
-
724
-
return nil
725
-
}
726
-
727
-
func (b *PostgresBackend) HandleCreateStarterPack(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
728
-
if !b.anyRelevantIdents(repo.Did) {
729
-
return nil
730
-
}
731
-
var rec bsky.GraphStarterpack
732
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
733
-
return err
734
-
}
735
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
736
-
if err != nil {
737
-
return fmt.Errorf("invalid timestamp: %w", err)
738
-
}
739
-
740
-
list, err := b.getOrCreateList(ctx, rec.List)
741
-
if err != nil {
742
-
return err
743
-
}
744
-
745
-
if err := b.db.Create(&StarterPack{
746
-
Created: created.Time(),
747
-
Indexed: time.Now(),
748
-
Author: repo.ID,
749
-
Rkey: rkey,
750
-
Raw: recb,
751
-
List: list.ID,
752
-
}).Error; err != nil {
753
-
return err
754
-
}
755
-
756
-
return nil
757
-
}
758
-
759
-
func (b *PostgresBackend) HandleUpdate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
760
-
start := time.Now()
761
-
762
-
rr, err := b.getOrCreateRepo(ctx, repo)
763
-
if err != nil {
764
-
return fmt.Errorf("get user failed: %w", err)
765
-
}
766
-
767
-
lrev, err := b.revForRepo(rr)
768
-
if err != nil {
769
-
return err
770
-
}
771
-
if lrev != "" {
772
-
if rev < lrev {
773
-
//slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
774
-
return nil
775
-
}
776
-
}
777
-
778
-
parts := strings.Split(path, "/")
779
-
if len(parts) != 2 {
780
-
return fmt.Errorf("invalid path in HandleCreate: %q", path)
781
-
}
782
-
col := parts[0]
783
-
rkey := parts[1]
784
-
785
-
defer func() {
786
-
handleOpHist.WithLabelValues("update", col).Observe(float64(time.Since(start).Milliseconds()))
787
-
}()
788
-
789
-
if rkey == "" {
790
-
fmt.Printf("messed up path: %q\n", rkey)
791
-
}
792
-
793
-
switch col {
794
-
/*
795
-
case "app.bsky.feed.post":
796
-
if err := s.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
797
-
return err
798
-
}
799
-
case "app.bsky.feed.like":
800
-
if err := s.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
801
-
return err
802
-
}
803
-
case "app.bsky.feed.repost":
804
-
if err := s.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
805
-
return err
806
-
}
807
-
case "app.bsky.graph.follow":
808
-
if err := s.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
809
-
return err
810
-
}
811
-
case "app.bsky.graph.block":
812
-
if err := s.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
813
-
return err
814
-
}
815
-
case "app.bsky.graph.list":
816
-
if err := s.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
817
-
return err
818
-
}
819
-
case "app.bsky.graph.listitem":
820
-
if err := s.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
821
-
return err
822
-
}
823
-
case "app.bsky.graph.listblock":
824
-
if err := s.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
825
-
return err
826
-
}
827
-
*/
828
-
case "app.bsky.actor.profile":
829
-
if err := b.HandleUpdateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
830
-
return err
831
-
}
832
-
/*
833
-
case "app.bsky.feed.generator":
834
-
if err := s.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
835
-
return err
836
-
}
837
-
case "app.bsky.feed.threadgate":
838
-
if err := s.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
839
-
return err
840
-
}
841
-
case "chat.bsky.actor.declaration":
842
-
if err := s.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
843
-
return err
844
-
}
845
-
*/
846
-
default:
847
-
slog.Debug("unrecognized record type in update", "repo", repo, "path", path, "rev", rev)
848
-
}
849
-
850
-
return nil
851
-
}
852
-
853
-
func (b *PostgresBackend) HandleDelete(ctx context.Context, repo string, rev string, path string) error {
854
-
start := time.Now()
855
-
856
-
rr, err := b.getOrCreateRepo(ctx, repo)
857
-
if err != nil {
858
-
return fmt.Errorf("get user failed: %w", err)
859
-
}
860
-
861
-
lrev, ok := b.revCache.Get(rr.ID)
862
-
if ok {
863
-
if rev < lrev {
864
-
//slog.Info("skipping old rev delete", "did", rr.Did, "rev", rev, "oldrev", lrev)
865
-
return nil
866
-
}
867
-
}
868
-
869
-
parts := strings.Split(path, "/")
870
-
if len(parts) != 2 {
871
-
return fmt.Errorf("invalid path in HandleDelete: %q", path)
872
-
}
873
-
col := parts[0]
874
-
rkey := parts[1]
875
-
876
-
defer func() {
877
-
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
878
-
}()
879
-
880
-
switch col {
881
-
case "app.bsky.feed.post":
882
-
if err := b.HandleDeletePost(ctx, rr, rkey); err != nil {
883
-
return err
884
-
}
885
-
case "app.bsky.feed.like":
886
-
if err := b.HandleDeleteLike(ctx, rr, rkey); err != nil {
887
-
return err
888
-
}
889
-
case "app.bsky.feed.repost":
890
-
if err := b.HandleDeleteRepost(ctx, rr, rkey); err != nil {
891
-
return err
892
-
}
893
-
case "app.bsky.graph.follow":
894
-
if err := b.HandleDeleteFollow(ctx, rr, rkey); err != nil {
895
-
return err
896
-
}
897
-
case "app.bsky.graph.block":
898
-
if err := b.HandleDeleteBlock(ctx, rr, rkey); err != nil {
899
-
return err
900
-
}
901
-
case "app.bsky.graph.list":
902
-
if err := b.HandleDeleteList(ctx, rr, rkey); err != nil {
903
-
return err
904
-
}
905
-
case "app.bsky.graph.listitem":
906
-
if err := b.HandleDeleteListitem(ctx, rr, rkey); err != nil {
907
-
return err
908
-
}
909
-
case "app.bsky.graph.listblock":
910
-
if err := b.HandleDeleteListblock(ctx, rr, rkey); err != nil {
911
-
return err
912
-
}
913
-
case "app.bsky.actor.profile":
914
-
if err := b.HandleDeleteProfile(ctx, rr, rkey); err != nil {
915
-
return err
916
-
}
917
-
case "app.bsky.feed.generator":
918
-
if err := b.HandleDeleteFeedGenerator(ctx, rr, rkey); err != nil {
919
-
return err
920
-
}
921
-
case "app.bsky.feed.threadgate":
922
-
if err := b.HandleDeleteThreadgate(ctx, rr, rkey); err != nil {
923
-
return err
924
-
}
925
-
default:
926
-
slog.Warn("delete unrecognized record type", "repo", repo, "path", path, "rev", rev)
927
-
}
928
-
929
-
b.revCache.Add(rr.ID, rev)
930
-
return nil
931
-
}
932
-
933
-
func (b *PostgresBackend) HandleDeletePost(ctx context.Context, repo *Repo, rkey string) error {
934
-
var p Post
935
-
if err := b.db.Find(&p, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
936
-
return err
937
-
}
938
-
939
-
if p.ID == 0 {
940
-
//slog.Warn("delete of unknown post record", "repo", repo.Did, "rkey", rkey)
941
-
return nil
942
-
}
943
-
944
-
if err := b.db.Delete(&Post{}, p.ID).Error; err != nil {
945
-
return err
946
-
}
947
-
948
-
return nil
949
-
}
950
-
951
-
func (b *PostgresBackend) HandleDeleteLike(ctx context.Context, repo *Repo, rkey string) error {
952
-
var like Like
953
-
if err := b.db.Find(&like, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
954
-
return err
955
-
}
956
-
957
-
if like.ID == 0 {
958
-
//slog.Warn("delete of missing like", "repo", repo.Did, "rkey", rkey)
959
-
return nil
960
-
}
961
-
962
-
if err := b.db.Exec("DELETE FROM likes WHERE id = ?", like.ID).Error; err != nil {
963
-
return err
964
-
}
965
-
966
-
return nil
967
-
}
968
-
969
-
func (b *PostgresBackend) HandleDeleteRepost(ctx context.Context, repo *Repo, rkey string) error {
970
-
var repost Repost
971
-
if err := b.db.Find(&repost, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
972
-
return err
973
-
}
974
-
975
-
if repost.ID == 0 {
976
-
//return fmt.Errorf("delete of missing repost: %s %s", repo.Did, rkey)
977
-
return nil
978
-
}
979
-
980
-
if err := b.db.Exec("DELETE FROM reposts WHERE id = ?", repost.ID).Error; err != nil {
981
-
return err
982
-
}
983
-
984
-
return nil
985
-
}
986
-
987
-
func (b *PostgresBackend) HandleDeleteFollow(ctx context.Context, repo *Repo, rkey string) error {
988
-
var follow Follow
989
-
if err := b.db.Find(&follow, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
990
-
return err
991
-
}
992
-
993
-
if follow.ID == 0 {
994
-
//slog.Warn("delete of missing follow", "repo", repo.Did, "rkey", rkey)
995
-
return nil
996
-
}
997
-
998
-
if err := b.db.Exec("DELETE FROM follows WHERE id = ?", follow.ID).Error; err != nil {
999
-
return err
1000
-
}
1001
-
1002
-
return nil
1003
-
}
1004
-
1005
-
func (b *PostgresBackend) HandleDeleteBlock(ctx context.Context, repo *Repo, rkey string) error {
1006
-
var block Block
1007
-
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1008
-
return err
1009
-
}
1010
-
1011
-
if block.ID == 0 {
1012
-
//slog.Warn("delete of missing block", "repo", repo.Did, "rkey", rkey)
1013
-
return nil
1014
-
}
1015
-
1016
-
if err := b.db.Exec("DELETE FROM blocks WHERE id = ?", block.ID).Error; err != nil {
1017
-
return err
1018
-
}
1019
-
1020
-
return nil
1021
-
}
1022
-
1023
-
func (b *PostgresBackend) HandleDeleteList(ctx context.Context, repo *Repo, rkey string) error {
1024
-
var list List
1025
-
if err := b.db.Find(&list, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1026
-
return err
1027
-
}
1028
-
1029
-
if list.ID == 0 {
1030
-
return nil
1031
-
//return fmt.Errorf("delete of missing list: %s %s", repo.Did, rkey)
1032
-
}
1033
-
1034
-
if err := b.db.Exec("DELETE FROM lists WHERE id = ?", list.ID).Error; err != nil {
1035
-
return err
1036
-
}
1037
-
1038
-
return nil
1039
-
}
1040
-
1041
-
func (b *PostgresBackend) HandleDeleteListitem(ctx context.Context, repo *Repo, rkey string) error {
1042
-
var item ListItem
1043
-
if err := b.db.Find(&item, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1044
-
return err
1045
-
}
1046
-
1047
-
if item.ID == 0 {
1048
-
return nil
1049
-
//return fmt.Errorf("delete of missing listitem: %s %s", repo.Did, rkey)
1050
-
}
1051
-
1052
-
if err := b.db.Exec("DELETE FROM list_items WHERE id = ?", item.ID).Error; err != nil {
1053
-
return err
1054
-
}
1055
-
1056
-
return nil
1057
-
}
1058
-
1059
-
func (b *PostgresBackend) HandleDeleteListblock(ctx context.Context, repo *Repo, rkey string) error {
1060
-
var block ListBlock
1061
-
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1062
-
return err
1063
-
}
1064
-
1065
-
if block.ID == 0 {
1066
-
return nil
1067
-
//return fmt.Errorf("delete of missing listblock: %s %s", repo.Did, rkey)
1068
-
}
1069
-
1070
-
if err := b.db.Exec("DELETE FROM list_blocks WHERE id = ?", block.ID).Error; err != nil {
1071
-
return err
1072
-
}
1073
-
1074
-
return nil
1075
-
}
1076
-
1077
-
func (b *PostgresBackend) HandleDeleteFeedGenerator(ctx context.Context, repo *Repo, rkey string) error {
1078
-
var feedgen FeedGenerator
1079
-
if err := b.db.Find(&feedgen, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1080
-
return err
1081
-
}
1082
-
1083
-
if feedgen.ID == 0 {
1084
-
return nil
1085
-
//return fmt.Errorf("delete of missing feedgen: %s %s", repo.Did, rkey)
1086
-
}
1087
-
1088
-
if err := b.db.Exec("DELETE FROM feed_generators WHERE id = ?", feedgen.ID).Error; err != nil {
1089
-
return err
1090
-
}
1091
-
1092
-
return nil
1093
-
}
1094
-
1095
-
func (b *PostgresBackend) HandleDeleteThreadgate(ctx context.Context, repo *Repo, rkey string) error {
1096
-
var threadgate ThreadGate
1097
-
if err := b.db.Find(&threadgate, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1098
-
return err
1099
-
}
1100
-
1101
-
if threadgate.ID == 0 {
1102
-
return nil
1103
-
//return fmt.Errorf("delete of missing threadgate: %s %s", repo.Did, rkey)
1104
-
}
1105
-
1106
-
if err := b.db.Exec("DELETE FROM thread_gates WHERE id = ?", threadgate.ID).Error; err != nil {
1107
-
return err
1108
-
}
1109
-
1110
-
return nil
1111
-
}
1112
-
1113
-
func (b *PostgresBackend) HandleDeleteProfile(ctx context.Context, repo *Repo, rkey string) error {
1114
-
var profile Profile
1115
-
if err := b.db.Find(&profile, "repo = ?", repo.ID).Error; err != nil {
1116
-
return err
1117
-
}
1118
-
1119
-
if profile.ID == 0 {
1120
-
return nil
1121
-
}
1122
-
1123
-
if err := b.db.Exec("DELETE FROM profiles WHERE id = ?", profile.ID).Error; err != nil {
1124
-
return err
1125
-
}
1126
-
1127
-
return nil
1128
-
}
+18
-9
go.mod
+18
-9
go.mod
···
3
3
go 1.25.1
4
4
5
5
require (
6
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f
7
-
github.com/golang-jwt/jwt/v5 v5.2.2
6
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe
7
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1
8
8
github.com/gorilla/websocket v1.5.1
9
9
github.com/hashicorp/golang-lru/v2 v2.0.7
10
10
github.com/ipfs/go-cid v0.4.1
11
11
github.com/jackc/pgx/v5 v5.6.0
12
12
github.com/labstack/echo/v4 v4.11.3
13
13
github.com/labstack/gommon v0.4.1
14
+
github.com/lestrrat-go/jwx/v2 v2.0.12
15
+
github.com/multiformats/go-multihash v0.2.3
14
16
github.com/prometheus/client_golang v1.19.1
15
17
github.com/urfave/cli/v2 v2.27.7
18
+
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
16
19
github.com/whyrusleeping/market v0.0.0-20250711215409-cc684a207f15
20
+
go.opentelemetry.io/otel v1.34.0
21
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0
22
+
go.opentelemetry.io/otel/sdk v1.34.0
17
23
gorm.io/gorm v1.31.0
18
24
)
19
25
···
25
31
github.com/cespare/xxhash/v2 v2.3.0 // indirect
26
32
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
27
33
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
34
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
28
35
github.com/felixge/httpsnoop v1.0.4 // indirect
29
36
github.com/go-logr/logr v1.4.2 // indirect
30
37
github.com/go-logr/stdr v1.2.2 // indirect
38
+
github.com/go-redis/cache/v9 v9.0.0 // indirect
31
39
github.com/goccy/go-json v0.10.5 // indirect
32
40
github.com/gogo/protobuf v1.3.2 // indirect
33
41
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
···
53
61
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
54
62
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
55
63
github.com/ipfs/go-verifcid v0.0.3 // indirect
56
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect
64
+
github.com/ipld/go-car v0.6.2 // indirect
57
65
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
58
66
github.com/ipld/go-ipld-prime v0.21.0 // indirect
59
67
github.com/jackc/pgpassfile v1.0.0 // indirect
···
62
70
github.com/jbenet/goprocess v0.1.4 // indirect
63
71
github.com/jinzhu/inflection v1.0.0 // indirect
64
72
github.com/jinzhu/now v1.1.5 // indirect
73
+
github.com/klauspost/compress v1.17.9 // indirect
65
74
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
66
75
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
67
76
github.com/lestrrat-go/httpcc v1.0.1 // indirect
68
77
github.com/lestrrat-go/httprc v1.0.4 // indirect
69
78
github.com/lestrrat-go/iter v1.0.2 // indirect
70
-
github.com/lestrrat-go/jwx/v2 v2.0.12 // indirect
71
79
github.com/lestrrat-go/option v1.0.1 // indirect
72
80
github.com/libp2p/go-libp2p v0.25.1 // indirect
73
81
github.com/mattn/go-colorable v0.1.13 // indirect
···
79
87
github.com/multiformats/go-base36 v0.2.0 // indirect
80
88
github.com/multiformats/go-multiaddr v0.8.0 // indirect
81
89
github.com/multiformats/go-multibase v0.2.0 // indirect
82
-
github.com/multiformats/go-multihash v0.2.3 // indirect
83
90
github.com/multiformats/go-varint v0.0.7 // indirect
84
91
github.com/opentracing/opentracing-go v1.2.0 // indirect
85
92
github.com/orandin/slog-gorm v1.3.2 // indirect
86
93
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
87
94
github.com/prometheus/client_model v0.6.1 // indirect
88
-
github.com/prometheus/common v0.48.0 // indirect
89
-
github.com/prometheus/procfs v0.12.0 // indirect
95
+
github.com/prometheus/common v0.54.0 // indirect
96
+
github.com/prometheus/procfs v0.15.1 // indirect
97
+
github.com/redis/go-redis/v9 v9.3.0 // indirect
90
98
github.com/russross/blackfriday/v2 v2.1.0 // indirect
91
99
github.com/segmentio/asm v1.2.0 // indirect
92
100
github.com/spaolacci/murmur3 v1.1.0 // indirect
93
101
github.com/valyala/bytebufferpool v1.0.0 // indirect
94
102
github.com/valyala/fasttemplate v1.2.2 // indirect
95
-
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect
103
+
github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
104
+
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
105
+
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
96
106
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 // indirect
97
107
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
98
108
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
99
109
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
100
110
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
101
111
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
102
-
go.opentelemetry.io/otel v1.34.0 // indirect
103
112
go.opentelemetry.io/otel/metric v1.34.0 // indirect
104
113
go.opentelemetry.io/otel/trace v1.34.0 // indirect
105
114
go.uber.org/atomic v1.11.0 // indirect
+155
-10
go.sum
+155
-10
go.sum
···
6
6
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
7
7
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
8
8
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
9
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f h1:FugOoTzh0nCMTWGqNGsjttFWVPcwxaaGD3p/nE9V8qY=
10
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f/go.mod h1:n6QE1NDPFoi7PRbMUZmc2y7FibCqiVU4ePpsvhHUBR8=
9
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe h1:VBhaqE5ewQgXbY5SfSWFZC/AwHFo7cHxZKFYi2ce9Yo=
10
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe/go.mod h1:RuQVrCGm42QNsgumKaR6se+XkFKfCPNwdCiTvqKRUck=
11
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1 h1:ovcRKN1iXZnY5WApVg+0Hw2RkwMH0ziA7lSAA8vellU=
12
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1/go.mod h1:5PtGi4r/PjEVBBl+0xWuQn4mBEjr9h6xsfDBADS6cHs=
11
13
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=
12
14
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=
15
+
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
16
+
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
17
+
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
18
+
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
13
19
github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
14
20
github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
21
+
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
22
+
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
15
23
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
16
24
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
25
+
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
26
+
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
27
+
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
17
28
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
18
29
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
19
30
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
31
+
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
20
32
github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
21
33
github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
22
34
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
···
25
37
github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
26
38
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
27
39
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
40
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
41
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
28
42
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
29
43
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
30
44
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
31
45
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
46
+
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
47
+
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
48
+
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
49
+
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
32
50
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
51
+
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
33
52
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
34
53
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
35
54
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
36
55
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
56
+
github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0=
57
+
github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI=
37
58
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
38
59
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
60
+
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
39
61
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
40
62
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
41
63
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
···
44
66
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
45
67
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
46
68
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
47
-
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
48
-
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
69
+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
70
+
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
71
+
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
72
+
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
73
+
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
74
+
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
75
+
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
76
+
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
77
+
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
78
+
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
79
+
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
80
+
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
81
+
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
82
+
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
83
+
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
49
84
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
50
85
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
51
86
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
52
87
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
88
+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
53
89
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
54
90
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
55
91
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
···
69
105
github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno=
70
106
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
71
107
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
108
+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
72
109
github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
73
110
github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
111
+
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
74
112
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
75
113
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
76
114
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
···
122
160
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
123
161
github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs=
124
162
github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw=
125
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 h1:oFo19cBmcP0Cmg3XXbrr0V/c+xU9U1huEZp8+OgBzdI=
126
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4/go.mod h1:6nkFF8OmR5wLKBzRKi7/YFJpyYR7+oEn1DX+mMWnlLA=
163
+
github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc=
164
+
github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
127
165
github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4=
128
166
github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo=
129
167
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
···
151
189
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
152
190
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
153
191
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
192
+
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
193
+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
194
+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
154
195
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
155
196
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
156
197
github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8=
157
198
github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA=
158
199
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
200
+
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
159
201
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
160
202
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
161
203
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
···
231
273
github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
232
274
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
233
275
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
276
+
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
277
+
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
278
+
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
279
+
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
280
+
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
281
+
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
282
+
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
283
+
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
284
+
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
285
+
github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
286
+
github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
287
+
github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
288
+
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
289
+
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
290
+
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
291
+
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
292
+
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
293
+
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
294
+
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
295
+
github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
296
+
github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
297
+
github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
298
+
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
299
+
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
300
+
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
301
+
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
234
302
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
235
303
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
236
304
github.com/orandin/slog-gorm v1.3.2 h1:C0lKDQPAx/pF+8K2HL7bdShPwOEJpPM0Bn80zTzxU1g=
···
246
314
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
247
315
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
248
316
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
249
-
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
250
-
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
251
-
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
252
-
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
317
+
github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
318
+
github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
319
+
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
320
+
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
321
+
github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA=
322
+
github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
323
+
github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
253
324
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
325
+
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
254
326
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
255
327
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
256
328
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
···
267
339
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
268
340
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
269
341
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
342
+
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
270
343
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
271
344
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
272
345
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
273
346
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
347
+
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
274
348
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
275
349
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
276
350
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
277
351
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
352
+
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
278
353
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
279
354
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
280
355
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
···
285
360
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
286
361
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
287
362
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
363
+
github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI=
364
+
github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q=
365
+
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
366
+
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
367
+
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
368
+
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
369
+
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
288
370
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
289
371
github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y=
290
372
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
···
302
384
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
303
385
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
304
386
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
387
+
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
305
388
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
306
389
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=
307
390
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
···
313
396
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
314
397
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
315
398
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
399
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=
400
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
316
401
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
317
402
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
403
+
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
404
+
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
318
405
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
319
406
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
320
407
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
···
338
425
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
339
426
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
340
427
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
428
+
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
341
429
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
342
430
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
343
431
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
···
348
436
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
349
437
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
350
438
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
439
+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
351
440
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
441
+
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
442
+
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
352
443
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
353
444
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
354
445
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
446
+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
355
447
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
356
448
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
357
449
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
358
450
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
451
+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
359
452
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
360
453
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
361
454
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
455
+
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
456
+
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
457
+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
458
+
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
362
459
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
460
+
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
461
+
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
462
+
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
463
+
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
363
464
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
364
465
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
365
466
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
366
467
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
468
+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
367
469
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
368
470
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
369
471
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
372
474
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
373
475
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
374
476
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
477
+
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
375
478
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
376
479
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
480
+
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
481
+
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
482
+
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
483
+
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
484
+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
377
485
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
378
486
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
487
+
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
379
488
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
489
+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
380
490
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
381
491
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
382
492
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
493
+
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
494
+
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
495
+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
383
496
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
497
+
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
384
498
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
385
499
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
386
500
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
501
+
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
502
+
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
503
+
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
504
+
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
387
505
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
388
506
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
389
507
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
···
392
510
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
393
511
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
394
512
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
513
+
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
514
+
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
515
+
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
516
+
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
395
517
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
396
518
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
397
519
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
398
520
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
399
521
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
522
+
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
400
523
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
524
+
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
525
+
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
526
+
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
401
527
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
402
528
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
403
529
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
···
413
539
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
414
540
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
415
541
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
542
+
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
416
543
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
417
544
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
545
+
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
418
546
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
547
+
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
548
+
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
419
549
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
420
550
golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE=
421
551
golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588=
···
425
555
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
426
556
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
427
557
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
558
+
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
559
+
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
560
+
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
561
+
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
562
+
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
563
+
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
564
+
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
565
+
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
566
+
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
428
567
google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
429
568
google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
430
569
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
431
570
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
571
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
432
572
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
433
573
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
434
574
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
575
+
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
576
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
577
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
435
578
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
579
+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
436
580
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
581
+
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
437
582
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
438
583
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
439
584
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+59
-41
handlers.go
+59
-41
handlers.go
···
17
17
"github.com/labstack/gommon/log"
18
18
"github.com/whyrusleeping/market/models"
19
19
20
+
"github.com/whyrusleeping/konbini/backend"
20
21
. "github.com/whyrusleeping/konbini/models"
21
22
)
22
23
···
26
27
e.Use(middleware.CORS())
27
28
e.GET("/debug", s.handleGetDebugInfo)
28
29
e.GET("/reldids", s.handleGetRelevantDids)
30
+
e.GET("/rescan/:did", s.handleRescanDid)
29
31
30
32
views := e.Group("/api")
31
33
views.GET("/me", s.handleGetMe)
···
55
57
56
58
func (s *Server) handleGetRelevantDids(e echo.Context) error {
57
59
return e.JSON(200, map[string]any{
58
-
"dids": s.backend.relevantDids,
60
+
"dids": s.backend.GetRelevantDids(),
59
61
})
60
62
}
61
63
64
+
func (s *Server) handleRescanDid(e echo.Context) error {
65
+
didparam := e.Param("did")
66
+
67
+
ctx := e.Request().Context()
68
+
did, err := s.resolveAccountIdent(ctx, didparam)
69
+
if err != nil {
70
+
return err
71
+
}
72
+
73
+
if err := s.rescanRepo(ctx, did); err != nil {
74
+
return err
75
+
}
76
+
77
+
return e.JSON(200, map[string]any{"status": "ok"})
78
+
}
79
+
62
80
func (s *Server) handleGetMe(e echo.Context) error {
63
81
ctx := e.Request().Context()
64
82
···
88
106
89
107
postUri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey)
90
108
91
-
p, err := s.backend.getPostByUri(ctx, postUri, "*")
109
+
p, err := s.backend.GetPostByUri(ctx, postUri, "*")
92
110
if err != nil {
93
111
return err
94
112
}
···
117
135
return err
118
136
}
119
137
120
-
r, err := s.backend.getOrCreateRepo(ctx, accdid)
138
+
r, err := s.backend.GetOrCreateRepo(ctx, accdid)
121
139
if err != nil {
122
140
return err
123
141
}
124
142
125
143
var profile models.Profile
126
-
if err := s.backend.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
144
+
if err := s.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
127
145
return err
128
146
}
129
147
130
148
if profile.Raw == nil || len(profile.Raw) == 0 {
131
-
s.addMissingProfile(ctx, accdid)
149
+
s.backend.TrackMissingRecord(accdid, false)
132
150
return e.JSON(404, map[string]any{
133
151
"error": "missing profile info for user",
134
152
})
···
152
170
return err
153
171
}
154
172
155
-
r, err := s.backend.getOrCreateRepo(ctx, accdid)
173
+
r, err := s.backend.GetOrCreateRepo(ctx, accdid)
156
174
if err != nil {
157
175
return err
158
176
}
···
171
189
}
172
190
173
191
var dbposts []models.Post
174
-
if err := s.backend.db.Raw("SELECT * FROM posts WHERE author = ? AND created < ? ORDER BY created DESC LIMIT ?", r.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
192
+
if err := s.db.Raw("SELECT * FROM posts WHERE author = ? AND created < ? ORDER BY created DESC LIMIT ?", r.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
175
193
return err
176
194
}
177
195
···
241
259
func (s *Server) handleGetFollowingFeed(e echo.Context) error {
242
260
ctx := e.Request().Context()
243
261
244
-
myr, err := s.backend.getOrCreateRepo(ctx, s.mydid)
262
+
myr, err := s.backend.GetOrCreateRepo(ctx, s.mydid)
245
263
if err != nil {
246
264
return err
247
265
}
···
259
277
tcursor = t
260
278
}
261
279
var dbposts []models.Post
262
-
if err := s.backend.db.Raw("select * from posts where reply_to = 0 AND author IN (select subject from follows where author = ?) AND created < ? order by created DESC limit ?", myr.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
280
+
if err := s.db.Raw("select * from posts where reply_to = 0 AND author IN (select subject from follows where author = ?) AND created < ? order by created DESC limit ?", myr.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
263
281
return err
264
282
}
265
283
···
279
297
280
298
func (s *Server) getAuthorInfo(ctx context.Context, r *models.Repo) (*authorInfo, error) {
281
299
var profile models.Profile
282
-
if err := s.backend.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
300
+
if err := s.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
283
301
return nil, err
284
302
}
285
303
···
289
307
}
290
308
291
309
if profile.Raw == nil || len(profile.Raw) == 0 {
292
-
s.addMissingProfile(ctx, r.Did)
310
+
s.backend.TrackMissingRecord(r.Did, false)
293
311
return &authorInfo{
294
312
Handle: resp.Handle.String(),
295
313
Did: r.Did,
···
316
334
317
335
go func() {
318
336
defer wg.Done()
319
-
if err := s.backend.db.Raw("SELECT count(*) FROM likes WHERE subject = ?", pid).Scan(&pc.Likes).Error; err != nil {
337
+
if err := s.db.Raw("SELECT count(*) FROM likes WHERE subject = ?", pid).Scan(&pc.Likes).Error; err != nil {
320
338
slog.Error("failed to get likes count", "post", pid, "error", err)
321
339
}
322
340
}()
323
341
324
342
go func() {
325
343
defer wg.Done()
326
-
if err := s.backend.db.Raw("SELECT count(*) FROM reposts WHERE subject = ?", pid).Scan(&pc.Reposts).Error; err != nil {
344
+
if err := s.db.Raw("SELECT count(*) FROM reposts WHERE subject = ?", pid).Scan(&pc.Reposts).Error; err != nil {
327
345
slog.Error("failed to get reposts count", "post", pid, "error", err)
328
346
}
329
347
}()
330
348
331
349
go func() {
332
350
defer wg.Done()
333
-
if err := s.backend.db.Raw("SELECT count(*) FROM posts WHERE reply_to = ?", pid).Scan(&pc.Replies).Error; err != nil {
351
+
if err := s.db.Raw("SELECT count(*) FROM posts WHERE reply_to = ?", pid).Scan(&pc.Replies).Error; err != nil {
334
352
slog.Error("failed to get replies count", "post", pid, "error", err)
335
353
}
336
354
}()
···
349
367
go func(ix int) {
350
368
defer wg.Done()
351
369
p := dbposts[ix]
352
-
r, err := s.backend.getRepoByID(ctx, p.Author)
370
+
r, err := s.backend.GetRepoByID(ctx, p.Author)
353
371
if err != nil {
354
372
fmt.Println("failed to get repo: ", err)
355
373
posts[ix] = postResponse{
···
361
379
362
380
uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", r.Did, p.Rkey)
363
381
if len(p.Raw) == 0 || p.NotFound {
364
-
s.addMissingPost(ctx, uri)
382
+
s.backend.TrackMissingRecord(uri, false)
365
383
posts[ix] = postResponse{
366
384
Uri: uri,
367
385
Missing: true,
···
417
435
418
436
func (s *Server) checkViewerLike(ctx context.Context, pid uint) *viewerLike {
419
437
var like Like
420
-
if err := s.backend.db.Raw("SELECT * FROM likes WHERE subject = ? AND author = ?", pid, s.myrepo.ID).Scan(&like).Error; err != nil {
438
+
if err := s.db.Raw("SELECT * FROM likes WHERE subject = ? AND author = ?", pid, s.myrepo.ID).Scan(&like).Error; err != nil {
421
439
slog.Error("failed to lookup like", "error", err)
422
440
return nil
423
441
}
···
494
512
quotedURI := embedRecord.Record.Uri
495
513
quotedCid := embedRecord.Record.Cid
496
514
497
-
quotedPost, err := s.backend.getPostByUri(ctx, quotedURI, "*")
515
+
quotedPost, err := s.backend.GetPostByUri(ctx, quotedURI, "*")
498
516
if err != nil {
499
517
slog.Warn("failed to get quoted post", "uri", quotedURI, "error", err)
500
-
s.addMissingPost(ctx, quotedURI)
518
+
s.backend.TrackMissingRecord(quotedURI, false)
501
519
return s.buildQuoteFallback(quotedURI, quotedCid)
502
520
}
503
521
504
522
if quotedPost == nil || quotedPost.Raw == nil || len(quotedPost.Raw) == 0 || quotedPost.NotFound {
505
-
s.addMissingPost(ctx, quotedURI)
523
+
s.backend.TrackMissingRecord(quotedURI, false)
506
524
return s.buildQuoteFallback(quotedURI, quotedCid)
507
525
}
508
526
···
512
530
return s.buildQuoteFallback(quotedURI, quotedCid)
513
531
}
514
532
515
-
quotedRepo, err := s.backend.getRepoByID(ctx, quotedPost.Author)
533
+
quotedRepo, err := s.backend.GetRepoByID(ctx, quotedPost.Author)
516
534
if err != nil {
517
535
slog.Warn("failed to get quoted post author", "error", err)
518
536
return s.buildQuoteFallback(quotedURI, quotedCid)
···
559
577
560
578
// Get the requested post to find the thread root
561
579
var requestedPost models.Post
562
-
if err := s.backend.db.Find(&requestedPost, "id = ?", postID).Error; err != nil {
580
+
if err := s.db.Find(&requestedPost, "id = ?", postID).Error; err != nil {
563
581
return err
564
582
}
565
583
···
578
596
// Get all posts in this thread
579
597
var dbposts []models.Post
580
598
query := "SELECT * FROM posts WHERE id = ? OR in_thread = ? ORDER BY created ASC"
581
-
if err := s.backend.db.Raw(query, rootPostID, rootPostID).Scan(&dbposts).Error; err != nil {
599
+
if err := s.db.Raw(query, rootPostID, rootPostID).Scan(&dbposts).Error; err != nil {
582
600
return err
583
601
}
584
602
585
603
// Build response for each post
586
604
posts := []postResponse{}
587
605
for _, p := range dbposts {
588
-
r, err := s.backend.getRepoByID(ctx, p.Author)
606
+
r, err := s.backend.GetRepoByID(ctx, p.Author)
589
607
if err != nil {
590
608
return err
591
609
}
···
659
677
660
678
// Get all likes for this post
661
679
var likes []models.Like
662
-
if err := s.backend.db.Find(&likes, "subject = ?", postID).Error; err != nil {
680
+
if err := s.db.Find(&likes, "subject = ?", postID).Error; err != nil {
663
681
return err
664
682
}
665
683
666
684
users := []engagementUser{}
667
685
for _, like := range likes {
668
-
r, err := s.backend.getRepoByID(ctx, like.Author)
686
+
r, err := s.backend.GetRepoByID(ctx, like.Author)
669
687
if err != nil {
670
688
slog.Error("failed to get repo for like author", "error", err)
671
689
continue
···
680
698
681
699
// Get profile if available
682
700
var profile models.Profile
683
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
701
+
s.db.Find(&profile, "repo = ?", r.ID)
684
702
685
703
var prof *bsky.ActorProfile
686
704
if len(profile.Raw) > 0 {
···
689
707
prof = &p
690
708
}
691
709
} else {
692
-
s.addMissingProfile(ctx, r.Did)
710
+
s.backend.TrackMissingRecord(r.Did, false)
693
711
}
694
712
695
713
users = append(users, engagementUser{
···
719
737
720
738
// Get all reposts for this post
721
739
var reposts []models.Repost
722
-
if err := s.backend.db.Find(&reposts, "subject = ?", postID).Error; err != nil {
740
+
if err := s.db.Find(&reposts, "subject = ?", postID).Error; err != nil {
723
741
return err
724
742
}
725
743
726
744
users := []engagementUser{}
727
745
for _, repost := range reposts {
728
-
r, err := s.backend.getRepoByID(ctx, repost.Author)
746
+
r, err := s.backend.GetRepoByID(ctx, repost.Author)
729
747
if err != nil {
730
748
slog.Error("failed to get repo for repost author", "error", err)
731
749
continue
···
740
758
741
759
// Get profile if available
742
760
var profile models.Profile
743
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
761
+
s.db.Find(&profile, "repo = ?", r.ID)
744
762
745
763
var prof *bsky.ActorProfile
746
764
if len(profile.Raw) > 0 {
···
749
767
prof = &p
750
768
}
751
769
} else {
752
-
s.addMissingProfile(ctx, r.Did)
770
+
s.backend.TrackMissingRecord(r.Did, false)
753
771
}
754
772
755
773
users = append(users, engagementUser{
···
779
797
780
798
// Get all replies to this post
781
799
var replies []models.Post
782
-
if err := s.backend.db.Find(&replies, "reply_to = ?", postID).Error; err != nil {
800
+
if err := s.db.Find(&replies, "reply_to = ?", postID).Error; err != nil {
783
801
return err
784
802
}
785
803
···
793
811
}
794
812
seen[reply.Author] = true
795
813
796
-
r, err := s.backend.getRepoByID(ctx, reply.Author)
814
+
r, err := s.backend.GetRepoByID(ctx, reply.Author)
797
815
if err != nil {
798
816
slog.Error("failed to get repo for reply author", "error", err)
799
817
continue
···
808
826
809
827
// Get profile if available
810
828
var profile models.Profile
811
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
829
+
s.db.Find(&profile, "repo = ?", r.ID)
812
830
813
831
var prof *bsky.ActorProfile
814
832
if len(profile.Raw) > 0 {
···
817
835
prof = &p
818
836
}
819
837
} else {
820
-
s.addMissingProfile(ctx, r.Did)
838
+
s.backend.TrackMissingRecord(r.Did, false)
821
839
}
822
840
823
841
users = append(users, engagementUser{
···
915
933
query := `SELECT * FROM notifications WHERE "for" = ?`
916
934
if cursorID > 0 {
917
935
query += ` AND id < ?`
918
-
if err := s.backend.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, cursorID, limit).Scan(¬ifications).Error; err != nil {
936
+
if err := s.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, cursorID, limit).Scan(¬ifications).Error; err != nil {
919
937
return err
920
938
}
921
939
} else {
922
-
if err := s.backend.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, limit).Scan(¬ifications).Error; err != nil {
940
+
if err := s.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, limit).Scan(¬ifications).Error; err != nil {
923
941
return err
924
942
}
925
943
}
···
928
946
results := []notificationResponse{}
929
947
for _, notif := range notifications {
930
948
// Get author info
931
-
author, err := s.backend.getRepoByID(ctx, notif.Author)
949
+
author, err := s.backend.GetRepoByID(ctx, notif.Author)
932
950
if err != nil {
933
951
slog.Error("failed to get repo for notification author", "error", err)
934
952
continue
···
949
967
}
950
968
951
969
// Try to get source post preview for reply/mention notifications
952
-
if notif.Kind == NotifKindReply || notif.Kind == NotifKindMention {
970
+
if notif.Kind == backend.NotifKindReply || notif.Kind == backend.NotifKindMention {
953
971
// Parse URI to get post
954
-
p, err := s.backend.getPostByUri(ctx, notif.Source, "*")
972
+
p, err := s.backend.GetPostByUri(ctx, notif.Source, "*")
955
973
if err == nil && p.Raw != nil && len(p.Raw) > 0 {
956
974
var fp bsky.FeedPost
957
975
if err := fp.UnmarshalCBOR(bytes.NewReader(p.Raw)); err == nil {
+113
-11
hydration/actor.go
+113
-11
hydration/actor.go
···
10
10
11
11
"github.com/bluesky-social/indigo/api/bsky"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/whyrusleeping/market/models"
13
14
)
14
15
15
16
// ActorInfo contains hydrated actor information
···
21
22
22
23
// HydrateActor hydrates full actor information
23
24
func (h *Hydrator) HydrateActor(ctx context.Context, did string) (*ActorInfo, error) {
25
+
ctx, span := tracer.Start(ctx, "hydrateActor")
26
+
defer span.End()
27
+
24
28
// Look up handle
25
29
resp, err := h.dir.LookupDID(ctx, syntax.DID(did))
26
30
if err != nil {
···
60
64
FollowCount int64
61
65
FollowerCount int64
62
66
PostCount int64
67
+
ViewerState *bsky.ActorDefs_ViewerState
63
68
}
64
69
65
-
func (h *Hydrator) HydrateActorDetailed(ctx context.Context, did string) (*ActorInfoDetailed, error) {
70
+
func (h *Hydrator) HydrateActorDetailed(ctx context.Context, did string, viewer string) (*ActorInfoDetailed, error) {
66
71
act, err := h.HydrateActor(ctx, did)
67
72
if err != nil {
68
73
return nil, err
···
73
78
}
74
79
75
80
var wg sync.WaitGroup
76
-
wg.Add(3)
77
-
go func() {
78
-
defer wg.Done()
81
+
wg.Go(func() {
79
82
c, err := h.getFollowCountForUser(ctx, did)
80
83
if err != nil {
81
84
slog.Error("failed to get follow count", "did", did, "error", err)
82
85
}
83
86
actd.FollowCount = c
84
-
}()
85
-
go func() {
86
-
defer wg.Done()
87
+
})
88
+
wg.Go(func() {
87
89
c, err := h.getFollowerCountForUser(ctx, did)
88
90
if err != nil {
89
91
slog.Error("failed to get follower count", "did", did, "error", err)
90
92
}
91
93
actd.FollowerCount = c
92
-
}()
93
-
go func() {
94
-
defer wg.Done()
94
+
})
95
+
wg.Go(func() {
95
96
c, err := h.getPostCountForUser(ctx, did)
96
97
if err != nil {
97
98
slog.Error("failed to get post count", "did", did, "error", err)
98
99
}
99
100
actd.PostCount = c
100
-
}()
101
+
})
102
+
103
+
if viewer != "" {
104
+
wg.Go(func() {
105
+
vs, err := h.getProfileViewerState(ctx, did, viewer)
106
+
if err != nil {
107
+
slog.Error("failed to get viewer state", "did", did, "viewer", viewer, "error", err)
108
+
}
109
+
actd.ViewerState = vs
110
+
})
111
+
}
112
+
101
113
wg.Wait()
102
114
103
115
return &actd, nil
116
+
}
117
+
118
+
func (h *Hydrator) getProfileViewerState(ctx context.Context, did, viewer string) (*bsky.ActorDefs_ViewerState, error) {
119
+
vs := &bsky.ActorDefs_ViewerState{}
120
+
121
+
var wg sync.WaitGroup
122
+
123
+
// Check if viewer is blocked by the target account
124
+
wg.Go(func() {
125
+
blockedBy, err := h.getBlockPair(ctx, did, viewer)
126
+
if err != nil {
127
+
slog.Error("failed to get blockedBy relationship", "did", did, "viewer", viewer, "error", err)
128
+
return
129
+
}
130
+
131
+
if blockedBy != nil {
132
+
v := true
133
+
vs.BlockedBy = &v
134
+
}
135
+
})
136
+
137
+
// Check if viewer is blocking the target account
138
+
wg.Go(func() {
139
+
blocking, err := h.getBlockPair(ctx, viewer, did)
140
+
if err != nil {
141
+
slog.Error("failed to get blocking relationship", "did", did, "viewer", viewer, "error", err)
142
+
return
143
+
}
144
+
145
+
if blocking != nil {
146
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.block/%s", viewer, blocking.Rkey)
147
+
vs.Blocking = &uri
148
+
}
149
+
})
150
+
151
+
// Check if viewer is following the target account
152
+
wg.Go(func() {
153
+
following, err := h.getFollowPair(ctx, viewer, did)
154
+
if err != nil {
155
+
slog.Error("failed to get following relationship", "did", did, "viewer", viewer, "error", err)
156
+
return
157
+
}
158
+
159
+
if following != nil {
160
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.follow/%s", viewer, following.Rkey)
161
+
vs.Following = &uri
162
+
}
163
+
})
164
+
165
+
// Check if target account is following the viewer
166
+
wg.Go(func() {
167
+
followedBy, err := h.getFollowPair(ctx, did, viewer)
168
+
if err != nil {
169
+
slog.Error("failed to get followedBy relationship", "did", did, "viewer", viewer, "error", err)
170
+
return
171
+
}
172
+
173
+
if followedBy != nil {
174
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.follow/%s", did, followedBy.Rkey)
175
+
vs.FollowedBy = &uri
176
+
}
177
+
})
178
+
179
+
wg.Wait()
180
+
181
+
return vs, nil
182
+
}
183
+
184
+
func (h *Hydrator) getBlockPair(ctx context.Context, a, b string) (*models.Block, error) {
185
+
var blk models.Block
186
+
if err := h.db.Raw("SELECT * FROM blocks WHERE author = (SELECT id FROM repos WHERE did = ?) AND subject = (SELECT id FROM repos WHERE did = ?)", a, b).Scan(&blk).Error; err != nil {
187
+
return nil, err
188
+
}
189
+
if blk.ID == 0 {
190
+
return nil, nil
191
+
}
192
+
193
+
return &blk, nil
194
+
}
195
+
196
+
func (h *Hydrator) getFollowPair(ctx context.Context, a, b string) (*models.Follow, error) {
197
+
var fol models.Follow
198
+
if err := h.db.Raw("SELECT * FROM follows WHERE author = (SELECT id FROM repos WHERE did = ?) AND subject = (SELECT id FROM repos WHERE did = ?)", a, b).Scan(&fol).Error; err != nil {
199
+
return nil, err
200
+
}
201
+
if fol.ID == 0 {
202
+
return nil, nil
203
+
}
204
+
205
+
return &fol, nil
104
206
}
105
207
106
208
func (h *Hydrator) getFollowCountForUser(ctx context.Context, did string) (int64, error) {
+15
-24
hydration/hydrator.go
+15
-24
hydration/hydrator.go
···
2
2
3
3
import (
4
4
"github.com/bluesky-social/indigo/atproto/identity"
5
+
"github.com/whyrusleeping/konbini/backend"
5
6
"gorm.io/gorm"
6
7
)
7
8
8
9
// Hydrator handles data hydration from the database
9
10
type Hydrator struct {
10
-
db *gorm.DB
11
-
dir identity.Directory
12
-
13
-
missingActorCallback func(string)
14
-
missingPostCallback func(string)
15
-
missingFeedGeneratorCallback func(string)
11
+
db *gorm.DB
12
+
dir identity.Directory
13
+
backend *backend.PostgresBackend
16
14
}
17
15
18
16
// NewHydrator creates a new Hydrator
19
-
func NewHydrator(db *gorm.DB, dir identity.Directory) *Hydrator {
17
+
func NewHydrator(db *gorm.DB, dir identity.Directory, backend *backend.PostgresBackend) *Hydrator {
20
18
return &Hydrator{
21
-
db: db,
22
-
dir: dir,
19
+
db: db,
20
+
dir: dir,
21
+
backend: backend,
23
22
}
24
23
}
25
24
26
-
func (h *Hydrator) SetMissingActorCallback(fn func(string)) {
27
-
h.missingActorCallback = fn
25
+
// AddMissingRecord reports a missing record that needs to be fetched
26
+
func (h *Hydrator) AddMissingRecord(identifier string, wait bool) {
27
+
if h.backend != nil {
28
+
h.backend.TrackMissingRecord(identifier, wait)
29
+
}
28
30
}
29
31
32
+
// addMissingActor is a convenience method for adding missing actors
30
33
func (h *Hydrator) addMissingActor(did string) {
31
-
if h.missingActorCallback != nil {
32
-
h.missingActorCallback(did)
33
-
}
34
-
}
35
-
36
-
func (h *Hydrator) SetMissingFeedGeneratorCallback(fn func(string)) {
37
-
h.missingFeedGeneratorCallback = fn
38
-
}
39
-
40
-
func (h *Hydrator) AddMissingFeedGenerator(uri string) {
41
-
if h.missingFeedGeneratorCallback != nil {
42
-
h.missingFeedGeneratorCallback(uri)
43
-
}
34
+
h.AddMissingRecord(did, false)
44
35
}
45
36
46
37
// HydrateCtx contains context for hydration operations
+388
-39
hydration/post.go
+388
-39
hydration/post.go
···
5
5
"context"
6
6
"fmt"
7
7
"log/slog"
8
+
"sync"
8
9
9
10
"github.com/bluesky-social/indigo/api/bsky"
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
"github.com/whyrusleeping/market/models"
13
+
"go.opentelemetry.io/otel"
10
14
)
11
15
16
+
var tracer = otel.Tracer("hydrator")
17
+
12
18
// PostInfo contains hydrated post information
13
19
type PostInfo struct {
20
+
ID uint
14
21
URI string
15
22
Cid string
16
23
Post *bsky.FeedPost
···
22
29
RepostCount int
23
30
ReplyCount int
24
31
ViewerLike string // URI of viewer's like, if any
32
+
33
+
EmbedInfo *bsky.FeedDefs_PostView_Embed
25
34
}
26
35
27
36
const fakeCid = "bafyreiapw4hagb5ehqgoeho4v23vf7fhlqey4b7xvjpy76krgkqx7xlolu"
28
37
29
38
// HydratePost hydrates a single post by URI
30
39
func (h *Hydrator) HydratePost(ctx context.Context, uri string, viewerDID string) (*PostInfo, error) {
31
-
// Query post from database
32
-
var dbPost struct {
33
-
ID uint
34
-
Cid string
35
-
Raw []byte
36
-
NotFound bool
37
-
ReplyTo uint
38
-
ReplyToUsr uint
39
-
InThread uint
40
-
AuthorID uint
40
+
ctx, span := tracer.Start(ctx, "hydratePost")
41
+
defer span.End()
42
+
43
+
p, err := h.backend.GetPostByUri(ctx, uri, "*")
44
+
if err != nil {
45
+
return nil, err
41
46
}
42
47
43
-
err := h.db.Raw(`
44
-
SELECT p.id, p.cid, p.raw, p.not_found, p.reply_to, p.reply_to_usr, p.in_thread, p.author as author_id
45
-
FROM posts p
46
-
WHERE p.id = (
47
-
SELECT id FROM posts
48
-
WHERE author = (SELECT id FROM repos WHERE did = ?)
49
-
AND rkey = ?
50
-
)
51
-
`, extractDIDFromURI(uri), extractRkeyFromURI(uri)).Scan(&dbPost).Error
48
+
return h.HydratePostDB(ctx, uri, p, viewerDID)
49
+
}
52
50
51
+
func (h *Hydrator) HydratePostDB(ctx context.Context, uri string, dbPost *models.Post, viewerDID string) (*PostInfo, error) {
52
+
autoFetch, _ := ctx.Value("auto-fetch").(bool)
53
+
54
+
authorDid := extractDIDFromURI(uri)
55
+
r, err := h.backend.GetOrCreateRepo(ctx, authorDid)
53
56
if err != nil {
54
-
return nil, fmt.Errorf("failed to query post: %w", err)
57
+
return nil, err
55
58
}
56
59
57
60
if dbPost.NotFound || len(dbPost.Raw) == 0 {
58
-
return nil, fmt.Errorf("post not found")
61
+
if autoFetch {
62
+
h.AddMissingRecord(uri, true)
63
+
if err := h.db.Raw(`SELECT * FROM posts WHERE author = ? AND rkey = ?`, r.ID, extractRkeyFromURI(uri)).Scan(&dbPost).Error; err != nil {
64
+
return nil, fmt.Errorf("failed to query post: %w", err)
65
+
}
66
+
if dbPost.NotFound || len(dbPost.Raw) == 0 {
67
+
return nil, fmt.Errorf("post not found")
68
+
}
69
+
} else {
70
+
return nil, fmt.Errorf("post not found")
71
+
}
59
72
}
60
73
61
74
// Unmarshal post record
···
64
77
return nil, fmt.Errorf("failed to unmarshal post: %w", err)
65
78
}
66
79
67
-
// Get author DID
68
-
var authorDID string
69
-
h.db.Raw("SELECT did FROM repos WHERE id = ?", dbPost.AuthorID).Scan(&authorDID)
80
+
var wg sync.WaitGroup
81
+
82
+
authorDID := r.Did
70
83
71
84
// Get engagement counts
72
85
var likes, reposts, replies int
73
-
h.db.Raw("SELECT COUNT(*) FROM likes WHERE subject = ?", dbPost.ID).Scan(&likes)
74
-
h.db.Raw("SELECT COUNT(*) FROM reposts WHERE subject = ?", dbPost.ID).Scan(&reposts)
75
-
h.db.Raw("SELECT COUNT(*) FROM posts WHERE reply_to = ?", dbPost.ID).Scan(&replies)
86
+
wg.Go(func() {
87
+
_, span := tracer.Start(ctx, "likeCounts")
88
+
defer span.End()
89
+
h.db.Raw("SELECT COUNT(*) FROM likes WHERE subject = ?", dbPost.ID).Scan(&likes)
90
+
})
91
+
wg.Go(func() {
92
+
_, span := tracer.Start(ctx, "repostCounts")
93
+
defer span.End()
94
+
h.db.Raw("SELECT COUNT(*) FROM reposts WHERE subject = ?", dbPost.ID).Scan(&reposts)
95
+
})
96
+
wg.Go(func() {
97
+
_, span := tracer.Start(ctx, "replyCounts")
98
+
defer span.End()
99
+
h.db.Raw("SELECT COUNT(*) FROM posts WHERE reply_to = ?", dbPost.ID).Scan(&replies)
100
+
})
101
+
102
+
// Check if viewer liked this post
103
+
var likeRkey string
104
+
if viewerDID != "" {
105
+
wg.Go(func() {
106
+
_, span := tracer.Start(ctx, "viewerLikeState")
107
+
defer span.End()
108
+
h.db.Raw(`
109
+
SELECT l.rkey FROM likes l
110
+
WHERE l.subject = ?
111
+
AND l.author = (SELECT id FROM repos WHERE did = ?)
112
+
`, dbPost.ID, viewerDID).Scan(&likeRkey)
113
+
})
114
+
}
115
+
116
+
var ei *bsky.FeedDefs_PostView_Embed
117
+
if feedPost.Embed != nil {
118
+
wg.Go(func() {
119
+
ei = h.formatEmbed(ctx, feedPost.Embed, authorDID, viewerDID)
120
+
})
121
+
}
122
+
123
+
wg.Wait()
76
124
77
125
info := &PostInfo{
126
+
ID: dbPost.ID,
78
127
URI: uri,
79
128
Cid: dbPost.Cid,
80
129
Post: &feedPost,
···
85
134
LikeCount: likes,
86
135
RepostCount: reposts,
87
136
ReplyCount: replies,
137
+
EmbedInfo: ei,
138
+
}
139
+
140
+
if likeRkey != "" {
141
+
info.ViewerLike = fmt.Sprintf("at://%s/app.bsky.feed.like/%s", viewerDID, likeRkey)
88
142
}
89
143
90
144
if info.Cid == "" {
···
92
146
info.Cid = fakeCid
93
147
}
94
148
95
-
// Check if viewer liked this post
96
-
if viewerDID != "" {
97
-
var likeRkey string
98
-
h.db.Raw(`
99
-
SELECT l.rkey FROM likes l
100
-
WHERE l.subject = ?
101
-
AND l.author = (SELECT id FROM repos WHERE did = ?)
102
-
`, dbPost.ID, viewerDID).Scan(&likeRkey)
103
-
if likeRkey != "" {
104
-
info.ViewerLike = fmt.Sprintf("at://%s/app.bsky.feed.like/%s", viewerDID, likeRkey)
105
-
}
106
-
}
149
+
// Hydrate embed
107
150
108
151
return info, nil
109
152
}
···
150
193
}
151
194
return ""
152
195
}
196
+
197
+
func (h *Hydrator) formatEmbed(ctx context.Context, embed *bsky.FeedPost_Embed, authorDID string, viewerDID string) *bsky.FeedDefs_PostView_Embed {
198
+
if embed == nil {
199
+
return nil
200
+
}
201
+
_, span := tracer.Start(ctx, "formatEmbed")
202
+
defer span.End()
203
+
204
+
result := &bsky.FeedDefs_PostView_Embed{}
205
+
206
+
// Handle images
207
+
if embed.EmbedImages != nil {
208
+
viewImages := make([]*bsky.EmbedImages_ViewImage, len(embed.EmbedImages.Images))
209
+
for i, img := range embed.EmbedImages.Images {
210
+
// Convert blob to CDN URLs
211
+
fullsize := ""
212
+
thumb := ""
213
+
if img.Image != nil {
214
+
// CDN URL format for feed images
215
+
cid := img.Image.Ref.String()
216
+
fullsize = fmt.Sprintf("https://cdn.bsky.app/img/feed_fullsize/plain/%s/%s@jpeg", authorDID, cid)
217
+
thumb = fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
218
+
}
219
+
220
+
viewImages[i] = &bsky.EmbedImages_ViewImage{
221
+
Alt: img.Alt,
222
+
AspectRatio: img.AspectRatio,
223
+
Fullsize: fullsize,
224
+
Thumb: thumb,
225
+
}
226
+
}
227
+
result.EmbedImages_View = &bsky.EmbedImages_View{
228
+
LexiconTypeID: "app.bsky.embed.images#view",
229
+
Images: viewImages,
230
+
}
231
+
return result
232
+
}
233
+
234
+
// Handle external links
235
+
if embed.EmbedExternal != nil && embed.EmbedExternal.External != nil {
236
+
// Convert blob thumb to CDN URL if present
237
+
var thumbURL *string
238
+
if embed.EmbedExternal.External.Thumb != nil {
239
+
// CDN URL for external link thumbnails
240
+
cid := embed.EmbedExternal.External.Thumb.Ref.String()
241
+
url := fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
242
+
thumbURL = &url
243
+
}
244
+
245
+
result.EmbedExternal_View = &bsky.EmbedExternal_View{
246
+
LexiconTypeID: "app.bsky.embed.external#view",
247
+
External: &bsky.EmbedExternal_ViewExternal{
248
+
Uri: embed.EmbedExternal.External.Uri,
249
+
Title: embed.EmbedExternal.External.Title,
250
+
Description: embed.EmbedExternal.External.Description,
251
+
Thumb: thumbURL,
252
+
},
253
+
}
254
+
return result
255
+
}
256
+
257
+
// Handle video
258
+
if embed.EmbedVideo != nil && embed.EmbedVideo.Video != nil {
259
+
cid := embed.EmbedVideo.Video.Ref.String()
260
+
// URL-encode the DID (replace : with %3A)
261
+
encodedDID := ""
262
+
for _, ch := range authorDID {
263
+
if ch == ':' {
264
+
encodedDID += "%3A"
265
+
} else {
266
+
encodedDID += string(ch)
267
+
}
268
+
}
269
+
270
+
playlist := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/playlist.m3u8", encodedDID, cid)
271
+
thumbnail := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/thumbnail.jpg", encodedDID, cid)
272
+
273
+
result.EmbedVideo_View = &bsky.EmbedVideo_View{
274
+
LexiconTypeID: "app.bsky.embed.video#view",
275
+
Cid: cid,
276
+
Playlist: playlist,
277
+
Thumbnail: &thumbnail,
278
+
Alt: embed.EmbedVideo.Alt,
279
+
AspectRatio: embed.EmbedVideo.AspectRatio,
280
+
}
281
+
return result
282
+
}
283
+
284
+
// Handle record (quote posts, etc.)
285
+
if embed.EmbedRecord != nil && embed.EmbedRecord.Record != nil {
286
+
rec := embed.EmbedRecord.Record
287
+
288
+
result.EmbedRecord_View = &bsky.EmbedRecord_View{
289
+
LexiconTypeID: "app.bsky.embed.record#view",
290
+
Record: h.hydrateEmbeddedRecord(ctx, rec.Uri, viewerDID),
291
+
}
292
+
return result
293
+
}
294
+
295
+
// Handle record with media (quote post with images/external)
296
+
if embed.EmbedRecordWithMedia != nil {
297
+
recordView := &bsky.EmbedRecordWithMedia_View{
298
+
LexiconTypeID: "app.bsky.embed.recordWithMedia#view",
299
+
}
300
+
301
+
// Hydrate the record part
302
+
if embed.EmbedRecordWithMedia.Record != nil && embed.EmbedRecordWithMedia.Record.Record != nil {
303
+
recordView.Record = &bsky.EmbedRecord_View{
304
+
LexiconTypeID: "app.bsky.embed.record#view",
305
+
Record: h.hydrateEmbeddedRecord(ctx, embed.EmbedRecordWithMedia.Record.Record.Uri, viewerDID),
306
+
}
307
+
}
308
+
309
+
// Hydrate the media part (images or external)
310
+
if embed.EmbedRecordWithMedia.Media != nil {
311
+
if embed.EmbedRecordWithMedia.Media.EmbedImages != nil {
312
+
viewImages := make([]*bsky.EmbedImages_ViewImage, len(embed.EmbedRecordWithMedia.Media.EmbedImages.Images))
313
+
for i, img := range embed.EmbedRecordWithMedia.Media.EmbedImages.Images {
314
+
fullsize := ""
315
+
thumb := ""
316
+
if img.Image != nil {
317
+
cid := img.Image.Ref.String()
318
+
fullsize = fmt.Sprintf("https://cdn.bsky.app/img/feed_fullsize/plain/%s/%s@jpeg", authorDID, cid)
319
+
thumb = fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
320
+
}
321
+
322
+
viewImages[i] = &bsky.EmbedImages_ViewImage{
323
+
Alt: img.Alt,
324
+
AspectRatio: img.AspectRatio,
325
+
Fullsize: fullsize,
326
+
Thumb: thumb,
327
+
}
328
+
}
329
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
330
+
EmbedImages_View: &bsky.EmbedImages_View{
331
+
LexiconTypeID: "app.bsky.embed.images#view",
332
+
Images: viewImages,
333
+
},
334
+
}
335
+
} else if embed.EmbedRecordWithMedia.Media.EmbedExternal != nil && embed.EmbedRecordWithMedia.Media.EmbedExternal.External != nil {
336
+
var thumbURL *string
337
+
if embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Thumb != nil {
338
+
cid := embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Thumb.Ref.String()
339
+
url := fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
340
+
thumbURL = &url
341
+
}
342
+
343
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
344
+
EmbedExternal_View: &bsky.EmbedExternal_View{
345
+
LexiconTypeID: "app.bsky.embed.external#view",
346
+
External: &bsky.EmbedExternal_ViewExternal{
347
+
Uri: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Uri,
348
+
Title: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Title,
349
+
Description: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Description,
350
+
Thumb: thumbURL,
351
+
},
352
+
},
353
+
}
354
+
} else if embed.EmbedRecordWithMedia.Media.EmbedVideo != nil && embed.EmbedRecordWithMedia.Media.EmbedVideo.Video != nil {
355
+
cid := embed.EmbedRecordWithMedia.Media.EmbedVideo.Video.Ref.String()
356
+
// URL-encode the DID (replace : with %3A)
357
+
encodedDID := ""
358
+
for _, ch := range authorDID {
359
+
if ch == ':' {
360
+
encodedDID += "%3A"
361
+
} else {
362
+
encodedDID += string(ch)
363
+
}
364
+
}
365
+
366
+
playlist := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/playlist.m3u8", encodedDID, cid)
367
+
thumbnail := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/thumbnail.jpg", encodedDID, cid)
368
+
369
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
370
+
EmbedVideo_View: &bsky.EmbedVideo_View{
371
+
LexiconTypeID: "app.bsky.embed.video#view",
372
+
Cid: cid,
373
+
Playlist: playlist,
374
+
Thumbnail: &thumbnail,
375
+
Alt: embed.EmbedRecordWithMedia.Media.EmbedVideo.Alt,
376
+
AspectRatio: embed.EmbedRecordWithMedia.Media.EmbedVideo.AspectRatio,
377
+
},
378
+
}
379
+
}
380
+
}
381
+
382
+
result.EmbedRecordWithMedia_View = recordView
383
+
return result
384
+
}
385
+
386
+
return nil
387
+
}
388
+
389
+
// hydrateEmbeddedRecord hydrates an embedded record (for quote posts, etc.)
390
+
func (h *Hydrator) hydrateEmbeddedRecord(ctx context.Context, uri string, viewerDID string) *bsky.EmbedRecord_View_Record {
391
+
ctx, span := tracer.Start(ctx, "hydrateEmbeddedRecord")
392
+
defer span.End()
393
+
394
+
// Check if it's a post URI
395
+
if !isPostURI(uri) {
396
+
// Could be a feed generator, list, labeler, or starter pack
397
+
// For now, return not found for non-post embeds
398
+
return &bsky.EmbedRecord_View_Record{
399
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
400
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
401
+
Uri: uri,
402
+
},
403
+
}
404
+
}
405
+
406
+
// Try to hydrate the post
407
+
quotedPost, err := h.HydratePost(ctx, uri, viewerDID)
408
+
if err != nil {
409
+
// Post not found
410
+
return &bsky.EmbedRecord_View_Record{
411
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
412
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
413
+
Uri: uri,
414
+
NotFound: true,
415
+
},
416
+
}
417
+
}
418
+
419
+
// Hydrate the author
420
+
authorInfo, err := h.HydrateActor(ctx, quotedPost.Author)
421
+
if err != nil {
422
+
// Author not found, treat as not found
423
+
return &bsky.EmbedRecord_View_Record{
424
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
425
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
426
+
Uri: uri,
427
+
NotFound: true,
428
+
},
429
+
}
430
+
}
431
+
432
+
// TODO: Check if viewer has blocked or is blocked by the author
433
+
// For now, just return the record view
434
+
435
+
// Build the author profile view
436
+
authorView := &bsky.ActorDefs_ProfileViewBasic{
437
+
Did: authorInfo.DID,
438
+
Handle: authorInfo.Handle,
439
+
}
440
+
if authorInfo.Profile != nil {
441
+
if authorInfo.Profile.DisplayName != nil && *authorInfo.Profile.DisplayName != "" {
442
+
authorView.DisplayName = authorInfo.Profile.DisplayName
443
+
}
444
+
if authorInfo.Profile.Avatar != nil {
445
+
avatarURL := fmt.Sprintf("https://cdn.bsky.app/img/avatar_thumbnail/plain/%s/%s@jpeg", authorInfo.DID, authorInfo.Profile.Avatar.Ref.String())
446
+
authorView.Avatar = &avatarURL
447
+
}
448
+
}
449
+
450
+
// Build the embedded post view
451
+
embedView := &bsky.EmbedRecord_ViewRecord{
452
+
LexiconTypeID: "app.bsky.embed.record#viewRecord",
453
+
Uri: quotedPost.URI,
454
+
Cid: quotedPost.Cid,
455
+
Author: authorView,
456
+
Value: &util.LexiconTypeDecoder{
457
+
Val: quotedPost.Post,
458
+
},
459
+
IndexedAt: quotedPost.Post.CreatedAt,
460
+
}
461
+
462
+
// Add engagement counts
463
+
if quotedPost.LikeCount > 0 {
464
+
lc := int64(quotedPost.LikeCount)
465
+
embedView.LikeCount = &lc
466
+
}
467
+
if quotedPost.RepostCount > 0 {
468
+
rc := int64(quotedPost.RepostCount)
469
+
embedView.RepostCount = &rc
470
+
}
471
+
if quotedPost.ReplyCount > 0 {
472
+
rpc := int64(quotedPost.ReplyCount)
473
+
embedView.ReplyCount = &rpc
474
+
}
475
+
476
+
// Note: We don't recursively hydrate embeds for quoted posts to avoid deep nesting
477
+
// The official app also doesn't show embeds within quoted posts
478
+
479
+
return &bsky.EmbedRecord_View_Record{
480
+
EmbedRecord_ViewRecord: embedView,
481
+
}
482
+
}
483
+
484
+
// isPostURI checks if a URI is a post URI
485
+
func isPostURI(uri string) bool {
486
+
return len(uri) > 5 && uri[:5] == "at://" && (
487
+
// Check if it contains /app.bsky.feed.post/
488
+
len(uri) > 25 && uri[len(uri)-25:len(uri)-12] == "/app.bsky.feed.post/" ||
489
+
// More flexible check
490
+
contains(uri, "/app.bsky.feed.post/"))
491
+
}
492
+
493
+
// contains checks if a string contains a substring
494
+
func contains(s, substr string) bool {
495
+
for i := 0; i <= len(s)-len(substr); i++ {
496
+
if s[i:i+len(substr)] == substr {
497
+
return true
498
+
}
499
+
}
500
+
return false
501
+
}
+10
hydration/utils.go
+10
hydration/utils.go
···
5
5
"fmt"
6
6
7
7
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
"github.com/whyrusleeping/market/models"
8
9
)
9
10
10
11
func (h *Hydrator) NormalizeUri(ctx context.Context, uri string) (string, error) {
···
27
28
28
29
return fmt.Sprintf("at://%s/%s/%s", did, puri.Collection().String(), puri.RecordKey().String()), nil
29
30
}
31
+
32
+
func (h *Hydrator) UriForPost(ctx context.Context, p *models.Post) (string, error) {
33
+
did, err := h.backend.DidFromID(ctx, p.Author)
34
+
if err != nil {
35
+
return "", err
36
+
}
37
+
38
+
return fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, p.Rkey), nil
39
+
}
+105
-141
main.go
+105
-141
main.go
···
3
3
import (
4
4
"bytes"
5
5
"context"
6
+
"encoding/json"
6
7
"fmt"
7
8
"log"
8
9
"log/slog"
···
17
18
18
19
"github.com/bluesky-social/indigo/api/atproto"
19
20
"github.com/bluesky-social/indigo/atproto/identity"
21
+
"github.com/bluesky-social/indigo/atproto/identity/redisdir"
20
22
"github.com/bluesky-social/indigo/atproto/syntax"
21
-
"github.com/bluesky-social/indigo/cmd/relay/stream"
22
-
"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
23
23
"github.com/bluesky-social/indigo/repo"
24
24
"github.com/bluesky-social/indigo/util/cliutil"
25
25
xrpclib "github.com/bluesky-social/indigo/xrpc"
26
-
"github.com/gorilla/websocket"
27
-
lru "github.com/hashicorp/golang-lru/v2"
28
26
"github.com/ipfs/go-cid"
29
27
"github.com/jackc/pgx/v5/pgxpool"
30
28
"github.com/prometheus/client_golang/prometheus"
31
29
"github.com/prometheus/client_golang/prometheus/promauto"
32
30
"github.com/urfave/cli/v2"
31
+
"github.com/whyrusleeping/konbini/backend"
33
32
"github.com/whyrusleeping/konbini/xrpc"
33
+
"go.opentelemetry.io/otel"
34
+
"go.opentelemetry.io/otel/attribute"
35
+
"go.opentelemetry.io/otel/exporters/jaeger"
36
+
"go.opentelemetry.io/otel/sdk/resource"
37
+
tracesdk "go.opentelemetry.io/otel/sdk/trace"
38
+
semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
39
+
"gorm.io/gorm"
34
40
"gorm.io/gorm/logger"
35
41
36
42
. "github.com/whyrusleeping/konbini/models"
37
43
)
38
44
39
-
var handleOpHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
40
-
Name: "handle_op_duration",
41
-
Help: "A histogram of op handling durations",
42
-
Buckets: prometheus.ExponentialBuckets(1, 2, 15),
43
-
}, []string{"op", "collection"})
44
-
45
45
var firehoseCursorGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
46
46
Name: "firehose_cursor",
47
47
}, []string{"stage"})
···
56
56
Name: "db-url",
57
57
EnvVars: []string{"DATABASE_URL"},
58
58
},
59
+
&cli.BoolFlag{
60
+
Name: "jaeger",
61
+
},
59
62
&cli.StringFlag{
60
63
Name: "handle",
61
64
},
···
63
66
Name: "max-db-connections",
64
67
Value: runtime.NumCPU(),
65
68
},
69
+
&cli.StringFlag{
70
+
Name: "redis-url",
71
+
},
72
+
&cli.StringFlag{
73
+
Name: "sync-config",
74
+
},
66
75
}
67
76
app.Action = func(cctx *cli.Context) error {
68
77
db, err := cliutil.SetupDatabase(cctx.String("db-url"), cctx.Int("max-db-connections"))
···
77
86
Colorful: true,
78
87
})
79
88
89
+
if cctx.Bool("jaeger") {
90
+
// Use Jaeger native exporter sending to port 14268
91
+
jaegerUrl := "http://localhost:14268/api/traces"
92
+
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl)))
93
+
if err != nil {
94
+
return err
95
+
}
96
+
97
+
env := os.Getenv("ENV")
98
+
if env == "" {
99
+
env = "development"
100
+
}
101
+
102
+
tp := tracesdk.NewTracerProvider(
103
+
// Always be sure to batch in production.
104
+
tracesdk.WithBatcher(exp),
105
+
// Record information about this application in a Resource.
106
+
tracesdk.WithResource(resource.NewWithAttributes(
107
+
semconv.SchemaURL,
108
+
semconv.ServiceNameKey.String("konbini"),
109
+
attribute.String("env", env), // DataDog
110
+
attribute.String("environment", env), // Others
111
+
attribute.Int64("ID", 1),
112
+
)),
113
+
)
114
+
115
+
otel.SetTracerProvider(tp)
116
+
}
117
+
80
118
db.AutoMigrate(Repo{})
81
119
db.AutoMigrate(Post{})
82
120
db.AutoMigrate(Follow{})
···
92
130
db.AutoMigrate(Image{})
93
131
db.AutoMigrate(PostGate{})
94
132
db.AutoMigrate(StarterPack{})
95
-
db.AutoMigrate(SyncInfo{})
133
+
db.AutoMigrate(backend.SyncInfo{})
96
134
db.AutoMigrate(Notification{})
135
+
db.AutoMigrate(NotificationSeen{})
97
136
db.AutoMigrate(SequenceTracker{})
137
+
db.Exec("CREATE INDEX IF NOT EXISTS reposts_subject_idx ON reposts (subject)")
138
+
db.Exec("CREATE INDEX IF NOT EXISTS posts_reply_to_idx ON posts (reply_to)")
139
+
db.Exec("CREATE INDEX IF NOT EXISTS posts_in_thread_idx ON posts (in_thread)")
98
140
99
141
ctx := context.TODO()
100
-
101
-
rc, _ := lru.New2Q[string, *Repo](1_000_000)
102
-
pc, _ := lru.New2Q[string, cachedPostInfo](1_000_000)
103
-
revc, _ := lru.New2Q[uint, string](1_000_000)
104
142
105
143
cfg, err := pgxpool.ParseConfig(cctx.String("db-url"))
106
144
if err != nil {
···
125
163
126
164
dir := identity.DefaultDirectory()
127
165
166
+
if redisURL := cctx.String("redis-url"); redisURL != "" {
167
+
rdir, err := redisdir.NewRedisDirectory(dir, redisURL, time.Minute, time.Second*10, time.Second*10, 100_000)
168
+
if err != nil {
169
+
return err
170
+
}
171
+
dir = rdir
172
+
}
173
+
128
174
resp, err := dir.LookupHandle(ctx, syntax.Handle(handle))
129
175
if err != nil {
130
176
return err
···
155
201
client: cc,
156
202
dir: dir,
157
203
158
-
missingRecords: make(chan MissingRecord, 1024),
204
+
db: db,
159
205
}
160
-
fmt.Println("MY DID: ", s.mydid)
161
206
162
-
pgb := &PostgresBackend{
163
-
relevantDids: make(map[string]bool),
164
-
s: s,
165
-
db: db,
166
-
postInfoCache: pc,
167
-
repoCache: rc,
168
-
revCache: revc,
169
-
pgx: pool,
207
+
pgb, err := backend.NewPostgresBackend(mydid, db, pool, cc, dir)
208
+
if err != nil {
209
+
return err
170
210
}
211
+
171
212
s.backend = pgb
172
213
173
-
myrepo, err := s.backend.getOrCreateRepo(ctx, mydid)
214
+
myrepo, err := s.backend.GetOrCreateRepo(ctx, mydid)
174
215
if err != nil {
175
216
return fmt.Errorf("failed to get repo record for our own did: %w", err)
176
217
}
177
218
s.myrepo = myrepo
178
219
179
-
if err := s.backend.loadRelevantDids(); err != nil {
220
+
if err := s.backend.LoadRelevantDids(); err != nil {
180
221
return fmt.Errorf("failed to load relevant dids set: %w", err)
181
222
}
182
223
···
200
241
http.ListenAndServe(":4445", nil)
201
242
}()
202
243
203
-
go s.missingRecordFetcher()
244
+
sc := SyncConfig{
245
+
Backends: []SyncBackend{
246
+
{
247
+
Type: "firehose",
248
+
Host: "bsky.network",
249
+
},
250
+
},
251
+
}
204
252
205
-
seqno, err := loadLastSeq(db, "firehose_seq")
206
-
if err != nil {
207
-
fmt.Println("failed to load sequence number, starting over", err)
253
+
if scfn := cctx.String("sync-config"); scfn != "" {
254
+
{
255
+
scfi, err := os.Open(scfn)
256
+
if err != nil {
257
+
return err
258
+
}
259
+
defer scfi.Close()
260
+
261
+
var lsc SyncConfig
262
+
if err := json.NewDecoder(scfi).Decode(&lsc); err != nil {
263
+
return err
264
+
}
265
+
sc = lsc
266
+
}
208
267
}
209
268
210
-
return s.startLiveTail(ctx, int(seqno), 10, 20)
269
+
/*
270
+
sc.Backends[0] = SyncBackend{
271
+
Type: "jetstream",
272
+
Host: "jetstream1.us-west.bsky.network",
273
+
}
274
+
*/
275
+
276
+
return s.StartSyncEngine(ctx, &sc)
277
+
211
278
}
212
279
213
280
app.RunAndExitOnError()
214
281
}
215
282
216
283
type Server struct {
217
-
backend *PostgresBackend
284
+
backend *backend.PostgresBackend
218
285
219
286
dir identity.Directory
220
287
···
225
292
seqLk sync.Mutex
226
293
lastSeq int64
227
294
228
-
mpLk sync.Mutex
229
-
missingRecords chan MissingRecord
295
+
mpLk sync.Mutex
296
+
297
+
db *gorm.DB
230
298
}
231
299
232
300
func (s *Server) getXrpcClient() (*xrpclib.Client, error) {
···
234
302
return s.client, nil
235
303
}
236
304
237
-
func (s *Server) startLiveTail(ctx context.Context, curs int, parWorkers, maxQ int) error {
238
-
slog.Info("starting live tail")
239
-
240
-
// Connect to the Relay websocket
241
-
urlStr := fmt.Sprintf("wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", curs)
242
-
243
-
d := websocket.DefaultDialer
244
-
con, _, err := d.Dial(urlStr, http.Header{
245
-
"User-Agent": []string{"market/0.0.1"},
246
-
})
247
-
if err != nil {
248
-
return fmt.Errorf("failed to connect to relay: %w", err)
249
-
}
250
-
251
-
var lelk sync.Mutex
252
-
lastEvent := time.Now()
253
-
254
-
go func() {
255
-
for range time.Tick(time.Second) {
256
-
lelk.Lock()
257
-
let := lastEvent
258
-
lelk.Unlock()
259
-
260
-
if time.Since(let) > time.Second*30 {
261
-
slog.Error("firehose connection timed out")
262
-
con.Close()
263
-
return
264
-
}
265
-
266
-
}
267
-
268
-
}()
269
-
270
-
var cclk sync.Mutex
271
-
var completeCursor int64
272
-
273
-
rsc := &stream.RepoStreamCallbacks{
274
-
RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
275
-
ctx := context.Background()
276
-
277
-
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
278
-
279
-
s.seqLk.Lock()
280
-
if evt.Seq > s.lastSeq {
281
-
curs = int(evt.Seq)
282
-
s.lastSeq = evt.Seq
283
-
284
-
if evt.Seq%1000 == 0 {
285
-
if err := storeLastSeq(s.backend.db, "firehose_seq", evt.Seq); err != nil {
286
-
fmt.Println("failed to store seqno: ", err)
287
-
}
288
-
}
289
-
}
290
-
s.seqLk.Unlock()
291
-
292
-
lelk.Lock()
293
-
lastEvent = time.Now()
294
-
lelk.Unlock()
295
-
296
-
if err := s.backend.HandleEvent(ctx, evt); err != nil {
297
-
return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
298
-
}
299
-
300
-
cclk.Lock()
301
-
if evt.Seq > completeCursor {
302
-
completeCursor = evt.Seq
303
-
firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
304
-
}
305
-
cclk.Unlock()
306
-
307
-
return nil
308
-
},
309
-
RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
310
-
return nil
311
-
},
312
-
// TODO: all the other event types
313
-
Error: func(errf *stream.ErrorFrame) error {
314
-
return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
315
-
},
316
-
}
317
-
318
-
sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
319
-
320
-
//s.eventScheduler = sched
321
-
//s.streamFinished = make(chan struct{})
322
-
323
-
return stream.HandleRepoStream(ctx, con, sched, slog.Default())
324
-
}
325
-
326
305
func (s *Server) resolveAccountIdent(ctx context.Context, acc string) (string, error) {
327
306
unesc, err := url.PathUnescape(acc)
328
307
if err != nil {
···
342
321
return resp.DID.String(), nil
343
322
}
344
323
345
-
const (
346
-
NotifKindReply = "reply"
347
-
NotifKindLike = "like"
348
-
NotifKindMention = "mention"
349
-
NotifKindRepost = "repost"
350
-
)
351
-
352
-
func (s *Server) AddNotification(ctx context.Context, forUser, author uint, recordUri string, recordCid cid.Cid, kind string) error {
353
-
return s.backend.db.Create(&Notification{
354
-
For: forUser,
355
-
Author: author,
356
-
Source: recordUri,
357
-
SourceCid: recordCid.String(),
358
-
Kind: kind,
359
-
}).Error
360
-
}
361
-
362
324
func (s *Server) rescanRepo(ctx context.Context, did string) error {
363
325
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
364
326
if err != nil {
365
327
return err
366
328
}
329
+
330
+
s.backend.AddRelevantDid(did)
367
331
368
332
c := &xrpclib.Client{
369
333
Host: resp.PDSEndpoint(),
-211
missing.go
-211
missing.go
···
1
-
package main
2
-
3
-
import (
4
-
"bytes"
5
-
"context"
6
-
"fmt"
7
-
"log/slog"
8
-
9
-
"github.com/bluesky-social/indigo/api/atproto"
10
-
"github.com/bluesky-social/indigo/api/bsky"
11
-
"github.com/bluesky-social/indigo/atproto/syntax"
12
-
xrpclib "github.com/bluesky-social/indigo/xrpc"
13
-
"github.com/ipfs/go-cid"
14
-
)
15
-
16
-
type MissingRecordType string
17
-
18
-
const (
19
-
MissingRecordTypeProfile MissingRecordType = "profile"
20
-
MissingRecordTypePost MissingRecordType = "post"
21
-
MissingRecordTypeFeedGenerator MissingRecordType = "feedgenerator"
22
-
)
23
-
24
-
type MissingRecord struct {
25
-
Type MissingRecordType
26
-
Identifier string // DID for profiles, AT-URI for posts/feedgens
27
-
}
28
-
29
-
func (s *Server) addMissingRecord(ctx context.Context, rec MissingRecord) {
30
-
select {
31
-
case s.missingRecords <- rec:
32
-
case <-ctx.Done():
33
-
}
34
-
}
35
-
36
-
// Legacy methods for backward compatibility
37
-
func (s *Server) addMissingProfile(ctx context.Context, did string) {
38
-
s.addMissingRecord(ctx, MissingRecord{
39
-
Type: MissingRecordTypeProfile,
40
-
Identifier: did,
41
-
})
42
-
}
43
-
44
-
func (s *Server) addMissingPost(ctx context.Context, uri string) {
45
-
slog.Info("adding missing post to fetch queue", "uri", uri)
46
-
s.addMissingRecord(ctx, MissingRecord{
47
-
Type: MissingRecordTypePost,
48
-
Identifier: uri,
49
-
})
50
-
}
51
-
52
-
func (s *Server) addMissingFeedGenerator(ctx context.Context, uri string) {
53
-
slog.Info("adding missing feed generator to fetch queue", "uri", uri)
54
-
s.addMissingRecord(ctx, MissingRecord{
55
-
Type: MissingRecordTypeFeedGenerator,
56
-
Identifier: uri,
57
-
})
58
-
}
59
-
60
-
func (s *Server) missingRecordFetcher() {
61
-
for rec := range s.missingRecords {
62
-
var err error
63
-
switch rec.Type {
64
-
case MissingRecordTypeProfile:
65
-
err = s.fetchMissingProfile(context.TODO(), rec.Identifier)
66
-
case MissingRecordTypePost:
67
-
err = s.fetchMissingPost(context.TODO(), rec.Identifier)
68
-
case MissingRecordTypeFeedGenerator:
69
-
err = s.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
70
-
default:
71
-
slog.Error("unknown missing record type", "type", rec.Type)
72
-
continue
73
-
}
74
-
75
-
if err != nil {
76
-
slog.Warn("failed to fetch missing record", "type", rec.Type, "identifier", rec.Identifier, "error", err)
77
-
}
78
-
}
79
-
}
80
-
81
-
func (s *Server) fetchMissingProfile(ctx context.Context, did string) error {
82
-
repo, err := s.backend.getOrCreateRepo(ctx, did)
83
-
if err != nil {
84
-
return err
85
-
}
86
-
87
-
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
88
-
if err != nil {
89
-
return err
90
-
}
91
-
92
-
c := &xrpclib.Client{
93
-
Host: resp.PDSEndpoint(),
94
-
}
95
-
96
-
rec, err := atproto.RepoGetRecord(ctx, c, "", "app.bsky.actor.profile", did, "self")
97
-
if err != nil {
98
-
return err
99
-
}
100
-
101
-
prof, ok := rec.Value.Val.(*bsky.ActorProfile)
102
-
if !ok {
103
-
return fmt.Errorf("record we got back wasnt a profile somehow")
104
-
}
105
-
106
-
buf := new(bytes.Buffer)
107
-
if err := prof.MarshalCBOR(buf); err != nil {
108
-
return err
109
-
}
110
-
111
-
cc, err := cid.Decode(*rec.Cid)
112
-
if err != nil {
113
-
return err
114
-
}
115
-
116
-
return s.backend.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
117
-
}
118
-
119
-
func (s *Server) fetchMissingPost(ctx context.Context, uri string) error {
120
-
puri, err := syntax.ParseATURI(uri)
121
-
if err != nil {
122
-
return fmt.Errorf("invalid AT URI: %s", uri)
123
-
}
124
-
125
-
did := puri.Authority().String()
126
-
collection := puri.Collection().String()
127
-
rkey := puri.RecordKey().String()
128
-
129
-
repo, err := s.backend.getOrCreateRepo(ctx, did)
130
-
if err != nil {
131
-
return err
132
-
}
133
-
134
-
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
135
-
if err != nil {
136
-
return err
137
-
}
138
-
139
-
c := &xrpclib.Client{
140
-
Host: resp.PDSEndpoint(),
141
-
}
142
-
143
-
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
144
-
if err != nil {
145
-
return err
146
-
}
147
-
148
-
post, ok := rec.Value.Val.(*bsky.FeedPost)
149
-
if !ok {
150
-
return fmt.Errorf("record we got back wasn't a post somehow")
151
-
}
152
-
153
-
buf := new(bytes.Buffer)
154
-
if err := post.MarshalCBOR(buf); err != nil {
155
-
return err
156
-
}
157
-
158
-
cc, err := cid.Decode(*rec.Cid)
159
-
if err != nil {
160
-
return err
161
-
}
162
-
163
-
return s.backend.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
164
-
}
165
-
166
-
func (s *Server) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
167
-
puri, err := syntax.ParseATURI(uri)
168
-
if err != nil {
169
-
return fmt.Errorf("invalid AT URI: %s", uri)
170
-
}
171
-
172
-
did := puri.Authority().String()
173
-
collection := puri.Collection().String()
174
-
rkey := puri.RecordKey().String()
175
-
176
-
repo, err := s.backend.getOrCreateRepo(ctx, did)
177
-
if err != nil {
178
-
return err
179
-
}
180
-
181
-
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
182
-
if err != nil {
183
-
return err
184
-
}
185
-
186
-
c := &xrpclib.Client{
187
-
Host: resp.PDSEndpoint(),
188
-
}
189
-
190
-
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
191
-
if err != nil {
192
-
return err
193
-
}
194
-
195
-
feedGen, ok := rec.Value.Val.(*bsky.FeedGenerator)
196
-
if !ok {
197
-
return fmt.Errorf("record we got back wasn't a feed generator somehow")
198
-
}
199
-
200
-
buf := new(bytes.Buffer)
201
-
if err := feedGen.MarshalCBOR(buf); err != nil {
202
-
return err
203
-
}
204
-
205
-
cc, err := cid.Decode(*rec.Cid)
206
-
if err != nil {
207
-
return err
208
-
}
209
-
210
-
return s.backend.HandleCreateFeedGenerator(ctx, repo, rkey, buf.Bytes(), cc)
211
-
}
+5
models/models.go
+5
models/models.go
-412
pgbackend.go
-412
pgbackend.go
···
1
-
package main
2
-
3
-
import (
4
-
"context"
5
-
"errors"
6
-
"fmt"
7
-
"strings"
8
-
"time"
9
-
10
-
"github.com/bluesky-social/indigo/api/atproto"
11
-
"github.com/bluesky-social/indigo/api/bsky"
12
-
"github.com/bluesky-social/indigo/atproto/syntax"
13
-
"github.com/bluesky-social/indigo/util"
14
-
"github.com/jackc/pgx/v5"
15
-
"github.com/jackc/pgx/v5/pgconn"
16
-
"github.com/whyrusleeping/market/models"
17
-
"gorm.io/gorm"
18
-
"gorm.io/gorm/clause"
19
-
"gorm.io/gorm/logger"
20
-
21
-
. "github.com/whyrusleeping/konbini/models"
22
-
)
23
-
24
-
func (b *PostgresBackend) getOrCreateRepo(ctx context.Context, did string) (*Repo, error) {
25
-
r, ok := b.repoCache.Get(did)
26
-
if !ok {
27
-
b.reposLk.Lock()
28
-
29
-
r, ok = b.repoCache.Get(did)
30
-
if !ok {
31
-
r = &Repo{}
32
-
r.Did = did
33
-
b.repoCache.Add(did, r)
34
-
}
35
-
36
-
b.reposLk.Unlock()
37
-
}
38
-
39
-
r.Lk.Lock()
40
-
defer r.Lk.Unlock()
41
-
if r.Setup {
42
-
return r, nil
43
-
}
44
-
45
-
row := b.pgx.QueryRow(ctx, "SELECT id, created_at, did FROM repos WHERE did = $1", did)
46
-
47
-
err := row.Scan(&r.ID, &r.CreatedAt, &r.Did)
48
-
if err == nil {
49
-
// found it!
50
-
r.Setup = true
51
-
return r, nil
52
-
}
53
-
54
-
if err != pgx.ErrNoRows {
55
-
return nil, err
56
-
}
57
-
58
-
r.Did = did
59
-
if err := b.db.Create(r).Error; err != nil {
60
-
return nil, err
61
-
}
62
-
63
-
r.Setup = true
64
-
65
-
return r, nil
66
-
}
67
-
68
-
func (b *PostgresBackend) getOrCreateList(ctx context.Context, uri string) (*List, error) {
69
-
puri, err := util.ParseAtUri(uri)
70
-
if err != nil {
71
-
return nil, err
72
-
}
73
-
74
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
75
-
if err != nil {
76
-
return nil, err
77
-
}
78
-
79
-
// TODO: needs upsert treatment when we actually find the list
80
-
var list List
81
-
if err := b.db.FirstOrCreate(&list, map[string]any{
82
-
"author": r.ID,
83
-
"rkey": puri.Rkey,
84
-
}).Error; err != nil {
85
-
return nil, err
86
-
}
87
-
return &list, nil
88
-
}
89
-
90
-
type cachedPostInfo struct {
91
-
ID uint
92
-
Author uint
93
-
}
94
-
95
-
func (b *PostgresBackend) postIDForUri(ctx context.Context, uri string) (uint, error) {
96
-
// getPostByUri implicitly fills the cache
97
-
p, err := b.postInfoForUri(ctx, uri)
98
-
if err != nil {
99
-
return 0, err
100
-
}
101
-
102
-
return p.ID, nil
103
-
}
104
-
105
-
func (b *PostgresBackend) postInfoForUri(ctx context.Context, uri string) (cachedPostInfo, error) {
106
-
v, ok := b.postInfoCache.Get(uri)
107
-
if ok {
108
-
return v, nil
109
-
}
110
-
111
-
// getPostByUri implicitly fills the cache
112
-
p, err := b.getOrCreatePostBare(ctx, uri)
113
-
if err != nil {
114
-
return cachedPostInfo{}, err
115
-
}
116
-
117
-
return cachedPostInfo{ID: p.ID, Author: p.Author}, nil
118
-
}
119
-
120
-
func (b *PostgresBackend) tryLoadPostInfo(ctx context.Context, uid uint, rkey string) (*Post, error) {
121
-
var p Post
122
-
q := "SELECT id, author FROM posts WHERE author = $1 AND rkey = $2"
123
-
if err := b.pgx.QueryRow(ctx, q, uid, rkey).Scan(&p.ID, &p.Author); err != nil {
124
-
if errors.Is(err, pgx.ErrNoRows) {
125
-
return nil, nil
126
-
}
127
-
return nil, err
128
-
}
129
-
130
-
return &p, nil
131
-
}
132
-
133
-
func (b *PostgresBackend) getOrCreatePostBare(ctx context.Context, uri string) (*Post, error) {
134
-
puri, err := util.ParseAtUri(uri)
135
-
if err != nil {
136
-
return nil, err
137
-
}
138
-
139
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
140
-
if err != nil {
141
-
return nil, err
142
-
}
143
-
144
-
post, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
145
-
if err != nil {
146
-
return nil, err
147
-
}
148
-
149
-
if post == nil {
150
-
post = &Post{
151
-
Rkey: puri.Rkey,
152
-
Author: r.ID,
153
-
NotFound: true,
154
-
}
155
-
156
-
err := b.pgx.QueryRow(ctx, "INSERT INTO posts (rkey, author, not_found) VALUES ($1, $2, $3) RETURNING id", puri.Rkey, r.ID, true).Scan(&post.ID)
157
-
if err != nil {
158
-
pgErr, ok := err.(*pgconn.PgError)
159
-
if !ok || pgErr.Code != "23505" {
160
-
return nil, err
161
-
}
162
-
163
-
out, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
164
-
if err != nil {
165
-
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
166
-
}
167
-
if out == nil {
168
-
return nil, fmt.Errorf("postgres is lying to us: %d %s", r.ID, puri.Rkey)
169
-
}
170
-
171
-
post = out
172
-
}
173
-
174
-
}
175
-
176
-
b.postInfoCache.Add(uri, cachedPostInfo{
177
-
ID: post.ID,
178
-
Author: post.Author,
179
-
})
180
-
181
-
return post, nil
182
-
}
183
-
184
-
func (b *PostgresBackend) getPostByUri(ctx context.Context, uri string, fields string) (*Post, error) {
185
-
puri, err := util.ParseAtUri(uri)
186
-
if err != nil {
187
-
return nil, err
188
-
}
189
-
190
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
191
-
if err != nil {
192
-
return nil, err
193
-
}
194
-
195
-
q := "SELECT " + fields + " FROM posts WHERE author = ? AND rkey = ?"
196
-
197
-
var post Post
198
-
if err := b.db.Raw(q, r.ID, puri.Rkey).Scan(&post).Error; err != nil {
199
-
return nil, err
200
-
}
201
-
202
-
if post.ID == 0 {
203
-
post.Rkey = puri.Rkey
204
-
post.Author = r.ID
205
-
post.NotFound = true
206
-
207
-
if err := b.db.Session(&gorm.Session{
208
-
Logger: logger.Default.LogMode(logger.Silent),
209
-
}).Create(&post).Error; err != nil {
210
-
if !errors.Is(err, gorm.ErrDuplicatedKey) {
211
-
return nil, err
212
-
}
213
-
if err := b.db.Find(&post, "author = ? AND rkey = ?", r.ID, puri.Rkey).Error; err != nil {
214
-
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
215
-
}
216
-
}
217
-
218
-
}
219
-
220
-
b.postInfoCache.Add(uri, cachedPostInfo{
221
-
ID: post.ID,
222
-
Author: post.Author,
223
-
})
224
-
225
-
return &post, nil
226
-
}
227
-
228
-
func (b *PostgresBackend) revForRepo(rr *Repo) (string, error) {
229
-
lrev, ok := b.revCache.Get(rr.ID)
230
-
if ok {
231
-
return lrev, nil
232
-
}
233
-
234
-
var rev string
235
-
if err := b.pgx.QueryRow(context.TODO(), "SELECT COALESCE(rev, '') FROM sync_infos WHERE repo = $1", rr.ID).Scan(&rev); err != nil {
236
-
if errors.Is(err, pgx.ErrNoRows) {
237
-
return "", nil
238
-
}
239
-
return "", err
240
-
}
241
-
242
-
if rev != "" {
243
-
b.revCache.Add(rr.ID, rev)
244
-
}
245
-
return rev, nil
246
-
}
247
-
248
-
func (b *PostgresBackend) ensureFollowsScraped(ctx context.Context, user string) error {
249
-
r, err := b.getOrCreateRepo(ctx, user)
250
-
if err != nil {
251
-
return err
252
-
}
253
-
254
-
var si SyncInfo
255
-
if err := b.db.Find(&si, "repo = ?", r.ID).Error; err != nil {
256
-
return err
257
-
}
258
-
259
-
// not found
260
-
if si.Repo == 0 {
261
-
if err := b.db.Create(&SyncInfo{
262
-
Repo: r.ID,
263
-
}).Error; err != nil {
264
-
return err
265
-
}
266
-
}
267
-
268
-
if si.FollowsSynced {
269
-
return nil
270
-
}
271
-
272
-
var follows []Follow
273
-
var cursor string
274
-
for {
275
-
resp, err := atproto.RepoListRecords(ctx, b.s.client, "app.bsky.graph.follow", cursor, 100, b.s.mydid, false)
276
-
if err != nil {
277
-
return err
278
-
}
279
-
280
-
for _, rec := range resp.Records {
281
-
if fol, ok := rec.Value.Val.(*bsky.GraphFollow); ok {
282
-
fr, err := b.getOrCreateRepo(ctx, fol.Subject)
283
-
if err != nil {
284
-
return err
285
-
}
286
-
287
-
puri, err := syntax.ParseATURI(rec.Uri)
288
-
if err != nil {
289
-
return err
290
-
}
291
-
292
-
follows = append(follows, Follow{
293
-
Created: time.Now(),
294
-
Indexed: time.Now(),
295
-
Rkey: puri.RecordKey().String(),
296
-
Author: r.ID,
297
-
Subject: fr.ID,
298
-
})
299
-
}
300
-
}
301
-
302
-
if resp.Cursor == nil || len(resp.Records) == 0 {
303
-
break
304
-
}
305
-
cursor = *resp.Cursor
306
-
}
307
-
308
-
if err := b.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(follows, 200).Error; err != nil {
309
-
return err
310
-
}
311
-
312
-
if err := b.db.Model(SyncInfo{}).Where("repo = ?", r.ID).Update("follows_synced", true).Error; err != nil {
313
-
return err
314
-
}
315
-
316
-
fmt.Println("Got follows: ", len(follows))
317
-
318
-
return nil
319
-
}
320
-
321
-
func (b *PostgresBackend) loadRelevantDids() error {
322
-
ctx := context.TODO()
323
-
324
-
if err := b.ensureFollowsScraped(ctx, b.s.mydid); err != nil {
325
-
return fmt.Errorf("failed to scrape follows: %w", err)
326
-
}
327
-
328
-
r, err := b.getOrCreateRepo(ctx, b.s.mydid)
329
-
if err != nil {
330
-
return err
331
-
}
332
-
333
-
var dids []string
334
-
if err := b.db.Raw("select did from follows left join repos on follows.subject = repos.id where follows.author = ?", r.ID).Scan(&dids).Error; err != nil {
335
-
return err
336
-
}
337
-
338
-
b.relevantDids[b.s.mydid] = true
339
-
for _, d := range dids {
340
-
fmt.Println("adding did: ", d)
341
-
b.relevantDids[d] = true
342
-
}
343
-
344
-
return nil
345
-
}
346
-
347
-
type SyncInfo struct {
348
-
Repo uint `gorm:"index"`
349
-
FollowsSynced bool
350
-
Rev string
351
-
}
352
-
353
-
func (b *PostgresBackend) checkPostExists(ctx context.Context, repo *Repo, rkey string) (bool, error) {
354
-
var id uint
355
-
var notfound bool
356
-
if err := b.pgx.QueryRow(ctx, "SELECT id, not_found FROM posts WHERE author = $1 AND rkey = $2", repo.ID, rkey).Scan(&id, ¬found); err != nil {
357
-
if errors.Is(err, pgx.ErrNoRows) {
358
-
return false, nil
359
-
}
360
-
return false, err
361
-
}
362
-
363
-
if id != 0 && !notfound {
364
-
return true, nil
365
-
}
366
-
367
-
return false, nil
368
-
}
369
-
370
-
func (b *PostgresBackend) didIsRelevant(did string) bool {
371
-
b.rdLk.Lock()
372
-
defer b.rdLk.Unlock()
373
-
return b.relevantDids[did]
374
-
}
375
-
376
-
func (b *PostgresBackend) anyRelevantIdents(idents ...string) bool {
377
-
for _, id := range idents {
378
-
if strings.HasPrefix(id, "did:") {
379
-
if b.didIsRelevant(id) {
380
-
return true
381
-
}
382
-
} else if strings.HasPrefix(id, "at://") {
383
-
puri, err := syntax.ParseATURI(id)
384
-
if err != nil {
385
-
continue
386
-
}
387
-
388
-
if b.didIsRelevant(puri.Authority().String()) {
389
-
return true
390
-
}
391
-
}
392
-
}
393
-
394
-
return false
395
-
}
396
-
397
-
func (b *PostgresBackend) getRepoByID(ctx context.Context, id uint) (*models.Repo, error) {
398
-
var r models.Repo
399
-
if err := b.db.Find(&r, "id = ?", id).Error; err != nil {
400
-
return nil, err
401
-
}
402
-
403
-
return &r, nil
404
-
}
405
-
406
-
func (b *PostgresBackend) TrackMissingActor(did string) {
407
-
b.s.addMissingProfile(context.TODO(), did)
408
-
}
409
-
410
-
func (b *PostgresBackend) TrackMissingFeedGenerator(uri string) {
411
-
b.s.addMissingFeedGenerator(context.TODO(), uri)
412
-
}
+8
sync-config-jetstream.json
+8
sync-config-jetstream.json
+281
sync.go
+281
sync.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"net/http"
8
+
"sync"
9
+
"time"
10
+
11
+
"github.com/bluesky-social/indigo/api/atproto"
12
+
"github.com/bluesky-social/indigo/cmd/relay/stream"
13
+
"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
14
+
jsclient "github.com/bluesky-social/jetstream/pkg/client"
15
+
jsparallel "github.com/bluesky-social/jetstream/pkg/client/schedulers/parallel"
16
+
"github.com/bluesky-social/jetstream/pkg/models"
17
+
"github.com/gorilla/websocket"
18
+
)
19
+
20
+
type SyncConfig struct {
21
+
Backends []SyncBackend `json:"backends"`
22
+
}
23
+
24
+
type SyncBackend struct {
25
+
Type string `json:"type"`
26
+
Host string `json:"host"`
27
+
MaxWorkers int `json:"max_workers,omitempty"`
28
+
}
29
+
30
+
func (s *Server) StartSyncEngine(ctx context.Context, sc *SyncConfig) error {
31
+
for _, be := range sc.Backends {
32
+
switch be.Type {
33
+
case "firehose":
34
+
go s.runSyncFirehose(ctx, be)
35
+
case "jetstream":
36
+
go s.runSyncJetstream(ctx, be)
37
+
default:
38
+
return fmt.Errorf("unrecognized sync backend type: %q", be.Type)
39
+
}
40
+
}
41
+
42
+
<-ctx.Done()
43
+
return fmt.Errorf("exiting sync routine")
44
+
}
45
+
46
+
const failureTimeInterval = time.Second * 5
47
+
48
+
func (s *Server) runSyncFirehose(ctx context.Context, be SyncBackend) {
49
+
var failures int
50
+
for {
51
+
seqno, err := loadLastSeq(s.db, be.Host)
52
+
if err != nil {
53
+
fmt.Println("failed to load sequence number, starting over", err)
54
+
}
55
+
56
+
maxWorkers := 10
57
+
if be.MaxWorkers != 0 {
58
+
maxWorkers = be.MaxWorkers
59
+
}
60
+
61
+
start := time.Now()
62
+
if err := s.startLiveTail(ctx, be.Host, int(seqno), maxWorkers, 20); err != nil {
63
+
slog.Error("firehose connection lost", "host", be.Host, "error", err)
64
+
}
65
+
66
+
elapsed := time.Since(start)
67
+
68
+
if elapsed > failureTimeInterval {
69
+
failures = 0
70
+
continue
71
+
}
72
+
failures++
73
+
74
+
delay := delayForFailureCount(failures)
75
+
slog.Warn("retrying connection after delay", "host", be.Host, "delay", delay)
76
+
}
77
+
}
78
+
79
+
func (s *Server) runSyncJetstream(ctx context.Context, be SyncBackend) {
80
+
var failures int
81
+
for {
82
+
// Load last cursor (stored as sequence number in same table)
83
+
cursor, err := loadLastSeq(s.db, be.Host)
84
+
if err != nil {
85
+
slog.Warn("failed to load jetstream cursor, starting from live", "error", err)
86
+
cursor = 0
87
+
}
88
+
89
+
maxWorkers := 10
90
+
if be.MaxWorkers != 0 {
91
+
maxWorkers = be.MaxWorkers
92
+
}
93
+
94
+
start := time.Now()
95
+
if err := s.startJetstreamTail(ctx, be.Host, cursor, maxWorkers); err != nil {
96
+
slog.Error("jetstream connection lost", "host", be.Host, "error", err)
97
+
}
98
+
99
+
elapsed := time.Since(start)
100
+
101
+
if elapsed > failureTimeInterval {
102
+
failures = 0
103
+
continue
104
+
}
105
+
failures++
106
+
107
+
delay := delayForFailureCount(failures)
108
+
slog.Warn("retrying jetstream connection after delay", "host", be.Host, "delay", delay)
109
+
time.Sleep(delay)
110
+
}
111
+
}
112
+
113
+
func delayForFailureCount(n int) time.Duration {
114
+
if n < 5 {
115
+
return (time.Second * 5) + (time.Second * 2 * time.Duration(n))
116
+
}
117
+
118
+
return time.Second * 30
119
+
}
120
+
121
+
func (s *Server) startLiveTail(ctx context.Context, host string, curs int, parWorkers, maxQ int) error {
122
+
ctx, cancel := context.WithCancel(ctx)
123
+
defer cancel()
124
+
125
+
slog.Info("starting live tail")
126
+
127
+
// Connect to the Relay websocket
128
+
urlStr := fmt.Sprintf("wss://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", host, curs)
129
+
130
+
d := websocket.DefaultDialer
131
+
con, _, err := d.Dial(urlStr, http.Header{
132
+
"User-Agent": []string{"konbini/0.0.1"},
133
+
})
134
+
if err != nil {
135
+
return fmt.Errorf("failed to connect to relay: %w", err)
136
+
}
137
+
138
+
var lelk sync.Mutex
139
+
lastEvent := time.Now()
140
+
141
+
go func() {
142
+
tick := time.NewTicker(time.Second)
143
+
defer tick.Stop()
144
+
for {
145
+
select {
146
+
case <-tick.C:
147
+
lelk.Lock()
148
+
let := lastEvent
149
+
lelk.Unlock()
150
+
151
+
if time.Since(let) > time.Second*30 {
152
+
slog.Error("firehose connection timed out")
153
+
con.Close()
154
+
return
155
+
}
156
+
case <-ctx.Done():
157
+
return
158
+
}
159
+
}
160
+
}()
161
+
162
+
var cclk sync.Mutex
163
+
var completeCursor int64
164
+
165
+
rsc := &stream.RepoStreamCallbacks{
166
+
RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
167
+
ctx := context.Background()
168
+
169
+
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
170
+
171
+
s.seqLk.Lock()
172
+
if evt.Seq > s.lastSeq {
173
+
curs = int(evt.Seq)
174
+
s.lastSeq = evt.Seq
175
+
176
+
if evt.Seq%1000 == 0 {
177
+
if err := storeLastSeq(s.db, host, evt.Seq); err != nil {
178
+
fmt.Println("failed to store seqno: ", err)
179
+
}
180
+
}
181
+
}
182
+
s.seqLk.Unlock()
183
+
184
+
lelk.Lock()
185
+
lastEvent = time.Now()
186
+
lelk.Unlock()
187
+
188
+
if err := s.backend.HandleEvent(ctx, evt); err != nil {
189
+
return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
190
+
}
191
+
192
+
cclk.Lock()
193
+
if evt.Seq > completeCursor {
194
+
completeCursor = evt.Seq
195
+
firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
196
+
}
197
+
cclk.Unlock()
198
+
199
+
return nil
200
+
},
201
+
RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
202
+
return nil
203
+
},
204
+
// TODO: all the other event types
205
+
Error: func(errf *stream.ErrorFrame) error {
206
+
return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
207
+
},
208
+
}
209
+
210
+
sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
211
+
212
+
return stream.HandleRepoStream(ctx, con, sched, slog.Default())
213
+
}
214
+
215
+
func (s *Server) startJetstreamTail(ctx context.Context, host string, cursor int64, parWorkers int) error {
216
+
ctx, cancel := context.WithCancel(ctx)
217
+
defer cancel()
218
+
219
+
slog.Info("starting jetstream tail", "host", host, "cursor", cursor)
220
+
221
+
// Create a scheduler for parallel processing
222
+
lastStored := int64(0)
223
+
sched := jsparallel.NewScheduler(
224
+
parWorkers,
225
+
host,
226
+
slog.Default(),
227
+
func(ctx context.Context, event *models.Event) error {
228
+
// Update cursor tracking
229
+
s.seqLk.Lock()
230
+
if event.TimeUS > s.lastSeq {
231
+
s.lastSeq = event.TimeUS
232
+
if event.TimeUS-lastStored > 1_000_000 {
233
+
// Store checkpoint periodically
234
+
if err := storeLastSeq(s.db, host, event.TimeUS); err != nil {
235
+
slog.Error("failed to store jetstream cursor", "error", err)
236
+
}
237
+
lastStored = event.TimeUS
238
+
}
239
+
}
240
+
s.seqLk.Unlock()
241
+
242
+
// Update metrics
243
+
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(event.TimeUS))
244
+
245
+
// Convert Jetstream event to ATProto event format
246
+
if event.Commit != nil {
247
+
248
+
if err := s.backend.HandleEventJetstream(ctx, event); err != nil {
249
+
return fmt.Errorf("handle event (%s,%d): %w", event.Did, event.TimeUS, err)
250
+
}
251
+
252
+
firehoseCursorGauge.WithLabelValues("complete").Set(float64(event.TimeUS))
253
+
}
254
+
255
+
return nil
256
+
},
257
+
)
258
+
259
+
// Configure Jetstream client
260
+
config := jsclient.DefaultClientConfig()
261
+
config.WebsocketURL = fmt.Sprintf("wss://%s/subscribe", host)
262
+
263
+
// Prepare cursor pointer
264
+
var cursorPtr *int64
265
+
if cursor > 0 {
266
+
cursorPtr = &cursor
267
+
}
268
+
269
+
// Create and connect client
270
+
client, err := jsclient.NewClient(
271
+
config,
272
+
slog.Default(),
273
+
sched,
274
+
)
275
+
if err != nil {
276
+
return fmt.Errorf("create jetstream client: %w", err)
277
+
}
278
+
279
+
// Start reading from Jetstream
280
+
return client.ConnectAndRead(ctx, cursorPtr)
281
+
}
+5
views/actor.go
+5
views/actor.go
+3
-86
views/feed.go
+3
-86
views/feed.go
···
42
42
}
43
43
}
44
44
45
-
// Add embed handling
46
-
if post.Post.Embed != nil {
47
-
view.Embed = formatEmbed(post.Post.Embed, post.Author)
45
+
// Add embed if it was hydrated
46
+
if post.EmbedInfo != nil {
47
+
view.Embed = post.EmbedInfo
48
48
}
49
49
50
50
return view
···
68
68
// For now leaving them as interface{} to be handled by handlers
69
69
70
70
return view
71
-
}
72
-
73
-
func formatEmbed(embed *bsky.FeedPost_Embed, authorDID string) *bsky.FeedDefs_PostView_Embed {
74
-
if embed == nil {
75
-
return nil
76
-
}
77
-
78
-
result := &bsky.FeedDefs_PostView_Embed{}
79
-
80
-
// Handle images
81
-
if embed.EmbedImages != nil {
82
-
viewImages := make([]*bsky.EmbedImages_ViewImage, len(embed.EmbedImages.Images))
83
-
for i, img := range embed.EmbedImages.Images {
84
-
// Convert blob to CDN URLs
85
-
fullsize := ""
86
-
thumb := ""
87
-
if img.Image != nil {
88
-
// CDN URL format for feed images
89
-
cid := img.Image.Ref.String()
90
-
fullsize = fmt.Sprintf("https://cdn.bsky.app/img/feed_fullsize/plain/%s/%s@jpeg", authorDID, cid)
91
-
thumb = fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
92
-
}
93
-
94
-
viewImages[i] = &bsky.EmbedImages_ViewImage{
95
-
Alt: img.Alt,
96
-
AspectRatio: img.AspectRatio,
97
-
Fullsize: fullsize,
98
-
Thumb: thumb,
99
-
}
100
-
}
101
-
result.EmbedImages_View = &bsky.EmbedImages_View{
102
-
LexiconTypeID: "app.bsky.embed.images#view",
103
-
Images: viewImages,
104
-
}
105
-
return result
106
-
}
107
-
108
-
// Handle external links
109
-
if embed.EmbedExternal != nil && embed.EmbedExternal.External != nil {
110
-
// Convert blob thumb to CDN URL if present
111
-
var thumbURL *string
112
-
if embed.EmbedExternal.External.Thumb != nil {
113
-
// CDN URL for external link thumbnails
114
-
cid := embed.EmbedExternal.External.Thumb.Ref.String()
115
-
url := fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
116
-
thumbURL = &url
117
-
}
118
-
119
-
result.EmbedExternal_View = &bsky.EmbedExternal_View{
120
-
LexiconTypeID: "app.bsky.embed.external#view",
121
-
External: &bsky.EmbedExternal_ViewExternal{
122
-
Uri: embed.EmbedExternal.External.Uri,
123
-
Title: embed.EmbedExternal.External.Title,
124
-
Description: embed.EmbedExternal.External.Description,
125
-
Thumb: thumbURL,
126
-
},
127
-
}
128
-
return result
129
-
}
130
-
131
-
// Handle video
132
-
if embed.EmbedVideo != nil {
133
-
// TODO: Implement video embed view
134
-
// This would require converting video blob to CDN URLs and playlist URLs
135
-
return nil
136
-
}
137
-
138
-
// Handle record (quote posts, etc.)
139
-
if embed.EmbedRecord != nil {
140
-
// TODO: Implement record embed view
141
-
// This requires hydrating the embedded record, which is complex
142
-
// For now, return nil to skip these embeds
143
-
return nil
144
-
}
145
-
146
-
// Handle record with media (quote post with images/external)
147
-
if embed.EmbedRecordWithMedia != nil {
148
-
// TODO: Implement record with media embed view
149
-
// This combines record hydration with media conversion
150
-
return nil
151
-
}
152
-
153
-
return nil
154
71
}
155
72
156
73
// GeneratorView builds a feed generator view (app.bsky.feed.defs#generatorView)
+3
-1
xrpc/actor/getProfile.go
+3
-1
xrpc/actor/getProfile.go
···
20
20
21
21
ctx := c.Request().Context()
22
22
23
+
viewer, _ := c.Get("viewer").(string)
24
+
23
25
// Resolve actor to DID
24
26
did, err := hydrator.ResolveDID(ctx, actorParam)
25
27
if err != nil {
···
30
32
}
31
33
32
34
// Hydrate actor info
33
-
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did)
35
+
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did, viewer)
34
36
if err != nil {
35
37
return c.JSON(http.StatusNotFound, map[string]interface{}{
36
38
"error": "ActorNotFound",
+2
-1
xrpc/actor/getProfiles.go
+2
-1
xrpc/actor/getProfiles.go
···
27
27
}
28
28
29
29
ctx := c.Request().Context()
30
+
viewer, _ := c.Get("viewer").(string)
30
31
31
32
// Resolve all actors to DIDs and hydrate profiles
32
33
profiles := make([]*bsky.ActorDefs_ProfileViewDetailed, 0, len(actors))
···
39
40
}
40
41
41
42
// Hydrate actor info
42
-
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did)
43
+
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did, viewer)
43
44
if err != nil {
44
45
// Skip actors that can't be hydrated
45
46
continue
+42
-6
xrpc/feed/getAuthorFeed.go
+42
-6
xrpc/feed/getAuthorFeed.go
···
5
5
"log/slog"
6
6
"net/http"
7
7
"strconv"
8
+
"strings"
8
9
"sync"
9
10
"time"
10
11
11
12
"github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
14
"github.com/labstack/echo/v4"
13
15
"github.com/whyrusleeping/konbini/hydration"
14
16
"github.com/whyrusleeping/konbini/views"
···
126
128
}
127
129
128
130
func hydratePostRows(ctx context.Context, hydrator *hydration.Hydrator, viewer string, rows []postRow) []*bsky.FeedDefs_FeedViewPost {
131
+
ctx, span := tracer.Start(ctx, "hydratePostRows")
132
+
defer span.End()
133
+
129
134
// Hydrate posts
130
135
var wg sync.WaitGroup
131
136
···
136
141
go func(i int, row postRow) {
137
142
defer wg.Done()
138
143
139
-
postInfo, err := hydrator.HydratePost(ctx, row.URI, viewer)
144
+
puri, err := syntax.ParseATURI(row.URI)
140
145
if err != nil {
141
-
slog.Error("failed to hydrate post", "uri", row.URI, "error", err)
146
+
slog.Error("row had invalid uri", "uri", row.URI, "error", err)
142
147
return
143
148
}
144
149
145
-
// Hydrate author
146
-
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
147
-
if err != nil {
148
-
slog.Error("failed to hydrate actor", "actor", postInfo.Author, "error", err)
150
+
var subwg sync.WaitGroup
151
+
152
+
var postInfo *hydration.PostInfo
153
+
subwg.Go(func() {
154
+
pi, err := hydrator.HydratePost(ctx, row.URI, viewer)
155
+
if err != nil {
156
+
if strings.Contains(err.Error(), "post not found") {
157
+
hydrator.AddMissingRecord(row.URI, true)
158
+
pi, err = hydrator.HydratePost(ctx, row.URI, viewer)
159
+
if err != nil {
160
+
slog.Error("failed to hydrate post after fetch missing", "uri", row.URI, "error", err)
161
+
return
162
+
}
163
+
} else {
164
+
slog.Warn("failed to hydrate post", "uri", row.URI, "error", err)
165
+
return
166
+
}
167
+
}
168
+
postInfo = pi
169
+
})
170
+
171
+
var authorInfo *hydration.ActorInfo
172
+
subwg.Go(func() {
173
+
ai, err := hydrator.HydrateActor(ctx, puri.Authority().String())
174
+
if err != nil {
175
+
hydrator.AddMissingRecord(postInfo.Author, false)
176
+
slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
177
+
return
178
+
}
179
+
authorInfo = ai
180
+
})
181
+
182
+
subwg.Wait()
183
+
184
+
if postInfo == nil || authorInfo == nil {
149
185
return
150
186
}
151
187
+38
-19
xrpc/feed/getFeed.go
+38
-19
xrpc/feed/getFeed.go
···
5
5
"log/slog"
6
6
"net/http"
7
7
"strconv"
8
+
"strings"
9
+
"sync"
8
10
9
11
"github.com/bluesky-social/indigo/api/bsky"
10
12
"github.com/bluesky-social/indigo/atproto/identity"
···
64
66
}
65
67
66
68
if feedGen.ID == 0 {
67
-
hydrator.AddMissingFeedGenerator(feedURI)
69
+
hydrator.AddMissingRecord(feedURI, true)
68
70
return c.JSON(http.StatusNotFound, map[string]any{
69
71
"error": "NotFound",
70
72
"message": "feed generator not found",
···
149
151
}
150
152
151
153
// Hydrate the posts from the skeleton
152
-
posts := make([]*bsky.FeedDefs_FeedViewPost, 0, len(skeleton.Feed))
153
-
for _, skeletonPost := range skeleton.Feed {
154
-
postURI, err := syntax.ParseATURI(skeletonPost.Post)
155
-
if err != nil {
156
-
slog.Warn("invalid post URI in skeleton", "uri", skeletonPost.Post, "error", err)
157
-
continue
158
-
}
154
+
posts := make([]*bsky.FeedDefs_FeedViewPost, len(skeleton.Feed))
155
+
var wg sync.WaitGroup
156
+
for i := range skeleton.Feed {
157
+
wg.Add(1)
158
+
go func(ix int) {
159
+
defer wg.Done()
160
+
skeletonPost := skeleton.Feed[ix]
161
+
postURI, err := syntax.ParseATURI(skeletonPost.Post)
162
+
if err != nil {
163
+
slog.Warn("invalid post URI in skeleton", "uri", skeletonPost.Post, "error", err)
164
+
return
165
+
}
159
166
160
-
postInfo, err := hydrator.HydratePost(ctx, string(postURI), viewer)
161
-
if err != nil {
162
-
slog.Warn("failed to hydrate post", "uri", postURI, "error", err)
163
-
continue
164
-
}
167
+
postInfo, err := hydrator.HydratePost(ctx, postURI.String(), viewer)
168
+
if err != nil {
169
+
if strings.Contains(err.Error(), "post not found") {
170
+
hydrator.AddMissingRecord(postURI.String(), true)
171
+
postInfo, err = hydrator.HydratePost(ctx, postURI.String(), viewer)
172
+
if err != nil {
173
+
slog.Error("failed to hydrate post after fetch missing", "uri", postURI, "error", err)
174
+
return
175
+
}
176
+
} else {
177
+
slog.Warn("failed to hydrate post", "uri", postURI, "error", err)
178
+
return
179
+
}
180
+
}
165
181
166
-
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
167
-
if err != nil {
168
-
slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
169
-
continue
170
-
}
182
+
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
183
+
if err != nil {
184
+
hydrator.AddMissingRecord(postInfo.Author, false)
185
+
slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
186
+
return
187
+
}
171
188
172
-
posts = append(posts, views.FeedViewPost(postInfo, authorInfo))
189
+
posts[ix] = views.FeedViewPost(postInfo, authorInfo)
190
+
}(i)
173
191
}
192
+
wg.Wait()
174
193
175
194
output := &bsky.FeedGetFeed_Output{
176
195
Feed: posts,
+1
-1
xrpc/feed/getFeedGenerator.go
+1
-1
xrpc/feed/getFeedGenerator.go
+11
-11
xrpc/feed/getPostThread.go
+11
-11
xrpc/feed/getPostThread.go
···
15
15
func HandleGetPostThread(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
16
16
uriParam := c.QueryParam("uri")
17
17
if uriParam == "" {
18
-
return c.JSON(http.StatusBadRequest, map[string]interface{}{
18
+
return c.JSON(http.StatusBadRequest, map[string]any{
19
19
"error": "InvalidRequest",
20
20
"message": "uri parameter is required",
21
21
})
···
27
27
// Hydrate the requested post
28
28
postInfo, err := hydrator.HydratePost(ctx, uriParam, viewer)
29
29
if err != nil {
30
-
return c.JSON(http.StatusNotFound, map[string]interface{}{
30
+
return c.JSON(http.StatusNotFound, map[string]any{
31
31
"error": "NotFound",
32
32
"message": "post not found",
33
33
})
···
74
74
uri: uri,
75
75
replyTo: tp.ReplyTo,
76
76
inThread: tp.InThread,
77
-
replies: []interface{}{},
77
+
replies: []any{},
78
78
}
79
79
}
80
80
···
98
98
}
99
99
100
100
if rootNode == nil {
101
-
return c.JSON(http.StatusNotFound, map[string]interface{}{
101
+
return c.JSON(http.StatusNotFound, map[string]any{
102
102
"error": "NotFound",
103
103
"message": "thread root not found",
104
104
})
···
107
107
// Build the response by traversing the tree
108
108
thread := buildThreadView(ctx, db, rootNode, postsByID, hydrator, viewer, nil)
109
109
110
-
return c.JSON(http.StatusOK, map[string]interface{}{
110
+
return c.JSON(http.StatusOK, map[string]any{
111
111
"thread": thread,
112
112
})
113
113
}
···
117
117
uri string
118
118
replyTo uint
119
119
inThread uint
120
-
replies []interface{}
120
+
replies []any
121
121
}
122
122
123
-
func buildThreadView(ctx context.Context, db *gorm.DB, node *threadPostNode, allNodes map[uint]*threadPostNode, hydrator *hydration.Hydrator, viewer string, parent interface{}) interface{} {
123
+
func buildThreadView(ctx context.Context, db *gorm.DB, node *threadPostNode, allNodes map[uint]*threadPostNode, hydrator *hydration.Hydrator, viewer string, parent any) any {
124
124
// Hydrate this post
125
125
postInfo, err := hydrator.HydratePost(ctx, node.uri, viewer)
126
126
if err != nil {
127
127
// Return a notFound post
128
-
return map[string]interface{}{
128
+
return map[string]any{
129
129
"$type": "app.bsky.feed.defs#notFoundPost",
130
130
"uri": node.uri,
131
131
}
···
134
134
// Hydrate author
135
135
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
136
136
if err != nil {
137
-
return map[string]interface{}{
137
+
return map[string]any{
138
138
"$type": "app.bsky.feed.defs#notFoundPost",
139
139
"uri": node.uri,
140
140
}
141
141
}
142
142
143
143
// Build replies
144
-
var replies []interface{}
144
+
var replies []any
145
145
for _, replyNode := range node.replies {
146
146
if rn, ok := replyNode.(*threadPostNode); ok {
147
147
replyView := buildThreadView(ctx, db, rn, allNodes, hydrator, viewer, nil)
···
150
150
}
151
151
152
152
// Build the thread view post
153
-
var repliesForView interface{}
153
+
var repliesForView any
154
154
if len(replies) > 0 {
155
155
repliesForView = replies
156
156
}
+34
-16
xrpc/feed/getTimeline.go
+34
-16
xrpc/feed/getTimeline.go
···
1
1
package feed
2
2
3
3
import (
4
+
"context"
4
5
"net/http"
5
6
"strconv"
6
7
"time"
7
8
8
9
"github.com/labstack/echo/v4"
9
10
"github.com/whyrusleeping/konbini/hydration"
11
+
"go.opentelemetry.io/otel"
10
12
"gorm.io/gorm"
11
13
)
14
+
15
+
var tracer = otel.Tracer("xrpc/feed")
12
16
13
17
// HandleGetTimeline implements app.bsky.feed.getTimeline
14
18
func HandleGetTimeline(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
19
+
ctx := c.Request().Context()
20
+
ctx, span := tracer.Start(ctx, "getTimeline")
21
+
defer span.End()
22
+
15
23
viewer := getUserDID(c)
16
24
if viewer == "" {
17
25
return c.JSON(http.StatusUnauthorized, map[string]any{
···
36
44
}
37
45
}
38
46
39
-
ctx := c.Request().Context()
40
-
41
47
// Get viewer's repo ID
42
48
var viewerRepoID uint
43
49
if err := db.Raw("SELECT id FROM repos WHERE did = ?", viewer).Scan(&viewerRepoID).Error; err != nil {
···
48
54
}
49
55
50
56
// Query posts from followed users
51
-
var rows []postRow
52
-
err := db.Raw(`
53
-
SELECT
54
-
'at://' || r.did || '/app.bsky.feed.post/' || p.rkey as uri,
55
-
p.author as author_id
56
-
FROM posts p
57
-
JOIN repos r ON r.id = p.author
58
-
WHERE p.reply_to = 0
59
-
AND p.author IN (SELECT subject FROM follows WHERE author = ?)
60
-
AND p.created < ?
61
-
AND p.not_found = false
62
-
ORDER BY p.created DESC
63
-
LIMIT ?
64
-
`, viewerRepoID, cursor, limit).Scan(&rows).Error
65
57
58
+
rows, err := getTimelinePosts(ctx, db, viewerRepoID, cursor, limit)
66
59
if err != nil {
67
60
return c.JSON(http.StatusInternalServerError, map[string]any{
68
61
"error": "InternalError",
···
94
87
"cursor": nextCursor,
95
88
})
96
89
}
90
+
91
+
func getTimelinePosts(ctx context.Context, db *gorm.DB, uid uint, cursor time.Time, limit int) ([]postRow, error) {
92
+
ctx, span := tracer.Start(ctx, "getTimelineQuery")
93
+
defer span.End()
94
+
95
+
var rows []postRow
96
+
err := db.Raw(`
97
+
SELECT
98
+
'at://' || r.did || '/app.bsky.feed.post/' || p.rkey as uri,
99
+
p.author as author_id
100
+
FROM posts p
101
+
JOIN repos r ON r.id = p.author
102
+
WHERE p.reply_to = 0
103
+
AND p.author IN (SELECT subject FROM follows WHERE author = ?)
104
+
AND p.created < ?
105
+
AND p.not_found = false
106
+
ORDER BY p.created DESC
107
+
LIMIT ?
108
+
`, uid, cursor, limit).Scan(&rows).Error
109
+
110
+
if err != nil {
111
+
return nil, err
112
+
}
113
+
return rows, nil
114
+
}
+92
-11
xrpc/notification/listNotifications.go
+92
-11
xrpc/notification/listNotifications.go
···
13
13
lexutil "github.com/bluesky-social/indigo/lex/util"
14
14
"github.com/labstack/echo/v4"
15
15
"github.com/whyrusleeping/konbini/hydration"
16
+
models "github.com/whyrusleeping/konbini/models"
16
17
"github.com/whyrusleeping/konbini/views"
17
-
"github.com/whyrusleeping/market/models"
18
18
"gorm.io/gorm"
19
+
"gorm.io/gorm/clause"
19
20
)
20
21
21
22
// HandleListNotifications implements app.bsky.notification.listNotifications
22
23
func HandleListNotifications(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
23
24
viewer := getUserDID(c)
24
25
if viewer == "" {
25
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
26
+
return c.JSON(http.StatusUnauthorized, map[string]any{
26
27
"error": "AuthenticationRequired",
27
28
"message": "authentication required",
28
29
})
···
77
78
}
78
79
query += ` ORDER BY n.created_at DESC LIMIT ?`
79
80
80
-
var queryArgs []interface{}
81
+
var queryArgs []any
81
82
queryArgs = append(queryArgs, viewer)
82
83
if cursor > 0 {
83
84
queryArgs = append(queryArgs, cursor)
···
85
86
queryArgs = append(queryArgs, limit)
86
87
87
88
if err := db.Raw(query, queryArgs...).Scan(&rows).Error; err != nil {
88
-
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
89
+
return c.JSON(http.StatusInternalServerError, map[string]any{
89
90
"error": "InternalError",
90
91
"message": "failed to query notifications",
91
92
})
···
130
131
cursorPtr = &cursor
131
132
}
132
133
134
+
var lastSeen time.Time
135
+
if err := db.Raw("SELECT seen_at FROM notification_seens WHERE repo = (select id from repos where did = ?)", viewer).Scan(&lastSeen).Error; err != nil {
136
+
return err
137
+
}
138
+
139
+
var lastSeenStr *string
140
+
if !lastSeen.IsZero() {
141
+
s := lastSeen.Format(time.RFC3339)
142
+
lastSeenStr = &s
143
+
}
144
+
133
145
output := &bsky.NotificationListNotifications_Output{
134
146
Notifications: notifications,
135
147
Cursor: cursorPtr,
148
+
SeenAt: lastSeenStr,
136
149
}
137
150
138
151
return c.JSON(http.StatusOK, output)
···
142
155
func HandleGetUnreadCount(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
143
156
viewer := getUserDID(c)
144
157
if viewer == "" {
145
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
158
+
return c.JSON(http.StatusUnauthorized, map[string]any{
146
159
"error": "AuthenticationRequired",
147
160
"message": "authentication required",
148
161
})
149
162
}
150
163
151
-
// For now, return 0 - we'd need to track read state in the database
152
-
return c.JSON(http.StatusOK, map[string]interface{}{
153
-
"count": 0,
164
+
var repo models.Repo
165
+
if err := db.Find(&repo, "did = ?", viewer).Error; err != nil {
166
+
return err
167
+
}
168
+
169
+
var lastSeen time.Time
170
+
if err := db.Raw("SELECT seen_at FROM notification_seens WHERE repo = ?", repo.ID).Scan(&lastSeen).Error; err != nil {
171
+
return err
172
+
}
173
+
174
+
var count int
175
+
query := `SELECT count(*) FROM notifications WHERE created_at > ? AND for = ?`
176
+
if err := db.Raw(query, lastSeen, repo.ID).Scan(&count).Error; err != nil {
177
+
return c.JSON(http.StatusInternalServerError, map[string]any{
178
+
"error": "InternalError",
179
+
"message": "failed to count unread notifications",
180
+
})
181
+
}
182
+
183
+
return c.JSON(http.StatusOK, map[string]any{
184
+
"count": count,
154
185
})
155
186
}
156
187
···
158
189
func HandleUpdateSeen(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
159
190
viewer := getUserDID(c)
160
191
if viewer == "" {
161
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
192
+
return c.JSON(http.StatusUnauthorized, map[string]any{
162
193
"error": "AuthenticationRequired",
163
194
"message": "authentication required",
164
195
})
165
196
}
166
197
167
-
// For now, just return success - we'd need to track seen timestamps in the database
168
-
return c.JSON(http.StatusOK, map[string]interface{}{})
198
+
var body bsky.NotificationUpdateSeen_Input
199
+
if err := c.Bind(&body); err != nil {
200
+
return c.JSON(http.StatusBadRequest, map[string]any{
201
+
"error": "InvalidRequest",
202
+
"message": "invalid request body",
203
+
})
204
+
}
205
+
206
+
// Parse the seenAt timestamp
207
+
seenAt, err := time.Parse(time.RFC3339, body.SeenAt)
208
+
if err != nil {
209
+
return c.JSON(http.StatusBadRequest, map[string]any{
210
+
"error": "InvalidRequest",
211
+
"message": "invalid seenAt timestamp",
212
+
})
213
+
}
214
+
215
+
// Get the viewer's repo ID
216
+
var repoID uint
217
+
if err := db.Raw("SELECT id FROM repos WHERE did = ?", viewer).Scan(&repoID).Error; err != nil {
218
+
return c.JSON(http.StatusInternalServerError, map[string]any{
219
+
"error": "InternalError",
220
+
"message": "failed to find viewer repo",
221
+
})
222
+
}
223
+
224
+
if repoID == 0 {
225
+
return c.JSON(http.StatusInternalServerError, map[string]any{
226
+
"error": "InternalError",
227
+
"message": "viewer repo not found",
228
+
})
229
+
}
230
+
231
+
// Upsert the NotificationSeen record
232
+
notifSeen := models.NotificationSeen{
233
+
Repo: repoID,
234
+
SeenAt: seenAt,
235
+
}
236
+
237
+
err = db.Clauses(clause.OnConflict{
238
+
Columns: []clause.Column{{Name: "repo"}},
239
+
DoUpdates: clause.AssignmentColumns([]string{"seen_at"}),
240
+
}).Create(¬ifSeen).Error
241
+
242
+
if err != nil {
243
+
return c.JSON(http.StatusInternalServerError, map[string]any{
244
+
"error": "InternalError",
245
+
"message": "failed to update seen timestamp",
246
+
})
247
+
}
248
+
249
+
return c.JSON(http.StatusOK, map[string]any{})
169
250
}
170
251
171
252
func getUserDID(c echo.Context) string {
+15
-10
xrpc/server.go
+15
-10
xrpc/server.go
···
1
1
package xrpc
2
2
3
3
import (
4
+
"context"
4
5
"log/slog"
5
6
"net/http"
6
7
7
8
"github.com/bluesky-social/indigo/atproto/identity"
8
9
"github.com/labstack/echo/v4"
9
10
"github.com/labstack/echo/v4/middleware"
11
+
"github.com/whyrusleeping/konbini/backend"
10
12
"github.com/whyrusleeping/konbini/hydration"
13
+
"github.com/whyrusleeping/konbini/models"
11
14
"github.com/whyrusleeping/konbini/xrpc/actor"
12
15
"github.com/whyrusleeping/konbini/xrpc/feed"
13
16
"github.com/whyrusleeping/konbini/xrpc/graph"
···
31
34
type Backend interface {
32
35
// Add methods as needed for data access
33
36
34
-
TrackMissingActor(did string)
35
-
TrackMissingFeedGenerator(uri string)
37
+
TrackMissingRecord(identifier string, wait bool)
38
+
GetOrCreateRepo(ctx context.Context, did string) (*models.Repo, error)
36
39
}
37
40
38
41
// NewServer creates a new XRPC server
39
-
func NewServer(db *gorm.DB, dir identity.Directory, backend Backend) *Server {
42
+
func NewServer(db *gorm.DB, dir identity.Directory, backend *backend.PostgresBackend) *Server {
40
43
e := echo.New()
41
44
e.HidePort = true
42
45
e.HideBanner = true
···
57
60
db: db,
58
61
dir: dir,
59
62
backend: backend,
60
-
hydrator: hydration.NewHydrator(db, dir),
63
+
hydrator: hydration.NewHydrator(db, dir, backend),
61
64
}
62
65
63
-
s.hydrator.SetMissingActorCallback(backend.TrackMissingActor)
64
-
s.hydrator.SetMissingFeedGeneratorCallback(backend.TrackMissingFeedGenerator)
65
-
66
66
// Register XRPC endpoints
67
67
s.registerEndpoints()
68
68
···
78
78
// registerEndpoints registers all XRPC endpoints
79
79
func (s *Server) registerEndpoints() {
80
80
// XRPC endpoints follow the pattern: /xrpc/<namespace>.<method>
81
+
82
+
s.e.GET("/.well-known/did.json", func(c echo.Context) error {
83
+
return c.File("did.json")
84
+
})
85
+
81
86
xrpcGroup := s.e.Group("/xrpc")
82
87
83
88
// com.atproto.identity.*
···
91
96
// app.bsky.actor.*
92
97
xrpcGroup.GET("/app.bsky.actor.getProfile", func(c echo.Context) error {
93
98
return actor.HandleGetProfile(c, s.hydrator)
94
-
})
99
+
}, s.optionalAuth)
95
100
xrpcGroup.GET("/app.bsky.actor.getProfiles", func(c echo.Context) error {
96
101
return actor.HandleGetProfiles(c, s.db, s.hydrator)
97
-
})
102
+
}, s.optionalAuth)
98
103
xrpcGroup.GET("/app.bsky.actor.getPreferences", func(c echo.Context) error {
99
104
return actor.HandleGetPreferences(c, s.db, s.hydrator)
100
105
}, s.requireAuth)
···
128
133
}, s.requireAuth)
129
134
xrpcGroup.GET("/app.bsky.feed.getFeed", func(c echo.Context) error {
130
135
return feed.HandleGetFeed(c, s.db, s.hydrator, s.dir)
131
-
})
136
+
}, s.optionalAuth)
132
137
xrpcGroup.GET("/app.bsky.feed.getFeedGenerator", func(c echo.Context) error {
133
138
return feed.HandleGetFeedGenerator(c, s.db, s.hydrator, s.dir)
134
139
})
+160
-131
xrpc/unspecced/getPostThreadV2.go
+160
-131
xrpc/unspecced/getPostThreadV2.go
···
1
1
package unspecced
2
2
3
3
import (
4
+
"bytes"
4
5
"context"
5
6
"fmt"
6
7
"log/slog"
7
8
"net/http"
8
9
"strconv"
10
+
"sync"
9
11
10
12
"github.com/bluesky-social/indigo/api/bsky"
11
13
"github.com/labstack/echo/v4"
12
14
"github.com/whyrusleeping/konbini/hydration"
13
15
"github.com/whyrusleeping/konbini/views"
16
+
"github.com/whyrusleeping/market/models"
17
+
"go.opentelemetry.io/otel"
14
18
"gorm.io/gorm"
15
19
)
16
20
21
+
var tracer = otel.Tracer("xrpc/unspecced")
22
+
17
23
// HandleGetPostThreadV2 implements app.bsky.unspecced.getPostThreadV2
18
24
func HandleGetPostThreadV2(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
19
-
ctx := c.Request().Context()
25
+
ctx, span := tracer.Start(c.Request().Context(), "getPostThreadV2")
26
+
defer span.End()
27
+
ctx = context.WithValue(ctx, "auto-fetch", true)
20
28
21
29
// Parse parameters
22
30
anchorRaw := c.QueryParam("anchor")
···
68
76
})
69
77
}
70
78
71
-
// Determine the root post ID for the thread
72
-
rootPostID := anchorPostInfo.InThread
73
-
if rootPostID == 0 {
74
-
// This post is the root - get its ID
75
-
var postID uint
76
-
db.Raw(`
77
-
SELECT id FROM posts
78
-
WHERE author = (SELECT id FROM repos WHERE did = ?)
79
-
AND rkey = ?
80
-
`, extractDIDFromURI(anchorUri), extractRkeyFromURI(anchorUri)).Scan(&postID)
81
-
rootPostID = postID
79
+
threadID := anchorPostInfo.InThread
80
+
if threadID == 0 {
81
+
threadID = anchorPostInfo.ID
82
82
}
83
83
84
-
// Query all posts in this thread
85
-
type threadPostRow struct {
86
-
ID uint
87
-
Rkey string
88
-
ReplyTo uint
89
-
InThread uint
90
-
AuthorDid string
84
+
var threadPosts []*models.Post
85
+
if err := db.Raw("SELECT * FROM posts WHERE in_thread = ? OR id = ?", threadID, anchorPostInfo.ID).Scan(&threadPosts).Error; err != nil {
86
+
return err
91
87
}
92
-
var threadPosts []threadPostRow
93
-
db.Raw(`
94
-
SELECT p.id, p.rkey, p.reply_to, p.in_thread, r.did as author_did
95
-
FROM posts p
96
-
JOIN repos r ON r.id = p.author
97
-
WHERE (p.id = ? OR p.in_thread = ?)
98
-
AND p.not_found = false
99
-
ORDER BY p.created ASC
100
-
`, rootPostID, rootPostID).Scan(&threadPosts)
101
88
102
-
// Build a map of posts by ID
103
-
postsByID := make(map[uint]*threadNode)
104
-
for _, tp := range threadPosts {
105
-
uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", tp.AuthorDid, tp.Rkey)
106
-
postsByID[tp.ID] = &threadNode{
107
-
id: tp.ID,
108
-
uri: uri,
109
-
replyTo: tp.ReplyTo,
110
-
inThread: tp.InThread,
111
-
children: []*threadNode{},
112
-
}
113
-
}
114
-
115
-
// Build parent-child relationships
116
-
for _, node := range postsByID {
117
-
if node.replyTo != 0 {
118
-
parent := postsByID[node.replyTo]
119
-
if parent != nil {
120
-
parent.children = append(parent.children, node)
121
-
}
122
-
}
123
-
}
124
-
125
-
// Find the anchor node
126
-
anchorID := uint(0)
127
-
for id, node := range postsByID {
128
-
if node.uri == anchorUri {
129
-
anchorID = id
130
-
break
131
-
}
132
-
}
89
+
fmt.Println("GOT THREAD POSTS: ", len(threadPosts))
133
90
134
-
if anchorID == 0 {
135
-
return c.JSON(http.StatusNotFound, map[string]interface{}{
136
-
"error": "NotFound",
137
-
"message": "anchor post not found in thread",
138
-
})
91
+
treeNodes, err := buildThreadTree(ctx, hydrator, db, threadPosts)
92
+
if err != nil {
93
+
return fmt.Errorf("failed to construct tree: %w", err)
139
94
}
140
95
141
-
anchorNode := postsByID[anchorID]
96
+
anchor := treeNodes[anchorPostInfo.ID]
142
97
143
98
// Build flat thread items list
144
99
var threadItems []*bsky.UnspeccedGetPostThreadV2_ThreadItem
···
146
101
147
102
// Add parents if requested
148
103
if above {
149
-
parents := collectParents(anchorNode, postsByID)
150
-
for i := len(parents) - 1; i >= 0; i-- {
151
-
depth := int64(-(len(parents) - i))
152
-
item := buildThreadItem(ctx, hydrator, parents[i], depth, viewer)
104
+
parent := anchor.parent
105
+
depth := int64(-1)
106
+
for parent != nil {
107
+
if parent.missing {
108
+
fmt.Println("Parent missing: ", depth)
109
+
item := &bsky.UnspeccedGetPostThreadV2_ThreadItem{
110
+
Depth: depth,
111
+
Uri: parent.uri,
112
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
113
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
114
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
115
+
},
116
+
},
117
+
}
118
+
119
+
threadItems = append(threadItems, item)
120
+
break
121
+
}
122
+
123
+
item := buildThreadItem(ctx, hydrator, parent, depth, viewer)
153
124
if item != nil {
154
125
threadItems = append(threadItems, item)
155
126
}
127
+
128
+
parent = parent.parent
129
+
depth--
156
130
}
157
131
}
158
132
159
133
// Add anchor post (depth 0)
160
-
anchorItem := buildThreadItem(ctx, hydrator, anchorNode, 0, viewer)
134
+
anchorItem := buildThreadItem(ctx, hydrator, anchor, 0, viewer)
161
135
if anchorItem != nil {
162
136
threadItems = append(threadItems, anchorItem)
163
137
}
164
138
165
139
// Add replies below anchor
166
140
if below > 0 {
167
-
replies, hasMore := collectReplies(ctx, hydrator, anchorNode, 1, below, branchingFactor, sort, viewer)
141
+
replies, err := collectReplies(ctx, hydrator, anchor, 0, below, branchingFactor, sort, viewer)
142
+
if err != nil {
143
+
return err
144
+
}
168
145
threadItems = append(threadItems, replies...)
169
-
hasOtherReplies = hasMore
146
+
//hasOtherReplies = hasMore
170
147
}
171
148
172
149
return c.JSON(http.StatusOK, &bsky.UnspeccedGetPostThreadV2_Output{
···
175
152
})
176
153
}
177
154
178
-
type threadNode struct {
179
-
id uint
180
-
uri string
181
-
replyTo uint
182
-
inThread uint
183
-
children []*threadNode
184
-
}
155
+
func collectReplies(ctx context.Context, hydrator *hydration.Hydrator, curnode *threadTree, depth int64, below int64, branchingFactor int64, sort string, viewer string) ([]*bsky.UnspeccedGetPostThreadV2_ThreadItem, error) {
156
+
if below == 0 {
157
+
return nil, nil
158
+
}
185
159
186
-
func collectParents(node *threadNode, allNodes map[uint]*threadNode) []*threadNode {
187
-
var parents []*threadNode
188
-
current := node
189
-
for current.replyTo != 0 {
190
-
parent := allNodes[current.replyTo]
191
-
if parent == nil {
192
-
break
193
-
}
194
-
parents = append(parents, parent)
195
-
current = parent
160
+
type parThreadResults struct {
161
+
node *bsky.UnspeccedGetPostThreadV2_ThreadItem
162
+
children []*bsky.UnspeccedGetPostThreadV2_ThreadItem
196
163
}
197
-
return parents
198
-
}
164
+
165
+
results := make([]parThreadResults, len(curnode.children))
166
+
167
+
var wg sync.WaitGroup
168
+
for i := range curnode.children {
169
+
ix := i
170
+
wg.Go(func() {
171
+
child := curnode.children[ix]
172
+
173
+
results[ix].node = buildThreadItem(ctx, hydrator, child, depth+1, viewer)
174
+
if child.missing {
175
+
return
176
+
}
199
177
200
-
func collectReplies(ctx context.Context, hydrator *hydration.Hydrator, node *threadNode, currentDepth, maxDepth, branchingFactor int64, sort string, viewer string) ([]*bsky.UnspeccedGetPostThreadV2_ThreadItem, bool) {
201
-
var items []*bsky.UnspeccedGetPostThreadV2_ThreadItem
202
-
hasMore := false
178
+
sub, err := collectReplies(ctx, hydrator, child, depth+1, below-1, branchingFactor, sort, viewer)
179
+
if err != nil {
180
+
slog.Error("failed to collect replies", "node", child.uri, "error", err)
181
+
return
182
+
}
203
183
204
-
if currentDepth > maxDepth {
205
-
return items, false
184
+
results[ix].children = sub
185
+
})
206
186
}
207
187
208
-
// Sort children based on sort parameter
209
-
children := node.children
210
-
// TODO: Actually sort based on the sort parameter (newest/oldest/top)
211
-
// For now, just use the order we have
188
+
wg.Wait()
212
189
213
-
// Limit to branchingFactor
214
-
limit := int(branchingFactor)
215
-
if len(children) > limit {
216
-
hasMore = true
217
-
children = children[:limit]
190
+
var out []*bsky.UnspeccedGetPostThreadV2_ThreadItem
191
+
for _, res := range results {
192
+
out = append(out, res.node)
193
+
out = append(out, res.children...)
218
194
}
219
195
220
-
for _, child := range children {
221
-
item := buildThreadItem(ctx, hydrator, child, currentDepth, viewer)
222
-
if item != nil {
223
-
items = append(items, item)
196
+
return out, nil
197
+
}
224
198
225
-
// Recursively collect replies
226
-
if currentDepth < maxDepth {
227
-
childReplies, childHasMore := collectReplies(ctx, hydrator, child, currentDepth+1, maxDepth, branchingFactor, sort, viewer)
228
-
items = append(items, childReplies...)
229
-
if childHasMore {
230
-
hasMore = true
231
-
}
232
-
}
199
+
func buildThreadItem(ctx context.Context, hydrator *hydration.Hydrator, node *threadTree, depth int64, viewer string) *bsky.UnspeccedGetPostThreadV2_ThreadItem {
200
+
if node.missing {
201
+
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
202
+
Depth: depth,
203
+
Uri: node.uri,
204
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
205
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
206
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
207
+
},
208
+
},
233
209
}
234
210
}
235
211
236
-
return items, hasMore
237
-
}
238
-
239
-
func buildThreadItem(ctx context.Context, hydrator *hydration.Hydrator, node *threadNode, depth int64, viewer string) *bsky.UnspeccedGetPostThreadV2_ThreadItem {
240
212
// Hydrate the post
241
-
postInfo, err := hydrator.HydratePost(ctx, node.uri, viewer)
213
+
postInfo, err := hydrator.HydratePostDB(ctx, node.uri, node.val, viewer)
242
214
if err != nil {
215
+
slog.Error("failed to hydrate post in thread item", "uri", node.uri, "error", err)
243
216
// Return not found item
244
217
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
245
218
Depth: depth,
···
255
228
// Hydrate author
256
229
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
257
230
if err != nil {
231
+
slog.Error("failed to hydrate actor in thread item", "author", postInfo.Author, "error", err)
258
232
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
259
233
Depth: depth,
260
234
Uri: node.uri,
···
318
292
return string(parts)
319
293
}
320
294
321
-
func extractRkeyFromURI(uri string) string {
322
-
// URI format: at://did:plc:xxx/collection/rkey
323
-
if len(uri) < 5 || uri[:5] != "at://" {
324
-
return ""
295
+
type threadTree struct {
296
+
parent *threadTree
297
+
children []*threadTree
298
+
299
+
val *models.Post
300
+
301
+
missing bool
302
+
303
+
uri string
304
+
cid string
305
+
}
306
+
307
+
func buildThreadTree(ctx context.Context, hydrator *hydration.Hydrator, db *gorm.DB, posts []*models.Post) (map[uint]*threadTree, error) {
308
+
nodes := make(map[uint]*threadTree)
309
+
for _, p := range posts {
310
+
puri, err := hydrator.UriForPost(ctx, p)
311
+
if err != nil {
312
+
return nil, err
313
+
}
314
+
315
+
t := &threadTree{
316
+
val: p,
317
+
uri: puri,
318
+
}
319
+
320
+
nodes[p.ID] = t
325
321
}
326
-
// Find last slash
327
-
for i := len(uri) - 1; i >= 5; i-- {
328
-
if uri[i] == '/' {
329
-
return uri[i+1:]
322
+
323
+
missing := make(map[uint]*threadTree)
324
+
for _, node := range nodes {
325
+
if node.val.ReplyTo == 0 {
326
+
continue
330
327
}
328
+
329
+
pnode, ok := nodes[node.val.ReplyTo]
330
+
if !ok {
331
+
pnode = &threadTree{
332
+
missing: true,
333
+
}
334
+
missing[node.val.ReplyTo] = pnode
335
+
336
+
var bspost bsky.FeedPost
337
+
if err := bspost.UnmarshalCBOR(bytes.NewReader(node.val.Raw)); err != nil {
338
+
return nil, err
339
+
}
340
+
341
+
if bspost.Reply == nil || bspost.Reply.Parent == nil {
342
+
return nil, fmt.Errorf("node with parent had no parent in object")
343
+
}
344
+
345
+
pnode.uri = bspost.Reply.Parent.Uri
346
+
pnode.cid = bspost.Reply.Parent.Cid
347
+
348
+
/* Maybe we could force hydrate these?
349
+
hydrator.AddMissingRecord(puri, true)
350
+
*/
351
+
}
352
+
353
+
pnode.children = append(pnode.children, node)
354
+
node.parent = pnode
331
355
}
332
-
return ""
356
+
357
+
for k, v := range missing {
358
+
nodes[k] = v
359
+
}
360
+
361
+
return nodes, nil
333
362
}