why.bsky.team
+19
LICENSE-MIT
+19
LICENSE-MIT
···
1
+
MIT License
2
+
3
+
Permission is hereby granted, free of charge, to any person obtaining a copy
4
+
of this software and associated documentation files (the "Software"), to deal
5
+
in the Software without restriction, including without limitation the rights
6
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+
copies of the Software, and to permit persons to whom the Software is
8
+
furnished to do so, subject to the following conditions:
9
+
10
+
The above copyright notice and this permission notice shall be included in all
11
+
copies or substantial portions of the Software.
12
+
13
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+
SOFTWARE.
+134
README.md
+134
README.md
···
109
109
110
110
The frontend will be available at http://localhost:3000 and will connect to the API at http://localhost:4444.
111
111
112
+
## Running the Bluesky App against Konbini
113
+
114
+
Konbini implements a large portion of the app.bsky.\* appview endpoints that
115
+
are required for pointing the main app to it and having it work reasonably
116
+
well.
117
+
118
+
To accomplish this you will need a few things:
119
+
120
+
### Service DID
121
+
122
+
You will need a DID, preferably a did:web for your appview that points at a
123
+
public endpoint where your appview is accessible via https.
124
+
I'll get into the https proxy next, but for the did, I've just pointed a domain
125
+
I own (in my case appview1.bluesky.day) to a VPS, and used caddy to host a file
126
+
at `/.well-known/did.json`.
127
+
That file should look like this:
128
+
129
+
```json
130
+
{
131
+
"@context": [
132
+
"https://www.w3.org/ns/did/v1",
133
+
"https://w3id.org/security/multikey/v1"
134
+
],
135
+
"id": "did:web:appview1.bluesky.day",
136
+
"verificationMethod": [
137
+
{
138
+
"id": "did:web:api.bsky.app#atproto",
139
+
"type": "Multikey",
140
+
"controller": "did:web:api.bsky.app",
141
+
"publicKeyMultibase": "zQ3shpRzb2NDriwCSSsce6EqGxG23kVktHZc57C3NEcuNy1jg"
142
+
}
143
+
],
144
+
"service": [
145
+
{
146
+
"id": "#bsky_notif",
147
+
"type": "BskyNotificationService",
148
+
"serviceEndpoint": "YOUR APPVIEW HTTPS URL"
149
+
},
150
+
{
151
+
"id": "#bsky_appview",
152
+
"type": "BskyAppView",
153
+
"serviceEndpoint": "YOUR APPVIEW HTTPS URL"
154
+
}
155
+
]
156
+
}
157
+
```
158
+
159
+
The verificationMethod isn't used but i'm not sure if _something_ is required
160
+
there or not, so i'm just leaving that there, it works on my machine.
161
+
162
+
### HTTPS Endpoint
163
+
164
+
I've been using ngrok to proxy traffic from a publicly accessible https url to my appview.
165
+
You can simply run `ngrok http 4446` and it will give you an https url that you
166
+
can then put in your DID doc above.
167
+
168
+
### The Social App
169
+
170
+
Now, clone and build the social app:
171
+
172
+
```
173
+
git clone https://github.com/bluesky-social/social-app
174
+
cd social-app
175
+
yarn
176
+
```
177
+
178
+
And then set this environment variable that tells it to use your appview:
179
+
180
+
```
181
+
export EXPO_PUBLIC_BLUESKY_PROXY_DID=did:web:YOURDIDWEB
182
+
```
183
+
184
+
And finally run the app:
185
+
186
+
```
187
+
yarn web
188
+
```
189
+
190
+
This takes a while on first load since its building everything.
191
+
After that, load the localhost url it gives you and it _should_ work.
192
+
193
+
## Selective Backfill
194
+
195
+
If you'd like to backfill a particular repo, just hit the following endpoint:
196
+
197
+
```
198
+
curl http://localhost:4444/rescan/<DID OR HANDLE>
199
+
200
+
```
201
+
202
+
It will take a minute but it should pull all records from that user.
203
+
204
+
## Upstream Firehose Configuration
205
+
206
+
Konbini supports both standard firehose endpoints as well as jetstream. If
207
+
bandwidth and CPU usage is a concern, and you trust the jetstream endpoint,
208
+
then it may be worth trying that out.
209
+
210
+
The configuration file is formatted as follows:
211
+
212
+
```json
213
+
{
214
+
"backends": [
215
+
{
216
+
"type": "jetstream",
217
+
"host": "jetstream1.us-west.bsky.network"
218
+
}
219
+
]
220
+
}
221
+
```
222
+
223
+
The default (implicit) configuration file looks like this:
224
+
225
+
```json
226
+
{
227
+
"backends": [
228
+
{
229
+
"type": "firehose",
230
+
"host": "bsky.network"
231
+
}
232
+
]
233
+
}
234
+
```
235
+
236
+
Note that this is an array of backends, you can specify multiple upstreams, and
237
+
konbini will read from all of them. The main intended purpose of this is to be
238
+
able to subscribe directly to PDSs. PDSs currently only support the full
239
+
firehose endpoint, not jetstream, so be sure to specify a type of "firehose"
240
+
for individual PDS endpoints.
241
+
112
242
## License
113
243
114
244
MIT (whyrusleeping)
245
+
246
+
```
247
+
248
+
```
+536
backend/backend.go
+536
backend/backend.go
···
1
+
package backend
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
"strings"
8
+
"sync"
9
+
"time"
10
+
11
+
"github.com/bluesky-social/indigo/api/atproto"
12
+
"github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/bluesky-social/indigo/atproto/identity"
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
"github.com/bluesky-social/indigo/util"
16
+
"github.com/bluesky-social/indigo/xrpc"
17
+
lru "github.com/hashicorp/golang-lru/v2"
18
+
"github.com/jackc/pgx/v5"
19
+
"github.com/jackc/pgx/v5/pgconn"
20
+
"github.com/jackc/pgx/v5/pgxpool"
21
+
. "github.com/whyrusleeping/konbini/models"
22
+
"github.com/whyrusleeping/market/models"
23
+
"gorm.io/gorm"
24
+
"gorm.io/gorm/clause"
25
+
"gorm.io/gorm/logger"
26
+
)
27
+
28
+
// PostgresBackend handles database operations
29
+
type PostgresBackend struct {
30
+
db *gorm.DB
31
+
pgx *pgxpool.Pool
32
+
33
+
dir identity.Directory
34
+
35
+
client *xrpc.Client
36
+
37
+
mydid string
38
+
myrepo *models.Repo
39
+
40
+
relevantDids map[string]bool
41
+
rdLk sync.Mutex
42
+
43
+
revCache *lru.TwoQueueCache[uint, string]
44
+
45
+
repoCache *lru.TwoQueueCache[string, *Repo]
46
+
reposLk sync.Mutex
47
+
48
+
didByIDCache *lru.TwoQueueCache[uint, string]
49
+
50
+
postInfoCache *lru.TwoQueueCache[string, cachedPostInfo]
51
+
52
+
missingRecords chan MissingRecord
53
+
}
54
+
55
+
type cachedPostInfo struct {
56
+
ID uint
57
+
Author uint
58
+
}
59
+
60
+
// NewPostgresBackend creates a new PostgresBackend
61
+
func NewPostgresBackend(mydid string, db *gorm.DB, pgx *pgxpool.Pool, client *xrpc.Client, dir identity.Directory) (*PostgresBackend, error) {
62
+
rc, _ := lru.New2Q[string, *Repo](1_000_000)
63
+
pc, _ := lru.New2Q[string, cachedPostInfo](1_000_000)
64
+
revc, _ := lru.New2Q[uint, string](1_000_000)
65
+
dbic, _ := lru.New2Q[uint, string](1_000_000)
66
+
67
+
b := &PostgresBackend{
68
+
client: client,
69
+
mydid: mydid,
70
+
db: db,
71
+
pgx: pgx,
72
+
relevantDids: make(map[string]bool),
73
+
repoCache: rc,
74
+
postInfoCache: pc,
75
+
revCache: revc,
76
+
didByIDCache: dbic,
77
+
dir: dir,
78
+
79
+
missingRecords: make(chan MissingRecord, 1000),
80
+
}
81
+
82
+
r, err := b.GetOrCreateRepo(context.TODO(), mydid)
83
+
if err != nil {
84
+
return nil, err
85
+
}
86
+
87
+
b.myrepo = r
88
+
89
+
go b.missingRecordFetcher()
90
+
return b, nil
91
+
}
92
+
93
+
// TrackMissingRecord implements the RecordTracker interface
94
+
func (b *PostgresBackend) TrackMissingRecord(identifier string, wait bool) {
95
+
mr := MissingRecord{
96
+
Type: mrTypeFromIdent(identifier),
97
+
Identifier: identifier,
98
+
Wait: wait,
99
+
}
100
+
101
+
b.addMissingRecord(context.TODO(), mr)
102
+
}
103
+
104
+
func mrTypeFromIdent(ident string) MissingRecordType {
105
+
if strings.HasPrefix(ident, "did:") {
106
+
return MissingRecordTypeProfile
107
+
}
108
+
109
+
puri, _ := syntax.ParseATURI(ident)
110
+
switch puri.Collection().String() {
111
+
case "app.bsky.feed.post":
112
+
return MissingRecordTypePost
113
+
case "app.bsky.feed.generator":
114
+
return MissingRecordTypeFeedGenerator
115
+
default:
116
+
return MissingRecordTypeUnknown
117
+
}
118
+
119
+
}
120
+
121
+
// DidToID converts a DID to a database ID
122
+
func (b *PostgresBackend) DidToID(ctx context.Context, did string) (uint, error) {
123
+
r, err := b.GetOrCreateRepo(ctx, did)
124
+
if err != nil {
125
+
return 0, err
126
+
}
127
+
return r.ID, nil
128
+
}
129
+
130
+
func (b *PostgresBackend) GetOrCreateRepo(ctx context.Context, did string) (*Repo, error) {
131
+
r, ok := b.repoCache.Get(did)
132
+
if !ok {
133
+
b.reposLk.Lock()
134
+
135
+
r, ok = b.repoCache.Get(did)
136
+
if !ok {
137
+
r = &Repo{}
138
+
r.Did = did
139
+
b.repoCache.Add(did, r)
140
+
}
141
+
142
+
b.reposLk.Unlock()
143
+
}
144
+
145
+
r.Lk.Lock()
146
+
defer r.Lk.Unlock()
147
+
if r.Setup {
148
+
return r, nil
149
+
}
150
+
151
+
row := b.pgx.QueryRow(ctx, "SELECT id, created_at, did FROM repos WHERE did = $1", did)
152
+
153
+
err := row.Scan(&r.ID, &r.CreatedAt, &r.Did)
154
+
if err == nil {
155
+
// found it!
156
+
r.Setup = true
157
+
return r, nil
158
+
}
159
+
160
+
if err != pgx.ErrNoRows {
161
+
return nil, err
162
+
}
163
+
164
+
r.Did = did
165
+
if err := b.db.Create(r).Error; err != nil {
166
+
return nil, err
167
+
}
168
+
169
+
r.Setup = true
170
+
171
+
return r, nil
172
+
}
173
+
174
+
func (b *PostgresBackend) GetOrCreateList(ctx context.Context, uri string) (*List, error) {
175
+
puri, err := util.ParseAtUri(uri)
176
+
if err != nil {
177
+
return nil, err
178
+
}
179
+
180
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
181
+
if err != nil {
182
+
return nil, err
183
+
}
184
+
185
+
// TODO: needs upsert treatment when we actually find the list
186
+
var list List
187
+
if err := b.db.FirstOrCreate(&list, map[string]any{
188
+
"author": r.ID,
189
+
"rkey": puri.Rkey,
190
+
}).Error; err != nil {
191
+
return nil, err
192
+
}
193
+
return &list, nil
194
+
}
195
+
196
+
func (b *PostgresBackend) postIDForUri(ctx context.Context, uri string) (uint, error) {
197
+
// getPostByUri implicitly fills the cache
198
+
p, err := b.postInfoForUri(ctx, uri)
199
+
if err != nil {
200
+
return 0, err
201
+
}
202
+
203
+
return p.ID, nil
204
+
}
205
+
206
+
func (b *PostgresBackend) postInfoForUri(ctx context.Context, uri string) (cachedPostInfo, error) {
207
+
v, ok := b.postInfoCache.Get(uri)
208
+
if ok {
209
+
return v, nil
210
+
}
211
+
212
+
// getPostByUri implicitly fills the cache
213
+
p, err := b.getOrCreatePostBare(ctx, uri)
214
+
if err != nil {
215
+
return cachedPostInfo{}, err
216
+
}
217
+
218
+
return cachedPostInfo{ID: p.ID, Author: p.Author}, nil
219
+
}
220
+
221
+
func (b *PostgresBackend) tryLoadPostInfo(ctx context.Context, uid uint, rkey string) (*Post, error) {
222
+
var p Post
223
+
q := "SELECT id, author FROM posts WHERE author = $1 AND rkey = $2"
224
+
if err := b.pgx.QueryRow(ctx, q, uid, rkey).Scan(&p.ID, &p.Author); err != nil {
225
+
if errors.Is(err, pgx.ErrNoRows) {
226
+
return nil, nil
227
+
}
228
+
return nil, err
229
+
}
230
+
231
+
return &p, nil
232
+
}
233
+
234
+
func (b *PostgresBackend) getOrCreatePostBare(ctx context.Context, uri string) (*Post, error) {
235
+
puri, err := util.ParseAtUri(uri)
236
+
if err != nil {
237
+
return nil, err
238
+
}
239
+
240
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
241
+
if err != nil {
242
+
return nil, err
243
+
}
244
+
245
+
post, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
246
+
if err != nil {
247
+
return nil, err
248
+
}
249
+
250
+
if post == nil {
251
+
post = &Post{
252
+
Rkey: puri.Rkey,
253
+
Author: r.ID,
254
+
NotFound: true,
255
+
}
256
+
257
+
err := b.pgx.QueryRow(ctx, "INSERT INTO posts (rkey, author, not_found) VALUES ($1, $2, $3) RETURNING id", puri.Rkey, r.ID, true).Scan(&post.ID)
258
+
if err != nil {
259
+
pgErr, ok := err.(*pgconn.PgError)
260
+
if !ok || pgErr.Code != "23505" {
261
+
return nil, err
262
+
}
263
+
264
+
out, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
265
+
if err != nil {
266
+
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
267
+
}
268
+
if out == nil {
269
+
return nil, fmt.Errorf("postgres is lying to us: %d %s", r.ID, puri.Rkey)
270
+
}
271
+
272
+
post = out
273
+
}
274
+
275
+
}
276
+
277
+
b.postInfoCache.Add(uri, cachedPostInfo{
278
+
ID: post.ID,
279
+
Author: post.Author,
280
+
})
281
+
282
+
return post, nil
283
+
}
284
+
285
+
func (b *PostgresBackend) GetPostByUri(ctx context.Context, uri string, fields string) (*Post, error) {
286
+
puri, err := util.ParseAtUri(uri)
287
+
if err != nil {
288
+
return nil, err
289
+
}
290
+
291
+
r, err := b.GetOrCreateRepo(ctx, puri.Did)
292
+
if err != nil {
293
+
return nil, err
294
+
}
295
+
296
+
q := "SELECT " + fields + " FROM posts WHERE author = ? AND rkey = ?"
297
+
298
+
var post Post
299
+
if err := b.db.Raw(q, r.ID, puri.Rkey).Scan(&post).Error; err != nil {
300
+
return nil, err
301
+
}
302
+
303
+
if post.ID == 0 {
304
+
post.Rkey = puri.Rkey
305
+
post.Author = r.ID
306
+
post.NotFound = true
307
+
308
+
if err := b.db.Session(&gorm.Session{
309
+
Logger: logger.Default.LogMode(logger.Silent),
310
+
}).Create(&post).Error; err != nil {
311
+
if !errors.Is(err, gorm.ErrDuplicatedKey) {
312
+
return nil, err
313
+
}
314
+
if err := b.db.Find(&post, "author = ? AND rkey = ?", r.ID, puri.Rkey).Error; err != nil {
315
+
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
316
+
}
317
+
}
318
+
319
+
}
320
+
321
+
b.postInfoCache.Add(uri, cachedPostInfo{
322
+
ID: post.ID,
323
+
Author: post.Author,
324
+
})
325
+
326
+
return &post, nil
327
+
}
328
+
329
+
func (b *PostgresBackend) revForRepo(rr *Repo) (string, error) {
330
+
lrev, ok := b.revCache.Get(rr.ID)
331
+
if ok {
332
+
return lrev, nil
333
+
}
334
+
335
+
var rev string
336
+
if err := b.pgx.QueryRow(context.TODO(), "SELECT COALESCE(rev, '') FROM sync_infos WHERE repo = $1", rr.ID).Scan(&rev); err != nil {
337
+
if errors.Is(err, pgx.ErrNoRows) {
338
+
return "", nil
339
+
}
340
+
return "", err
341
+
}
342
+
343
+
if rev != "" {
344
+
b.revCache.Add(rr.ID, rev)
345
+
}
346
+
return rev, nil
347
+
}
348
+
349
+
func (b *PostgresBackend) AddRelevantDid(did string) {
350
+
b.rdLk.Lock()
351
+
defer b.rdLk.Unlock()
352
+
b.relevantDids[did] = true
353
+
}
354
+
355
+
func (b *PostgresBackend) DidIsRelevant(did string) bool {
356
+
b.rdLk.Lock()
357
+
defer b.rdLk.Unlock()
358
+
return b.relevantDids[did]
359
+
}
360
+
361
+
func (b *PostgresBackend) anyRelevantIdents(idents ...string) bool {
362
+
for _, id := range idents {
363
+
if strings.HasPrefix(id, "did:") {
364
+
if b.DidIsRelevant(id) {
365
+
return true
366
+
}
367
+
} else if strings.HasPrefix(id, "at://") {
368
+
puri, err := syntax.ParseATURI(id)
369
+
if err != nil {
370
+
continue
371
+
}
372
+
373
+
if b.DidIsRelevant(puri.Authority().String()) {
374
+
return true
375
+
}
376
+
}
377
+
}
378
+
379
+
return false
380
+
}
381
+
382
+
func (b *PostgresBackend) GetRelevantDids() []string {
383
+
b.rdLk.Lock()
384
+
var out []string
385
+
for k := range b.relevantDids {
386
+
out = append(out, k)
387
+
}
388
+
b.rdLk.Unlock()
389
+
return out
390
+
}
391
+
392
+
func (b *PostgresBackend) GetRepoByID(ctx context.Context, id uint) (*models.Repo, error) {
393
+
var r models.Repo
394
+
if err := b.db.Find(&r, "id = ?", id).Error; err != nil {
395
+
return nil, err
396
+
}
397
+
398
+
return &r, nil
399
+
}
400
+
401
+
func (b *PostgresBackend) DidFromID(ctx context.Context, uid uint) (string, error) {
402
+
val, ok := b.didByIDCache.Get(uid)
403
+
if ok {
404
+
return val, nil
405
+
}
406
+
407
+
r, err := b.GetRepoByID(ctx, uid)
408
+
if err != nil {
409
+
return "", err
410
+
}
411
+
412
+
b.didByIDCache.Add(uid, r.Did)
413
+
return r.Did, nil
414
+
}
415
+
416
+
func (b *PostgresBackend) checkPostExists(ctx context.Context, repo *Repo, rkey string) (bool, error) {
417
+
var id uint
418
+
var notfound bool
419
+
if err := b.pgx.QueryRow(ctx, "SELECT id, not_found FROM posts WHERE author = $1 AND rkey = $2", repo.ID, rkey).Scan(&id, ¬found); err != nil {
420
+
if errors.Is(err, pgx.ErrNoRows) {
421
+
return false, nil
422
+
}
423
+
return false, err
424
+
}
425
+
426
+
if id != 0 && !notfound {
427
+
return true, nil
428
+
}
429
+
430
+
return false, nil
431
+
}
432
+
433
+
func (b *PostgresBackend) LoadRelevantDids() error {
434
+
ctx := context.TODO()
435
+
436
+
if err := b.ensureFollowsScraped(ctx, b.mydid); err != nil {
437
+
return fmt.Errorf("failed to scrape follows: %w", err)
438
+
}
439
+
440
+
r, err := b.GetOrCreateRepo(ctx, b.mydid)
441
+
if err != nil {
442
+
return err
443
+
}
444
+
445
+
var dids []string
446
+
if err := b.db.Raw("select did from follows left join repos on follows.subject = repos.id where follows.author = ?", r.ID).Scan(&dids).Error; err != nil {
447
+
return err
448
+
}
449
+
450
+
b.relevantDids[b.mydid] = true
451
+
for _, d := range dids {
452
+
fmt.Println("adding did: ", d)
453
+
b.relevantDids[d] = true
454
+
}
455
+
456
+
return nil
457
+
}
458
+
459
+
type SyncInfo struct {
460
+
Repo uint `gorm:"index"`
461
+
FollowsSynced bool
462
+
Rev string
463
+
}
464
+
465
+
func (b *PostgresBackend) ensureFollowsScraped(ctx context.Context, user string) error {
466
+
r, err := b.GetOrCreateRepo(ctx, user)
467
+
if err != nil {
468
+
return err
469
+
}
470
+
471
+
var si SyncInfo
472
+
if err := b.db.Find(&si, "repo = ?", r.ID).Error; err != nil {
473
+
return err
474
+
}
475
+
476
+
// not found
477
+
if si.Repo == 0 {
478
+
if err := b.db.Create(&SyncInfo{
479
+
Repo: r.ID,
480
+
}).Error; err != nil {
481
+
return err
482
+
}
483
+
}
484
+
485
+
if si.FollowsSynced {
486
+
return nil
487
+
}
488
+
489
+
var follows []Follow
490
+
var cursor string
491
+
for {
492
+
resp, err := atproto.RepoListRecords(ctx, b.client, "app.bsky.graph.follow", cursor, 100, b.mydid, false)
493
+
if err != nil {
494
+
return err
495
+
}
496
+
497
+
for _, rec := range resp.Records {
498
+
if fol, ok := rec.Value.Val.(*bsky.GraphFollow); ok {
499
+
fr, err := b.GetOrCreateRepo(ctx, fol.Subject)
500
+
if err != nil {
501
+
return err
502
+
}
503
+
504
+
puri, err := syntax.ParseATURI(rec.Uri)
505
+
if err != nil {
506
+
return err
507
+
}
508
+
509
+
follows = append(follows, Follow{
510
+
Created: time.Now(),
511
+
Indexed: time.Now(),
512
+
Rkey: puri.RecordKey().String(),
513
+
Author: r.ID,
514
+
Subject: fr.ID,
515
+
})
516
+
}
517
+
}
518
+
519
+
if resp.Cursor == nil || len(resp.Records) == 0 {
520
+
break
521
+
}
522
+
cursor = *resp.Cursor
523
+
}
524
+
525
+
if err := b.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(follows, 200).Error; err != nil {
526
+
return err
527
+
}
528
+
529
+
if err := b.db.Model(SyncInfo{}).Where("repo = ?", r.ID).Update("follows_synced", true).Error; err != nil {
530
+
return err
531
+
}
532
+
533
+
fmt.Println("Got follows: ", len(follows))
534
+
535
+
return nil
536
+
}
+1208
backend/events.go
+1208
backend/events.go
···
1
+
package backend
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"encoding/json"
7
+
"fmt"
8
+
"log/slog"
9
+
"strings"
10
+
"time"
11
+
12
+
"github.com/bluesky-social/indigo/api/atproto"
13
+
"github.com/bluesky-social/indigo/api/bsky"
14
+
"github.com/bluesky-social/indigo/atproto/syntax"
15
+
lexutil "github.com/bluesky-social/indigo/lex/util"
16
+
"github.com/bluesky-social/indigo/repo"
17
+
jsmodels "github.com/bluesky-social/jetstream/pkg/models"
18
+
"github.com/ipfs/go-cid"
19
+
"github.com/jackc/pgx/v5/pgconn"
20
+
"github.com/prometheus/client_golang/prometheus"
21
+
"github.com/prometheus/client_golang/prometheus/promauto"
22
+
23
+
. "github.com/whyrusleeping/konbini/models"
24
+
)
25
+
26
+
var handleOpHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
27
+
Name: "handle_op_duration",
28
+
Help: "A histogram of op handling durations",
29
+
Buckets: prometheus.ExponentialBuckets(1, 2, 15),
30
+
}, []string{"op", "collection"})
31
+
32
+
func (b *PostgresBackend) HandleEvent(ctx context.Context, evt *atproto.SyncSubscribeRepos_Commit) error {
33
+
r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks))
34
+
if err != nil {
35
+
return fmt.Errorf("failed to read event repo: %w", err)
36
+
}
37
+
38
+
for _, op := range evt.Ops {
39
+
switch op.Action {
40
+
case "create":
41
+
c, rec, err := r.GetRecordBytes(ctx, op.Path)
42
+
if err != nil {
43
+
return err
44
+
}
45
+
if err := b.HandleCreate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
46
+
return fmt.Errorf("create record failed: %w", err)
47
+
}
48
+
case "update":
49
+
c, rec, err := r.GetRecordBytes(ctx, op.Path)
50
+
if err != nil {
51
+
return err
52
+
}
53
+
if err := b.HandleUpdate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
54
+
return fmt.Errorf("update record failed: %w", err)
55
+
}
56
+
case "delete":
57
+
if err := b.HandleDelete(ctx, evt.Repo, evt.Rev, op.Path); err != nil {
58
+
return fmt.Errorf("delete record failed: %w", err)
59
+
}
60
+
}
61
+
}
62
+
63
+
// TODO: sync with the Since field to make sure we don't miss events we care about
64
+
/*
65
+
if err := bf.Store.UpdateRev(ctx, evt.Repo, evt.Rev); err != nil {
66
+
return fmt.Errorf("failed to update rev: %w", err)
67
+
}
68
+
*/
69
+
70
+
return nil
71
+
}
72
+
73
+
func cborBytesFromEvent(evt *jsmodels.Event) ([]byte, error) {
74
+
val, err := lexutil.NewFromType(evt.Commit.Collection)
75
+
if err != nil {
76
+
return nil, fmt.Errorf("failed to load event record type: %w", err)
77
+
}
78
+
79
+
if err := json.Unmarshal(evt.Commit.Record, val); err != nil {
80
+
return nil, err
81
+
}
82
+
83
+
cval, ok := val.(lexutil.CBOR)
84
+
if !ok {
85
+
return nil, fmt.Errorf("decoded type was not cbor marshalable")
86
+
}
87
+
88
+
buf := new(bytes.Buffer)
89
+
if err := cval.MarshalCBOR(buf); err != nil {
90
+
return nil, fmt.Errorf("failed to marshal event to cbor: %w", err)
91
+
}
92
+
93
+
rec := buf.Bytes()
94
+
return rec, nil
95
+
}
96
+
97
+
func (b *PostgresBackend) HandleEventJetstream(ctx context.Context, evt *jsmodels.Event) error {
98
+
99
+
path := evt.Commit.Collection + "/" + evt.Commit.RKey
100
+
switch evt.Commit.Operation {
101
+
case jsmodels.CommitOperationCreate:
102
+
rec, err := cborBytesFromEvent(evt)
103
+
if err != nil {
104
+
return err
105
+
}
106
+
107
+
c, err := cid.Decode(evt.Commit.CID)
108
+
if err != nil {
109
+
return err
110
+
}
111
+
112
+
if err := b.HandleCreate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
113
+
return fmt.Errorf("create record failed: %w", err)
114
+
}
115
+
case jsmodels.CommitOperationUpdate:
116
+
rec, err := cborBytesFromEvent(evt)
117
+
if err != nil {
118
+
return err
119
+
}
120
+
121
+
c, err := cid.Decode(evt.Commit.CID)
122
+
if err != nil {
123
+
return err
124
+
}
125
+
126
+
if err := b.HandleUpdate(ctx, evt.Did, evt.Commit.Rev, path, &rec, &c); err != nil {
127
+
return fmt.Errorf("update record failed: %w", err)
128
+
}
129
+
case jsmodels.CommitOperationDelete:
130
+
if err := b.HandleDelete(ctx, evt.Did, evt.Commit.Rev, path); err != nil {
131
+
return fmt.Errorf("delete record failed: %w", err)
132
+
}
133
+
}
134
+
135
+
return nil
136
+
}
137
+
138
+
func (b *PostgresBackend) HandleCreate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
139
+
start := time.Now()
140
+
141
+
rr, err := b.GetOrCreateRepo(ctx, repo)
142
+
if err != nil {
143
+
return fmt.Errorf("get user failed: %w", err)
144
+
}
145
+
146
+
lrev, err := b.revForRepo(rr)
147
+
if err != nil {
148
+
return err
149
+
}
150
+
if lrev != "" {
151
+
if rev < lrev {
152
+
slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
153
+
return nil
154
+
}
155
+
}
156
+
157
+
parts := strings.Split(path, "/")
158
+
if len(parts) != 2 {
159
+
return fmt.Errorf("invalid path in HandleCreate: %q", path)
160
+
}
161
+
col := parts[0]
162
+
rkey := parts[1]
163
+
164
+
defer func() {
165
+
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
166
+
}()
167
+
168
+
if rkey == "" {
169
+
fmt.Printf("messed up path: %q\n", rkey)
170
+
}
171
+
172
+
switch col {
173
+
case "app.bsky.feed.post":
174
+
if err := b.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
175
+
return err
176
+
}
177
+
case "app.bsky.feed.like":
178
+
if err := b.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
179
+
return err
180
+
}
181
+
case "app.bsky.feed.repost":
182
+
if err := b.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
183
+
return err
184
+
}
185
+
case "app.bsky.graph.follow":
186
+
if err := b.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
187
+
return err
188
+
}
189
+
case "app.bsky.graph.block":
190
+
if err := b.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
191
+
return err
192
+
}
193
+
case "app.bsky.graph.list":
194
+
if err := b.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
195
+
return err
196
+
}
197
+
case "app.bsky.graph.listitem":
198
+
if err := b.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
199
+
return err
200
+
}
201
+
case "app.bsky.graph.listblock":
202
+
if err := b.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
203
+
return err
204
+
}
205
+
case "app.bsky.actor.profile":
206
+
if err := b.HandleCreateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
207
+
return err
208
+
}
209
+
case "app.bsky.feed.generator":
210
+
if err := b.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
211
+
return err
212
+
}
213
+
case "app.bsky.feed.threadgate":
214
+
if err := b.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
215
+
return err
216
+
}
217
+
case "chat.bsky.actor.declaration":
218
+
if err := b.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
219
+
return err
220
+
}
221
+
case "app.bsky.feed.postgate":
222
+
if err := b.HandleCreatePostGate(ctx, rr, rkey, *rec, *cid); err != nil {
223
+
return err
224
+
}
225
+
case "app.bsky.graph.starterpack":
226
+
if err := b.HandleCreateStarterPack(ctx, rr, rkey, *rec, *cid); err != nil {
227
+
return err
228
+
}
229
+
default:
230
+
slog.Debug("unrecognized record type", "repo", repo, "path", path, "rev", rev)
231
+
}
232
+
233
+
b.revCache.Add(rr.ID, rev)
234
+
return nil
235
+
}
236
+
237
+
func (b *PostgresBackend) HandleCreatePost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
238
+
exists, err := b.checkPostExists(ctx, repo, rkey)
239
+
if err != nil {
240
+
return err
241
+
}
242
+
243
+
// still technically a race condition if two creates for the same post happen concurrently... probably fine
244
+
if exists {
245
+
return nil
246
+
}
247
+
248
+
var rec bsky.FeedPost
249
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
250
+
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
251
+
slog.Warn("skipping post with malformed data", "uri", uri, "error", err)
252
+
return nil // Skip this post rather than failing the entire event
253
+
}
254
+
255
+
reldids := []string{repo.Did}
256
+
// care about a post if its in a thread of a user we are interested in
257
+
if rec.Reply != nil && rec.Reply.Parent != nil && rec.Reply.Root != nil {
258
+
reldids = append(reldids, rec.Reply.Parent.Uri, rec.Reply.Root.Uri)
259
+
}
260
+
// TODO: maybe also care if its mentioning a user we care about or quoting a user we care about?
261
+
if !b.anyRelevantIdents(reldids...) {
262
+
return nil
263
+
}
264
+
265
+
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
266
+
slog.Warn("adding post", "uri", uri)
267
+
268
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
269
+
if err != nil {
270
+
return fmt.Errorf("invalid timestamp: %w", err)
271
+
}
272
+
273
+
p := Post{
274
+
Created: created.Time(),
275
+
Indexed: time.Now(),
276
+
Author: repo.ID,
277
+
Rkey: rkey,
278
+
Raw: recb,
279
+
Cid: cc.String(),
280
+
}
281
+
282
+
if rec.Reply != nil && rec.Reply.Parent != nil {
283
+
if rec.Reply.Root == nil {
284
+
return fmt.Errorf("post reply had nil root")
285
+
}
286
+
287
+
pinfo, err := b.postInfoForUri(ctx, rec.Reply.Parent.Uri)
288
+
if err != nil {
289
+
return fmt.Errorf("getting reply parent: %w", err)
290
+
}
291
+
292
+
p.ReplyTo = pinfo.ID
293
+
p.ReplyToUsr = pinfo.Author
294
+
295
+
thread, err := b.postIDForUri(ctx, rec.Reply.Root.Uri)
296
+
if err != nil {
297
+
return fmt.Errorf("getting thread root: %w", err)
298
+
}
299
+
300
+
p.InThread = thread
301
+
302
+
r, err := b.GetOrCreateRepo(ctx, b.mydid)
303
+
if err != nil {
304
+
return err
305
+
}
306
+
307
+
if p.ReplyToUsr == r.ID {
308
+
if err := b.AddNotification(ctx, r.ID, p.Author, uri, cc, NotifKindReply); err != nil {
309
+
slog.Warn("failed to create notification", "uri", uri, "error", err)
310
+
}
311
+
}
312
+
}
313
+
314
+
if rec.Embed != nil {
315
+
var rpref string
316
+
if rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {
317
+
rpref = rec.Embed.EmbedRecord.Record.Uri
318
+
}
319
+
if rec.Embed.EmbedRecordWithMedia != nil &&
320
+
rec.Embed.EmbedRecordWithMedia.Record != nil &&
321
+
rec.Embed.EmbedRecordWithMedia.Record.Record != nil {
322
+
rpref = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri
323
+
}
324
+
325
+
if rpref != "" && strings.Contains(rpref, "app.bsky.feed.post") {
326
+
rp, err := b.postIDForUri(ctx, rpref)
327
+
if err != nil {
328
+
return fmt.Errorf("getting quote subject: %w", err)
329
+
}
330
+
331
+
p.Reposting = rp
332
+
}
333
+
}
334
+
335
+
if err := b.doPostCreate(ctx, &p); err != nil {
336
+
return err
337
+
}
338
+
339
+
// Check for mentions and create notifications
340
+
if rec.Facets != nil {
341
+
for _, facet := range rec.Facets {
342
+
for _, feature := range facet.Features {
343
+
if feature.RichtextFacet_Mention != nil {
344
+
mentionDid := feature.RichtextFacet_Mention.Did
345
+
// This is a mention
346
+
mentionedRepo, err := b.GetOrCreateRepo(ctx, mentionDid)
347
+
if err != nil {
348
+
slog.Warn("failed to get repo for mention", "did", mentionDid, "error", err)
349
+
continue
350
+
}
351
+
352
+
// Create notification if the mentioned user is the current user
353
+
if mentionedRepo.ID == b.myrepo.ID {
354
+
if err := b.AddNotification(ctx, b.myrepo.ID, p.Author, uri, cc, NotifKindMention); err != nil {
355
+
slog.Warn("failed to create mention notification", "uri", uri, "error", err)
356
+
}
357
+
}
358
+
}
359
+
}
360
+
}
361
+
}
362
+
363
+
b.postInfoCache.Add(uri, cachedPostInfo{
364
+
ID: p.ID,
365
+
Author: p.Author,
366
+
})
367
+
368
+
return nil
369
+
}
370
+
371
+
func (b *PostgresBackend) doPostCreate(ctx context.Context, p *Post) error {
372
+
/*
373
+
if err := b.db.Clauses(clause.OnConflict{
374
+
Columns: []clause.Column{{Name: "author"}, {Name: "rkey"}},
375
+
DoUpdates: clause.AssignmentColumns([]string{"cid", "not_found", "raw", "created", "indexed"}),
376
+
}).Create(p).Error; err != nil {
377
+
return err
378
+
}
379
+
*/
380
+
381
+
query := `
382
+
INSERT INTO posts (author, rkey, cid, not_found, raw, created, indexed, reposting, reply_to, reply_to_usr, in_thread)
383
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
384
+
ON CONFLICT (author, rkey)
385
+
DO UPDATE SET
386
+
cid = $3,
387
+
not_found = $4,
388
+
raw = $5,
389
+
created = $6,
390
+
indexed = $7,
391
+
reposting = $8,
392
+
reply_to = $9,
393
+
reply_to_usr = $10,
394
+
in_thread = $11
395
+
RETURNING id
396
+
`
397
+
398
+
// Execute the query with parameters from the Post struct
399
+
if err := b.pgx.QueryRow(
400
+
ctx,
401
+
query,
402
+
p.Author,
403
+
p.Rkey,
404
+
p.Cid,
405
+
p.NotFound,
406
+
p.Raw,
407
+
p.Created,
408
+
p.Indexed,
409
+
p.Reposting,
410
+
p.ReplyTo,
411
+
p.ReplyToUsr,
412
+
p.InThread,
413
+
).Scan(&p.ID); err != nil {
414
+
return err
415
+
}
416
+
417
+
return nil
418
+
}
419
+
420
+
func (b *PostgresBackend) HandleCreateLike(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
421
+
var rec bsky.FeedLike
422
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
423
+
return err
424
+
}
425
+
426
+
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
427
+
return nil
428
+
}
429
+
430
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
431
+
if err != nil {
432
+
return fmt.Errorf("invalid timestamp: %w", err)
433
+
}
434
+
435
+
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
436
+
if err != nil {
437
+
return fmt.Errorf("getting like subject: %w", err)
438
+
}
439
+
440
+
if _, err := b.pgx.Exec(ctx, `INSERT INTO "likes" ("created","indexed","author","rkey","subject","cid") VALUES ($1, $2, $3, $4, $5, $6)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID, cc.String()); err != nil {
441
+
pgErr, ok := err.(*pgconn.PgError)
442
+
if ok && pgErr.Code == "23505" {
443
+
return nil
444
+
}
445
+
return err
446
+
}
447
+
448
+
// Create notification if the liked post belongs to the current user
449
+
if pinfo.Author == b.myrepo.ID {
450
+
uri := fmt.Sprintf("at://%s/app.bsky.feed.like/%s", repo.Did, rkey)
451
+
if err := b.AddNotification(ctx, b.myrepo.ID, repo.ID, uri, cc, NotifKindLike); err != nil {
452
+
slog.Warn("failed to create like notification", "uri", uri, "error", err)
453
+
}
454
+
}
455
+
456
+
return nil
457
+
}
458
+
459
+
func (b *PostgresBackend) HandleCreateRepost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
460
+
var rec bsky.FeedRepost
461
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
462
+
return err
463
+
}
464
+
465
+
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
466
+
return nil
467
+
}
468
+
469
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
470
+
if err != nil {
471
+
return fmt.Errorf("invalid timestamp: %w", err)
472
+
}
473
+
474
+
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
475
+
if err != nil {
476
+
return fmt.Errorf("getting repost subject: %w", err)
477
+
}
478
+
479
+
if _, err := b.pgx.Exec(ctx, `INSERT INTO "reposts" ("created","indexed","author","rkey","subject") VALUES ($1, $2, $3, $4, $5)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID); err != nil {
480
+
pgErr, ok := err.(*pgconn.PgError)
481
+
if ok && pgErr.Code == "23505" {
482
+
return nil
483
+
}
484
+
return err
485
+
}
486
+
487
+
// Create notification if the reposted post belongs to the current user
488
+
if pinfo.Author == b.myrepo.ID {
489
+
uri := fmt.Sprintf("at://%s/app.bsky.feed.repost/%s", repo.Did, rkey)
490
+
if err := b.AddNotification(ctx, b.myrepo.ID, repo.ID, uri, cc, NotifKindRepost); err != nil {
491
+
slog.Warn("failed to create repost notification", "uri", uri, "error", err)
492
+
}
493
+
}
494
+
495
+
return nil
496
+
}
497
+
498
+
func (b *PostgresBackend) HandleCreateFollow(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
499
+
var rec bsky.GraphFollow
500
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
501
+
return err
502
+
}
503
+
504
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
505
+
return nil
506
+
}
507
+
508
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
509
+
if err != nil {
510
+
return fmt.Errorf("invalid timestamp: %w", err)
511
+
}
512
+
513
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
514
+
if err != nil {
515
+
return err
516
+
}
517
+
518
+
if _, err := b.pgx.Exec(ctx, "INSERT INTO follows (created, indexed, author, rkey, subject) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", created.Time(), time.Now(), repo.ID, rkey, subj.ID); err != nil {
519
+
return err
520
+
}
521
+
522
+
return nil
523
+
}
524
+
525
+
func (b *PostgresBackend) HandleCreateBlock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
526
+
var rec bsky.GraphBlock
527
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
528
+
return err
529
+
}
530
+
531
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
532
+
return nil
533
+
}
534
+
535
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
536
+
if err != nil {
537
+
return fmt.Errorf("invalid timestamp: %w", err)
538
+
}
539
+
540
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
541
+
if err != nil {
542
+
return err
543
+
}
544
+
545
+
if err := b.db.Create(&Block{
546
+
Created: created.Time(),
547
+
Indexed: time.Now(),
548
+
Author: repo.ID,
549
+
Rkey: rkey,
550
+
Subject: subj.ID,
551
+
}).Error; err != nil {
552
+
return err
553
+
}
554
+
555
+
return nil
556
+
}
557
+
558
+
func (b *PostgresBackend) HandleCreateList(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
559
+
var rec bsky.GraphList
560
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
561
+
return err
562
+
}
563
+
564
+
if !b.anyRelevantIdents(repo.Did) {
565
+
return nil
566
+
}
567
+
568
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
569
+
if err != nil {
570
+
return fmt.Errorf("invalid timestamp: %w", err)
571
+
}
572
+
573
+
if err := b.db.Create(&List{
574
+
Created: created.Time(),
575
+
Indexed: time.Now(),
576
+
Author: repo.ID,
577
+
Rkey: rkey,
578
+
Raw: recb,
579
+
}).Error; err != nil {
580
+
return err
581
+
}
582
+
583
+
return nil
584
+
}
585
+
586
+
func (b *PostgresBackend) HandleCreateListitem(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
587
+
var rec bsky.GraphListitem
588
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
589
+
return err
590
+
}
591
+
if !b.anyRelevantIdents(repo.Did) {
592
+
return nil
593
+
}
594
+
595
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
596
+
if err != nil {
597
+
return fmt.Errorf("invalid timestamp: %w", err)
598
+
}
599
+
600
+
subj, err := b.GetOrCreateRepo(ctx, rec.Subject)
601
+
if err != nil {
602
+
return err
603
+
}
604
+
605
+
list, err := b.GetOrCreateList(ctx, rec.List)
606
+
if err != nil {
607
+
return err
608
+
}
609
+
610
+
if err := b.db.Create(&ListItem{
611
+
Created: created.Time(),
612
+
Indexed: time.Now(),
613
+
Author: repo.ID,
614
+
Rkey: rkey,
615
+
Subject: subj.ID,
616
+
List: list.ID,
617
+
}).Error; err != nil {
618
+
return err
619
+
}
620
+
621
+
return nil
622
+
}
623
+
624
+
func (b *PostgresBackend) HandleCreateListblock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
625
+
var rec bsky.GraphListblock
626
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
627
+
return err
628
+
}
629
+
630
+
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
631
+
return nil
632
+
}
633
+
634
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
635
+
if err != nil {
636
+
return fmt.Errorf("invalid timestamp: %w", err)
637
+
}
638
+
639
+
list, err := b.GetOrCreateList(ctx, rec.Subject)
640
+
if err != nil {
641
+
return err
642
+
}
643
+
644
+
if err := b.db.Create(&ListBlock{
645
+
Created: created.Time(),
646
+
Indexed: time.Now(),
647
+
Author: repo.ID,
648
+
Rkey: rkey,
649
+
List: list.ID,
650
+
}).Error; err != nil {
651
+
return err
652
+
}
653
+
654
+
return nil
655
+
}
656
+
657
+
func (b *PostgresBackend) HandleCreateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
658
+
if !b.anyRelevantIdents(repo.Did) {
659
+
return nil
660
+
}
661
+
662
+
if err := b.db.Create(&Profile{
663
+
//Created: created.Time(),
664
+
Indexed: time.Now(),
665
+
Repo: repo.ID,
666
+
Raw: recb,
667
+
Rev: rev,
668
+
}).Error; err != nil {
669
+
return err
670
+
}
671
+
672
+
return nil
673
+
}
674
+
675
+
func (b *PostgresBackend) HandleUpdateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
676
+
if !b.anyRelevantIdents(repo.Did) {
677
+
return nil
678
+
}
679
+
680
+
if err := b.db.Create(&Profile{
681
+
Indexed: time.Now(),
682
+
Repo: repo.ID,
683
+
Raw: recb,
684
+
Rev: rev,
685
+
}).Error; err != nil {
686
+
return err
687
+
}
688
+
689
+
return nil
690
+
}
691
+
692
+
func (b *PostgresBackend) HandleCreateFeedGenerator(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
693
+
if !b.anyRelevantIdents(repo.Did) {
694
+
return nil
695
+
}
696
+
697
+
var rec bsky.FeedGenerator
698
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
699
+
return err
700
+
}
701
+
702
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
703
+
if err != nil {
704
+
return fmt.Errorf("invalid timestamp: %w", err)
705
+
}
706
+
707
+
if err := b.db.Create(&FeedGenerator{
708
+
Created: created.Time(),
709
+
Indexed: time.Now(),
710
+
Author: repo.ID,
711
+
Rkey: rkey,
712
+
Did: rec.Did,
713
+
Raw: recb,
714
+
}).Error; err != nil {
715
+
return err
716
+
}
717
+
718
+
return nil
719
+
}
720
+
721
+
func (b *PostgresBackend) HandleCreateThreadgate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
722
+
if !b.anyRelevantIdents(repo.Did) {
723
+
return nil
724
+
}
725
+
var rec bsky.FeedThreadgate
726
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
727
+
return err
728
+
}
729
+
730
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
731
+
if err != nil {
732
+
return fmt.Errorf("invalid timestamp: %w", err)
733
+
}
734
+
735
+
pid, err := b.postIDForUri(ctx, rec.Post)
736
+
if err != nil {
737
+
return err
738
+
}
739
+
740
+
if err := b.db.Create(&ThreadGate{
741
+
Created: created.Time(),
742
+
Indexed: time.Now(),
743
+
Author: repo.ID,
744
+
Rkey: rkey,
745
+
Post: pid,
746
+
}).Error; err != nil {
747
+
return err
748
+
}
749
+
750
+
return nil
751
+
}
752
+
753
+
func (b *PostgresBackend) HandleCreateChatDeclaration(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
754
+
// TODO: maybe track these?
755
+
return nil
756
+
}
757
+
758
+
func (b *PostgresBackend) HandleCreatePostGate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
759
+
if !b.anyRelevantIdents(repo.Did) {
760
+
return nil
761
+
}
762
+
var rec bsky.FeedPostgate
763
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
764
+
return err
765
+
}
766
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
767
+
if err != nil {
768
+
return fmt.Errorf("invalid timestamp: %w", err)
769
+
}
770
+
771
+
refPost, err := b.postInfoForUri(ctx, rec.Post)
772
+
if err != nil {
773
+
return err
774
+
}
775
+
776
+
if err := b.db.Create(&PostGate{
777
+
Created: created.Time(),
778
+
Indexed: time.Now(),
779
+
Author: repo.ID,
780
+
Rkey: rkey,
781
+
Subject: refPost.ID,
782
+
Raw: recb,
783
+
}).Error; err != nil {
784
+
return err
785
+
}
786
+
787
+
return nil
788
+
}
789
+
790
+
func (b *PostgresBackend) HandleCreateStarterPack(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
791
+
if !b.anyRelevantIdents(repo.Did) {
792
+
return nil
793
+
}
794
+
var rec bsky.GraphStarterpack
795
+
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
796
+
return err
797
+
}
798
+
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
799
+
if err != nil {
800
+
return fmt.Errorf("invalid timestamp: %w", err)
801
+
}
802
+
803
+
list, err := b.GetOrCreateList(ctx, rec.List)
804
+
if err != nil {
805
+
return err
806
+
}
807
+
808
+
if err := b.db.Create(&StarterPack{
809
+
Created: created.Time(),
810
+
Indexed: time.Now(),
811
+
Author: repo.ID,
812
+
Rkey: rkey,
813
+
Raw: recb,
814
+
List: list.ID,
815
+
}).Error; err != nil {
816
+
return err
817
+
}
818
+
819
+
return nil
820
+
}
821
+
822
+
func (b *PostgresBackend) HandleUpdate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
823
+
start := time.Now()
824
+
825
+
rr, err := b.GetOrCreateRepo(ctx, repo)
826
+
if err != nil {
827
+
return fmt.Errorf("get user failed: %w", err)
828
+
}
829
+
830
+
lrev, err := b.revForRepo(rr)
831
+
if err != nil {
832
+
return err
833
+
}
834
+
if lrev != "" {
835
+
if rev < lrev {
836
+
//slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
837
+
return nil
838
+
}
839
+
}
840
+
841
+
parts := strings.Split(path, "/")
842
+
if len(parts) != 2 {
843
+
return fmt.Errorf("invalid path in HandleCreate: %q", path)
844
+
}
845
+
col := parts[0]
846
+
rkey := parts[1]
847
+
848
+
defer func() {
849
+
handleOpHist.WithLabelValues("update", col).Observe(float64(time.Since(start).Milliseconds()))
850
+
}()
851
+
852
+
if rkey == "" {
853
+
fmt.Printf("messed up path: %q\n", rkey)
854
+
}
855
+
856
+
switch col {
857
+
/*
858
+
case "app.bsky.feed.post":
859
+
if err := s.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
860
+
return err
861
+
}
862
+
case "app.bsky.feed.like":
863
+
if err := s.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
864
+
return err
865
+
}
866
+
case "app.bsky.feed.repost":
867
+
if err := s.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
868
+
return err
869
+
}
870
+
case "app.bsky.graph.follow":
871
+
if err := s.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
872
+
return err
873
+
}
874
+
case "app.bsky.graph.block":
875
+
if err := s.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
876
+
return err
877
+
}
878
+
case "app.bsky.graph.list":
879
+
if err := s.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
880
+
return err
881
+
}
882
+
case "app.bsky.graph.listitem":
883
+
if err := s.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
884
+
return err
885
+
}
886
+
case "app.bsky.graph.listblock":
887
+
if err := s.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
888
+
return err
889
+
}
890
+
*/
891
+
case "app.bsky.actor.profile":
892
+
if err := b.HandleUpdateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
893
+
return err
894
+
}
895
+
/*
896
+
case "app.bsky.feed.generator":
897
+
if err := s.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
898
+
return err
899
+
}
900
+
case "app.bsky.feed.threadgate":
901
+
if err := s.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
902
+
return err
903
+
}
904
+
case "chat.bsky.actor.declaration":
905
+
if err := s.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
906
+
return err
907
+
}
908
+
*/
909
+
default:
910
+
slog.Debug("unrecognized record type in update", "repo", repo, "path", path, "rev", rev)
911
+
}
912
+
913
+
return nil
914
+
}
915
+
916
+
func (b *PostgresBackend) HandleDelete(ctx context.Context, repo string, rev string, path string) error {
917
+
start := time.Now()
918
+
919
+
rr, err := b.GetOrCreateRepo(ctx, repo)
920
+
if err != nil {
921
+
return fmt.Errorf("get user failed: %w", err)
922
+
}
923
+
924
+
lrev, ok := b.revCache.Get(rr.ID)
925
+
if ok {
926
+
if rev < lrev {
927
+
//slog.Info("skipping old rev delete", "did", rr.Did, "rev", rev, "oldrev", lrev)
928
+
return nil
929
+
}
930
+
}
931
+
932
+
parts := strings.Split(path, "/")
933
+
if len(parts) != 2 {
934
+
return fmt.Errorf("invalid path in HandleDelete: %q", path)
935
+
}
936
+
col := parts[0]
937
+
rkey := parts[1]
938
+
939
+
defer func() {
940
+
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
941
+
}()
942
+
943
+
switch col {
944
+
case "app.bsky.feed.post":
945
+
if err := b.HandleDeletePost(ctx, rr, rkey); err != nil {
946
+
return err
947
+
}
948
+
case "app.bsky.feed.like":
949
+
if err := b.HandleDeleteLike(ctx, rr, rkey); err != nil {
950
+
return err
951
+
}
952
+
case "app.bsky.feed.repost":
953
+
if err := b.HandleDeleteRepost(ctx, rr, rkey); err != nil {
954
+
return err
955
+
}
956
+
case "app.bsky.graph.follow":
957
+
if err := b.HandleDeleteFollow(ctx, rr, rkey); err != nil {
958
+
return err
959
+
}
960
+
case "app.bsky.graph.block":
961
+
if err := b.HandleDeleteBlock(ctx, rr, rkey); err != nil {
962
+
return err
963
+
}
964
+
case "app.bsky.graph.list":
965
+
if err := b.HandleDeleteList(ctx, rr, rkey); err != nil {
966
+
return err
967
+
}
968
+
case "app.bsky.graph.listitem":
969
+
if err := b.HandleDeleteListitem(ctx, rr, rkey); err != nil {
970
+
return err
971
+
}
972
+
case "app.bsky.graph.listblock":
973
+
if err := b.HandleDeleteListblock(ctx, rr, rkey); err != nil {
974
+
return err
975
+
}
976
+
case "app.bsky.actor.profile":
977
+
if err := b.HandleDeleteProfile(ctx, rr, rkey); err != nil {
978
+
return err
979
+
}
980
+
case "app.bsky.feed.generator":
981
+
if err := b.HandleDeleteFeedGenerator(ctx, rr, rkey); err != nil {
982
+
return err
983
+
}
984
+
case "app.bsky.feed.threadgate":
985
+
if err := b.HandleDeleteThreadgate(ctx, rr, rkey); err != nil {
986
+
return err
987
+
}
988
+
default:
989
+
slog.Warn("delete unrecognized record type", "repo", repo, "path", path, "rev", rev)
990
+
}
991
+
992
+
b.revCache.Add(rr.ID, rev)
993
+
return nil
994
+
}
995
+
996
+
func (b *PostgresBackend) HandleDeletePost(ctx context.Context, repo *Repo, rkey string) error {
997
+
var p Post
998
+
if err := b.db.Find(&p, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
999
+
return err
1000
+
}
1001
+
1002
+
if p.ID == 0 {
1003
+
//slog.Warn("delete of unknown post record", "repo", repo.Did, "rkey", rkey)
1004
+
return nil
1005
+
}
1006
+
1007
+
if err := b.db.Delete(&Post{}, p.ID).Error; err != nil {
1008
+
return err
1009
+
}
1010
+
1011
+
return nil
1012
+
}
1013
+
1014
+
func (b *PostgresBackend) HandleDeleteLike(ctx context.Context, repo *Repo, rkey string) error {
1015
+
var like Like
1016
+
if err := b.db.Find(&like, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1017
+
return err
1018
+
}
1019
+
1020
+
if like.ID == 0 {
1021
+
//slog.Warn("delete of missing like", "repo", repo.Did, "rkey", rkey)
1022
+
return nil
1023
+
}
1024
+
1025
+
if err := b.db.Exec("DELETE FROM likes WHERE id = ?", like.ID).Error; err != nil {
1026
+
return err
1027
+
}
1028
+
1029
+
return nil
1030
+
}
1031
+
1032
+
func (b *PostgresBackend) HandleDeleteRepost(ctx context.Context, repo *Repo, rkey string) error {
1033
+
var repost Repost
1034
+
if err := b.db.Find(&repost, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1035
+
return err
1036
+
}
1037
+
1038
+
if repost.ID == 0 {
1039
+
//return fmt.Errorf("delete of missing repost: %s %s", repo.Did, rkey)
1040
+
return nil
1041
+
}
1042
+
1043
+
if err := b.db.Exec("DELETE FROM reposts WHERE id = ?", repost.ID).Error; err != nil {
1044
+
return err
1045
+
}
1046
+
1047
+
return nil
1048
+
}
1049
+
1050
+
func (b *PostgresBackend) HandleDeleteFollow(ctx context.Context, repo *Repo, rkey string) error {
1051
+
var follow Follow
1052
+
if err := b.db.Find(&follow, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1053
+
return err
1054
+
}
1055
+
1056
+
if follow.ID == 0 {
1057
+
//slog.Warn("delete of missing follow", "repo", repo.Did, "rkey", rkey)
1058
+
return nil
1059
+
}
1060
+
1061
+
if err := b.db.Exec("DELETE FROM follows WHERE id = ?", follow.ID).Error; err != nil {
1062
+
return err
1063
+
}
1064
+
1065
+
return nil
1066
+
}
1067
+
1068
+
func (b *PostgresBackend) HandleDeleteBlock(ctx context.Context, repo *Repo, rkey string) error {
1069
+
var block Block
1070
+
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1071
+
return err
1072
+
}
1073
+
1074
+
if block.ID == 0 {
1075
+
//slog.Warn("delete of missing block", "repo", repo.Did, "rkey", rkey)
1076
+
return nil
1077
+
}
1078
+
1079
+
if err := b.db.Exec("DELETE FROM blocks WHERE id = ?", block.ID).Error; err != nil {
1080
+
return err
1081
+
}
1082
+
1083
+
return nil
1084
+
}
1085
+
1086
+
func (b *PostgresBackend) HandleDeleteList(ctx context.Context, repo *Repo, rkey string) error {
1087
+
var list List
1088
+
if err := b.db.Find(&list, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1089
+
return err
1090
+
}
1091
+
1092
+
if list.ID == 0 {
1093
+
return nil
1094
+
//return fmt.Errorf("delete of missing list: %s %s", repo.Did, rkey)
1095
+
}
1096
+
1097
+
if err := b.db.Exec("DELETE FROM lists WHERE id = ?", list.ID).Error; err != nil {
1098
+
return err
1099
+
}
1100
+
1101
+
return nil
1102
+
}
1103
+
1104
+
func (b *PostgresBackend) HandleDeleteListitem(ctx context.Context, repo *Repo, rkey string) error {
1105
+
var item ListItem
1106
+
if err := b.db.Find(&item, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1107
+
return err
1108
+
}
1109
+
1110
+
if item.ID == 0 {
1111
+
return nil
1112
+
//return fmt.Errorf("delete of missing listitem: %s %s", repo.Did, rkey)
1113
+
}
1114
+
1115
+
if err := b.db.Exec("DELETE FROM list_items WHERE id = ?", item.ID).Error; err != nil {
1116
+
return err
1117
+
}
1118
+
1119
+
return nil
1120
+
}
1121
+
1122
+
func (b *PostgresBackend) HandleDeleteListblock(ctx context.Context, repo *Repo, rkey string) error {
1123
+
var block ListBlock
1124
+
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1125
+
return err
1126
+
}
1127
+
1128
+
if block.ID == 0 {
1129
+
return nil
1130
+
//return fmt.Errorf("delete of missing listblock: %s %s", repo.Did, rkey)
1131
+
}
1132
+
1133
+
if err := b.db.Exec("DELETE FROM list_blocks WHERE id = ?", block.ID).Error; err != nil {
1134
+
return err
1135
+
}
1136
+
1137
+
return nil
1138
+
}
1139
+
1140
+
func (b *PostgresBackend) HandleDeleteFeedGenerator(ctx context.Context, repo *Repo, rkey string) error {
1141
+
var feedgen FeedGenerator
1142
+
if err := b.db.Find(&feedgen, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1143
+
return err
1144
+
}
1145
+
1146
+
if feedgen.ID == 0 {
1147
+
return nil
1148
+
//return fmt.Errorf("delete of missing feedgen: %s %s", repo.Did, rkey)
1149
+
}
1150
+
1151
+
if err := b.db.Exec("DELETE FROM feed_generators WHERE id = ?", feedgen.ID).Error; err != nil {
1152
+
return err
1153
+
}
1154
+
1155
+
return nil
1156
+
}
1157
+
1158
+
func (b *PostgresBackend) HandleDeleteThreadgate(ctx context.Context, repo *Repo, rkey string) error {
1159
+
var threadgate ThreadGate
1160
+
if err := b.db.Find(&threadgate, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1161
+
return err
1162
+
}
1163
+
1164
+
if threadgate.ID == 0 {
1165
+
return nil
1166
+
//return fmt.Errorf("delete of missing threadgate: %s %s", repo.Did, rkey)
1167
+
}
1168
+
1169
+
if err := b.db.Exec("DELETE FROM thread_gates WHERE id = ?", threadgate.ID).Error; err != nil {
1170
+
return err
1171
+
}
1172
+
1173
+
return nil
1174
+
}
1175
+
1176
+
func (b *PostgresBackend) HandleDeleteProfile(ctx context.Context, repo *Repo, rkey string) error {
1177
+
var profile Profile
1178
+
if err := b.db.Find(&profile, "repo = ?", repo.ID).Error; err != nil {
1179
+
return err
1180
+
}
1181
+
1182
+
if profile.ID == 0 {
1183
+
return nil
1184
+
}
1185
+
1186
+
if err := b.db.Exec("DELETE FROM profiles WHERE id = ?", profile.ID).Error; err != nil {
1187
+
return err
1188
+
}
1189
+
1190
+
return nil
1191
+
}
1192
+
1193
+
const (
1194
+
NotifKindReply = "reply"
1195
+
NotifKindLike = "like"
1196
+
NotifKindMention = "mention"
1197
+
NotifKindRepost = "repost"
1198
+
)
1199
+
1200
+
func (b *PostgresBackend) AddNotification(ctx context.Context, forUser, author uint, recordUri string, recordCid cid.Cid, kind string) error {
1201
+
return b.db.Create(&Notification{
1202
+
For: forUser,
1203
+
Author: author,
1204
+
Source: recordUri,
1205
+
SourceCid: recordCid.String(),
1206
+
Kind: kind,
1207
+
}).Error
1208
+
}
+12
backend/interface.go
+12
backend/interface.go
···
1
+
package backend
2
+
3
+
// RecordTracker is an interface for tracking missing records that need to be fetched
4
+
type RecordTracker interface {
5
+
// TrackMissingRecord queues a missing record for fetching
6
+
// identifier can be:
7
+
// - A DID (e.g., "did:plc:...") for actors/profiles
8
+
// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.post/...") for posts
9
+
// - An AT-URI (e.g., "at://did:plc:.../app.bsky.feed.generator/...") for feed generators
10
+
// wait: if true, blocks until the record is fetched
11
+
TrackMissingRecord(identifier string, wait bool)
12
+
}
+211
backend/missing.go
+211
backend/missing.go
···
1
+
package backend
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"log/slog"
8
+
9
+
"github.com/bluesky-social/indigo/api/atproto"
10
+
"github.com/bluesky-social/indigo/api/bsky"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
xrpclib "github.com/bluesky-social/indigo/xrpc"
13
+
"github.com/ipfs/go-cid"
14
+
)
15
+
16
+
type MissingRecordType string
17
+
18
+
const (
19
+
MissingRecordTypeProfile MissingRecordType = "profile"
20
+
MissingRecordTypePost MissingRecordType = "post"
21
+
MissingRecordTypeFeedGenerator MissingRecordType = "feedgenerator"
22
+
MissingRecordTypeUnknown MissingRecordType = "unknown"
23
+
)
24
+
25
+
type MissingRecord struct {
26
+
Type MissingRecordType
27
+
Identifier string // DID for profiles, AT-URI for posts/feedgens
28
+
Wait bool
29
+
30
+
waitch chan struct{}
31
+
}
32
+
33
+
func (b *PostgresBackend) addMissingRecord(ctx context.Context, rec MissingRecord) {
34
+
if rec.Wait {
35
+
rec.waitch = make(chan struct{})
36
+
}
37
+
38
+
select {
39
+
case b.missingRecords <- rec:
40
+
case <-ctx.Done():
41
+
}
42
+
43
+
if rec.Wait {
44
+
select {
45
+
case <-rec.waitch:
46
+
case <-ctx.Done():
47
+
}
48
+
}
49
+
}
50
+
51
+
func (b *PostgresBackend) missingRecordFetcher() {
52
+
for rec := range b.missingRecords {
53
+
var err error
54
+
switch rec.Type {
55
+
case MissingRecordTypeProfile:
56
+
err = b.fetchMissingProfile(context.TODO(), rec.Identifier)
57
+
case MissingRecordTypePost:
58
+
err = b.fetchMissingPost(context.TODO(), rec.Identifier)
59
+
case MissingRecordTypeFeedGenerator:
60
+
err = b.fetchMissingFeedGenerator(context.TODO(), rec.Identifier)
61
+
default:
62
+
slog.Error("unknown missing record type", "type", rec.Type)
63
+
continue
64
+
}
65
+
66
+
if err != nil {
67
+
slog.Warn("failed to fetch missing record", "type", rec.Type, "identifier", rec.Identifier, "error", err)
68
+
}
69
+
70
+
if rec.Wait {
71
+
close(rec.waitch)
72
+
}
73
+
}
74
+
}
75
+
76
+
func (b *PostgresBackend) fetchMissingProfile(ctx context.Context, did string) error {
77
+
b.AddRelevantDid(did)
78
+
79
+
repo, err := b.GetOrCreateRepo(ctx, did)
80
+
if err != nil {
81
+
return err
82
+
}
83
+
84
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
85
+
if err != nil {
86
+
return err
87
+
}
88
+
89
+
c := &xrpclib.Client{
90
+
Host: resp.PDSEndpoint(),
91
+
}
92
+
93
+
rec, err := atproto.RepoGetRecord(ctx, c, "", "app.bsky.actor.profile", did, "self")
94
+
if err != nil {
95
+
return err
96
+
}
97
+
98
+
prof, ok := rec.Value.Val.(*bsky.ActorProfile)
99
+
if !ok {
100
+
return fmt.Errorf("record we got back wasnt a profile somehow")
101
+
}
102
+
103
+
buf := new(bytes.Buffer)
104
+
if err := prof.MarshalCBOR(buf); err != nil {
105
+
return err
106
+
}
107
+
108
+
cc, err := cid.Decode(*rec.Cid)
109
+
if err != nil {
110
+
return err
111
+
}
112
+
113
+
return b.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
114
+
}
115
+
116
+
func (b *PostgresBackend) fetchMissingPost(ctx context.Context, uri string) error {
117
+
puri, err := syntax.ParseATURI(uri)
118
+
if err != nil {
119
+
return fmt.Errorf("invalid AT URI: %s", uri)
120
+
}
121
+
122
+
did := puri.Authority().String()
123
+
collection := puri.Collection().String()
124
+
rkey := puri.RecordKey().String()
125
+
126
+
b.AddRelevantDid(did)
127
+
128
+
repo, err := b.GetOrCreateRepo(ctx, did)
129
+
if err != nil {
130
+
return err
131
+
}
132
+
133
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
134
+
if err != nil {
135
+
return err
136
+
}
137
+
138
+
c := &xrpclib.Client{
139
+
Host: resp.PDSEndpoint(),
140
+
}
141
+
142
+
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
143
+
if err != nil {
144
+
return err
145
+
}
146
+
147
+
post, ok := rec.Value.Val.(*bsky.FeedPost)
148
+
if !ok {
149
+
return fmt.Errorf("record we got back wasn't a post somehow")
150
+
}
151
+
152
+
buf := new(bytes.Buffer)
153
+
if err := post.MarshalCBOR(buf); err != nil {
154
+
return err
155
+
}
156
+
157
+
cc, err := cid.Decode(*rec.Cid)
158
+
if err != nil {
159
+
return err
160
+
}
161
+
162
+
return b.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
163
+
}
164
+
165
+
func (b *PostgresBackend) fetchMissingFeedGenerator(ctx context.Context, uri string) error {
166
+
puri, err := syntax.ParseATURI(uri)
167
+
if err != nil {
168
+
return fmt.Errorf("invalid AT URI: %s", uri)
169
+
}
170
+
171
+
did := puri.Authority().String()
172
+
collection := puri.Collection().String()
173
+
rkey := puri.RecordKey().String()
174
+
b.AddRelevantDid(did)
175
+
176
+
repo, err := b.GetOrCreateRepo(ctx, did)
177
+
if err != nil {
178
+
return err
179
+
}
180
+
181
+
resp, err := b.dir.LookupDID(ctx, syntax.DID(did))
182
+
if err != nil {
183
+
return err
184
+
}
185
+
186
+
c := &xrpclib.Client{
187
+
Host: resp.PDSEndpoint(),
188
+
}
189
+
190
+
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
191
+
if err != nil {
192
+
return err
193
+
}
194
+
195
+
feedGen, ok := rec.Value.Val.(*bsky.FeedGenerator)
196
+
if !ok {
197
+
return fmt.Errorf("record we got back wasn't a feed generator somehow")
198
+
}
199
+
200
+
buf := new(bytes.Buffer)
201
+
if err := feedGen.MarshalCBOR(buf); err != nil {
202
+
return err
203
+
}
204
+
205
+
cc, err := cid.Decode(*rec.Cid)
206
+
if err != nil {
207
+
return err
208
+
}
209
+
210
+
return b.HandleCreateFeedGenerator(ctx, repo, rkey, buf.Bytes(), cc)
211
+
}
+2
-4
docker-compose.yml
+2
-4
docker-compose.yml
···
1
-
version: '3.8'
2
-
3
1
services:
4
2
postgres:
5
3
image: postgres:15-alpine
···
25
23
container_name: konbini-backend
26
24
environment:
27
25
- DATABASE_URL=postgres://konbini:konbini_password@postgres:5432/konbini?sslmode=disable
28
-
- BSKY_HANDLE=${BSKY_HANDLE}
29
-
- BSKY_PASSWORD=${BSKY_PASSWORD}
26
+
- BSKY_HANDLE=${BSKY_HANDLE:?}
27
+
- BSKY_PASSWORD=${BSKY_PASSWORD:?}
30
28
ports:
31
29
- "4444:4444"
32
30
depends_on:
-1125
events.go
-1125
events.go
···
1
-
package main
2
-
3
-
import (
4
-
"bytes"
5
-
"context"
6
-
"fmt"
7
-
"log/slog"
8
-
"strings"
9
-
"sync"
10
-
"time"
11
-
12
-
"github.com/bluesky-social/indigo/api/atproto"
13
-
"github.com/bluesky-social/indigo/api/bsky"
14
-
"github.com/bluesky-social/indigo/atproto/syntax"
15
-
"github.com/bluesky-social/indigo/repo"
16
-
lru "github.com/hashicorp/golang-lru/v2"
17
-
"github.com/ipfs/go-cid"
18
-
"github.com/jackc/pgx/v5/pgconn"
19
-
"github.com/jackc/pgx/v5/pgxpool"
20
-
"gorm.io/gorm"
21
-
)
22
-
23
-
type PostgresBackend struct {
24
-
db *gorm.DB
25
-
pgx *pgxpool.Pool
26
-
s *Server
27
-
28
-
relevantDids map[string]bool
29
-
rdLk sync.Mutex
30
-
31
-
revCache *lru.TwoQueueCache[uint, string]
32
-
33
-
repoCache *lru.TwoQueueCache[string, *Repo]
34
-
reposLk sync.Mutex
35
-
36
-
postInfoCache *lru.TwoQueueCache[string, cachedPostInfo]
37
-
}
38
-
39
-
func (b *PostgresBackend) HandleEvent(ctx context.Context, evt *atproto.SyncSubscribeRepos_Commit) error {
40
-
r, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(evt.Blocks))
41
-
if err != nil {
42
-
return fmt.Errorf("failed to read event repo: %w", err)
43
-
}
44
-
45
-
for _, op := range evt.Ops {
46
-
switch op.Action {
47
-
case "create":
48
-
c, rec, err := r.GetRecordBytes(ctx, op.Path)
49
-
if err != nil {
50
-
return err
51
-
}
52
-
if err := b.HandleCreate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
53
-
return fmt.Errorf("create record failed: %w", err)
54
-
}
55
-
case "update":
56
-
c, rec, err := r.GetRecordBytes(ctx, op.Path)
57
-
if err != nil {
58
-
return err
59
-
}
60
-
if err := b.HandleUpdate(ctx, evt.Repo, evt.Rev, op.Path, rec, &c); err != nil {
61
-
return fmt.Errorf("update record failed: %w", err)
62
-
}
63
-
case "delete":
64
-
if err := b.HandleDelete(ctx, evt.Repo, evt.Rev, op.Path); err != nil {
65
-
return fmt.Errorf("delete record failed: %w", err)
66
-
}
67
-
}
68
-
}
69
-
70
-
// TODO: sync with the Since field to make sure we don't miss events we care about
71
-
/*
72
-
if err := bf.Store.UpdateRev(ctx, evt.Repo, evt.Rev); err != nil {
73
-
return fmt.Errorf("failed to update rev: %w", err)
74
-
}
75
-
*/
76
-
77
-
return nil
78
-
}
79
-
80
-
func (b *PostgresBackend) HandleCreate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
81
-
start := time.Now()
82
-
83
-
rr, err := b.getOrCreateRepo(ctx, repo)
84
-
if err != nil {
85
-
return fmt.Errorf("get user failed: %w", err)
86
-
}
87
-
88
-
lrev, err := b.revForRepo(rr)
89
-
if err != nil {
90
-
return err
91
-
}
92
-
if lrev != "" {
93
-
if rev < lrev {
94
-
slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
95
-
return nil
96
-
}
97
-
}
98
-
99
-
parts := strings.Split(path, "/")
100
-
if len(parts) != 2 {
101
-
return fmt.Errorf("invalid path in HandleCreate: %q", path)
102
-
}
103
-
col := parts[0]
104
-
rkey := parts[1]
105
-
106
-
defer func() {
107
-
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
108
-
}()
109
-
110
-
if rkey == "" {
111
-
fmt.Printf("messed up path: %q\n", rkey)
112
-
}
113
-
114
-
switch col {
115
-
case "app.bsky.feed.post":
116
-
if err := b.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
117
-
return err
118
-
}
119
-
case "app.bsky.feed.like":
120
-
if err := b.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
121
-
return err
122
-
}
123
-
case "app.bsky.feed.repost":
124
-
if err := b.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
125
-
return err
126
-
}
127
-
case "app.bsky.graph.follow":
128
-
if err := b.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
129
-
return err
130
-
}
131
-
case "app.bsky.graph.block":
132
-
if err := b.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
133
-
return err
134
-
}
135
-
case "app.bsky.graph.list":
136
-
if err := b.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
137
-
return err
138
-
}
139
-
case "app.bsky.graph.listitem":
140
-
if err := b.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
141
-
return err
142
-
}
143
-
case "app.bsky.graph.listblock":
144
-
if err := b.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
145
-
return err
146
-
}
147
-
case "app.bsky.actor.profile":
148
-
if err := b.HandleCreateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
149
-
return err
150
-
}
151
-
case "app.bsky.feed.generator":
152
-
if err := b.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
153
-
return err
154
-
}
155
-
case "app.bsky.feed.threadgate":
156
-
if err := b.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
157
-
return err
158
-
}
159
-
case "chat.bsky.actor.declaration":
160
-
if err := b.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
161
-
return err
162
-
}
163
-
case "app.bsky.feed.postgate":
164
-
if err := b.HandleCreatePostGate(ctx, rr, rkey, *rec, *cid); err != nil {
165
-
return err
166
-
}
167
-
case "app.bsky.graph.starterpack":
168
-
if err := b.HandleCreateStarterPack(ctx, rr, rkey, *rec, *cid); err != nil {
169
-
return err
170
-
}
171
-
default:
172
-
slog.Debug("unrecognized record type", "repo", repo, "path", path, "rev", rev)
173
-
}
174
-
175
-
b.revCache.Add(rr.ID, rev)
176
-
return nil
177
-
}
178
-
179
-
func (b *PostgresBackend) HandleCreatePost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
180
-
exists, err := b.checkPostExists(ctx, repo, rkey)
181
-
if err != nil {
182
-
return err
183
-
}
184
-
185
-
// still technically a race condition if two creates for the same post happen concurrently... probably fine
186
-
if exists {
187
-
return nil
188
-
}
189
-
190
-
var rec bsky.FeedPost
191
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
192
-
return err
193
-
}
194
-
195
-
reldids := []string{repo.Did}
196
-
// care about a post if its in a thread of a user we are interested in
197
-
if rec.Reply != nil && rec.Reply.Parent != nil && rec.Reply.Root != nil {
198
-
reldids = append(reldids, rec.Reply.Parent.Uri, rec.Reply.Root.Uri)
199
-
}
200
-
// TODO: maybe also care if its mentioning a user we care about or quoting a user we care about?
201
-
if !b.anyRelevantIdents(reldids...) {
202
-
return nil
203
-
}
204
-
205
-
uri := "at://" + repo.Did + "/app.bsky.feed.post/" + rkey
206
-
slog.Warn("adding post", "uri", uri)
207
-
208
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
209
-
if err != nil {
210
-
return fmt.Errorf("invalid timestamp: %w", err)
211
-
}
212
-
213
-
p := Post{
214
-
Created: created.Time(),
215
-
Indexed: time.Now(),
216
-
Author: repo.ID,
217
-
Rkey: rkey,
218
-
Raw: recb,
219
-
Cid: cc.String(),
220
-
}
221
-
222
-
if rec.Reply != nil && rec.Reply.Parent != nil {
223
-
if rec.Reply.Root == nil {
224
-
return fmt.Errorf("post reply had nil root")
225
-
}
226
-
227
-
pinfo, err := b.postInfoForUri(ctx, rec.Reply.Parent.Uri)
228
-
if err != nil {
229
-
return fmt.Errorf("getting reply parent: %w", err)
230
-
}
231
-
232
-
p.ReplyTo = pinfo.ID
233
-
p.ReplyToUsr = pinfo.Author
234
-
235
-
thread, err := b.postIDForUri(ctx, rec.Reply.Root.Uri)
236
-
if err != nil {
237
-
return fmt.Errorf("getting thread root: %w", err)
238
-
}
239
-
240
-
p.InThread = thread
241
-
242
-
if p.ReplyToUsr == b.s.myrepo.ID {
243
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, p.Author, uri, cc, NotifKindReply); err != nil {
244
-
slog.Warn("failed to create notification", "uri", uri, "error", err)
245
-
}
246
-
}
247
-
}
248
-
249
-
if rec.Embed != nil {
250
-
var rpref string
251
-
if rec.Embed.EmbedRecord != nil && rec.Embed.EmbedRecord.Record != nil {
252
-
rpref = rec.Embed.EmbedRecord.Record.Uri
253
-
}
254
-
if rec.Embed.EmbedRecordWithMedia != nil &&
255
-
rec.Embed.EmbedRecordWithMedia.Record != nil &&
256
-
rec.Embed.EmbedRecordWithMedia.Record.Record != nil {
257
-
rpref = rec.Embed.EmbedRecordWithMedia.Record.Record.Uri
258
-
}
259
-
260
-
if rpref != "" && strings.Contains(rpref, "app.bsky.feed.post") {
261
-
rp, err := b.postIDForUri(ctx, rpref)
262
-
if err != nil {
263
-
return fmt.Errorf("getting quote subject: %w", err)
264
-
}
265
-
266
-
p.Reposting = rp
267
-
}
268
-
}
269
-
270
-
if err := b.doPostCreate(ctx, &p); err != nil {
271
-
return err
272
-
}
273
-
274
-
// Check for mentions and create notifications
275
-
if rec.Facets != nil {
276
-
for _, facet := range rec.Facets {
277
-
for _, feature := range facet.Features {
278
-
if feature.RichtextFacet_Mention != nil {
279
-
mentionDid := feature.RichtextFacet_Mention.Did
280
-
// This is a mention
281
-
mentionedRepo, err := b.getOrCreateRepo(ctx, mentionDid)
282
-
if err != nil {
283
-
slog.Warn("failed to get repo for mention", "did", mentionDid, "error", err)
284
-
continue
285
-
}
286
-
287
-
// Create notification if the mentioned user is the current user
288
-
if mentionedRepo.ID == b.s.myrepo.ID {
289
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, p.Author, uri, cc, NotifKindMention); err != nil {
290
-
slog.Warn("failed to create mention notification", "uri", uri, "error", err)
291
-
}
292
-
}
293
-
}
294
-
}
295
-
}
296
-
}
297
-
298
-
b.postInfoCache.Add(uri, cachedPostInfo{
299
-
ID: p.ID,
300
-
Author: p.Author,
301
-
})
302
-
303
-
return nil
304
-
}
305
-
306
-
func (b *PostgresBackend) doPostCreate(ctx context.Context, p *Post) error {
307
-
/*
308
-
if err := b.db.Clauses(clause.OnConflict{
309
-
Columns: []clause.Column{{Name: "author"}, {Name: "rkey"}},
310
-
DoUpdates: clause.AssignmentColumns([]string{"cid", "not_found", "raw", "created", "indexed"}),
311
-
}).Create(p).Error; err != nil {
312
-
return err
313
-
}
314
-
*/
315
-
316
-
query := `
317
-
INSERT INTO posts (author, rkey, cid, not_found, raw, created, indexed, reposting, reply_to, reply_to_usr, in_thread)
318
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
319
-
ON CONFLICT (author, rkey)
320
-
DO UPDATE SET
321
-
cid = $3,
322
-
not_found = $4,
323
-
raw = $5,
324
-
created = $6,
325
-
indexed = $7,
326
-
reposting = $8,
327
-
reply_to = $9,
328
-
reply_to_usr = $10,
329
-
in_thread = $11
330
-
RETURNING id
331
-
`
332
-
333
-
// Execute the query with parameters from the Post struct
334
-
if err := b.pgx.QueryRow(
335
-
ctx,
336
-
query,
337
-
p.Author,
338
-
p.Rkey,
339
-
p.Cid,
340
-
p.NotFound,
341
-
p.Raw,
342
-
p.Created,
343
-
p.Indexed,
344
-
p.Reposting,
345
-
p.ReplyTo,
346
-
p.ReplyToUsr,
347
-
p.InThread,
348
-
).Scan(&p.ID); err != nil {
349
-
return err
350
-
}
351
-
352
-
return nil
353
-
}
354
-
355
-
func (b *PostgresBackend) HandleCreateLike(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
356
-
var rec bsky.FeedLike
357
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
358
-
return err
359
-
}
360
-
361
-
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
362
-
return nil
363
-
}
364
-
365
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
366
-
if err != nil {
367
-
return fmt.Errorf("invalid timestamp: %w", err)
368
-
}
369
-
370
-
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
371
-
if err != nil {
372
-
return fmt.Errorf("getting like subject: %w", err)
373
-
}
374
-
375
-
if _, err := b.pgx.Exec(ctx, `INSERT INTO "likes" ("created","indexed","author","rkey","subject","cid") VALUES ($1, $2, $3, $4, $5, $6)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID, cc.String()); err != nil {
376
-
pgErr, ok := err.(*pgconn.PgError)
377
-
if ok && pgErr.Code == "23505" {
378
-
return nil
379
-
}
380
-
return err
381
-
}
382
-
383
-
// Create notification if the liked post belongs to the current user
384
-
if pinfo.Author == b.s.myrepo.ID {
385
-
uri := fmt.Sprintf("at://%s/app.bsky.feed.like/%s", repo.Did, rkey)
386
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, repo.ID, uri, cc, NotifKindLike); err != nil {
387
-
slog.Warn("failed to create like notification", "uri", uri, "error", err)
388
-
}
389
-
}
390
-
391
-
return nil
392
-
}
393
-
394
-
func (b *PostgresBackend) HandleCreateRepost(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
395
-
var rec bsky.FeedRepost
396
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
397
-
return err
398
-
}
399
-
400
-
if !b.anyRelevantIdents(repo.Did, rec.Subject.Uri) {
401
-
return nil
402
-
}
403
-
404
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
405
-
if err != nil {
406
-
return fmt.Errorf("invalid timestamp: %w", err)
407
-
}
408
-
409
-
pinfo, err := b.postInfoForUri(ctx, rec.Subject.Uri)
410
-
if err != nil {
411
-
return fmt.Errorf("getting repost subject: %w", err)
412
-
}
413
-
414
-
if _, err := b.pgx.Exec(ctx, `INSERT INTO "reposts" ("created","indexed","author","rkey","subject") VALUES ($1, $2, $3, $4, $5)`, created.Time(), time.Now(), repo.ID, rkey, pinfo.ID); err != nil {
415
-
pgErr, ok := err.(*pgconn.PgError)
416
-
if ok && pgErr.Code == "23505" {
417
-
return nil
418
-
}
419
-
return err
420
-
}
421
-
422
-
// Create notification if the reposted post belongs to the current user
423
-
if pinfo.Author == b.s.myrepo.ID {
424
-
uri := fmt.Sprintf("at://%s/app.bsky.feed.repost/%s", repo.Did, rkey)
425
-
if err := b.s.AddNotification(ctx, b.s.myrepo.ID, repo.ID, uri, cc, NotifKindRepost); err != nil {
426
-
slog.Warn("failed to create repost notification", "uri", uri, "error", err)
427
-
}
428
-
}
429
-
430
-
return nil
431
-
}
432
-
433
-
func (b *PostgresBackend) HandleCreateFollow(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
434
-
var rec bsky.GraphFollow
435
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
436
-
return err
437
-
}
438
-
439
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
440
-
return nil
441
-
}
442
-
443
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
444
-
if err != nil {
445
-
return fmt.Errorf("invalid timestamp: %w", err)
446
-
}
447
-
448
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
449
-
if err != nil {
450
-
return err
451
-
}
452
-
453
-
if _, err := b.pgx.Exec(ctx, "INSERT INTO follows (created, indexed, author, rkey, subject) VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING", created.Time(), time.Now(), repo.ID, rkey, subj.ID); err != nil {
454
-
return err
455
-
}
456
-
457
-
return nil
458
-
}
459
-
460
-
func (b *PostgresBackend) HandleCreateBlock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
461
-
var rec bsky.GraphBlock
462
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
463
-
return err
464
-
}
465
-
466
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
467
-
return nil
468
-
}
469
-
470
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
471
-
if err != nil {
472
-
return fmt.Errorf("invalid timestamp: %w", err)
473
-
}
474
-
475
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
476
-
if err != nil {
477
-
return err
478
-
}
479
-
480
-
if err := b.db.Create(&Block{
481
-
Created: created.Time(),
482
-
Indexed: time.Now(),
483
-
Author: repo.ID,
484
-
Rkey: rkey,
485
-
Subject: subj.ID,
486
-
}).Error; err != nil {
487
-
return err
488
-
}
489
-
490
-
return nil
491
-
}
492
-
493
-
func (b *PostgresBackend) HandleCreateList(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
494
-
var rec bsky.GraphList
495
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
496
-
return err
497
-
}
498
-
499
-
if !b.anyRelevantIdents(repo.Did) {
500
-
return nil
501
-
}
502
-
503
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
504
-
if err != nil {
505
-
return fmt.Errorf("invalid timestamp: %w", err)
506
-
}
507
-
508
-
if err := b.db.Create(&List{
509
-
Created: created.Time(),
510
-
Indexed: time.Now(),
511
-
Author: repo.ID,
512
-
Rkey: rkey,
513
-
Raw: recb,
514
-
}).Error; err != nil {
515
-
return err
516
-
}
517
-
518
-
return nil
519
-
}
520
-
521
-
func (b *PostgresBackend) HandleCreateListitem(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
522
-
var rec bsky.GraphListitem
523
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
524
-
return err
525
-
}
526
-
if !b.anyRelevantIdents(repo.Did) {
527
-
return nil
528
-
}
529
-
530
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
531
-
if err != nil {
532
-
return fmt.Errorf("invalid timestamp: %w", err)
533
-
}
534
-
535
-
subj, err := b.getOrCreateRepo(ctx, rec.Subject)
536
-
if err != nil {
537
-
return err
538
-
}
539
-
540
-
list, err := b.getOrCreateList(ctx, rec.List)
541
-
if err != nil {
542
-
return err
543
-
}
544
-
545
-
if err := b.db.Create(&ListItem{
546
-
Created: created.Time(),
547
-
Indexed: time.Now(),
548
-
Author: repo.ID,
549
-
Rkey: rkey,
550
-
Subject: subj.ID,
551
-
List: list.ID,
552
-
}).Error; err != nil {
553
-
return err
554
-
}
555
-
556
-
return nil
557
-
}
558
-
559
-
func (b *PostgresBackend) HandleCreateListblock(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
560
-
var rec bsky.GraphListblock
561
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
562
-
return err
563
-
}
564
-
565
-
if !b.anyRelevantIdents(repo.Did, rec.Subject) {
566
-
return nil
567
-
}
568
-
569
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
570
-
if err != nil {
571
-
return fmt.Errorf("invalid timestamp: %w", err)
572
-
}
573
-
574
-
list, err := b.getOrCreateList(ctx, rec.Subject)
575
-
if err != nil {
576
-
return err
577
-
}
578
-
579
-
if err := b.db.Create(&ListBlock{
580
-
Created: created.Time(),
581
-
Indexed: time.Now(),
582
-
Author: repo.ID,
583
-
Rkey: rkey,
584
-
List: list.ID,
585
-
}).Error; err != nil {
586
-
return err
587
-
}
588
-
589
-
return nil
590
-
}
591
-
592
-
func (b *PostgresBackend) HandleCreateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
593
-
if !b.anyRelevantIdents(repo.Did) {
594
-
return nil
595
-
}
596
-
597
-
if err := b.db.Create(&Profile{
598
-
//Created: created.Time(),
599
-
Indexed: time.Now(),
600
-
Repo: repo.ID,
601
-
Raw: recb,
602
-
Rev: rev,
603
-
}).Error; err != nil {
604
-
return err
605
-
}
606
-
607
-
return nil
608
-
}
609
-
610
-
func (b *PostgresBackend) HandleUpdateProfile(ctx context.Context, repo *Repo, rkey, rev string, recb []byte, cc cid.Cid) error {
611
-
if !b.anyRelevantIdents(repo.Did) {
612
-
return nil
613
-
}
614
-
615
-
if err := b.db.Create(&Profile{
616
-
Indexed: time.Now(),
617
-
Repo: repo.ID,
618
-
Raw: recb,
619
-
Rev: rev,
620
-
}).Error; err != nil {
621
-
return err
622
-
}
623
-
624
-
return nil
625
-
}
626
-
627
-
func (b *PostgresBackend) HandleCreateFeedGenerator(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
628
-
if !b.anyRelevantIdents(repo.Did) {
629
-
return nil
630
-
}
631
-
632
-
var rec bsky.FeedGenerator
633
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
634
-
return err
635
-
}
636
-
637
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
638
-
if err != nil {
639
-
return fmt.Errorf("invalid timestamp: %w", err)
640
-
}
641
-
642
-
if err := b.db.Create(&FeedGenerator{
643
-
Created: created.Time(),
644
-
Indexed: time.Now(),
645
-
Author: repo.ID,
646
-
Rkey: rkey,
647
-
Did: rec.Did,
648
-
}).Error; err != nil {
649
-
return err
650
-
}
651
-
652
-
return nil
653
-
}
654
-
655
-
func (b *PostgresBackend) HandleCreateThreadgate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
656
-
if !b.anyRelevantIdents(repo.Did) {
657
-
return nil
658
-
}
659
-
var rec bsky.FeedThreadgate
660
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
661
-
return err
662
-
}
663
-
664
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
665
-
if err != nil {
666
-
return fmt.Errorf("invalid timestamp: %w", err)
667
-
}
668
-
669
-
pid, err := b.postIDForUri(ctx, rec.Post)
670
-
if err != nil {
671
-
return err
672
-
}
673
-
674
-
if err := b.db.Create(&ThreadGate{
675
-
Created: created.Time(),
676
-
Indexed: time.Now(),
677
-
Author: repo.ID,
678
-
Rkey: rkey,
679
-
Post: pid,
680
-
}).Error; err != nil {
681
-
return err
682
-
}
683
-
684
-
return nil
685
-
}
686
-
687
-
func (b *PostgresBackend) HandleCreateChatDeclaration(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
688
-
// TODO: maybe track these?
689
-
return nil
690
-
}
691
-
692
-
func (b *PostgresBackend) HandleCreatePostGate(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
693
-
if !b.anyRelevantIdents(repo.Did) {
694
-
return nil
695
-
}
696
-
var rec bsky.FeedPostgate
697
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
698
-
return err
699
-
}
700
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
701
-
if err != nil {
702
-
return fmt.Errorf("invalid timestamp: %w", err)
703
-
}
704
-
705
-
refPost, err := b.postInfoForUri(ctx, rec.Post)
706
-
if err != nil {
707
-
return err
708
-
}
709
-
710
-
if err := b.db.Create(&PostGate{
711
-
Created: created.Time(),
712
-
Indexed: time.Now(),
713
-
Author: repo.ID,
714
-
Rkey: rkey,
715
-
Subject: refPost.ID,
716
-
Raw: recb,
717
-
}).Error; err != nil {
718
-
return err
719
-
}
720
-
721
-
return nil
722
-
}
723
-
724
-
func (b *PostgresBackend) HandleCreateStarterPack(ctx context.Context, repo *Repo, rkey string, recb []byte, cc cid.Cid) error {
725
-
if !b.anyRelevantIdents(repo.Did) {
726
-
return nil
727
-
}
728
-
var rec bsky.GraphStarterpack
729
-
if err := rec.UnmarshalCBOR(bytes.NewReader(recb)); err != nil {
730
-
return err
731
-
}
732
-
created, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
733
-
if err != nil {
734
-
return fmt.Errorf("invalid timestamp: %w", err)
735
-
}
736
-
737
-
list, err := b.getOrCreateList(ctx, rec.List)
738
-
if err != nil {
739
-
return err
740
-
}
741
-
742
-
if err := b.db.Create(&StarterPack{
743
-
Created: created.Time(),
744
-
Indexed: time.Now(),
745
-
Author: repo.ID,
746
-
Rkey: rkey,
747
-
Raw: recb,
748
-
List: list.ID,
749
-
}).Error; err != nil {
750
-
return err
751
-
}
752
-
753
-
return nil
754
-
}
755
-
756
-
func (b *PostgresBackend) HandleUpdate(ctx context.Context, repo string, rev string, path string, rec *[]byte, cid *cid.Cid) error {
757
-
start := time.Now()
758
-
759
-
rr, err := b.getOrCreateRepo(ctx, repo)
760
-
if err != nil {
761
-
return fmt.Errorf("get user failed: %w", err)
762
-
}
763
-
764
-
lrev, err := b.revForRepo(rr)
765
-
if err != nil {
766
-
return err
767
-
}
768
-
if lrev != "" {
769
-
if rev < lrev {
770
-
//slog.Info("skipping old rev create", "did", rr.Did, "rev", rev, "oldrev", lrev, "path", path)
771
-
return nil
772
-
}
773
-
}
774
-
775
-
parts := strings.Split(path, "/")
776
-
if len(parts) != 2 {
777
-
return fmt.Errorf("invalid path in HandleCreate: %q", path)
778
-
}
779
-
col := parts[0]
780
-
rkey := parts[1]
781
-
782
-
defer func() {
783
-
handleOpHist.WithLabelValues("update", col).Observe(float64(time.Since(start).Milliseconds()))
784
-
}()
785
-
786
-
if rkey == "" {
787
-
fmt.Printf("messed up path: %q\n", rkey)
788
-
}
789
-
790
-
switch col {
791
-
/*
792
-
case "app.bsky.feed.post":
793
-
if err := s.HandleCreatePost(ctx, rr, rkey, *rec, *cid); err != nil {
794
-
return err
795
-
}
796
-
case "app.bsky.feed.like":
797
-
if err := s.HandleCreateLike(ctx, rr, rkey, *rec, *cid); err != nil {
798
-
return err
799
-
}
800
-
case "app.bsky.feed.repost":
801
-
if err := s.HandleCreateRepost(ctx, rr, rkey, *rec, *cid); err != nil {
802
-
return err
803
-
}
804
-
case "app.bsky.graph.follow":
805
-
if err := s.HandleCreateFollow(ctx, rr, rkey, *rec, *cid); err != nil {
806
-
return err
807
-
}
808
-
case "app.bsky.graph.block":
809
-
if err := s.HandleCreateBlock(ctx, rr, rkey, *rec, *cid); err != nil {
810
-
return err
811
-
}
812
-
case "app.bsky.graph.list":
813
-
if err := s.HandleCreateList(ctx, rr, rkey, *rec, *cid); err != nil {
814
-
return err
815
-
}
816
-
case "app.bsky.graph.listitem":
817
-
if err := s.HandleCreateListitem(ctx, rr, rkey, *rec, *cid); err != nil {
818
-
return err
819
-
}
820
-
case "app.bsky.graph.listblock":
821
-
if err := s.HandleCreateListblock(ctx, rr, rkey, *rec, *cid); err != nil {
822
-
return err
823
-
}
824
-
*/
825
-
case "app.bsky.actor.profile":
826
-
if err := b.HandleUpdateProfile(ctx, rr, rkey, rev, *rec, *cid); err != nil {
827
-
return err
828
-
}
829
-
/*
830
-
case "app.bsky.feed.generator":
831
-
if err := s.HandleCreateFeedGenerator(ctx, rr, rkey, *rec, *cid); err != nil {
832
-
return err
833
-
}
834
-
case "app.bsky.feed.threadgate":
835
-
if err := s.HandleCreateThreadgate(ctx, rr, rkey, *rec, *cid); err != nil {
836
-
return err
837
-
}
838
-
case "chat.bsky.actor.declaration":
839
-
if err := s.HandleCreateChatDeclaration(ctx, rr, rkey, *rec, *cid); err != nil {
840
-
return err
841
-
}
842
-
*/
843
-
default:
844
-
slog.Debug("unrecognized record type in update", "repo", repo, "path", path, "rev", rev)
845
-
}
846
-
847
-
return nil
848
-
}
849
-
850
-
func (b *PostgresBackend) HandleDelete(ctx context.Context, repo string, rev string, path string) error {
851
-
start := time.Now()
852
-
853
-
rr, err := b.getOrCreateRepo(ctx, repo)
854
-
if err != nil {
855
-
return fmt.Errorf("get user failed: %w", err)
856
-
}
857
-
858
-
lrev, ok := b.revCache.Get(rr.ID)
859
-
if ok {
860
-
if rev < lrev {
861
-
//slog.Info("skipping old rev delete", "did", rr.Did, "rev", rev, "oldrev", lrev)
862
-
return nil
863
-
}
864
-
}
865
-
866
-
parts := strings.Split(path, "/")
867
-
if len(parts) != 2 {
868
-
return fmt.Errorf("invalid path in HandleDelete: %q", path)
869
-
}
870
-
col := parts[0]
871
-
rkey := parts[1]
872
-
873
-
defer func() {
874
-
handleOpHist.WithLabelValues("create", col).Observe(float64(time.Since(start).Milliseconds()))
875
-
}()
876
-
877
-
switch col {
878
-
case "app.bsky.feed.post":
879
-
if err := b.HandleDeletePost(ctx, rr, rkey); err != nil {
880
-
return err
881
-
}
882
-
case "app.bsky.feed.like":
883
-
if err := b.HandleDeleteLike(ctx, rr, rkey); err != nil {
884
-
return err
885
-
}
886
-
case "app.bsky.feed.repost":
887
-
if err := b.HandleDeleteRepost(ctx, rr, rkey); err != nil {
888
-
return err
889
-
}
890
-
case "app.bsky.graph.follow":
891
-
if err := b.HandleDeleteFollow(ctx, rr, rkey); err != nil {
892
-
return err
893
-
}
894
-
case "app.bsky.graph.block":
895
-
if err := b.HandleDeleteBlock(ctx, rr, rkey); err != nil {
896
-
return err
897
-
}
898
-
case "app.bsky.graph.list":
899
-
if err := b.HandleDeleteList(ctx, rr, rkey); err != nil {
900
-
return err
901
-
}
902
-
case "app.bsky.graph.listitem":
903
-
if err := b.HandleDeleteListitem(ctx, rr, rkey); err != nil {
904
-
return err
905
-
}
906
-
case "app.bsky.graph.listblock":
907
-
if err := b.HandleDeleteListblock(ctx, rr, rkey); err != nil {
908
-
return err
909
-
}
910
-
case "app.bsky.actor.profile":
911
-
if err := b.HandleDeleteProfile(ctx, rr, rkey); err != nil {
912
-
return err
913
-
}
914
-
case "app.bsky.feed.generator":
915
-
if err := b.HandleDeleteFeedGenerator(ctx, rr, rkey); err != nil {
916
-
return err
917
-
}
918
-
case "app.bsky.feed.threadgate":
919
-
if err := b.HandleDeleteThreadgate(ctx, rr, rkey); err != nil {
920
-
return err
921
-
}
922
-
default:
923
-
slog.Warn("delete unrecognized record type", "repo", repo, "path", path, "rev", rev)
924
-
}
925
-
926
-
b.revCache.Add(rr.ID, rev)
927
-
return nil
928
-
}
929
-
930
-
func (b *PostgresBackend) HandleDeletePost(ctx context.Context, repo *Repo, rkey string) error {
931
-
var p Post
932
-
if err := b.db.Find(&p, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
933
-
return err
934
-
}
935
-
936
-
if p.ID == 0 {
937
-
//slog.Warn("delete of unknown post record", "repo", repo.Did, "rkey", rkey)
938
-
return nil
939
-
}
940
-
941
-
if err := b.db.Delete(&Post{}, p.ID).Error; err != nil {
942
-
return err
943
-
}
944
-
945
-
return nil
946
-
}
947
-
948
-
func (b *PostgresBackend) HandleDeleteLike(ctx context.Context, repo *Repo, rkey string) error {
949
-
var like Like
950
-
if err := b.db.Find(&like, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
951
-
return err
952
-
}
953
-
954
-
if like.ID == 0 {
955
-
//slog.Warn("delete of missing like", "repo", repo.Did, "rkey", rkey)
956
-
return nil
957
-
}
958
-
959
-
if err := b.db.Exec("DELETE FROM likes WHERE id = ?", like.ID).Error; err != nil {
960
-
return err
961
-
}
962
-
963
-
return nil
964
-
}
965
-
966
-
func (b *PostgresBackend) HandleDeleteRepost(ctx context.Context, repo *Repo, rkey string) error {
967
-
var repost Repost
968
-
if err := b.db.Find(&repost, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
969
-
return err
970
-
}
971
-
972
-
if repost.ID == 0 {
973
-
//return fmt.Errorf("delete of missing repost: %s %s", repo.Did, rkey)
974
-
return nil
975
-
}
976
-
977
-
if err := b.db.Exec("DELETE FROM reposts WHERE id = ?", repost.ID).Error; err != nil {
978
-
return err
979
-
}
980
-
981
-
return nil
982
-
}
983
-
984
-
func (b *PostgresBackend) HandleDeleteFollow(ctx context.Context, repo *Repo, rkey string) error {
985
-
var follow Follow
986
-
if err := b.db.Find(&follow, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
987
-
return err
988
-
}
989
-
990
-
if follow.ID == 0 {
991
-
//slog.Warn("delete of missing follow", "repo", repo.Did, "rkey", rkey)
992
-
return nil
993
-
}
994
-
995
-
if err := b.db.Exec("DELETE FROM follows WHERE id = ?", follow.ID).Error; err != nil {
996
-
return err
997
-
}
998
-
999
-
return nil
1000
-
}
1001
-
1002
-
func (b *PostgresBackend) HandleDeleteBlock(ctx context.Context, repo *Repo, rkey string) error {
1003
-
var block Block
1004
-
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1005
-
return err
1006
-
}
1007
-
1008
-
if block.ID == 0 {
1009
-
//slog.Warn("delete of missing block", "repo", repo.Did, "rkey", rkey)
1010
-
return nil
1011
-
}
1012
-
1013
-
if err := b.db.Exec("DELETE FROM blocks WHERE id = ?", block.ID).Error; err != nil {
1014
-
return err
1015
-
}
1016
-
1017
-
return nil
1018
-
}
1019
-
1020
-
func (b *PostgresBackend) HandleDeleteList(ctx context.Context, repo *Repo, rkey string) error {
1021
-
var list List
1022
-
if err := b.db.Find(&list, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1023
-
return err
1024
-
}
1025
-
1026
-
if list.ID == 0 {
1027
-
return nil
1028
-
//return fmt.Errorf("delete of missing list: %s %s", repo.Did, rkey)
1029
-
}
1030
-
1031
-
if err := b.db.Exec("DELETE FROM lists WHERE id = ?", list.ID).Error; err != nil {
1032
-
return err
1033
-
}
1034
-
1035
-
return nil
1036
-
}
1037
-
1038
-
func (b *PostgresBackend) HandleDeleteListitem(ctx context.Context, repo *Repo, rkey string) error {
1039
-
var item ListItem
1040
-
if err := b.db.Find(&item, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1041
-
return err
1042
-
}
1043
-
1044
-
if item.ID == 0 {
1045
-
return nil
1046
-
//return fmt.Errorf("delete of missing listitem: %s %s", repo.Did, rkey)
1047
-
}
1048
-
1049
-
if err := b.db.Exec("DELETE FROM list_items WHERE id = ?", item.ID).Error; err != nil {
1050
-
return err
1051
-
}
1052
-
1053
-
return nil
1054
-
}
1055
-
1056
-
func (b *PostgresBackend) HandleDeleteListblock(ctx context.Context, repo *Repo, rkey string) error {
1057
-
var block ListBlock
1058
-
if err := b.db.Find(&block, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1059
-
return err
1060
-
}
1061
-
1062
-
if block.ID == 0 {
1063
-
return nil
1064
-
//return fmt.Errorf("delete of missing listblock: %s %s", repo.Did, rkey)
1065
-
}
1066
-
1067
-
if err := b.db.Exec("DELETE FROM list_blocks WHERE id = ?", block.ID).Error; err != nil {
1068
-
return err
1069
-
}
1070
-
1071
-
return nil
1072
-
}
1073
-
1074
-
func (b *PostgresBackend) HandleDeleteFeedGenerator(ctx context.Context, repo *Repo, rkey string) error {
1075
-
var feedgen FeedGenerator
1076
-
if err := b.db.Find(&feedgen, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1077
-
return err
1078
-
}
1079
-
1080
-
if feedgen.ID == 0 {
1081
-
return nil
1082
-
//return fmt.Errorf("delete of missing feedgen: %s %s", repo.Did, rkey)
1083
-
}
1084
-
1085
-
if err := b.db.Exec("DELETE FROM feed_generators WHERE id = ?", feedgen.ID).Error; err != nil {
1086
-
return err
1087
-
}
1088
-
1089
-
return nil
1090
-
}
1091
-
1092
-
func (b *PostgresBackend) HandleDeleteThreadgate(ctx context.Context, repo *Repo, rkey string) error {
1093
-
var threadgate ThreadGate
1094
-
if err := b.db.Find(&threadgate, "author = ? AND rkey = ?", repo.ID, rkey).Error; err != nil {
1095
-
return err
1096
-
}
1097
-
1098
-
if threadgate.ID == 0 {
1099
-
return nil
1100
-
//return fmt.Errorf("delete of missing threadgate: %s %s", repo.Did, rkey)
1101
-
}
1102
-
1103
-
if err := b.db.Exec("DELETE FROM thread_gates WHERE id = ?", threadgate.ID).Error; err != nil {
1104
-
return err
1105
-
}
1106
-
1107
-
return nil
1108
-
}
1109
-
1110
-
func (b *PostgresBackend) HandleDeleteProfile(ctx context.Context, repo *Repo, rkey string) error {
1111
-
var profile Profile
1112
-
if err := b.db.Find(&profile, "repo = ?", repo.ID).Error; err != nil {
1113
-
return err
1114
-
}
1115
-
1116
-
if profile.ID == 0 {
1117
-
return nil
1118
-
}
1119
-
1120
-
if err := b.db.Exec("DELETE FROM profiles WHERE id = ?", profile.ID).Error; err != nil {
1121
-
return err
1122
-
}
1123
-
1124
-
return nil
1125
-
}
+1
-1
frontend/Dockerfile
+1
-1
frontend/Dockerfile
+18
-9
go.mod
+18
-9
go.mod
···
3
3
go 1.25.1
4
4
5
5
require (
6
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f
7
-
github.com/golang-jwt/jwt/v5 v5.2.2
6
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe
7
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1
8
8
github.com/gorilla/websocket v1.5.1
9
9
github.com/hashicorp/golang-lru/v2 v2.0.7
10
10
github.com/ipfs/go-cid v0.4.1
11
11
github.com/jackc/pgx/v5 v5.6.0
12
12
github.com/labstack/echo/v4 v4.11.3
13
13
github.com/labstack/gommon v0.4.1
14
+
github.com/lestrrat-go/jwx/v2 v2.0.12
15
+
github.com/multiformats/go-multihash v0.2.3
14
16
github.com/prometheus/client_golang v1.19.1
15
17
github.com/urfave/cli/v2 v2.27.7
18
+
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e
16
19
github.com/whyrusleeping/market v0.0.0-20250711215409-cc684a207f15
20
+
go.opentelemetry.io/otel v1.34.0
21
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0
22
+
go.opentelemetry.io/otel/sdk v1.34.0
17
23
gorm.io/gorm v1.31.0
18
24
)
19
25
···
25
31
github.com/cespare/xxhash/v2 v2.3.0 // indirect
26
32
github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
27
33
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
34
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
28
35
github.com/felixge/httpsnoop v1.0.4 // indirect
29
36
github.com/go-logr/logr v1.4.2 // indirect
30
37
github.com/go-logr/stdr v1.2.2 // indirect
38
+
github.com/go-redis/cache/v9 v9.0.0 // indirect
31
39
github.com/goccy/go-json v0.10.5 // indirect
32
40
github.com/gogo/protobuf v1.3.2 // indirect
33
41
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
···
53
61
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
54
62
github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
55
63
github.com/ipfs/go-verifcid v0.0.3 // indirect
56
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect
64
+
github.com/ipld/go-car v0.6.2 // indirect
57
65
github.com/ipld/go-codec-dagpb v1.6.0 // indirect
58
66
github.com/ipld/go-ipld-prime v0.21.0 // indirect
59
67
github.com/jackc/pgpassfile v1.0.0 // indirect
···
62
70
github.com/jbenet/goprocess v0.1.4 // indirect
63
71
github.com/jinzhu/inflection v1.0.0 // indirect
64
72
github.com/jinzhu/now v1.1.5 // indirect
73
+
github.com/klauspost/compress v1.17.9 // indirect
65
74
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
66
75
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
67
76
github.com/lestrrat-go/httpcc v1.0.1 // indirect
68
77
github.com/lestrrat-go/httprc v1.0.4 // indirect
69
78
github.com/lestrrat-go/iter v1.0.2 // indirect
70
-
github.com/lestrrat-go/jwx/v2 v2.0.12 // indirect
71
79
github.com/lestrrat-go/option v1.0.1 // indirect
72
80
github.com/libp2p/go-libp2p v0.25.1 // indirect
73
81
github.com/mattn/go-colorable v0.1.13 // indirect
···
79
87
github.com/multiformats/go-base36 v0.2.0 // indirect
80
88
github.com/multiformats/go-multiaddr v0.8.0 // indirect
81
89
github.com/multiformats/go-multibase v0.2.0 // indirect
82
-
github.com/multiformats/go-multihash v0.2.3 // indirect
83
90
github.com/multiformats/go-varint v0.0.7 // indirect
84
91
github.com/opentracing/opentracing-go v1.2.0 // indirect
85
92
github.com/orandin/slog-gorm v1.3.2 // indirect
86
93
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
87
94
github.com/prometheus/client_model v0.6.1 // indirect
88
-
github.com/prometheus/common v0.48.0 // indirect
89
-
github.com/prometheus/procfs v0.12.0 // indirect
95
+
github.com/prometheus/common v0.54.0 // indirect
96
+
github.com/prometheus/procfs v0.15.1 // indirect
97
+
github.com/redis/go-redis/v9 v9.3.0 // indirect
90
98
github.com/russross/blackfriday/v2 v2.1.0 // indirect
91
99
github.com/segmentio/asm v1.2.0 // indirect
92
100
github.com/spaolacci/murmur3 v1.1.0 // indirect
93
101
github.com/valyala/bytebufferpool v1.0.0 // indirect
94
102
github.com/valyala/fasttemplate v1.2.2 // indirect
95
-
github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e // indirect
103
+
github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
104
+
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
105
+
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
96
106
github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 // indirect
97
107
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
98
108
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect
99
109
gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect
100
110
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
101
111
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
102
-
go.opentelemetry.io/otel v1.34.0 // indirect
103
112
go.opentelemetry.io/otel/metric v1.34.0 // indirect
104
113
go.opentelemetry.io/otel/trace v1.34.0 // indirect
105
114
go.uber.org/atomic v1.11.0 // indirect
+155
-10
go.sum
+155
-10
go.sum
···
6
6
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
7
7
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
8
8
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
9
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f h1:FugOoTzh0nCMTWGqNGsjttFWVPcwxaaGD3p/nE9V8qY=
10
-
github.com/bluesky-social/indigo v0.0.0-20250909204019-c5eaa30f683f/go.mod h1:n6QE1NDPFoi7PRbMUZmc2y7FibCqiVU4ePpsvhHUBR8=
9
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe h1:VBhaqE5ewQgXbY5SfSWFZC/AwHFo7cHxZKFYi2ce9Yo=
10
+
github.com/bluesky-social/indigo v0.0.0-20251009212240-20524de167fe/go.mod h1:RuQVrCGm42QNsgumKaR6se+XkFKfCPNwdCiTvqKRUck=
11
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1 h1:ovcRKN1iXZnY5WApVg+0Hw2RkwMH0ziA7lSAA8vellU=
12
+
github.com/bluesky-social/jetstream v0.0.0-20251009222037-7d7efa58d7f1/go.mod h1:5PtGi4r/PjEVBBl+0xWuQn4mBEjr9h6xsfDBADS6cHs=
11
13
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=
12
14
github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=
15
+
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
16
+
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
17
+
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
18
+
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
13
19
github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc=
14
20
github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8=
21
+
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
22
+
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
15
23
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
16
24
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
25
+
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
26
+
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
27
+
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
17
28
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
18
29
github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
19
30
github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
31
+
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
20
32
github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0=
21
33
github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis=
22
34
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
···
25
37
github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
26
38
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
27
39
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
40
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
41
+
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
28
42
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
29
43
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
30
44
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
31
45
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
46
+
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
47
+
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
48
+
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
49
+
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
32
50
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
51
+
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
33
52
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
34
53
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
35
54
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
36
55
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
56
+
github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0=
57
+
github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI=
37
58
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
38
59
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
60
+
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
39
61
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
40
62
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
41
63
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
···
44
66
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
45
67
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
46
68
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
47
-
github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
48
-
github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
69
+
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
70
+
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
71
+
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
72
+
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
73
+
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
74
+
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
75
+
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
76
+
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
77
+
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
78
+
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
79
+
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
80
+
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
81
+
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
82
+
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
83
+
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
49
84
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
50
85
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
51
86
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
52
87
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
88
+
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
53
89
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
54
90
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
55
91
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
···
69
105
github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno=
70
106
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
71
107
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
108
+
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
72
109
github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ=
73
110
github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y=
111
+
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
74
112
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
75
113
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
76
114
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
···
122
160
github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
123
161
github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs=
124
162
github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw=
125
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 h1:oFo19cBmcP0Cmg3XXbrr0V/c+xU9U1huEZp8+OgBzdI=
126
-
github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4/go.mod h1:6nkFF8OmR5wLKBzRKi7/YFJpyYR7+oEn1DX+mMWnlLA=
163
+
github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc=
164
+
github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8=
127
165
github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4=
128
166
github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo=
129
167
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
···
151
189
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
152
190
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
153
191
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
192
+
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
193
+
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
194
+
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
154
195
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
155
196
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
156
197
github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8=
157
198
github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA=
158
199
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
200
+
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
159
201
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
160
202
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
161
203
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
···
231
273
github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
232
274
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
233
275
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
276
+
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
277
+
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
278
+
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
279
+
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
280
+
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
281
+
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
282
+
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
283
+
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
284
+
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
285
+
github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
286
+
github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
287
+
github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
288
+
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
289
+
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
290
+
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
291
+
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
292
+
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
293
+
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
294
+
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
295
+
github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
296
+
github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
297
+
github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
298
+
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
299
+
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
300
+
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
301
+
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
234
302
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
235
303
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
236
304
github.com/orandin/slog-gorm v1.3.2 h1:C0lKDQPAx/pF+8K2HL7bdShPwOEJpPM0Bn80zTzxU1g=
···
246
314
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
247
315
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
248
316
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
249
-
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
250
-
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
251
-
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
252
-
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
317
+
github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8=
318
+
github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ=
319
+
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
320
+
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
321
+
github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA=
322
+
github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0=
323
+
github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
253
324
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
325
+
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
254
326
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
255
327
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
256
328
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
···
267
339
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
268
340
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
269
341
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
342
+
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
270
343
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
271
344
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
272
345
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
273
346
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
347
+
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
274
348
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
275
349
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
276
350
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
277
351
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
352
+
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
278
353
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
279
354
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
280
355
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
···
285
360
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
286
361
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
287
362
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
363
+
github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI=
364
+
github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q=
365
+
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
366
+
github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
367
+
github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
368
+
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
369
+
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
288
370
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
289
371
github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y=
290
372
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
···
302
384
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
303
385
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
304
386
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
387
+
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
305
388
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
306
389
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA=
307
390
gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8=
···
313
396
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q=
314
397
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
315
398
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
399
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=
400
+
go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
316
401
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
317
402
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
403
+
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
404
+
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
318
405
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
319
406
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
320
407
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
···
338
425
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
339
426
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
340
427
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
428
+
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
341
429
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
342
430
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
343
431
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
···
348
436
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
349
437
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
350
438
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
439
+
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
351
440
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
441
+
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
442
+
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
352
443
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
353
444
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
354
445
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
446
+
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
355
447
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
356
448
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
357
449
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
358
450
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
451
+
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
359
452
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
360
453
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
361
454
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
455
+
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
456
+
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
457
+
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
458
+
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
362
459
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
460
+
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
461
+
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
462
+
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
463
+
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
363
464
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
364
465
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
365
466
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
366
467
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
468
+
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
367
469
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
368
470
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
369
471
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
372
474
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
373
475
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
374
476
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
477
+
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
375
478
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
376
479
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
480
+
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
481
+
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
482
+
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
483
+
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
484
+
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
377
485
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
378
486
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
487
+
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
379
488
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
489
+
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
380
490
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
381
491
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
382
492
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
493
+
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
494
+
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
495
+
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
383
496
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
497
+
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
384
498
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
385
499
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
386
500
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
501
+
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
502
+
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
503
+
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
504
+
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
387
505
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
388
506
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
389
507
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
···
392
510
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
393
511
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
394
512
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
513
+
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
514
+
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
515
+
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
516
+
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
395
517
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
396
518
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
397
519
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
398
520
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
399
521
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
522
+
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
400
523
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
524
+
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
525
+
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
526
+
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
401
527
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
402
528
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
403
529
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
···
413
539
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
414
540
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
415
541
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
542
+
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
416
543
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
417
544
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
545
+
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
418
546
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
547
+
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
548
+
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
419
549
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
420
550
golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE=
421
551
golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588=
···
425
555
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
426
556
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
427
557
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
558
+
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
559
+
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
560
+
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
561
+
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
562
+
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
563
+
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
564
+
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
565
+
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
566
+
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
428
567
google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM=
429
568
google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
430
569
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
431
570
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
571
+
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
432
572
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
433
573
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
434
574
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
575
+
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
576
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
577
+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
435
578
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
579
+
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
436
580
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
581
+
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
437
582
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
438
583
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
439
584
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+61
-41
handlers.go
+61
-41
handlers.go
···
16
16
"github.com/labstack/echo/v4/middleware"
17
17
"github.com/labstack/gommon/log"
18
18
"github.com/whyrusleeping/market/models"
19
+
20
+
"github.com/whyrusleeping/konbini/backend"
21
+
. "github.com/whyrusleeping/konbini/models"
19
22
)
20
23
21
24
func (s *Server) runApiServer() error {
···
24
27
e.Use(middleware.CORS())
25
28
e.GET("/debug", s.handleGetDebugInfo)
26
29
e.GET("/reldids", s.handleGetRelevantDids)
30
+
e.GET("/rescan/:did", s.handleRescanDid)
27
31
28
32
views := e.Group("/api")
29
33
views.GET("/me", s.handleGetMe)
···
53
57
54
58
func (s *Server) handleGetRelevantDids(e echo.Context) error {
55
59
return e.JSON(200, map[string]any{
56
-
"dids": s.backend.relevantDids,
60
+
"dids": s.backend.GetRelevantDids(),
57
61
})
58
62
}
59
63
64
+
func (s *Server) handleRescanDid(e echo.Context) error {
65
+
didparam := e.Param("did")
66
+
67
+
ctx := e.Request().Context()
68
+
did, err := s.resolveAccountIdent(ctx, didparam)
69
+
if err != nil {
70
+
return err
71
+
}
72
+
73
+
if err := s.rescanRepo(ctx, did); err != nil {
74
+
return err
75
+
}
76
+
77
+
return e.JSON(200, map[string]any{"status": "ok"})
78
+
}
79
+
60
80
func (s *Server) handleGetMe(e echo.Context) error {
61
81
ctx := e.Request().Context()
62
82
···
86
106
87
107
postUri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey)
88
108
89
-
p, err := s.backend.getPostByUri(ctx, postUri, "*")
109
+
p, err := s.backend.GetPostByUri(ctx, postUri, "*")
90
110
if err != nil {
91
111
return err
92
112
}
···
115
135
return err
116
136
}
117
137
118
-
r, err := s.backend.getOrCreateRepo(ctx, accdid)
138
+
r, err := s.backend.GetOrCreateRepo(ctx, accdid)
119
139
if err != nil {
120
140
return err
121
141
}
122
142
123
143
var profile models.Profile
124
-
if err := s.backend.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
144
+
if err := s.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
125
145
return err
126
146
}
127
147
128
148
if profile.Raw == nil || len(profile.Raw) == 0 {
129
-
s.addMissingProfile(ctx, accdid)
149
+
s.backend.TrackMissingRecord(accdid, false)
130
150
return e.JSON(404, map[string]any{
131
151
"error": "missing profile info for user",
132
152
})
···
150
170
return err
151
171
}
152
172
153
-
r, err := s.backend.getOrCreateRepo(ctx, accdid)
173
+
r, err := s.backend.GetOrCreateRepo(ctx, accdid)
154
174
if err != nil {
155
175
return err
156
176
}
···
169
189
}
170
190
171
191
var dbposts []models.Post
172
-
if err := s.backend.db.Raw("SELECT * FROM posts WHERE author = ? AND created < ? ORDER BY created DESC LIMIT ?", r.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
192
+
if err := s.db.Raw("SELECT * FROM posts WHERE author = ? AND created < ? ORDER BY created DESC LIMIT ?", r.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
173
193
return err
174
194
}
175
195
···
239
259
func (s *Server) handleGetFollowingFeed(e echo.Context) error {
240
260
ctx := e.Request().Context()
241
261
242
-
myr, err := s.backend.getOrCreateRepo(ctx, s.mydid)
262
+
myr, err := s.backend.GetOrCreateRepo(ctx, s.mydid)
243
263
if err != nil {
244
264
return err
245
265
}
···
257
277
tcursor = t
258
278
}
259
279
var dbposts []models.Post
260
-
if err := s.backend.db.Raw("select * from posts where reply_to = 0 AND author IN (select subject from follows where author = ?) AND created < ? order by created DESC limit ?", myr.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
280
+
if err := s.db.Raw("select * from posts where reply_to = 0 AND author IN (select subject from follows where author = ?) AND created < ? order by created DESC limit ?", myr.ID, tcursor, limit).Scan(&dbposts).Error; err != nil {
261
281
return err
262
282
}
263
283
···
277
297
278
298
func (s *Server) getAuthorInfo(ctx context.Context, r *models.Repo) (*authorInfo, error) {
279
299
var profile models.Profile
280
-
if err := s.backend.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
300
+
if err := s.db.Find(&profile, "repo = ?", r.ID).Error; err != nil {
281
301
return nil, err
282
302
}
283
303
···
287
307
}
288
308
289
309
if profile.Raw == nil || len(profile.Raw) == 0 {
290
-
s.addMissingProfile(ctx, r.Did)
310
+
s.backend.TrackMissingRecord(r.Did, false)
291
311
return &authorInfo{
292
312
Handle: resp.Handle.String(),
293
313
Did: r.Did,
···
314
334
315
335
go func() {
316
336
defer wg.Done()
317
-
if err := s.backend.db.Raw("SELECT count(*) FROM likes WHERE subject = ?", pid).Scan(&pc.Likes).Error; err != nil {
337
+
if err := s.db.Raw("SELECT count(*) FROM likes WHERE subject = ?", pid).Scan(&pc.Likes).Error; err != nil {
318
338
slog.Error("failed to get likes count", "post", pid, "error", err)
319
339
}
320
340
}()
321
341
322
342
go func() {
323
343
defer wg.Done()
324
-
if err := s.backend.db.Raw("SELECT count(*) FROM reposts WHERE subject = ?", pid).Scan(&pc.Reposts).Error; err != nil {
344
+
if err := s.db.Raw("SELECT count(*) FROM reposts WHERE subject = ?", pid).Scan(&pc.Reposts).Error; err != nil {
325
345
slog.Error("failed to get reposts count", "post", pid, "error", err)
326
346
}
327
347
}()
328
348
329
349
go func() {
330
350
defer wg.Done()
331
-
if err := s.backend.db.Raw("SELECT count(*) FROM posts WHERE reply_to = ?", pid).Scan(&pc.Replies).Error; err != nil {
351
+
if err := s.db.Raw("SELECT count(*) FROM posts WHERE reply_to = ?", pid).Scan(&pc.Replies).Error; err != nil {
332
352
slog.Error("failed to get replies count", "post", pid, "error", err)
333
353
}
334
354
}()
···
347
367
go func(ix int) {
348
368
defer wg.Done()
349
369
p := dbposts[ix]
350
-
r, err := s.backend.getRepoByID(ctx, p.Author)
370
+
r, err := s.backend.GetRepoByID(ctx, p.Author)
351
371
if err != nil {
352
372
fmt.Println("failed to get repo: ", err)
353
373
posts[ix] = postResponse{
···
359
379
360
380
uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", r.Did, p.Rkey)
361
381
if len(p.Raw) == 0 || p.NotFound {
362
-
s.addMissingPost(ctx, uri)
382
+
s.backend.TrackMissingRecord(uri, false)
363
383
posts[ix] = postResponse{
364
384
Uri: uri,
365
385
Missing: true,
···
415
435
416
436
func (s *Server) checkViewerLike(ctx context.Context, pid uint) *viewerLike {
417
437
var like Like
418
-
if err := s.backend.db.Raw("SELECT * FROM likes WHERE subject = ? AND author = ?", pid, s.myrepo.ID).Scan(&like).Error; err != nil {
438
+
if err := s.db.Raw("SELECT * FROM likes WHERE subject = ? AND author = ?", pid, s.myrepo.ID).Scan(&like).Error; err != nil {
419
439
slog.Error("failed to lookup like", "error", err)
420
440
return nil
421
441
}
···
492
512
quotedURI := embedRecord.Record.Uri
493
513
quotedCid := embedRecord.Record.Cid
494
514
495
-
quotedPost, err := s.backend.getPostByUri(ctx, quotedURI, "*")
515
+
quotedPost, err := s.backend.GetPostByUri(ctx, quotedURI, "*")
496
516
if err != nil {
497
517
slog.Warn("failed to get quoted post", "uri", quotedURI, "error", err)
498
-
s.addMissingPost(ctx, quotedURI)
518
+
s.backend.TrackMissingRecord(quotedURI, false)
499
519
return s.buildQuoteFallback(quotedURI, quotedCid)
500
520
}
501
521
502
522
if quotedPost == nil || quotedPost.Raw == nil || len(quotedPost.Raw) == 0 || quotedPost.NotFound {
503
-
s.addMissingPost(ctx, quotedURI)
523
+
s.backend.TrackMissingRecord(quotedURI, false)
504
524
return s.buildQuoteFallback(quotedURI, quotedCid)
505
525
}
506
526
···
510
530
return s.buildQuoteFallback(quotedURI, quotedCid)
511
531
}
512
532
513
-
quotedRepo, err := s.backend.getRepoByID(ctx, quotedPost.Author)
533
+
quotedRepo, err := s.backend.GetRepoByID(ctx, quotedPost.Author)
514
534
if err != nil {
515
535
slog.Warn("failed to get quoted post author", "error", err)
516
536
return s.buildQuoteFallback(quotedURI, quotedCid)
···
557
577
558
578
// Get the requested post to find the thread root
559
579
var requestedPost models.Post
560
-
if err := s.backend.db.Find(&requestedPost, "id = ?", postID).Error; err != nil {
580
+
if err := s.db.Find(&requestedPost, "id = ?", postID).Error; err != nil {
561
581
return err
562
582
}
563
583
···
576
596
// Get all posts in this thread
577
597
var dbposts []models.Post
578
598
query := "SELECT * FROM posts WHERE id = ? OR in_thread = ? ORDER BY created ASC"
579
-
if err := s.backend.db.Raw(query, rootPostID, rootPostID).Scan(&dbposts).Error; err != nil {
599
+
if err := s.db.Raw(query, rootPostID, rootPostID).Scan(&dbposts).Error; err != nil {
580
600
return err
581
601
}
582
602
583
603
// Build response for each post
584
604
posts := []postResponse{}
585
605
for _, p := range dbposts {
586
-
r, err := s.backend.getRepoByID(ctx, p.Author)
606
+
r, err := s.backend.GetRepoByID(ctx, p.Author)
587
607
if err != nil {
588
608
return err
589
609
}
···
657
677
658
678
// Get all likes for this post
659
679
var likes []models.Like
660
-
if err := s.backend.db.Find(&likes, "subject = ?", postID).Error; err != nil {
680
+
if err := s.db.Find(&likes, "subject = ?", postID).Error; err != nil {
661
681
return err
662
682
}
663
683
664
684
users := []engagementUser{}
665
685
for _, like := range likes {
666
-
r, err := s.backend.getRepoByID(ctx, like.Author)
686
+
r, err := s.backend.GetRepoByID(ctx, like.Author)
667
687
if err != nil {
668
688
slog.Error("failed to get repo for like author", "error", err)
669
689
continue
···
678
698
679
699
// Get profile if available
680
700
var profile models.Profile
681
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
701
+
s.db.Find(&profile, "repo = ?", r.ID)
682
702
683
703
var prof *bsky.ActorProfile
684
704
if len(profile.Raw) > 0 {
···
687
707
prof = &p
688
708
}
689
709
} else {
690
-
s.addMissingProfile(ctx, r.Did)
710
+
s.backend.TrackMissingRecord(r.Did, false)
691
711
}
692
712
693
713
users = append(users, engagementUser{
···
717
737
718
738
// Get all reposts for this post
719
739
var reposts []models.Repost
720
-
if err := s.backend.db.Find(&reposts, "subject = ?", postID).Error; err != nil {
740
+
if err := s.db.Find(&reposts, "subject = ?", postID).Error; err != nil {
721
741
return err
722
742
}
723
743
724
744
users := []engagementUser{}
725
745
for _, repost := range reposts {
726
-
r, err := s.backend.getRepoByID(ctx, repost.Author)
746
+
r, err := s.backend.GetRepoByID(ctx, repost.Author)
727
747
if err != nil {
728
748
slog.Error("failed to get repo for repost author", "error", err)
729
749
continue
···
738
758
739
759
// Get profile if available
740
760
var profile models.Profile
741
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
761
+
s.db.Find(&profile, "repo = ?", r.ID)
742
762
743
763
var prof *bsky.ActorProfile
744
764
if len(profile.Raw) > 0 {
···
747
767
prof = &p
748
768
}
749
769
} else {
750
-
s.addMissingProfile(ctx, r.Did)
770
+
s.backend.TrackMissingRecord(r.Did, false)
751
771
}
752
772
753
773
users = append(users, engagementUser{
···
777
797
778
798
// Get all replies to this post
779
799
var replies []models.Post
780
-
if err := s.backend.db.Find(&replies, "reply_to = ?", postID).Error; err != nil {
800
+
if err := s.db.Find(&replies, "reply_to = ?", postID).Error; err != nil {
781
801
return err
782
802
}
783
803
···
791
811
}
792
812
seen[reply.Author] = true
793
813
794
-
r, err := s.backend.getRepoByID(ctx, reply.Author)
814
+
r, err := s.backend.GetRepoByID(ctx, reply.Author)
795
815
if err != nil {
796
816
slog.Error("failed to get repo for reply author", "error", err)
797
817
continue
···
806
826
807
827
// Get profile if available
808
828
var profile models.Profile
809
-
s.backend.db.Find(&profile, "repo = ?", r.ID)
829
+
s.db.Find(&profile, "repo = ?", r.ID)
810
830
811
831
var prof *bsky.ActorProfile
812
832
if len(profile.Raw) > 0 {
···
815
835
prof = &p
816
836
}
817
837
} else {
818
-
s.addMissingProfile(ctx, r.Did)
838
+
s.backend.TrackMissingRecord(r.Did, false)
819
839
}
820
840
821
841
users = append(users, engagementUser{
···
913
933
query := `SELECT * FROM notifications WHERE "for" = ?`
914
934
if cursorID > 0 {
915
935
query += ` AND id < ?`
916
-
if err := s.backend.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, cursorID, limit).Scan(¬ifications).Error; err != nil {
936
+
if err := s.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, cursorID, limit).Scan(¬ifications).Error; err != nil {
917
937
return err
918
938
}
919
939
} else {
920
-
if err := s.backend.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, limit).Scan(¬ifications).Error; err != nil {
940
+
if err := s.db.Raw(query+" ORDER BY created_at DESC LIMIT ?", s.myrepo.ID, limit).Scan(¬ifications).Error; err != nil {
921
941
return err
922
942
}
923
943
}
···
926
946
results := []notificationResponse{}
927
947
for _, notif := range notifications {
928
948
// Get author info
929
-
author, err := s.backend.getRepoByID(ctx, notif.Author)
949
+
author, err := s.backend.GetRepoByID(ctx, notif.Author)
930
950
if err != nil {
931
951
slog.Error("failed to get repo for notification author", "error", err)
932
952
continue
···
947
967
}
948
968
949
969
// Try to get source post preview for reply/mention notifications
950
-
if notif.Kind == NotifKindReply || notif.Kind == NotifKindMention {
970
+
if notif.Kind == backend.NotifKindReply || notif.Kind == backend.NotifKindMention {
951
971
// Parse URI to get post
952
-
p, err := s.backend.getPostByUri(ctx, notif.Source, "*")
972
+
p, err := s.backend.GetPostByUri(ctx, notif.Source, "*")
953
973
if err == nil && p.Raw != nil && len(p.Raw) > 0 {
954
974
var fp bsky.FeedPost
955
975
if err := fp.UnmarshalCBOR(bytes.NewReader(p.Raw)); err == nil {
+113
-11
hydration/actor.go
+113
-11
hydration/actor.go
···
10
10
11
11
"github.com/bluesky-social/indigo/api/bsky"
12
12
"github.com/bluesky-social/indigo/atproto/syntax"
13
+
"github.com/whyrusleeping/market/models"
13
14
)
14
15
15
16
// ActorInfo contains hydrated actor information
···
21
22
22
23
// HydrateActor hydrates full actor information
23
24
func (h *Hydrator) HydrateActor(ctx context.Context, did string) (*ActorInfo, error) {
25
+
ctx, span := tracer.Start(ctx, "hydrateActor")
26
+
defer span.End()
27
+
24
28
// Look up handle
25
29
resp, err := h.dir.LookupDID(ctx, syntax.DID(did))
26
30
if err != nil {
···
60
64
FollowCount int64
61
65
FollowerCount int64
62
66
PostCount int64
67
+
ViewerState *bsky.ActorDefs_ViewerState
63
68
}
64
69
65
-
func (h *Hydrator) HydrateActorDetailed(ctx context.Context, did string) (*ActorInfoDetailed, error) {
70
+
func (h *Hydrator) HydrateActorDetailed(ctx context.Context, did string, viewer string) (*ActorInfoDetailed, error) {
66
71
act, err := h.HydrateActor(ctx, did)
67
72
if err != nil {
68
73
return nil, err
···
73
78
}
74
79
75
80
var wg sync.WaitGroup
76
-
wg.Add(3)
77
-
go func() {
78
-
defer wg.Done()
81
+
wg.Go(func() {
79
82
c, err := h.getFollowCountForUser(ctx, did)
80
83
if err != nil {
81
84
slog.Error("failed to get follow count", "did", did, "error", err)
82
85
}
83
86
actd.FollowCount = c
84
-
}()
85
-
go func() {
86
-
defer wg.Done()
87
+
})
88
+
wg.Go(func() {
87
89
c, err := h.getFollowerCountForUser(ctx, did)
88
90
if err != nil {
89
91
slog.Error("failed to get follower count", "did", did, "error", err)
90
92
}
91
93
actd.FollowerCount = c
92
-
}()
93
-
go func() {
94
-
defer wg.Done()
94
+
})
95
+
wg.Go(func() {
95
96
c, err := h.getPostCountForUser(ctx, did)
96
97
if err != nil {
97
98
slog.Error("failed to get post count", "did", did, "error", err)
98
99
}
99
100
actd.PostCount = c
100
-
}()
101
+
})
102
+
103
+
if viewer != "" {
104
+
wg.Go(func() {
105
+
vs, err := h.getProfileViewerState(ctx, did, viewer)
106
+
if err != nil {
107
+
slog.Error("failed to get viewer state", "did", did, "viewer", viewer, "error", err)
108
+
}
109
+
actd.ViewerState = vs
110
+
})
111
+
}
112
+
101
113
wg.Wait()
102
114
103
115
return &actd, nil
116
+
}
117
+
118
+
func (h *Hydrator) getProfileViewerState(ctx context.Context, did, viewer string) (*bsky.ActorDefs_ViewerState, error) {
119
+
vs := &bsky.ActorDefs_ViewerState{}
120
+
121
+
var wg sync.WaitGroup
122
+
123
+
// Check if viewer is blocked by the target account
124
+
wg.Go(func() {
125
+
blockedBy, err := h.getBlockPair(ctx, did, viewer)
126
+
if err != nil {
127
+
slog.Error("failed to get blockedBy relationship", "did", did, "viewer", viewer, "error", err)
128
+
return
129
+
}
130
+
131
+
if blockedBy != nil {
132
+
v := true
133
+
vs.BlockedBy = &v
134
+
}
135
+
})
136
+
137
+
// Check if viewer is blocking the target account
138
+
wg.Go(func() {
139
+
blocking, err := h.getBlockPair(ctx, viewer, did)
140
+
if err != nil {
141
+
slog.Error("failed to get blocking relationship", "did", did, "viewer", viewer, "error", err)
142
+
return
143
+
}
144
+
145
+
if blocking != nil {
146
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.block/%s", viewer, blocking.Rkey)
147
+
vs.Blocking = &uri
148
+
}
149
+
})
150
+
151
+
// Check if viewer is following the target account
152
+
wg.Go(func() {
153
+
following, err := h.getFollowPair(ctx, viewer, did)
154
+
if err != nil {
155
+
slog.Error("failed to get following relationship", "did", did, "viewer", viewer, "error", err)
156
+
return
157
+
}
158
+
159
+
if following != nil {
160
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.follow/%s", viewer, following.Rkey)
161
+
vs.Following = &uri
162
+
}
163
+
})
164
+
165
+
// Check if target account is following the viewer
166
+
wg.Go(func() {
167
+
followedBy, err := h.getFollowPair(ctx, did, viewer)
168
+
if err != nil {
169
+
slog.Error("failed to get followedBy relationship", "did", did, "viewer", viewer, "error", err)
170
+
return
171
+
}
172
+
173
+
if followedBy != nil {
174
+
uri := fmt.Sprintf("at://%s/app.bsky.graph.follow/%s", did, followedBy.Rkey)
175
+
vs.FollowedBy = &uri
176
+
}
177
+
})
178
+
179
+
wg.Wait()
180
+
181
+
return vs, nil
182
+
}
183
+
184
+
func (h *Hydrator) getBlockPair(ctx context.Context, a, b string) (*models.Block, error) {
185
+
var blk models.Block
186
+
if err := h.db.Raw("SELECT * FROM blocks WHERE author = (SELECT id FROM repos WHERE did = ?) AND subject = (SELECT id FROM repos WHERE did = ?)", a, b).Scan(&blk).Error; err != nil {
187
+
return nil, err
188
+
}
189
+
if blk.ID == 0 {
190
+
return nil, nil
191
+
}
192
+
193
+
return &blk, nil
194
+
}
195
+
196
+
func (h *Hydrator) getFollowPair(ctx context.Context, a, b string) (*models.Follow, error) {
197
+
var fol models.Follow
198
+
if err := h.db.Raw("SELECT * FROM follows WHERE author = (SELECT id FROM repos WHERE did = ?) AND subject = (SELECT id FROM repos WHERE did = ?)", a, b).Scan(&fol).Error; err != nil {
199
+
return nil, err
200
+
}
201
+
if fol.ID == 0 {
202
+
return nil, nil
203
+
}
204
+
205
+
return &fol, nil
104
206
}
105
207
106
208
func (h *Hydrator) getFollowCountForUser(ctx context.Context, did string) (int64, error) {
+15
-13
hydration/hydrator.go
+15
-13
hydration/hydrator.go
···
2
2
3
3
import (
4
4
"github.com/bluesky-social/indigo/atproto/identity"
5
+
"github.com/whyrusleeping/konbini/backend"
5
6
"gorm.io/gorm"
6
7
)
7
8
8
9
// Hydrator handles data hydration from the database
9
10
type Hydrator struct {
10
-
db *gorm.DB
11
-
dir identity.Directory
12
-
13
-
missingActorCallback func(string)
14
-
missingPostCallback func(string)
11
+
db *gorm.DB
12
+
dir identity.Directory
13
+
backend *backend.PostgresBackend
15
14
}
16
15
17
16
// NewHydrator creates a new Hydrator
18
-
func NewHydrator(db *gorm.DB, dir identity.Directory) *Hydrator {
17
+
func NewHydrator(db *gorm.DB, dir identity.Directory, backend *backend.PostgresBackend) *Hydrator {
19
18
return &Hydrator{
20
-
db: db,
21
-
dir: dir,
19
+
db: db,
20
+
dir: dir,
21
+
backend: backend,
22
22
}
23
23
}
24
24
25
-
func (h *Hydrator) SetMissingActorCallback(fn func(string)) {
26
-
h.missingActorCallback = fn
25
+
// AddMissingRecord reports a missing record that needs to be fetched
26
+
func (h *Hydrator) AddMissingRecord(identifier string, wait bool) {
27
+
if h.backend != nil {
28
+
h.backend.TrackMissingRecord(identifier, wait)
29
+
}
27
30
}
28
31
32
+
// addMissingActor is a convenience method for adding missing actors
29
33
func (h *Hydrator) addMissingActor(did string) {
30
-
if h.missingActorCallback != nil {
31
-
h.missingActorCallback(did)
32
-
}
34
+
h.AddMissingRecord(did, false)
33
35
}
34
36
35
37
// HydrateCtx contains context for hydration operations
+388
-39
hydration/post.go
+388
-39
hydration/post.go
···
5
5
"context"
6
6
"fmt"
7
7
"log/slog"
8
+
"sync"
8
9
9
10
"github.com/bluesky-social/indigo/api/bsky"
11
+
"github.com/bluesky-social/indigo/lex/util"
12
+
"github.com/whyrusleeping/market/models"
13
+
"go.opentelemetry.io/otel"
10
14
)
11
15
16
+
var tracer = otel.Tracer("hydrator")
17
+
12
18
// PostInfo contains hydrated post information
13
19
type PostInfo struct {
20
+
ID uint
14
21
URI string
15
22
Cid string
16
23
Post *bsky.FeedPost
···
22
29
RepostCount int
23
30
ReplyCount int
24
31
ViewerLike string // URI of viewer's like, if any
32
+
33
+
EmbedInfo *bsky.FeedDefs_PostView_Embed
25
34
}
26
35
27
36
const fakeCid = "bafyreiapw4hagb5ehqgoeho4v23vf7fhlqey4b7xvjpy76krgkqx7xlolu"
28
37
29
38
// HydratePost hydrates a single post by URI
30
39
func (h *Hydrator) HydratePost(ctx context.Context, uri string, viewerDID string) (*PostInfo, error) {
31
-
// Query post from database
32
-
var dbPost struct {
33
-
ID uint
34
-
Cid string
35
-
Raw []byte
36
-
NotFound bool
37
-
ReplyTo uint
38
-
ReplyToUsr uint
39
-
InThread uint
40
-
AuthorID uint
40
+
ctx, span := tracer.Start(ctx, "hydratePost")
41
+
defer span.End()
42
+
43
+
p, err := h.backend.GetPostByUri(ctx, uri, "*")
44
+
if err != nil {
45
+
return nil, err
41
46
}
42
47
43
-
err := h.db.Raw(`
44
-
SELECT p.id, p.cid, p.raw, p.not_found, p.reply_to, p.reply_to_usr, p.in_thread, p.author as author_id
45
-
FROM posts p
46
-
WHERE p.id = (
47
-
SELECT id FROM posts
48
-
WHERE author = (SELECT id FROM repos WHERE did = ?)
49
-
AND rkey = ?
50
-
)
51
-
`, extractDIDFromURI(uri), extractRkeyFromURI(uri)).Scan(&dbPost).Error
48
+
return h.HydratePostDB(ctx, uri, p, viewerDID)
49
+
}
52
50
51
+
func (h *Hydrator) HydratePostDB(ctx context.Context, uri string, dbPost *models.Post, viewerDID string) (*PostInfo, error) {
52
+
autoFetch, _ := ctx.Value("auto-fetch").(bool)
53
+
54
+
authorDid := extractDIDFromURI(uri)
55
+
r, err := h.backend.GetOrCreateRepo(ctx, authorDid)
53
56
if err != nil {
54
-
return nil, fmt.Errorf("failed to query post: %w", err)
57
+
return nil, err
55
58
}
56
59
57
60
if dbPost.NotFound || len(dbPost.Raw) == 0 {
58
-
return nil, fmt.Errorf("post not found")
61
+
if autoFetch {
62
+
h.AddMissingRecord(uri, true)
63
+
if err := h.db.Raw(`SELECT * FROM posts WHERE author = ? AND rkey = ?`, r.ID, extractRkeyFromURI(uri)).Scan(&dbPost).Error; err != nil {
64
+
return nil, fmt.Errorf("failed to query post: %w", err)
65
+
}
66
+
if dbPost.NotFound || len(dbPost.Raw) == 0 {
67
+
return nil, fmt.Errorf("post not found")
68
+
}
69
+
} else {
70
+
return nil, fmt.Errorf("post not found")
71
+
}
59
72
}
60
73
61
74
// Unmarshal post record
···
64
77
return nil, fmt.Errorf("failed to unmarshal post: %w", err)
65
78
}
66
79
67
-
// Get author DID
68
-
var authorDID string
69
-
h.db.Raw("SELECT did FROM repos WHERE id = ?", dbPost.AuthorID).Scan(&authorDID)
80
+
var wg sync.WaitGroup
81
+
82
+
authorDID := r.Did
70
83
71
84
// Get engagement counts
72
85
var likes, reposts, replies int
73
-
h.db.Raw("SELECT COUNT(*) FROM likes WHERE subject = ?", dbPost.ID).Scan(&likes)
74
-
h.db.Raw("SELECT COUNT(*) FROM reposts WHERE subject = ?", dbPost.ID).Scan(&reposts)
75
-
h.db.Raw("SELECT COUNT(*) FROM posts WHERE reply_to = ?", dbPost.ID).Scan(&replies)
86
+
wg.Go(func() {
87
+
_, span := tracer.Start(ctx, "likeCounts")
88
+
defer span.End()
89
+
h.db.Raw("SELECT COUNT(*) FROM likes WHERE subject = ?", dbPost.ID).Scan(&likes)
90
+
})
91
+
wg.Go(func() {
92
+
_, span := tracer.Start(ctx, "repostCounts")
93
+
defer span.End()
94
+
h.db.Raw("SELECT COUNT(*) FROM reposts WHERE subject = ?", dbPost.ID).Scan(&reposts)
95
+
})
96
+
wg.Go(func() {
97
+
_, span := tracer.Start(ctx, "replyCounts")
98
+
defer span.End()
99
+
h.db.Raw("SELECT COUNT(*) FROM posts WHERE reply_to = ?", dbPost.ID).Scan(&replies)
100
+
})
101
+
102
+
// Check if viewer liked this post
103
+
var likeRkey string
104
+
if viewerDID != "" {
105
+
wg.Go(func() {
106
+
_, span := tracer.Start(ctx, "viewerLikeState")
107
+
defer span.End()
108
+
h.db.Raw(`
109
+
SELECT l.rkey FROM likes l
110
+
WHERE l.subject = ?
111
+
AND l.author = (SELECT id FROM repos WHERE did = ?)
112
+
`, dbPost.ID, viewerDID).Scan(&likeRkey)
113
+
})
114
+
}
115
+
116
+
var ei *bsky.FeedDefs_PostView_Embed
117
+
if feedPost.Embed != nil {
118
+
wg.Go(func() {
119
+
ei = h.formatEmbed(ctx, feedPost.Embed, authorDID, viewerDID)
120
+
})
121
+
}
122
+
123
+
wg.Wait()
76
124
77
125
info := &PostInfo{
126
+
ID: dbPost.ID,
78
127
URI: uri,
79
128
Cid: dbPost.Cid,
80
129
Post: &feedPost,
···
85
134
LikeCount: likes,
86
135
RepostCount: reposts,
87
136
ReplyCount: replies,
137
+
EmbedInfo: ei,
138
+
}
139
+
140
+
if likeRkey != "" {
141
+
info.ViewerLike = fmt.Sprintf("at://%s/app.bsky.feed.like/%s", viewerDID, likeRkey)
88
142
}
89
143
90
144
if info.Cid == "" {
···
92
146
info.Cid = fakeCid
93
147
}
94
148
95
-
// Check if viewer liked this post
96
-
if viewerDID != "" {
97
-
var likeRkey string
98
-
h.db.Raw(`
99
-
SELECT l.rkey FROM likes l
100
-
WHERE l.subject = ?
101
-
AND l.author = (SELECT id FROM repos WHERE did = ?)
102
-
`, dbPost.ID, viewerDID).Scan(&likeRkey)
103
-
if likeRkey != "" {
104
-
info.ViewerLike = fmt.Sprintf("at://%s/app.bsky.feed.like/%s", viewerDID, likeRkey)
105
-
}
106
-
}
149
+
// Hydrate embed
107
150
108
151
return info, nil
109
152
}
···
150
193
}
151
194
return ""
152
195
}
196
+
197
+
func (h *Hydrator) formatEmbed(ctx context.Context, embed *bsky.FeedPost_Embed, authorDID string, viewerDID string) *bsky.FeedDefs_PostView_Embed {
198
+
if embed == nil {
199
+
return nil
200
+
}
201
+
_, span := tracer.Start(ctx, "formatEmbed")
202
+
defer span.End()
203
+
204
+
result := &bsky.FeedDefs_PostView_Embed{}
205
+
206
+
// Handle images
207
+
if embed.EmbedImages != nil {
208
+
viewImages := make([]*bsky.EmbedImages_ViewImage, len(embed.EmbedImages.Images))
209
+
for i, img := range embed.EmbedImages.Images {
210
+
// Convert blob to CDN URLs
211
+
fullsize := ""
212
+
thumb := ""
213
+
if img.Image != nil {
214
+
// CDN URL format for feed images
215
+
cid := img.Image.Ref.String()
216
+
fullsize = fmt.Sprintf("https://cdn.bsky.app/img/feed_fullsize/plain/%s/%s@jpeg", authorDID, cid)
217
+
thumb = fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
218
+
}
219
+
220
+
viewImages[i] = &bsky.EmbedImages_ViewImage{
221
+
Alt: img.Alt,
222
+
AspectRatio: img.AspectRatio,
223
+
Fullsize: fullsize,
224
+
Thumb: thumb,
225
+
}
226
+
}
227
+
result.EmbedImages_View = &bsky.EmbedImages_View{
228
+
LexiconTypeID: "app.bsky.embed.images#view",
229
+
Images: viewImages,
230
+
}
231
+
return result
232
+
}
233
+
234
+
// Handle external links
235
+
if embed.EmbedExternal != nil && embed.EmbedExternal.External != nil {
236
+
// Convert blob thumb to CDN URL if present
237
+
var thumbURL *string
238
+
if embed.EmbedExternal.External.Thumb != nil {
239
+
// CDN URL for external link thumbnails
240
+
cid := embed.EmbedExternal.External.Thumb.Ref.String()
241
+
url := fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
242
+
thumbURL = &url
243
+
}
244
+
245
+
result.EmbedExternal_View = &bsky.EmbedExternal_View{
246
+
LexiconTypeID: "app.bsky.embed.external#view",
247
+
External: &bsky.EmbedExternal_ViewExternal{
248
+
Uri: embed.EmbedExternal.External.Uri,
249
+
Title: embed.EmbedExternal.External.Title,
250
+
Description: embed.EmbedExternal.External.Description,
251
+
Thumb: thumbURL,
252
+
},
253
+
}
254
+
return result
255
+
}
256
+
257
+
// Handle video
258
+
if embed.EmbedVideo != nil && embed.EmbedVideo.Video != nil {
259
+
cid := embed.EmbedVideo.Video.Ref.String()
260
+
// URL-encode the DID (replace : with %3A)
261
+
encodedDID := ""
262
+
for _, ch := range authorDID {
263
+
if ch == ':' {
264
+
encodedDID += "%3A"
265
+
} else {
266
+
encodedDID += string(ch)
267
+
}
268
+
}
269
+
270
+
playlist := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/playlist.m3u8", encodedDID, cid)
271
+
thumbnail := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/thumbnail.jpg", encodedDID, cid)
272
+
273
+
result.EmbedVideo_View = &bsky.EmbedVideo_View{
274
+
LexiconTypeID: "app.bsky.embed.video#view",
275
+
Cid: cid,
276
+
Playlist: playlist,
277
+
Thumbnail: &thumbnail,
278
+
Alt: embed.EmbedVideo.Alt,
279
+
AspectRatio: embed.EmbedVideo.AspectRatio,
280
+
}
281
+
return result
282
+
}
283
+
284
+
// Handle record (quote posts, etc.)
285
+
if embed.EmbedRecord != nil && embed.EmbedRecord.Record != nil {
286
+
rec := embed.EmbedRecord.Record
287
+
288
+
result.EmbedRecord_View = &bsky.EmbedRecord_View{
289
+
LexiconTypeID: "app.bsky.embed.record#view",
290
+
Record: h.hydrateEmbeddedRecord(ctx, rec.Uri, viewerDID),
291
+
}
292
+
return result
293
+
}
294
+
295
+
// Handle record with media (quote post with images/external)
296
+
if embed.EmbedRecordWithMedia != nil {
297
+
recordView := &bsky.EmbedRecordWithMedia_View{
298
+
LexiconTypeID: "app.bsky.embed.recordWithMedia#view",
299
+
}
300
+
301
+
// Hydrate the record part
302
+
if embed.EmbedRecordWithMedia.Record != nil && embed.EmbedRecordWithMedia.Record.Record != nil {
303
+
recordView.Record = &bsky.EmbedRecord_View{
304
+
LexiconTypeID: "app.bsky.embed.record#view",
305
+
Record: h.hydrateEmbeddedRecord(ctx, embed.EmbedRecordWithMedia.Record.Record.Uri, viewerDID),
306
+
}
307
+
}
308
+
309
+
// Hydrate the media part (images or external)
310
+
if embed.EmbedRecordWithMedia.Media != nil {
311
+
if embed.EmbedRecordWithMedia.Media.EmbedImages != nil {
312
+
viewImages := make([]*bsky.EmbedImages_ViewImage, len(embed.EmbedRecordWithMedia.Media.EmbedImages.Images))
313
+
for i, img := range embed.EmbedRecordWithMedia.Media.EmbedImages.Images {
314
+
fullsize := ""
315
+
thumb := ""
316
+
if img.Image != nil {
317
+
cid := img.Image.Ref.String()
318
+
fullsize = fmt.Sprintf("https://cdn.bsky.app/img/feed_fullsize/plain/%s/%s@jpeg", authorDID, cid)
319
+
thumb = fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
320
+
}
321
+
322
+
viewImages[i] = &bsky.EmbedImages_ViewImage{
323
+
Alt: img.Alt,
324
+
AspectRatio: img.AspectRatio,
325
+
Fullsize: fullsize,
326
+
Thumb: thumb,
327
+
}
328
+
}
329
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
330
+
EmbedImages_View: &bsky.EmbedImages_View{
331
+
LexiconTypeID: "app.bsky.embed.images#view",
332
+
Images: viewImages,
333
+
},
334
+
}
335
+
} else if embed.EmbedRecordWithMedia.Media.EmbedExternal != nil && embed.EmbedRecordWithMedia.Media.EmbedExternal.External != nil {
336
+
var thumbURL *string
337
+
if embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Thumb != nil {
338
+
cid := embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Thumb.Ref.String()
339
+
url := fmt.Sprintf("https://cdn.bsky.app/img/feed_thumbnail/plain/%s/%s@jpeg", authorDID, cid)
340
+
thumbURL = &url
341
+
}
342
+
343
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
344
+
EmbedExternal_View: &bsky.EmbedExternal_View{
345
+
LexiconTypeID: "app.bsky.embed.external#view",
346
+
External: &bsky.EmbedExternal_ViewExternal{
347
+
Uri: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Uri,
348
+
Title: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Title,
349
+
Description: embed.EmbedRecordWithMedia.Media.EmbedExternal.External.Description,
350
+
Thumb: thumbURL,
351
+
},
352
+
},
353
+
}
354
+
} else if embed.EmbedRecordWithMedia.Media.EmbedVideo != nil && embed.EmbedRecordWithMedia.Media.EmbedVideo.Video != nil {
355
+
cid := embed.EmbedRecordWithMedia.Media.EmbedVideo.Video.Ref.String()
356
+
// URL-encode the DID (replace : with %3A)
357
+
encodedDID := ""
358
+
for _, ch := range authorDID {
359
+
if ch == ':' {
360
+
encodedDID += "%3A"
361
+
} else {
362
+
encodedDID += string(ch)
363
+
}
364
+
}
365
+
366
+
playlist := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/playlist.m3u8", encodedDID, cid)
367
+
thumbnail := fmt.Sprintf("https://video.bsky.app/watch/%s/%s/thumbnail.jpg", encodedDID, cid)
368
+
369
+
recordView.Media = &bsky.EmbedRecordWithMedia_View_Media{
370
+
EmbedVideo_View: &bsky.EmbedVideo_View{
371
+
LexiconTypeID: "app.bsky.embed.video#view",
372
+
Cid: cid,
373
+
Playlist: playlist,
374
+
Thumbnail: &thumbnail,
375
+
Alt: embed.EmbedRecordWithMedia.Media.EmbedVideo.Alt,
376
+
AspectRatio: embed.EmbedRecordWithMedia.Media.EmbedVideo.AspectRatio,
377
+
},
378
+
}
379
+
}
380
+
}
381
+
382
+
result.EmbedRecordWithMedia_View = recordView
383
+
return result
384
+
}
385
+
386
+
return nil
387
+
}
388
+
389
+
// hydrateEmbeddedRecord hydrates an embedded record (for quote posts, etc.)
390
+
func (h *Hydrator) hydrateEmbeddedRecord(ctx context.Context, uri string, viewerDID string) *bsky.EmbedRecord_View_Record {
391
+
ctx, span := tracer.Start(ctx, "hydrateEmbeddedRecord")
392
+
defer span.End()
393
+
394
+
// Check if it's a post URI
395
+
if !isPostURI(uri) {
396
+
// Could be a feed generator, list, labeler, or starter pack
397
+
// For now, return not found for non-post embeds
398
+
return &bsky.EmbedRecord_View_Record{
399
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
400
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
401
+
Uri: uri,
402
+
},
403
+
}
404
+
}
405
+
406
+
// Try to hydrate the post
407
+
quotedPost, err := h.HydratePost(ctx, uri, viewerDID)
408
+
if err != nil {
409
+
// Post not found
410
+
return &bsky.EmbedRecord_View_Record{
411
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
412
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
413
+
Uri: uri,
414
+
NotFound: true,
415
+
},
416
+
}
417
+
}
418
+
419
+
// Hydrate the author
420
+
authorInfo, err := h.HydrateActor(ctx, quotedPost.Author)
421
+
if err != nil {
422
+
// Author not found, treat as not found
423
+
return &bsky.EmbedRecord_View_Record{
424
+
EmbedRecord_ViewNotFound: &bsky.EmbedRecord_ViewNotFound{
425
+
LexiconTypeID: "app.bsky.embed.record#viewNotFound",
426
+
Uri: uri,
427
+
NotFound: true,
428
+
},
429
+
}
430
+
}
431
+
432
+
// TODO: Check if viewer has blocked or is blocked by the author
433
+
// For now, just return the record view
434
+
435
+
// Build the author profile view
436
+
authorView := &bsky.ActorDefs_ProfileViewBasic{
437
+
Did: authorInfo.DID,
438
+
Handle: authorInfo.Handle,
439
+
}
440
+
if authorInfo.Profile != nil {
441
+
if authorInfo.Profile.DisplayName != nil && *authorInfo.Profile.DisplayName != "" {
442
+
authorView.DisplayName = authorInfo.Profile.DisplayName
443
+
}
444
+
if authorInfo.Profile.Avatar != nil {
445
+
avatarURL := fmt.Sprintf("https://cdn.bsky.app/img/avatar_thumbnail/plain/%s/%s@jpeg", authorInfo.DID, authorInfo.Profile.Avatar.Ref.String())
446
+
authorView.Avatar = &avatarURL
447
+
}
448
+
}
449
+
450
+
// Build the embedded post view
451
+
embedView := &bsky.EmbedRecord_ViewRecord{
452
+
LexiconTypeID: "app.bsky.embed.record#viewRecord",
453
+
Uri: quotedPost.URI,
454
+
Cid: quotedPost.Cid,
455
+
Author: authorView,
456
+
Value: &util.LexiconTypeDecoder{
457
+
Val: quotedPost.Post,
458
+
},
459
+
IndexedAt: quotedPost.Post.CreatedAt,
460
+
}
461
+
462
+
// Add engagement counts
463
+
if quotedPost.LikeCount > 0 {
464
+
lc := int64(quotedPost.LikeCount)
465
+
embedView.LikeCount = &lc
466
+
}
467
+
if quotedPost.RepostCount > 0 {
468
+
rc := int64(quotedPost.RepostCount)
469
+
embedView.RepostCount = &rc
470
+
}
471
+
if quotedPost.ReplyCount > 0 {
472
+
rpc := int64(quotedPost.ReplyCount)
473
+
embedView.ReplyCount = &rpc
474
+
}
475
+
476
+
// Note: We don't recursively hydrate embeds for quoted posts to avoid deep nesting
477
+
// The official app also doesn't show embeds within quoted posts
478
+
479
+
return &bsky.EmbedRecord_View_Record{
480
+
EmbedRecord_ViewRecord: embedView,
481
+
}
482
+
}
483
+
484
+
// isPostURI checks if a URI is a post URI
485
+
func isPostURI(uri string) bool {
486
+
return len(uri) > 5 && uri[:5] == "at://" && (
487
+
// Check if it contains /app.bsky.feed.post/
488
+
len(uri) > 25 && uri[len(uri)-25:len(uri)-12] == "/app.bsky.feed.post/" ||
489
+
// More flexible check
490
+
contains(uri, "/app.bsky.feed.post/"))
491
+
}
492
+
493
+
// contains checks if a string contains a substring
494
+
func contains(s, substr string) bool {
495
+
for i := 0; i <= len(s)-len(substr); i++ {
496
+
if s[i:i+len(substr)] == substr {
497
+
return true
498
+
}
499
+
}
500
+
return false
501
+
}
+39
hydration/utils.go
+39
hydration/utils.go
···
1
+
package hydration
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
"github.com/whyrusleeping/market/models"
9
+
)
10
+
11
+
func (h *Hydrator) NormalizeUri(ctx context.Context, uri string) (string, error) {
12
+
puri, err := syntax.ParseATURI(uri)
13
+
if err != nil {
14
+
return "", fmt.Errorf("invalid uri: %w", err)
15
+
}
16
+
17
+
var did string
18
+
if !puri.Authority().IsDID() {
19
+
resp, err := h.dir.LookupHandle(ctx, syntax.Handle(puri.Authority().String()))
20
+
if err != nil {
21
+
return "", err
22
+
}
23
+
24
+
did = resp.DID.String()
25
+
} else {
26
+
did = puri.Authority().String()
27
+
}
28
+
29
+
return fmt.Sprintf("at://%s/%s/%s", did, puri.Collection().String(), puri.RecordKey().String()), nil
30
+
}
31
+
32
+
func (h *Hydrator) UriForPost(ctx context.Context, p *models.Post) (string, error) {
33
+
did, err := h.backend.DidFromID(ctx, p.Author)
34
+
if err != nil {
35
+
return "", err
36
+
}
37
+
38
+
return fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, p.Rkey), nil
39
+
}
+136
-146
main.go
+136
-146
main.go
···
1
1
package main
2
2
3
3
import (
4
+
"bytes"
4
5
"context"
6
+
"encoding/json"
5
7
"fmt"
6
8
"log"
7
9
"log/slog"
···
16
18
17
19
"github.com/bluesky-social/indigo/api/atproto"
18
20
"github.com/bluesky-social/indigo/atproto/identity"
21
+
"github.com/bluesky-social/indigo/atproto/identity/redisdir"
19
22
"github.com/bluesky-social/indigo/atproto/syntax"
20
-
"github.com/bluesky-social/indigo/cmd/relay/stream"
21
-
"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
23
+
"github.com/bluesky-social/indigo/repo"
22
24
"github.com/bluesky-social/indigo/util/cliutil"
23
25
xrpclib "github.com/bluesky-social/indigo/xrpc"
24
-
"github.com/gorilla/websocket"
25
-
lru "github.com/hashicorp/golang-lru/v2"
26
26
"github.com/ipfs/go-cid"
27
27
"github.com/jackc/pgx/v5/pgxpool"
28
28
"github.com/prometheus/client_golang/prometheus"
29
29
"github.com/prometheus/client_golang/prometheus/promauto"
30
30
"github.com/urfave/cli/v2"
31
+
"github.com/whyrusleeping/konbini/backend"
31
32
"github.com/whyrusleeping/konbini/xrpc"
33
+
"go.opentelemetry.io/otel"
34
+
"go.opentelemetry.io/otel/attribute"
35
+
"go.opentelemetry.io/otel/exporters/jaeger"
36
+
"go.opentelemetry.io/otel/sdk/resource"
37
+
tracesdk "go.opentelemetry.io/otel/sdk/trace"
38
+
semconv "go.opentelemetry.io/otel/semconv/v1.20.0"
39
+
"gorm.io/gorm"
32
40
"gorm.io/gorm/logger"
33
-
)
34
41
35
-
var handleOpHist = promauto.NewHistogramVec(prometheus.HistogramOpts{
36
-
Name: "handle_op_duration",
37
-
Help: "A histogram of op handling durations",
38
-
Buckets: prometheus.ExponentialBuckets(1, 2, 15),
39
-
}, []string{"op", "collection"})
42
+
. "github.com/whyrusleeping/konbini/models"
43
+
)
40
44
41
45
var firehoseCursorGauge = promauto.NewGaugeVec(prometheus.GaugeOpts{
42
46
Name: "firehose_cursor",
···
51
55
&cli.StringFlag{
52
56
Name: "db-url",
53
57
EnvVars: []string{"DATABASE_URL"},
58
+
},
59
+
&cli.BoolFlag{
60
+
Name: "jaeger",
54
61
},
55
62
&cli.StringFlag{
56
63
Name: "handle",
···
59
66
Name: "max-db-connections",
60
67
Value: runtime.NumCPU(),
61
68
},
69
+
&cli.StringFlag{
70
+
Name: "redis-url",
71
+
},
72
+
&cli.StringFlag{
73
+
Name: "sync-config",
74
+
},
62
75
}
63
76
app.Action = func(cctx *cli.Context) error {
64
77
db, err := cliutil.SetupDatabase(cctx.String("db-url"), cctx.Int("max-db-connections"))
···
73
86
Colorful: true,
74
87
})
75
88
89
+
if cctx.Bool("jaeger") {
90
+
// Use Jaeger native exporter sending to port 14268
91
+
jaegerUrl := "http://localhost:14268/api/traces"
92
+
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(jaegerUrl)))
93
+
if err != nil {
94
+
return err
95
+
}
96
+
97
+
env := os.Getenv("ENV")
98
+
if env == "" {
99
+
env = "development"
100
+
}
101
+
102
+
tp := tracesdk.NewTracerProvider(
103
+
// Always be sure to batch in production.
104
+
tracesdk.WithBatcher(exp),
105
+
// Record information about this application in a Resource.
106
+
tracesdk.WithResource(resource.NewWithAttributes(
107
+
semconv.SchemaURL,
108
+
semconv.ServiceNameKey.String("konbini"),
109
+
attribute.String("env", env), // DataDog
110
+
attribute.String("environment", env), // Others
111
+
attribute.Int64("ID", 1),
112
+
)),
113
+
)
114
+
115
+
otel.SetTracerProvider(tp)
116
+
}
117
+
76
118
db.AutoMigrate(Repo{})
77
119
db.AutoMigrate(Post{})
78
120
db.AutoMigrate(Follow{})
···
88
130
db.AutoMigrate(Image{})
89
131
db.AutoMigrate(PostGate{})
90
132
db.AutoMigrate(StarterPack{})
91
-
db.AutoMigrate(SyncInfo{})
133
+
db.AutoMigrate(backend.SyncInfo{})
92
134
db.AutoMigrate(Notification{})
135
+
db.AutoMigrate(NotificationSeen{})
93
136
db.AutoMigrate(SequenceTracker{})
137
+
db.Exec("CREATE INDEX IF NOT EXISTS reposts_subject_idx ON reposts (subject)")
138
+
db.Exec("CREATE INDEX IF NOT EXISTS posts_reply_to_idx ON posts (reply_to)")
139
+
db.Exec("CREATE INDEX IF NOT EXISTS posts_in_thread_idx ON posts (in_thread)")
94
140
95
141
ctx := context.TODO()
96
142
97
-
rc, _ := lru.New2Q[string, *Repo](1_000_000)
98
-
pc, _ := lru.New2Q[string, cachedPostInfo](1_000_000)
99
-
revc, _ := lru.New2Q[uint, string](1_000_000)
100
-
101
143
cfg, err := pgxpool.ParseConfig(cctx.String("db-url"))
102
144
if err != nil {
103
145
return err
···
120
162
password := os.Getenv("BSKY_PASSWORD")
121
163
122
164
dir := identity.DefaultDirectory()
165
+
166
+
if redisURL := cctx.String("redis-url"); redisURL != "" {
167
+
rdir, err := redisdir.NewRedisDirectory(dir, redisURL, time.Minute, time.Second*10, time.Second*10, 100_000)
168
+
if err != nil {
169
+
return err
170
+
}
171
+
dir = rdir
172
+
}
123
173
124
174
resp, err := dir.LookupHandle(ctx, syntax.Handle(handle))
125
175
if err != nil {
···
151
201
client: cc,
152
202
dir: dir,
153
203
154
-
missingProfiles: make(chan string, 1024),
155
-
missingPosts: make(chan string, 1024),
204
+
db: db,
156
205
}
157
-
fmt.Println("MY DID: ", s.mydid)
158
206
159
-
pgb := &PostgresBackend{
160
-
relevantDids: make(map[string]bool),
161
-
s: s,
162
-
db: db,
163
-
postInfoCache: pc,
164
-
repoCache: rc,
165
-
revCache: revc,
166
-
pgx: pool,
207
+
pgb, err := backend.NewPostgresBackend(mydid, db, pool, cc, dir)
208
+
if err != nil {
209
+
return err
167
210
}
211
+
168
212
s.backend = pgb
169
213
170
-
myrepo, err := s.backend.getOrCreateRepo(ctx, mydid)
214
+
myrepo, err := s.backend.GetOrCreateRepo(ctx, mydid)
171
215
if err != nil {
172
216
return fmt.Errorf("failed to get repo record for our own did: %w", err)
173
217
}
174
218
s.myrepo = myrepo
175
219
176
-
if err := s.backend.loadRelevantDids(); err != nil {
220
+
if err := s.backend.LoadRelevantDids(); err != nil {
177
221
return fmt.Errorf("failed to load relevant dids set: %w", err)
178
222
}
179
223
···
197
241
http.ListenAndServe(":4445", nil)
198
242
}()
199
243
200
-
go s.missingProfileFetcher()
201
-
go s.missingPostFetcher()
244
+
sc := SyncConfig{
245
+
Backends: []SyncBackend{
246
+
{
247
+
Type: "firehose",
248
+
Host: "bsky.network",
249
+
},
250
+
},
251
+
}
202
252
203
-
seqno, err := loadLastSeq(db, "firehose_seq")
204
-
if err != nil {
205
-
fmt.Println("failed to load sequence number, starting over", err)
253
+
if scfn := cctx.String("sync-config"); scfn != "" {
254
+
{
255
+
scfi, err := os.Open(scfn)
256
+
if err != nil {
257
+
return err
258
+
}
259
+
defer scfi.Close()
260
+
261
+
var lsc SyncConfig
262
+
if err := json.NewDecoder(scfi).Decode(&lsc); err != nil {
263
+
return err
264
+
}
265
+
sc = lsc
266
+
}
206
267
}
207
268
208
-
return s.startLiveTail(ctx, int(seqno), 10, 20)
269
+
/*
270
+
sc.Backends[0] = SyncBackend{
271
+
Type: "jetstream",
272
+
Host: "jetstream1.us-west.bsky.network",
273
+
}
274
+
*/
275
+
276
+
return s.StartSyncEngine(ctx, &sc)
277
+
209
278
}
210
279
211
280
app.RunAndExitOnError()
212
281
}
213
282
214
283
type Server struct {
215
-
backend *PostgresBackend
284
+
backend *backend.PostgresBackend
216
285
217
286
dir identity.Directory
218
287
···
223
292
seqLk sync.Mutex
224
293
lastSeq int64
225
294
226
-
mpLk sync.Mutex
227
-
missingProfiles chan string
228
-
missingPosts chan string
295
+
mpLk sync.Mutex
296
+
297
+
db *gorm.DB
229
298
}
230
299
231
300
func (s *Server) getXrpcClient() (*xrpclib.Client, error) {
···
233
302
return s.client, nil
234
303
}
235
304
236
-
func (s *Server) startLiveTail(ctx context.Context, curs int, parWorkers, maxQ int) error {
237
-
slog.Info("starting live tail")
238
-
239
-
// Connect to the Relay websocket
240
-
urlStr := fmt.Sprintf("wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", curs)
241
-
242
-
d := websocket.DefaultDialer
243
-
con, _, err := d.Dial(urlStr, http.Header{
244
-
"User-Agent": []string{"market/0.0.1"},
245
-
})
246
-
if err != nil {
247
-
return fmt.Errorf("failed to connect to relay: %w", err)
248
-
}
249
-
250
-
var lelk sync.Mutex
251
-
lastEvent := time.Now()
252
-
253
-
go func() {
254
-
for range time.Tick(time.Second) {
255
-
lelk.Lock()
256
-
let := lastEvent
257
-
lelk.Unlock()
258
-
259
-
if time.Since(let) > time.Second*30 {
260
-
slog.Error("firehose connection timed out")
261
-
con.Close()
262
-
return
263
-
}
264
-
265
-
}
266
-
267
-
}()
268
-
269
-
var cclk sync.Mutex
270
-
var completeCursor int64
271
-
272
-
rsc := &stream.RepoStreamCallbacks{
273
-
RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
274
-
ctx := context.Background()
275
-
276
-
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
277
-
278
-
s.seqLk.Lock()
279
-
if evt.Seq > s.lastSeq {
280
-
curs = int(evt.Seq)
281
-
s.lastSeq = evt.Seq
282
-
283
-
if evt.Seq%1000 == 0 {
284
-
if err := storeLastSeq(s.backend.db, "firehose_seq", evt.Seq); err != nil {
285
-
fmt.Println("failed to store seqno: ", err)
286
-
}
287
-
}
288
-
}
289
-
s.seqLk.Unlock()
290
-
291
-
lelk.Lock()
292
-
lastEvent = time.Now()
293
-
lelk.Unlock()
294
-
295
-
if err := s.backend.HandleEvent(ctx, evt); err != nil {
296
-
return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
297
-
}
298
-
299
-
cclk.Lock()
300
-
if evt.Seq > completeCursor {
301
-
completeCursor = evt.Seq
302
-
firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
303
-
}
304
-
cclk.Unlock()
305
-
306
-
return nil
307
-
},
308
-
RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
309
-
return nil
310
-
},
311
-
// TODO: all the other event types
312
-
Error: func(errf *stream.ErrorFrame) error {
313
-
return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
314
-
},
315
-
}
316
-
317
-
sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
318
-
319
-
//s.eventScheduler = sched
320
-
//s.streamFinished = make(chan struct{})
321
-
322
-
return stream.HandleRepoStream(ctx, con, sched, slog.Default())
323
-
}
324
-
325
305
func (s *Server) resolveAccountIdent(ctx context.Context, acc string) (string, error) {
326
306
unesc, err := url.PathUnescape(acc)
327
307
if err != nil {
···
341
321
return resp.DID.String(), nil
342
322
}
343
323
344
-
const (
345
-
NotifKindReply = "reply"
346
-
NotifKindLike = "like"
347
-
NotifKindMention = "mention"
348
-
NotifKindRepost = "repost"
349
-
)
350
-
351
-
func (s *Server) AddNotification(ctx context.Context, forUser, author uint, recordUri string, recordCid cid.Cid, kind string) error {
352
-
return s.backend.db.Create(&Notification{
353
-
For: forUser,
354
-
Author: author,
355
-
Source: recordUri,
356
-
SourceCid: recordCid.String(),
357
-
Kind: kind,
358
-
}).Error
359
-
}
360
-
361
324
func (s *Server) rescanRepo(ctx context.Context, did string) error {
362
325
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
363
326
if err != nil {
364
327
return err
365
328
}
366
329
367
-
_ = resp
368
-
return nil
330
+
s.backend.AddRelevantDid(did)
331
+
332
+
c := &xrpclib.Client{
333
+
Host: resp.PDSEndpoint(),
334
+
}
335
+
336
+
repob, err := atproto.SyncGetRepo(ctx, c, did, "")
337
+
if err != nil {
338
+
return err
339
+
}
340
+
341
+
rep, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(repob))
342
+
if err != nil {
343
+
return err
344
+
}
345
+
346
+
return rep.ForEach(ctx, "", func(k string, v cid.Cid) error {
347
+
blk, err := rep.Blockstore().Get(ctx, v)
348
+
if err != nil {
349
+
slog.Error("record missing in repo", "path", k, "cid", v, "error", err)
350
+
return nil
351
+
}
352
+
353
+
d := blk.RawData()
354
+
if err := s.backend.HandleCreate(ctx, did, "", k, &d, &v); err != nil {
355
+
slog.Error("failed to index record", "path", k, "cid", v, "error", err)
356
+
}
357
+
return nil
358
+
})
369
359
370
360
}
-133
missing.go
-133
missing.go
···
1
-
package main
2
-
3
-
import (
4
-
"bytes"
5
-
"context"
6
-
"fmt"
7
-
"log/slog"
8
-
"strings"
9
-
10
-
"github.com/bluesky-social/indigo/api/atproto"
11
-
"github.com/bluesky-social/indigo/api/bsky"
12
-
"github.com/bluesky-social/indigo/atproto/syntax"
13
-
xrpclib "github.com/bluesky-social/indigo/xrpc"
14
-
"github.com/ipfs/go-cid"
15
-
"github.com/labstack/gommon/log"
16
-
)
17
-
18
-
func (s *Server) addMissingProfile(ctx context.Context, did string) {
19
-
select {
20
-
case s.missingProfiles <- did:
21
-
case <-ctx.Done():
22
-
}
23
-
}
24
-
25
-
func (s *Server) missingProfileFetcher() {
26
-
for did := range s.missingProfiles {
27
-
if err := s.fetchMissingProfile(context.TODO(), did); err != nil {
28
-
log.Warn("failed to fetch missing profile", "did", did, "error", err)
29
-
}
30
-
}
31
-
}
32
-
33
-
func (s *Server) fetchMissingProfile(ctx context.Context, did string) error {
34
-
repo, err := s.backend.getOrCreateRepo(ctx, did)
35
-
if err != nil {
36
-
return err
37
-
}
38
-
39
-
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
40
-
if err != nil {
41
-
return err
42
-
}
43
-
44
-
c := &xrpclib.Client{
45
-
Host: resp.PDSEndpoint(),
46
-
}
47
-
48
-
rec, err := atproto.RepoGetRecord(ctx, c, "", "app.bsky.actor.profile", did, "self")
49
-
if err != nil {
50
-
return err
51
-
}
52
-
53
-
prof, ok := rec.Value.Val.(*bsky.ActorProfile)
54
-
if !ok {
55
-
return fmt.Errorf("record we got back wasnt a profile somehow")
56
-
}
57
-
58
-
buf := new(bytes.Buffer)
59
-
if err := prof.MarshalCBOR(buf); err != nil {
60
-
return err
61
-
}
62
-
63
-
cc, err := cid.Decode(*rec.Cid)
64
-
if err != nil {
65
-
return err
66
-
}
67
-
68
-
return s.backend.HandleUpdateProfile(ctx, repo, "self", "", buf.Bytes(), cc)
69
-
}
70
-
71
-
func (s *Server) addMissingPost(ctx context.Context, uri string) {
72
-
slog.Info("adding missing post to fetch queue", "uri", uri)
73
-
select {
74
-
case s.missingPosts <- uri:
75
-
case <-ctx.Done():
76
-
}
77
-
}
78
-
79
-
func (s *Server) missingPostFetcher() {
80
-
for uri := range s.missingPosts {
81
-
if err := s.fetchMissingPost(context.TODO(), uri); err != nil {
82
-
log.Warn("failed to fetch missing post", "uri", uri, "error", err)
83
-
}
84
-
}
85
-
}
86
-
87
-
func (s *Server) fetchMissingPost(ctx context.Context, uri string) error {
88
-
// Parse AT URI: at://did:plc:xxx/app.bsky.feed.post/rkey
89
-
parts := strings.Split(uri, "/")
90
-
if len(parts) < 5 || !strings.HasPrefix(parts[2], "did:") {
91
-
return fmt.Errorf("invalid AT URI: %s", uri)
92
-
}
93
-
94
-
did := parts[2]
95
-
collection := parts[3]
96
-
rkey := parts[4]
97
-
98
-
repo, err := s.backend.getOrCreateRepo(ctx, did)
99
-
if err != nil {
100
-
return err
101
-
}
102
-
103
-
resp, err := s.dir.LookupDID(ctx, syntax.DID(did))
104
-
if err != nil {
105
-
return err
106
-
}
107
-
108
-
c := &xrpclib.Client{
109
-
Host: resp.PDSEndpoint(),
110
-
}
111
-
112
-
rec, err := atproto.RepoGetRecord(ctx, c, "", collection, did, rkey)
113
-
if err != nil {
114
-
return err
115
-
}
116
-
117
-
post, ok := rec.Value.Val.(*bsky.FeedPost)
118
-
if !ok {
119
-
return fmt.Errorf("record we got back wasn't a post somehow")
120
-
}
121
-
122
-
buf := new(bytes.Buffer)
123
-
if err := post.MarshalCBOR(buf); err != nil {
124
-
return err
125
-
}
126
-
127
-
cc, err := cid.Decode(*rec.Cid)
128
-
if err != nil {
129
-
return err
130
-
}
131
-
132
-
return s.backend.HandleCreatePost(ctx, repo, rkey, buf.Bytes(), cc)
133
-
}
+54
models/models.go
+54
models/models.go
···
1
+
package models
2
+
3
+
import (
4
+
"time"
5
+
6
+
"github.com/whyrusleeping/market/models"
7
+
"gorm.io/gorm"
8
+
)
9
+
10
+
type Repo = models.Repo
11
+
type Post = models.Post
12
+
type Follow = models.Follow
13
+
type Block = models.Block
14
+
type Repost = models.Repost
15
+
type List = models.List
16
+
type ListItem = models.ListItem
17
+
type ListBlock = models.ListBlock
18
+
type Profile = models.Profile
19
+
type ThreadGate = models.ThreadGate
20
+
type FeedGenerator = models.FeedGenerator
21
+
type Image = models.Image
22
+
type PostGate = models.PostGate
23
+
type StarterPack = models.StarterPack
24
+
25
+
type Like struct {
26
+
ID uint `gorm:"primarykey"`
27
+
Created time.Time
28
+
Indexed time.Time
29
+
Author uint `gorm:"uniqueIndex:idx_likes_rkeyauthor"`
30
+
Rkey string `gorm:"uniqueIndex:idx_likes_rkeyauthor"`
31
+
Subject uint
32
+
Cid string
33
+
}
34
+
35
+
type Notification struct {
36
+
gorm.Model
37
+
For uint
38
+
39
+
Author uint
40
+
Source string
41
+
SourceCid string
42
+
Kind string
43
+
}
44
+
45
+
type SequenceTracker struct {
46
+
ID uint `gorm:"primarykey"`
47
+
Key string `gorm:"uniqueIndex"`
48
+
IntVal int64
49
+
}
50
+
51
+
type NotificationSeen struct {
52
+
Repo uint `gorm:"uniqueindex"`
53
+
SeenAt time.Time
54
+
}
-49
models.go
-49
models.go
···
1
-
package main
2
-
3
-
import (
4
-
"time"
5
-
6
-
"github.com/whyrusleeping/market/models"
7
-
"gorm.io/gorm"
8
-
)
9
-
10
-
type Repo = models.Repo
11
-
type Post = models.Post
12
-
type Follow = models.Follow
13
-
type Block = models.Block
14
-
type Repost = models.Repost
15
-
type List = models.List
16
-
type ListItem = models.ListItem
17
-
type ListBlock = models.ListBlock
18
-
type Profile = models.Profile
19
-
type ThreadGate = models.ThreadGate
20
-
type FeedGenerator = models.FeedGenerator
21
-
type Image = models.Image
22
-
type PostGate = models.PostGate
23
-
type StarterPack = models.StarterPack
24
-
25
-
type Like struct {
26
-
ID uint `gorm:"primarykey"`
27
-
Created time.Time
28
-
Indexed time.Time
29
-
Author uint `gorm:"uniqueIndex:idx_likes_rkeyauthor"`
30
-
Rkey string `gorm:"uniqueIndex:idx_likes_rkeyauthor"`
31
-
Subject uint
32
-
Cid string
33
-
}
34
-
35
-
type Notification struct {
36
-
gorm.Model
37
-
For uint
38
-
39
-
Author uint
40
-
Source string
41
-
SourceCid string
42
-
Kind string
43
-
}
44
-
45
-
type SequenceTracker struct {
46
-
ID uint `gorm:"primarykey"`
47
-
Key string `gorm:"uniqueIndex"`
48
-
IntVal int64
49
-
}
-406
pgbackend.go
-406
pgbackend.go
···
1
-
package main
2
-
3
-
import (
4
-
"context"
5
-
"errors"
6
-
"fmt"
7
-
"strings"
8
-
"time"
9
-
10
-
"github.com/bluesky-social/indigo/api/atproto"
11
-
"github.com/bluesky-social/indigo/api/bsky"
12
-
"github.com/bluesky-social/indigo/atproto/syntax"
13
-
"github.com/bluesky-social/indigo/util"
14
-
"github.com/jackc/pgx/v5"
15
-
"github.com/jackc/pgx/v5/pgconn"
16
-
"github.com/whyrusleeping/market/models"
17
-
"gorm.io/gorm"
18
-
"gorm.io/gorm/clause"
19
-
"gorm.io/gorm/logger"
20
-
)
21
-
22
-
func (b *PostgresBackend) getOrCreateRepo(ctx context.Context, did string) (*Repo, error) {
23
-
r, ok := b.repoCache.Get(did)
24
-
if !ok {
25
-
b.reposLk.Lock()
26
-
27
-
r, ok = b.repoCache.Get(did)
28
-
if !ok {
29
-
r = &Repo{}
30
-
r.Did = did
31
-
b.repoCache.Add(did, r)
32
-
}
33
-
34
-
b.reposLk.Unlock()
35
-
}
36
-
37
-
r.Lk.Lock()
38
-
defer r.Lk.Unlock()
39
-
if r.Setup {
40
-
return r, nil
41
-
}
42
-
43
-
row := b.pgx.QueryRow(ctx, "SELECT id, created_at, did FROM repos WHERE did = $1", did)
44
-
45
-
err := row.Scan(&r.ID, &r.CreatedAt, &r.Did)
46
-
if err == nil {
47
-
// found it!
48
-
r.Setup = true
49
-
return r, nil
50
-
}
51
-
52
-
if err != pgx.ErrNoRows {
53
-
return nil, err
54
-
}
55
-
56
-
r.Did = did
57
-
if err := b.db.Create(r).Error; err != nil {
58
-
return nil, err
59
-
}
60
-
61
-
r.Setup = true
62
-
63
-
return r, nil
64
-
}
65
-
66
-
func (b *PostgresBackend) getOrCreateList(ctx context.Context, uri string) (*List, error) {
67
-
puri, err := util.ParseAtUri(uri)
68
-
if err != nil {
69
-
return nil, err
70
-
}
71
-
72
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
73
-
if err != nil {
74
-
return nil, err
75
-
}
76
-
77
-
// TODO: needs upsert treatment when we actually find the list
78
-
var list List
79
-
if err := b.db.FirstOrCreate(&list, map[string]any{
80
-
"author": r.ID,
81
-
"rkey": puri.Rkey,
82
-
}).Error; err != nil {
83
-
return nil, err
84
-
}
85
-
return &list, nil
86
-
}
87
-
88
-
type cachedPostInfo struct {
89
-
ID uint
90
-
Author uint
91
-
}
92
-
93
-
func (b *PostgresBackend) postIDForUri(ctx context.Context, uri string) (uint, error) {
94
-
// getPostByUri implicitly fills the cache
95
-
p, err := b.postInfoForUri(ctx, uri)
96
-
if err != nil {
97
-
return 0, err
98
-
}
99
-
100
-
return p.ID, nil
101
-
}
102
-
103
-
func (b *PostgresBackend) postInfoForUri(ctx context.Context, uri string) (cachedPostInfo, error) {
104
-
v, ok := b.postInfoCache.Get(uri)
105
-
if ok {
106
-
return v, nil
107
-
}
108
-
109
-
// getPostByUri implicitly fills the cache
110
-
p, err := b.getOrCreatePostBare(ctx, uri)
111
-
if err != nil {
112
-
return cachedPostInfo{}, err
113
-
}
114
-
115
-
return cachedPostInfo{ID: p.ID, Author: p.Author}, nil
116
-
}
117
-
118
-
func (b *PostgresBackend) tryLoadPostInfo(ctx context.Context, uid uint, rkey string) (*Post, error) {
119
-
var p Post
120
-
q := "SELECT id, author FROM posts WHERE author = $1 AND rkey = $2"
121
-
if err := b.pgx.QueryRow(ctx, q, uid, rkey).Scan(&p.ID, &p.Author); err != nil {
122
-
if errors.Is(err, pgx.ErrNoRows) {
123
-
return nil, nil
124
-
}
125
-
return nil, err
126
-
}
127
-
128
-
return &p, nil
129
-
}
130
-
131
-
func (b *PostgresBackend) getOrCreatePostBare(ctx context.Context, uri string) (*Post, error) {
132
-
puri, err := util.ParseAtUri(uri)
133
-
if err != nil {
134
-
return nil, err
135
-
}
136
-
137
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
138
-
if err != nil {
139
-
return nil, err
140
-
}
141
-
142
-
post, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
143
-
if err != nil {
144
-
return nil, err
145
-
}
146
-
147
-
if post == nil {
148
-
post = &Post{
149
-
Rkey: puri.Rkey,
150
-
Author: r.ID,
151
-
NotFound: true,
152
-
}
153
-
154
-
err := b.pgx.QueryRow(ctx, "INSERT INTO posts (rkey, author, not_found) VALUES ($1, $2, $3) RETURNING id", puri.Rkey, r.ID, true).Scan(&post.ID)
155
-
if err != nil {
156
-
pgErr, ok := err.(*pgconn.PgError)
157
-
if !ok || pgErr.Code != "23505" {
158
-
return nil, err
159
-
}
160
-
161
-
out, err := b.tryLoadPostInfo(ctx, r.ID, puri.Rkey)
162
-
if err != nil {
163
-
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
164
-
}
165
-
if out == nil {
166
-
return nil, fmt.Errorf("postgres is lying to us: %d %s", r.ID, puri.Rkey)
167
-
}
168
-
169
-
post = out
170
-
}
171
-
172
-
}
173
-
174
-
b.postInfoCache.Add(uri, cachedPostInfo{
175
-
ID: post.ID,
176
-
Author: post.Author,
177
-
})
178
-
179
-
return post, nil
180
-
}
181
-
182
-
func (b *PostgresBackend) getPostByUri(ctx context.Context, uri string, fields string) (*Post, error) {
183
-
puri, err := util.ParseAtUri(uri)
184
-
if err != nil {
185
-
return nil, err
186
-
}
187
-
188
-
r, err := b.getOrCreateRepo(ctx, puri.Did)
189
-
if err != nil {
190
-
return nil, err
191
-
}
192
-
193
-
q := "SELECT " + fields + " FROM posts WHERE author = ? AND rkey = ?"
194
-
195
-
var post Post
196
-
if err := b.db.Raw(q, r.ID, puri.Rkey).Scan(&post).Error; err != nil {
197
-
return nil, err
198
-
}
199
-
200
-
if post.ID == 0 {
201
-
post.Rkey = puri.Rkey
202
-
post.Author = r.ID
203
-
post.NotFound = true
204
-
205
-
if err := b.db.Session(&gorm.Session{
206
-
Logger: logger.Default.LogMode(logger.Silent),
207
-
}).Create(&post).Error; err != nil {
208
-
if !errors.Is(err, gorm.ErrDuplicatedKey) {
209
-
return nil, err
210
-
}
211
-
if err := b.db.Find(&post, "author = ? AND rkey = ?", r.ID, puri.Rkey).Error; err != nil {
212
-
return nil, fmt.Errorf("got duplicate post and still couldnt find it: %w", err)
213
-
}
214
-
}
215
-
216
-
}
217
-
218
-
b.postInfoCache.Add(uri, cachedPostInfo{
219
-
ID: post.ID,
220
-
Author: post.Author,
221
-
})
222
-
223
-
return &post, nil
224
-
}
225
-
226
-
func (b *PostgresBackend) revForRepo(rr *Repo) (string, error) {
227
-
lrev, ok := b.revCache.Get(rr.ID)
228
-
if ok {
229
-
return lrev, nil
230
-
}
231
-
232
-
var rev string
233
-
if err := b.pgx.QueryRow(context.TODO(), "SELECT COALESCE(rev, '') FROM sync_infos WHERE repo = $1", rr.ID).Scan(&rev); err != nil {
234
-
if errors.Is(err, pgx.ErrNoRows) {
235
-
return "", nil
236
-
}
237
-
return "", err
238
-
}
239
-
240
-
if rev != "" {
241
-
b.revCache.Add(rr.ID, rev)
242
-
}
243
-
return rev, nil
244
-
}
245
-
246
-
func (b *PostgresBackend) ensureFollowsScraped(ctx context.Context, user string) error {
247
-
r, err := b.getOrCreateRepo(ctx, user)
248
-
if err != nil {
249
-
return err
250
-
}
251
-
252
-
var si SyncInfo
253
-
if err := b.db.Find(&si, "repo = ?", r.ID).Error; err != nil {
254
-
return err
255
-
}
256
-
257
-
// not found
258
-
if si.Repo == 0 {
259
-
if err := b.db.Create(&SyncInfo{
260
-
Repo: r.ID,
261
-
}).Error; err != nil {
262
-
return err
263
-
}
264
-
}
265
-
266
-
if si.FollowsSynced {
267
-
return nil
268
-
}
269
-
270
-
var follows []Follow
271
-
var cursor string
272
-
for {
273
-
resp, err := atproto.RepoListRecords(ctx, b.s.client, "app.bsky.graph.follow", cursor, 100, b.s.mydid, false)
274
-
if err != nil {
275
-
return err
276
-
}
277
-
278
-
for _, rec := range resp.Records {
279
-
if fol, ok := rec.Value.Val.(*bsky.GraphFollow); ok {
280
-
fr, err := b.getOrCreateRepo(ctx, fol.Subject)
281
-
if err != nil {
282
-
return err
283
-
}
284
-
285
-
puri, err := syntax.ParseATURI(rec.Uri)
286
-
if err != nil {
287
-
return err
288
-
}
289
-
290
-
follows = append(follows, Follow{
291
-
Created: time.Now(),
292
-
Indexed: time.Now(),
293
-
Rkey: puri.RecordKey().String(),
294
-
Author: r.ID,
295
-
Subject: fr.ID,
296
-
})
297
-
}
298
-
}
299
-
300
-
if resp.Cursor == nil || len(resp.Records) == 0 {
301
-
break
302
-
}
303
-
cursor = *resp.Cursor
304
-
}
305
-
306
-
if err := b.db.Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(follows, 200).Error; err != nil {
307
-
return err
308
-
}
309
-
310
-
if err := b.db.Model(SyncInfo{}).Where("repo = ?", r.ID).Update("follows_synced", true).Error; err != nil {
311
-
return err
312
-
}
313
-
314
-
fmt.Println("Got follows: ", len(follows))
315
-
316
-
return nil
317
-
}
318
-
319
-
func (b *PostgresBackend) loadRelevantDids() error {
320
-
ctx := context.TODO()
321
-
322
-
if err := b.ensureFollowsScraped(ctx, b.s.mydid); err != nil {
323
-
return fmt.Errorf("failed to scrape follows: %w", err)
324
-
}
325
-
326
-
r, err := b.getOrCreateRepo(ctx, b.s.mydid)
327
-
if err != nil {
328
-
return err
329
-
}
330
-
331
-
var dids []string
332
-
if err := b.db.Raw("select did from follows left join repos on follows.subject = repos.id where follows.author = ?", r.ID).Scan(&dids).Error; err != nil {
333
-
return err
334
-
}
335
-
336
-
b.relevantDids[b.s.mydid] = true
337
-
for _, d := range dids {
338
-
fmt.Println("adding did: ", d)
339
-
b.relevantDids[d] = true
340
-
}
341
-
342
-
return nil
343
-
}
344
-
345
-
type SyncInfo struct {
346
-
Repo uint `gorm:"index"`
347
-
FollowsSynced bool
348
-
Rev string
349
-
}
350
-
351
-
func (b *PostgresBackend) checkPostExists(ctx context.Context, repo *Repo, rkey string) (bool, error) {
352
-
var id uint
353
-
var notfound bool
354
-
if err := b.pgx.QueryRow(ctx, "SELECT id, not_found FROM posts WHERE author = $1 AND rkey = $2", repo.ID, rkey).Scan(&id, ¬found); err != nil {
355
-
if errors.Is(err, pgx.ErrNoRows) {
356
-
return false, nil
357
-
}
358
-
return false, err
359
-
}
360
-
361
-
if id != 0 && !notfound {
362
-
return true, nil
363
-
}
364
-
365
-
return false, nil
366
-
}
367
-
368
-
func (b *PostgresBackend) didIsRelevant(did string) bool {
369
-
b.rdLk.Lock()
370
-
defer b.rdLk.Unlock()
371
-
return b.relevantDids[did]
372
-
}
373
-
374
-
func (b *PostgresBackend) anyRelevantIdents(idents ...string) bool {
375
-
for _, id := range idents {
376
-
if strings.HasPrefix(id, "did:") {
377
-
if b.didIsRelevant(id) {
378
-
return true
379
-
}
380
-
} else if strings.HasPrefix(id, "at://") {
381
-
puri, err := syntax.ParseATURI(id)
382
-
if err != nil {
383
-
continue
384
-
}
385
-
386
-
if b.didIsRelevant(puri.Authority().String()) {
387
-
return true
388
-
}
389
-
}
390
-
}
391
-
392
-
return false
393
-
}
394
-
395
-
func (b *PostgresBackend) getRepoByID(ctx context.Context, id uint) (*models.Repo, error) {
396
-
var r models.Repo
397
-
if err := b.db.Find(&r, "id = ?", id).Error; err != nil {
398
-
return nil, err
399
-
}
400
-
401
-
return &r, nil
402
-
}
403
-
404
-
func (b *PostgresBackend) TrackMissingActor(did string) {
405
-
b.s.addMissingProfile(context.TODO(), did)
406
-
}
+2
seqno.go
+2
seqno.go
+8
sync-config-jetstream.json
+8
sync-config-jetstream.json
+281
sync.go
+281
sync.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"log/slog"
7
+
"net/http"
8
+
"sync"
9
+
"time"
10
+
11
+
"github.com/bluesky-social/indigo/api/atproto"
12
+
"github.com/bluesky-social/indigo/cmd/relay/stream"
13
+
"github.com/bluesky-social/indigo/cmd/relay/stream/schedulers/parallel"
14
+
jsclient "github.com/bluesky-social/jetstream/pkg/client"
15
+
jsparallel "github.com/bluesky-social/jetstream/pkg/client/schedulers/parallel"
16
+
"github.com/bluesky-social/jetstream/pkg/models"
17
+
"github.com/gorilla/websocket"
18
+
)
19
+
20
+
type SyncConfig struct {
21
+
Backends []SyncBackend `json:"backends"`
22
+
}
23
+
24
+
type SyncBackend struct {
25
+
Type string `json:"type"`
26
+
Host string `json:"host"`
27
+
MaxWorkers int `json:"max_workers,omitempty"`
28
+
}
29
+
30
+
func (s *Server) StartSyncEngine(ctx context.Context, sc *SyncConfig) error {
31
+
for _, be := range sc.Backends {
32
+
switch be.Type {
33
+
case "firehose":
34
+
go s.runSyncFirehose(ctx, be)
35
+
case "jetstream":
36
+
go s.runSyncJetstream(ctx, be)
37
+
default:
38
+
return fmt.Errorf("unrecognized sync backend type: %q", be.Type)
39
+
}
40
+
}
41
+
42
+
<-ctx.Done()
43
+
return fmt.Errorf("exiting sync routine")
44
+
}
45
+
46
+
const failureTimeInterval = time.Second * 5
47
+
48
+
func (s *Server) runSyncFirehose(ctx context.Context, be SyncBackend) {
49
+
var failures int
50
+
for {
51
+
seqno, err := loadLastSeq(s.db, be.Host)
52
+
if err != nil {
53
+
fmt.Println("failed to load sequence number, starting over", err)
54
+
}
55
+
56
+
maxWorkers := 10
57
+
if be.MaxWorkers != 0 {
58
+
maxWorkers = be.MaxWorkers
59
+
}
60
+
61
+
start := time.Now()
62
+
if err := s.startLiveTail(ctx, be.Host, int(seqno), maxWorkers, 20); err != nil {
63
+
slog.Error("firehose connection lost", "host", be.Host, "error", err)
64
+
}
65
+
66
+
elapsed := time.Since(start)
67
+
68
+
if elapsed > failureTimeInterval {
69
+
failures = 0
70
+
continue
71
+
}
72
+
failures++
73
+
74
+
delay := delayForFailureCount(failures)
75
+
slog.Warn("retrying connection after delay", "host", be.Host, "delay", delay)
76
+
}
77
+
}
78
+
79
+
func (s *Server) runSyncJetstream(ctx context.Context, be SyncBackend) {
80
+
var failures int
81
+
for {
82
+
// Load last cursor (stored as sequence number in same table)
83
+
cursor, err := loadLastSeq(s.db, be.Host)
84
+
if err != nil {
85
+
slog.Warn("failed to load jetstream cursor, starting from live", "error", err)
86
+
cursor = 0
87
+
}
88
+
89
+
maxWorkers := 10
90
+
if be.MaxWorkers != 0 {
91
+
maxWorkers = be.MaxWorkers
92
+
}
93
+
94
+
start := time.Now()
95
+
if err := s.startJetstreamTail(ctx, be.Host, cursor, maxWorkers); err != nil {
96
+
slog.Error("jetstream connection lost", "host", be.Host, "error", err)
97
+
}
98
+
99
+
elapsed := time.Since(start)
100
+
101
+
if elapsed > failureTimeInterval {
102
+
failures = 0
103
+
continue
104
+
}
105
+
failures++
106
+
107
+
delay := delayForFailureCount(failures)
108
+
slog.Warn("retrying jetstream connection after delay", "host", be.Host, "delay", delay)
109
+
time.Sleep(delay)
110
+
}
111
+
}
112
+
113
+
func delayForFailureCount(n int) time.Duration {
114
+
if n < 5 {
115
+
return (time.Second * 5) + (time.Second * 2 * time.Duration(n))
116
+
}
117
+
118
+
return time.Second * 30
119
+
}
120
+
121
+
func (s *Server) startLiveTail(ctx context.Context, host string, curs int, parWorkers, maxQ int) error {
122
+
ctx, cancel := context.WithCancel(ctx)
123
+
defer cancel()
124
+
125
+
slog.Info("starting live tail")
126
+
127
+
// Connect to the Relay websocket
128
+
urlStr := fmt.Sprintf("wss://%s/xrpc/com.atproto.sync.subscribeRepos?cursor=%d", host, curs)
129
+
130
+
d := websocket.DefaultDialer
131
+
con, _, err := d.Dial(urlStr, http.Header{
132
+
"User-Agent": []string{"konbini/0.0.1"},
133
+
})
134
+
if err != nil {
135
+
return fmt.Errorf("failed to connect to relay: %w", err)
136
+
}
137
+
138
+
var lelk sync.Mutex
139
+
lastEvent := time.Now()
140
+
141
+
go func() {
142
+
tick := time.NewTicker(time.Second)
143
+
defer tick.Stop()
144
+
for {
145
+
select {
146
+
case <-tick.C:
147
+
lelk.Lock()
148
+
let := lastEvent
149
+
lelk.Unlock()
150
+
151
+
if time.Since(let) > time.Second*30 {
152
+
slog.Error("firehose connection timed out")
153
+
con.Close()
154
+
return
155
+
}
156
+
case <-ctx.Done():
157
+
return
158
+
}
159
+
}
160
+
}()
161
+
162
+
var cclk sync.Mutex
163
+
var completeCursor int64
164
+
165
+
rsc := &stream.RepoStreamCallbacks{
166
+
RepoCommit: func(evt *atproto.SyncSubscribeRepos_Commit) error {
167
+
ctx := context.Background()
168
+
169
+
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(evt.Seq))
170
+
171
+
s.seqLk.Lock()
172
+
if evt.Seq > s.lastSeq {
173
+
curs = int(evt.Seq)
174
+
s.lastSeq = evt.Seq
175
+
176
+
if evt.Seq%1000 == 0 {
177
+
if err := storeLastSeq(s.db, host, evt.Seq); err != nil {
178
+
fmt.Println("failed to store seqno: ", err)
179
+
}
180
+
}
181
+
}
182
+
s.seqLk.Unlock()
183
+
184
+
lelk.Lock()
185
+
lastEvent = time.Now()
186
+
lelk.Unlock()
187
+
188
+
if err := s.backend.HandleEvent(ctx, evt); err != nil {
189
+
return fmt.Errorf("handle event (%s,%d): %w", evt.Repo, evt.Seq, err)
190
+
}
191
+
192
+
cclk.Lock()
193
+
if evt.Seq > completeCursor {
194
+
completeCursor = evt.Seq
195
+
firehoseCursorGauge.WithLabelValues("complete").Set(float64(evt.Seq))
196
+
}
197
+
cclk.Unlock()
198
+
199
+
return nil
200
+
},
201
+
RepoInfo: func(info *atproto.SyncSubscribeRepos_Info) error {
202
+
return nil
203
+
},
204
+
// TODO: all the other event types
205
+
Error: func(errf *stream.ErrorFrame) error {
206
+
return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message)
207
+
},
208
+
}
209
+
210
+
sched := parallel.NewScheduler(parWorkers, maxQ, con.RemoteAddr().String(), rsc.EventHandler)
211
+
212
+
return stream.HandleRepoStream(ctx, con, sched, slog.Default())
213
+
}
214
+
215
+
func (s *Server) startJetstreamTail(ctx context.Context, host string, cursor int64, parWorkers int) error {
216
+
ctx, cancel := context.WithCancel(ctx)
217
+
defer cancel()
218
+
219
+
slog.Info("starting jetstream tail", "host", host, "cursor", cursor)
220
+
221
+
// Create a scheduler for parallel processing
222
+
lastStored := int64(0)
223
+
sched := jsparallel.NewScheduler(
224
+
parWorkers,
225
+
host,
226
+
slog.Default(),
227
+
func(ctx context.Context, event *models.Event) error {
228
+
// Update cursor tracking
229
+
s.seqLk.Lock()
230
+
if event.TimeUS > s.lastSeq {
231
+
s.lastSeq = event.TimeUS
232
+
if event.TimeUS-lastStored > 1_000_000 {
233
+
// Store checkpoint periodically
234
+
if err := storeLastSeq(s.db, host, event.TimeUS); err != nil {
235
+
slog.Error("failed to store jetstream cursor", "error", err)
236
+
}
237
+
lastStored = event.TimeUS
238
+
}
239
+
}
240
+
s.seqLk.Unlock()
241
+
242
+
// Update metrics
243
+
firehoseCursorGauge.WithLabelValues("ingest").Set(float64(event.TimeUS))
244
+
245
+
// Convert Jetstream event to ATProto event format
246
+
if event.Commit != nil {
247
+
248
+
if err := s.backend.HandleEventJetstream(ctx, event); err != nil {
249
+
return fmt.Errorf("handle event (%s,%d): %w", event.Did, event.TimeUS, err)
250
+
}
251
+
252
+
firehoseCursorGauge.WithLabelValues("complete").Set(float64(event.TimeUS))
253
+
}
254
+
255
+
return nil
256
+
},
257
+
)
258
+
259
+
// Configure Jetstream client
260
+
config := jsclient.DefaultClientConfig()
261
+
config.WebsocketURL = fmt.Sprintf("wss://%s/subscribe", host)
262
+
263
+
// Prepare cursor pointer
264
+
var cursorPtr *int64
265
+
if cursor > 0 {
266
+
cursorPtr = &cursor
267
+
}
268
+
269
+
// Create and connect client
270
+
client, err := jsclient.NewClient(
271
+
config,
272
+
slog.Default(),
273
+
sched,
274
+
)
275
+
if err != nil {
276
+
return fmt.Errorf("create jetstream client: %w", err)
277
+
}
278
+
279
+
// Start reading from Jetstream
280
+
return client.ConnectAndRead(ctx, cursorPtr)
281
+
}
+5
views/actor.go
+5
views/actor.go
+53
-5
views/feed.go
+53
-5
views/feed.go
···
1
1
package views
2
2
3
3
import (
4
+
"fmt"
5
+
4
6
"github.com/bluesky-social/indigo/api/bsky"
5
7
"github.com/bluesky-social/indigo/lex/util"
6
8
"github.com/whyrusleeping/konbini/hydration"
···
40
42
}
41
43
}
42
44
43
-
// TODO: Add embed handling - need to convert embed types to proper views
44
-
// if post.Post.Embed != nil {
45
-
// view.Embed = formatEmbed(post.Post.Embed)
46
-
// }
45
+
// Add embed if it was hydrated
46
+
if post.EmbedInfo != nil {
47
+
view.Embed = post.EmbedInfo
48
+
}
47
49
48
50
return view
49
51
}
···
56
58
}
57
59
58
60
// ThreadViewPost builds a thread view post (app.bsky.feed.defs#threadViewPost)
59
-
func ThreadViewPost(post *hydration.PostInfo, author *hydration.ActorInfo, parent, replies interface{}) *bsky.FeedDefs_ThreadViewPost {
61
+
func ThreadViewPost(post *hydration.PostInfo, author *hydration.ActorInfo, parent, replies any) *bsky.FeedDefs_ThreadViewPost {
60
62
view := &bsky.FeedDefs_ThreadViewPost{
61
63
LexiconTypeID: "app.bsky.feed.defs#threadViewPost",
62
64
Post: PostView(post, author),
···
67
69
68
70
return view
69
71
}
72
+
73
+
// GeneratorView builds a feed generator view (app.bsky.feed.defs#generatorView)
74
+
func GeneratorView(uri, cid string, record *bsky.FeedGenerator, creator *hydration.ActorInfo, likeCount int64, viewerLike string, indexedAt string) *bsky.FeedDefs_GeneratorView {
75
+
view := &bsky.FeedDefs_GeneratorView{
76
+
LexiconTypeID: "app.bsky.feed.defs#generatorView",
77
+
Uri: uri,
78
+
Cid: cid,
79
+
Did: record.Did,
80
+
Creator: ProfileView(creator),
81
+
DisplayName: record.DisplayName,
82
+
Description: record.Description,
83
+
IndexedAt: indexedAt,
84
+
}
85
+
86
+
// Add optional fields
87
+
if record.Avatar != nil {
88
+
avatarURL := fmt.Sprintf("https://cdn.bsky.app/img/avatar/plain/%s/%s@jpeg", creator.DID, record.Avatar.Ref.String())
89
+
view.Avatar = &avatarURL
90
+
}
91
+
92
+
if record.DescriptionFacets != nil && len(record.DescriptionFacets) > 0 {
93
+
view.DescriptionFacets = record.DescriptionFacets
94
+
}
95
+
96
+
if record.AcceptsInteractions != nil {
97
+
view.AcceptsInteractions = record.AcceptsInteractions
98
+
}
99
+
100
+
if record.ContentMode != nil {
101
+
view.ContentMode = record.ContentMode
102
+
}
103
+
104
+
// Add like count if present
105
+
if likeCount > 0 {
106
+
view.LikeCount = &likeCount
107
+
}
108
+
109
+
// Add viewer state if viewer has liked
110
+
if viewerLike != "" {
111
+
view.Viewer = &bsky.FeedDefs_GeneratorViewerState{
112
+
Like: &viewerLike,
113
+
}
114
+
}
115
+
116
+
return view
117
+
}
+3
-1
xrpc/actor/getProfile.go
+3
-1
xrpc/actor/getProfile.go
···
20
20
21
21
ctx := c.Request().Context()
22
22
23
+
viewer, _ := c.Get("viewer").(string)
24
+
23
25
// Resolve actor to DID
24
26
did, err := hydrator.ResolveDID(ctx, actorParam)
25
27
if err != nil {
···
30
32
}
31
33
32
34
// Hydrate actor info
33
-
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did)
35
+
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did, viewer)
34
36
if err != nil {
35
37
return c.JSON(http.StatusNotFound, map[string]interface{}{
36
38
"error": "ActorNotFound",
+4
-16
xrpc/actor/getProfiles.go
+4
-16
xrpc/actor/getProfiles.go
···
3
3
import (
4
4
"net/http"
5
5
6
+
"github.com/bluesky-social/indigo/api/bsky"
6
7
"github.com/labstack/echo/v4"
7
8
"github.com/whyrusleeping/konbini/hydration"
8
9
"github.com/whyrusleeping/konbini/views"
···
26
27
}
27
28
28
29
ctx := c.Request().Context()
30
+
viewer, _ := c.Get("viewer").(string)
29
31
30
32
// Resolve all actors to DIDs and hydrate profiles
31
-
profiles := make([]interface{}, 0)
33
+
profiles := make([]*bsky.ActorDefs_ProfileViewDetailed, 0, len(actors))
32
34
for _, actor := range actors {
33
35
// Resolve actor to DID
34
36
did, err := hydrator.ResolveDID(ctx, actor)
···
38
40
}
39
41
40
42
// Hydrate actor info
41
-
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did)
43
+
actorInfo, err := hydrator.HydrateActorDetailed(ctx, did, viewer)
42
44
if err != nil {
43
45
// Skip actors that can't be hydrated
44
46
continue
45
47
}
46
-
47
-
// Get counts for the profile
48
-
type counts struct {
49
-
Followers int
50
-
Follows int
51
-
Posts int
52
-
}
53
-
var c counts
54
-
db.Raw(`
55
-
SELECT
56
-
(SELECT COUNT(*) FROM follows WHERE subject = (SELECT id FROM repos WHERE did = ?)) as followers,
57
-
(SELECT COUNT(*) FROM follows WHERE author = (SELECT id FROM repos WHERE did = ?)) as follows,
58
-
(SELECT COUNT(*) FROM posts WHERE author = (SELECT id FROM repos WHERE did = ?)) as posts
59
-
`, did, did, did).Scan(&c)
60
48
61
49
profiles = append(profiles, views.ProfileViewDetailed(actorInfo))
62
50
}
+95
-25
xrpc/feed/getAuthorFeed.go
+95
-25
xrpc/feed/getAuthorFeed.go
···
1
1
package feed
2
2
3
3
import (
4
+
"context"
5
+
"log/slog"
4
6
"net/http"
5
7
"strconv"
8
+
"strings"
9
+
"sync"
6
10
"time"
7
11
12
+
"github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
14
"github.com/labstack/echo/v4"
9
15
"github.com/whyrusleeping/konbini/hydration"
10
16
"github.com/whyrusleeping/konbini/views"
11
17
"gorm.io/gorm"
12
18
)
13
19
20
+
type postRow struct {
21
+
URI string
22
+
AuthorID uint
23
+
}
24
+
14
25
// HandleGetAuthorFeed implements app.bsky.feed.getAuthorFeed
15
26
func HandleGetAuthorFeed(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
16
27
actorParam := c.QueryParam("actor")
17
28
if actorParam == "" {
18
-
return c.JSON(http.StatusBadRequest, map[string]interface{}{
29
+
return c.JSON(http.StatusBadRequest, map[string]any{
19
30
"error": "InvalidRequest",
20
31
"message": "actor parameter is required",
21
32
})
···
49
60
// Resolve actor to DID
50
61
did, err := hydrator.ResolveDID(ctx, actorParam)
51
62
if err != nil {
52
-
return c.JSON(http.StatusBadRequest, map[string]interface{}{
63
+
return c.JSON(http.StatusBadRequest, map[string]any{
53
64
"error": "ActorNotFound",
54
65
"message": "actor not found",
55
66
})
···
87
98
`
88
99
}
89
100
90
-
type postRow struct {
91
-
URI string
92
-
AuthorID uint
93
-
}
94
101
var rows []postRow
95
102
if err := db.Raw(query, did, cursor, limit).Scan(&rows).Error; err != nil {
96
-
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
103
+
return c.JSON(http.StatusInternalServerError, map[string]any{
97
104
"error": "InternalError",
98
105
"message": "failed to query author feed",
99
106
})
100
107
}
101
108
102
-
// Hydrate posts
103
-
feed := make([]interface{}, 0)
104
-
for _, row := range rows {
105
-
postInfo, err := hydrator.HydratePost(ctx, row.URI, viewer)
106
-
if err != nil {
107
-
continue
108
-
}
109
-
110
-
// Hydrate author
111
-
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
112
-
if err != nil {
113
-
continue
114
-
}
115
-
116
-
feedItem := views.FeedViewPost(postInfo, authorInfo)
117
-
feed = append(feed, feedItem)
118
-
}
109
+
feed := hydratePostRows(ctx, hydrator, viewer, rows)
119
110
120
111
// Generate next cursor
121
112
var nextCursor string
···
130
121
}
131
122
}
132
123
133
-
return c.JSON(http.StatusOK, map[string]interface{}{
124
+
return c.JSON(http.StatusOK, map[string]any{
134
125
"feed": feed,
135
126
"cursor": nextCursor,
136
127
})
137
128
}
129
+
130
+
func hydratePostRows(ctx context.Context, hydrator *hydration.Hydrator, viewer string, rows []postRow) []*bsky.FeedDefs_FeedViewPost {
131
+
ctx, span := tracer.Start(ctx, "hydratePostRows")
132
+
defer span.End()
133
+
134
+
// Hydrate posts
135
+
var wg sync.WaitGroup
136
+
137
+
var outLk sync.Mutex
138
+
feed := make([]*bsky.FeedDefs_FeedViewPost, len(rows))
139
+
for i, row := range rows {
140
+
wg.Add(1)
141
+
go func(i int, row postRow) {
142
+
defer wg.Done()
143
+
144
+
puri, err := syntax.ParseATURI(row.URI)
145
+
if err != nil {
146
+
slog.Error("row had invalid uri", "uri", row.URI, "error", err)
147
+
return
148
+
}
149
+
150
+
var subwg sync.WaitGroup
151
+
152
+
var postInfo *hydration.PostInfo
153
+
subwg.Go(func() {
154
+
pi, err := hydrator.HydratePost(ctx, row.URI, viewer)
155
+
if err != nil {
156
+
if strings.Contains(err.Error(), "post not found") {
157
+
hydrator.AddMissingRecord(row.URI, true)
158
+
pi, err = hydrator.HydratePost(ctx, row.URI, viewer)
159
+
if err != nil {
160
+
slog.Error("failed to hydrate post after fetch missing", "uri", row.URI, "error", err)
161
+
return
162
+
}
163
+
} else {
164
+
slog.Warn("failed to hydrate post", "uri", row.URI, "error", err)
165
+
return
166
+
}
167
+
}
168
+
postInfo = pi
169
+
})
170
+
171
+
var authorInfo *hydration.ActorInfo
172
+
subwg.Go(func() {
173
+
ai, err := hydrator.HydrateActor(ctx, puri.Authority().String())
174
+
if err != nil {
175
+
hydrator.AddMissingRecord(postInfo.Author, false)
176
+
slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
177
+
return
178
+
}
179
+
authorInfo = ai
180
+
})
181
+
182
+
subwg.Wait()
183
+
184
+
if postInfo == nil || authorInfo == nil {
185
+
return
186
+
}
187
+
188
+
feedItem := views.FeedViewPost(postInfo, authorInfo)
189
+
outLk.Lock()
190
+
feed[i] = feedItem
191
+
outLk.Unlock()
192
+
}(i, row)
193
+
}
194
+
wg.Wait()
195
+
196
+
x := 0
197
+
for i := 0; i < len(feed); i++ {
198
+
if feed[i] != nil {
199
+
feed[x] = feed[i]
200
+
x++
201
+
continue
202
+
}
203
+
}
204
+
feed = feed[:x]
205
+
206
+
return feed
207
+
}
+200
xrpc/feed/getFeed.go
+200
xrpc/feed/getFeed.go
···
1
+
package feed
2
+
3
+
import (
4
+
"bytes"
5
+
"log/slog"
6
+
"net/http"
7
+
"strconv"
8
+
"strings"
9
+
"sync"
10
+
11
+
"github.com/bluesky-social/indigo/api/bsky"
12
+
"github.com/bluesky-social/indigo/atproto/identity"
13
+
"github.com/bluesky-social/indigo/atproto/syntax"
14
+
"github.com/bluesky-social/indigo/xrpc"
15
+
"github.com/labstack/echo/v4"
16
+
"github.com/whyrusleeping/konbini/hydration"
17
+
"github.com/whyrusleeping/konbini/views"
18
+
"github.com/whyrusleeping/market/models"
19
+
"gorm.io/gorm"
20
+
)
21
+
22
+
// HandleGetFeed implements app.bsky.feed.getFeed
23
+
// Gets posts from a custom feed generator
24
+
func HandleGetFeed(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator, dir identity.Directory) error {
25
+
// Parse parameters
26
+
feedURI := c.QueryParam("feed")
27
+
if feedURI == "" {
28
+
return c.JSON(http.StatusBadRequest, map[string]any{
29
+
"error": "InvalidRequest",
30
+
"message": "feed parameter is required",
31
+
})
32
+
}
33
+
34
+
// Parse limit
35
+
limit := int64(50)
36
+
if limitParam := c.QueryParam("limit"); limitParam != "" {
37
+
if l, err := strconv.ParseInt(limitParam, 10, 64); err == nil && l > 0 && l <= 100 {
38
+
limit = l
39
+
}
40
+
}
41
+
42
+
// Parse cursor
43
+
cursor := c.QueryParam("cursor")
44
+
45
+
ctx := c.Request().Context()
46
+
viewer := getUserDID(c)
47
+
48
+
// Extract feed generator DID and rkey from URI
49
+
// URI format: at://did:plc:xxx/app.bsky.feed.generator/rkey
50
+
did := extractDIDFromURI(feedURI)
51
+
rkey := extractRkeyFromURI(feedURI)
52
+
53
+
if did == "" || rkey == "" {
54
+
return c.JSON(http.StatusBadRequest, map[string]any{
55
+
"error": "InvalidRequest",
56
+
"message": "invalid feed URI format",
57
+
})
58
+
}
59
+
60
+
// Check if feed generator exists in database
61
+
var feedGen models.FeedGenerator
62
+
if err := db.Raw(`
63
+
SELECT * FROM feed_generators fg WHERE fg.author = (select id from repos where did = ?) AND fg.rkey = ?
64
+
`, did, rkey).Scan(&feedGen).Error; err != nil {
65
+
return err
66
+
}
67
+
68
+
if feedGen.ID == 0 {
69
+
hydrator.AddMissingRecord(feedURI, true)
70
+
return c.JSON(http.StatusNotFound, map[string]any{
71
+
"error": "NotFound",
72
+
"message": "feed generator not found",
73
+
})
74
+
}
75
+
76
+
// Decode the feed generator record to get the service DID
77
+
var feedGenRecord bsky.FeedGenerator
78
+
if err := feedGenRecord.UnmarshalCBOR(bytes.NewReader(feedGen.Raw)); err != nil {
79
+
slog.Error("failed to decode feed generator record", "error", err)
80
+
return c.JSON(http.StatusInternalServerError, map[string]any{
81
+
"error": "InternalError",
82
+
"message": "failed to decode feed generator record",
83
+
})
84
+
}
85
+
86
+
// Parse the service DID
87
+
serviceDID, err := syntax.ParseDID(feedGenRecord.Did)
88
+
if err != nil {
89
+
slog.Error("invalid service DID in feed generator", "error", err, "did", feedGenRecord.Did)
90
+
return c.JSON(http.StatusInternalServerError, map[string]any{
91
+
"error": "InternalError",
92
+
"message": "invalid service DID",
93
+
})
94
+
}
95
+
96
+
// Resolve the service DID to get its endpoint
97
+
serviceIdent, err := dir.LookupDID(ctx, serviceDID)
98
+
if err != nil {
99
+
slog.Error("failed to resolve service DID", "error", err, "did", serviceDID)
100
+
return c.JSON(http.StatusInternalServerError, map[string]any{
101
+
"error": "InternalError",
102
+
"message": "failed to resolve service endpoint",
103
+
})
104
+
}
105
+
106
+
serviceEndpoint := serviceIdent.GetServiceEndpoint("bsky_fg")
107
+
if serviceEndpoint == "" {
108
+
slog.Error("service has no bsky_fg endpoint", "did", serviceDID)
109
+
return c.JSON(http.StatusInternalServerError, map[string]any{
110
+
"error": "InternalError",
111
+
"message": "service has no endpoint",
112
+
})
113
+
}
114
+
115
+
// Create XRPC client for the feed generator service
116
+
// Pass through headers from the original request so feed generators can
117
+
// customize feeds based on the viewer
118
+
headers := make(map[string]string)
119
+
120
+
// Set User-Agent to identify konbini
121
+
headers["User-Agent"] = "konbini/0.0.1"
122
+
123
+
// Pass through Authorization header if present (for authenticated feed requests)
124
+
if authHeader := c.Request().Header.Get("Authorization"); authHeader != "" {
125
+
headers["Authorization"] = authHeader
126
+
}
127
+
128
+
// Pass through Accept-Language header if present
129
+
if langHeader := c.Request().Header.Get("Accept-Language"); langHeader != "" {
130
+
headers["Accept-Language"] = langHeader
131
+
}
132
+
133
+
// Pass through X-Bsky-Topics header if present
134
+
if topicsHeader := c.Request().Header.Get("X-Bsky-Topics"); topicsHeader != "" {
135
+
headers["X-Bsky-Topics"] = topicsHeader
136
+
}
137
+
138
+
client := &xrpc.Client{
139
+
Host: serviceEndpoint,
140
+
Headers: headers,
141
+
}
142
+
143
+
// Call getFeedSkeleton on the service
144
+
skeleton, err := bsky.FeedGetFeedSkeleton(ctx, client, cursor, feedURI, limit)
145
+
if err != nil {
146
+
slog.Error("failed to call getFeedSkeleton", "error", err, "service", serviceEndpoint)
147
+
// Return empty feed on error rather than failing completely
148
+
return c.JSON(http.StatusOK, &bsky.FeedGetFeed_Output{
149
+
Feed: make([]*bsky.FeedDefs_FeedViewPost, 0),
150
+
})
151
+
}
152
+
153
+
// Hydrate the posts from the skeleton
154
+
posts := make([]*bsky.FeedDefs_FeedViewPost, len(skeleton.Feed))
155
+
var wg sync.WaitGroup
156
+
for i := range skeleton.Feed {
157
+
wg.Add(1)
158
+
go func(ix int) {
159
+
defer wg.Done()
160
+
skeletonPost := skeleton.Feed[ix]
161
+
postURI, err := syntax.ParseATURI(skeletonPost.Post)
162
+
if err != nil {
163
+
slog.Warn("invalid post URI in skeleton", "uri", skeletonPost.Post, "error", err)
164
+
return
165
+
}
166
+
167
+
postInfo, err := hydrator.HydratePost(ctx, postURI.String(), viewer)
168
+
if err != nil {
169
+
if strings.Contains(err.Error(), "post not found") {
170
+
hydrator.AddMissingRecord(postURI.String(), true)
171
+
postInfo, err = hydrator.HydratePost(ctx, postURI.String(), viewer)
172
+
if err != nil {
173
+
slog.Error("failed to hydrate post after fetch missing", "uri", postURI, "error", err)
174
+
return
175
+
}
176
+
} else {
177
+
slog.Warn("failed to hydrate post", "uri", postURI, "error", err)
178
+
return
179
+
}
180
+
}
181
+
182
+
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
183
+
if err != nil {
184
+
hydrator.AddMissingRecord(postInfo.Author, false)
185
+
slog.Warn("failed to hydrate author", "did", postInfo.Author, "error", err)
186
+
return
187
+
}
188
+
189
+
posts[ix] = views.FeedViewPost(postInfo, authorInfo)
190
+
}(i)
191
+
}
192
+
wg.Wait()
193
+
194
+
output := &bsky.FeedGetFeed_Output{
195
+
Feed: posts,
196
+
Cursor: skeleton.Cursor,
197
+
}
198
+
199
+
return c.JSON(http.StatusOK, output)
200
+
}
+161
xrpc/feed/getFeedGenerator.go
+161
xrpc/feed/getFeedGenerator.go
···
1
+
package feed
2
+
3
+
import (
4
+
"bytes"
5
+
"log/slog"
6
+
"net/http"
7
+
"time"
8
+
9
+
"github.com/bluesky-social/indigo/api/bsky"
10
+
"github.com/bluesky-social/indigo/atproto/identity"
11
+
"github.com/bluesky-social/indigo/atproto/syntax"
12
+
cid "github.com/ipfs/go-cid"
13
+
"github.com/labstack/echo/v4"
14
+
mh "github.com/multiformats/go-multihash"
15
+
"github.com/whyrusleeping/konbini/hydration"
16
+
"github.com/whyrusleeping/konbini/views"
17
+
"gorm.io/gorm"
18
+
)
19
+
20
+
// HandleGetFeedGenerator implements app.bsky.feed.getFeedGenerator
21
+
func HandleGetFeedGenerator(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator, dir identity.Directory) error {
22
+
ctx := c.Request().Context()
23
+
24
+
// Parse parameters
25
+
feedURI := c.QueryParam("feed")
26
+
if feedURI == "" {
27
+
return c.JSON(http.StatusBadRequest, map[string]any{
28
+
"error": "InvalidRequest",
29
+
"message": "feed parameter is required",
30
+
})
31
+
}
32
+
33
+
nu, err := hydrator.NormalizeUri(ctx, feedURI)
34
+
if err != nil {
35
+
return err
36
+
}
37
+
feedURI = nu
38
+
39
+
viewer := getUserDID(c)
40
+
_ = viewer
41
+
42
+
// Extract feed generator DID and rkey from URI
43
+
did := extractDIDFromURI(feedURI)
44
+
rkey := extractRkeyFromURI(feedURI)
45
+
46
+
if did == "" || rkey == "" {
47
+
return c.JSON(http.StatusBadRequest, map[string]any{
48
+
"error": "InvalidRequest",
49
+
"message": "invalid feed URI format",
50
+
})
51
+
}
52
+
53
+
// Query feed generator from database
54
+
type feedGenRow struct {
55
+
ID uint
56
+
Did string
57
+
Raw []byte
58
+
AuthorDid string
59
+
Indexed time.Time
60
+
}
61
+
var feedGen feedGenRow
62
+
err = db.Raw(`
63
+
SELECT fg.id, fg.did, fg.raw, r.did as author_did, indexed
64
+
FROM feed_generators fg
65
+
JOIN repos r ON r.id = fg.author
66
+
WHERE r.did = ? AND fg.rkey = ?
67
+
`, did, rkey).Scan(&feedGen).Error
68
+
69
+
if err != nil || feedGen.ID == 0 {
70
+
// Track this missing feed generator for fetching
71
+
hydrator.AddMissingRecord(feedURI, true)
72
+
73
+
return c.JSON(http.StatusNotFound, map[string]any{
74
+
"error": "NotFound",
75
+
"message": "feed generator not found",
76
+
})
77
+
}
78
+
79
+
// Decode the feed generator record
80
+
var feedGenRecord bsky.FeedGenerator
81
+
if err := feedGenRecord.UnmarshalCBOR(bytes.NewReader(feedGen.Raw)); err != nil {
82
+
slog.Error("failed to decode feed generator record", "error", err)
83
+
return c.JSON(http.StatusInternalServerError, map[string]any{
84
+
"error": "InternalError",
85
+
"message": "failed to decode feed generator record",
86
+
})
87
+
}
88
+
89
+
// Compute CID from raw bytes
90
+
hash, err := mh.Sum(feedGen.Raw, mh.SHA2_256, -1)
91
+
if err != nil {
92
+
slog.Error("failed to hash record", "error", err)
93
+
return c.JSON(http.StatusInternalServerError, map[string]any{
94
+
"error": "InternalError",
95
+
"message": "failed to compute CID",
96
+
})
97
+
}
98
+
recordCid := cid.NewCidV1(cid.DagCBOR, hash).String()
99
+
100
+
// Hydrate the creator
101
+
creatorInfo, err := hydrator.HydrateActor(ctx, feedGen.AuthorDid)
102
+
if err != nil {
103
+
slog.Error("failed to hydrate creator", "error", err, "did", feedGen.AuthorDid)
104
+
return c.JSON(http.StatusInternalServerError, map[string]any{
105
+
"error": "InternalError",
106
+
"message": "failed to hydrate creator",
107
+
})
108
+
}
109
+
110
+
// Count likes for this feed generator
111
+
var likeCount int64
112
+
113
+
// Check if viewer has liked this feed generator
114
+
viewerLike := ""
115
+
116
+
// Validate the service DID (check if it's resolvable)
117
+
serviceDID, err := syntax.ParseDID(feedGenRecord.Did)
118
+
if err != nil {
119
+
slog.Error("invalid service DID in feed generator", "error", err, "did", feedGenRecord.Did)
120
+
return c.JSON(http.StatusInternalServerError, map[string]any{
121
+
"error": "InternalError",
122
+
"message": "invalid service DID",
123
+
})
124
+
}
125
+
126
+
// Try to resolve the service DID to check if it's online/valid
127
+
isOnline := true
128
+
isValid := true
129
+
serviceIdent, err := dir.LookupDID(ctx, serviceDID)
130
+
if err != nil {
131
+
slog.Warn("failed to resolve service DID", "error", err, "did", serviceDID)
132
+
isOnline = false
133
+
isValid = false
134
+
} else {
135
+
// Check if service has an endpoint
136
+
serviceEndpoint := serviceIdent.PDSEndpoint()
137
+
if serviceEndpoint == "" {
138
+
slog.Warn("service has no PDS endpoint", "did", serviceDID)
139
+
isValid = false
140
+
}
141
+
}
142
+
143
+
// Build the generator view
144
+
generatorView := views.GeneratorView(
145
+
feedURI,
146
+
recordCid,
147
+
&feedGenRecord,
148
+
creatorInfo,
149
+
likeCount,
150
+
viewerLike,
151
+
feedGen.Indexed.Format(time.RFC3339),
152
+
)
153
+
154
+
output := &bsky.FeedGetFeedGenerator_Output{
155
+
View: generatorView,
156
+
IsOnline: isOnline,
157
+
IsValid: isValid,
158
+
}
159
+
160
+
return c.JSON(http.StatusOK, output)
161
+
}
+11
-11
xrpc/feed/getPostThread.go
+11
-11
xrpc/feed/getPostThread.go
···
15
15
func HandleGetPostThread(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
16
16
uriParam := c.QueryParam("uri")
17
17
if uriParam == "" {
18
-
return c.JSON(http.StatusBadRequest, map[string]interface{}{
18
+
return c.JSON(http.StatusBadRequest, map[string]any{
19
19
"error": "InvalidRequest",
20
20
"message": "uri parameter is required",
21
21
})
···
27
27
// Hydrate the requested post
28
28
postInfo, err := hydrator.HydratePost(ctx, uriParam, viewer)
29
29
if err != nil {
30
-
return c.JSON(http.StatusNotFound, map[string]interface{}{
30
+
return c.JSON(http.StatusNotFound, map[string]any{
31
31
"error": "NotFound",
32
32
"message": "post not found",
33
33
})
···
74
74
uri: uri,
75
75
replyTo: tp.ReplyTo,
76
76
inThread: tp.InThread,
77
-
replies: []interface{}{},
77
+
replies: []any{},
78
78
}
79
79
}
80
80
···
98
98
}
99
99
100
100
if rootNode == nil {
101
-
return c.JSON(http.StatusNotFound, map[string]interface{}{
101
+
return c.JSON(http.StatusNotFound, map[string]any{
102
102
"error": "NotFound",
103
103
"message": "thread root not found",
104
104
})
···
107
107
// Build the response by traversing the tree
108
108
thread := buildThreadView(ctx, db, rootNode, postsByID, hydrator, viewer, nil)
109
109
110
-
return c.JSON(http.StatusOK, map[string]interface{}{
110
+
return c.JSON(http.StatusOK, map[string]any{
111
111
"thread": thread,
112
112
})
113
113
}
···
117
117
uri string
118
118
replyTo uint
119
119
inThread uint
120
-
replies []interface{}
120
+
replies []any
121
121
}
122
122
123
-
func buildThreadView(ctx context.Context, db *gorm.DB, node *threadPostNode, allNodes map[uint]*threadPostNode, hydrator *hydration.Hydrator, viewer string, parent interface{}) interface{} {
123
+
func buildThreadView(ctx context.Context, db *gorm.DB, node *threadPostNode, allNodes map[uint]*threadPostNode, hydrator *hydration.Hydrator, viewer string, parent any) any {
124
124
// Hydrate this post
125
125
postInfo, err := hydrator.HydratePost(ctx, node.uri, viewer)
126
126
if err != nil {
127
127
// Return a notFound post
128
-
return map[string]interface{}{
128
+
return map[string]any{
129
129
"$type": "app.bsky.feed.defs#notFoundPost",
130
130
"uri": node.uri,
131
131
}
···
134
134
// Hydrate author
135
135
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
136
136
if err != nil {
137
-
return map[string]interface{}{
137
+
return map[string]any{
138
138
"$type": "app.bsky.feed.defs#notFoundPost",
139
139
"uri": node.uri,
140
140
}
141
141
}
142
142
143
143
// Build replies
144
-
var replies []interface{}
144
+
var replies []any
145
145
for _, replyNode := range node.replies {
146
146
if rn, ok := replyNode.(*threadPostNode); ok {
147
147
replyView := buildThreadView(ctx, db, rn, allNodes, hydrator, viewer, nil)
···
150
150
}
151
151
152
152
// Build the thread view post
153
-
var repliesForView interface{}
153
+
var repliesForView any
154
154
if len(replies) > 0 {
155
155
repliesForView = replies
156
156
}
+39
-43
xrpc/feed/getTimeline.go
+39
-43
xrpc/feed/getTimeline.go
···
1
1
package feed
2
2
3
3
import (
4
-
"log/slog"
4
+
"context"
5
5
"net/http"
6
6
"strconv"
7
7
"time"
8
8
9
9
"github.com/labstack/echo/v4"
10
10
"github.com/whyrusleeping/konbini/hydration"
11
-
"github.com/whyrusleeping/konbini/views"
11
+
"go.opentelemetry.io/otel"
12
12
"gorm.io/gorm"
13
13
)
14
+
15
+
var tracer = otel.Tracer("xrpc/feed")
14
16
15
17
// HandleGetTimeline implements app.bsky.feed.getTimeline
16
18
func HandleGetTimeline(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
19
+
ctx := c.Request().Context()
20
+
ctx, span := tracer.Start(ctx, "getTimeline")
21
+
defer span.End()
22
+
17
23
viewer := getUserDID(c)
18
24
if viewer == "" {
19
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
25
+
return c.JSON(http.StatusUnauthorized, map[string]any{
20
26
"error": "AuthenticationRequired",
21
27
"message": "authentication required",
22
28
})
···
38
44
}
39
45
}
40
46
41
-
ctx := c.Request().Context()
42
-
43
47
// Get viewer's repo ID
44
48
var viewerRepoID uint
45
49
if err := db.Raw("SELECT id FROM repos WHERE did = ?", viewer).Scan(&viewerRepoID).Error; err != nil {
46
-
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
50
+
return c.JSON(http.StatusInternalServerError, map[string]any{
47
51
"error": "InternalError",
48
52
"message": "failed to load viewer",
49
53
})
50
54
}
51
55
52
56
// Query posts from followed users
53
-
type postRow struct {
54
-
URI string
55
-
AuthorID uint
56
-
}
57
-
var rows []postRow
58
-
err := db.Raw(`
59
-
SELECT
60
-
'at://' || r.did || '/app.bsky.feed.post/' || p.rkey as uri,
61
-
p.author as author_id
62
-
FROM posts p
63
-
JOIN repos r ON r.id = p.author
64
-
WHERE p.reply_to = 0
65
-
AND p.author IN (SELECT subject FROM follows WHERE author = ?)
66
-
AND p.created < ?
67
-
AND p.not_found = false
68
-
ORDER BY p.created DESC
69
-
LIMIT ?
70
-
`, viewerRepoID, cursor, limit).Scan(&rows).Error
71
57
58
+
rows, err := getTimelinePosts(ctx, db, viewerRepoID, cursor, limit)
72
59
if err != nil {
73
-
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
60
+
return c.JSON(http.StatusInternalServerError, map[string]any{
74
61
"error": "InternalError",
75
62
"message": "failed to query timeline",
76
63
})
77
64
}
78
65
79
66
// Hydrate posts
80
-
feed := make([]interface{}, 0)
81
-
for _, row := range rows {
82
-
postInfo, err := hydrator.HydratePost(ctx, row.URI, viewer)
83
-
if err != nil {
84
-
continue
85
-
}
86
-
87
-
// Hydrate author
88
-
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
89
-
if err != nil {
90
-
slog.Error("failed to hydrate actor", "author", postInfo.Author, "error", err)
91
-
continue
92
-
}
93
-
94
-
feedItem := views.FeedViewPost(postInfo, authorInfo)
95
-
feed = append(feed, feedItem)
96
-
}
67
+
feed := hydratePostRows(ctx, hydrator, viewer, rows)
97
68
98
69
// Generate next cursor
99
70
var nextCursor string
···
111
82
}
112
83
}
113
84
114
-
return c.JSON(http.StatusOK, map[string]interface{}{
85
+
return c.JSON(http.StatusOK, map[string]any{
115
86
"feed": feed,
116
87
"cursor": nextCursor,
117
88
})
118
89
}
90
+
91
+
func getTimelinePosts(ctx context.Context, db *gorm.DB, uid uint, cursor time.Time, limit int) ([]postRow, error) {
92
+
ctx, span := tracer.Start(ctx, "getTimelineQuery")
93
+
defer span.End()
94
+
95
+
var rows []postRow
96
+
err := db.Raw(`
97
+
SELECT
98
+
'at://' || r.did || '/app.bsky.feed.post/' || p.rkey as uri,
99
+
p.author as author_id
100
+
FROM posts p
101
+
JOIN repos r ON r.id = p.author
102
+
WHERE p.reply_to = 0
103
+
AND p.author IN (SELECT subject FROM follows WHERE author = ?)
104
+
AND p.created < ?
105
+
AND p.not_found = false
106
+
ORDER BY p.created DESC
107
+
LIMIT ?
108
+
`, uid, cursor, limit).Scan(&rows).Error
109
+
110
+
if err != nil {
111
+
return nil, err
112
+
}
113
+
return rows, nil
114
+
}
+241
-30
xrpc/notification/listNotifications.go
+241
-30
xrpc/notification/listNotifications.go
···
1
1
package notification
2
2
3
3
import (
4
+
"bytes"
5
+
"fmt"
4
6
"net/http"
5
7
"strconv"
8
+
"time"
6
9
10
+
"github.com/bluesky-social/indigo/api/atproto"
11
+
"github.com/bluesky-social/indigo/api/bsky"
12
+
"github.com/bluesky-social/indigo/lex/util"
13
+
lexutil "github.com/bluesky-social/indigo/lex/util"
7
14
"github.com/labstack/echo/v4"
8
15
"github.com/whyrusleeping/konbini/hydration"
16
+
models "github.com/whyrusleeping/konbini/models"
9
17
"github.com/whyrusleeping/konbini/views"
10
18
"gorm.io/gorm"
19
+
"gorm.io/gorm/clause"
11
20
)
12
21
13
22
// HandleListNotifications implements app.bsky.notification.listNotifications
14
23
func HandleListNotifications(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
15
24
viewer := getUserDID(c)
16
25
if viewer == "" {
17
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
26
+
return c.JSON(http.StatusUnauthorized, map[string]any{
18
27
"error": "AuthenticationRequired",
19
28
"message": "authentication required",
20
29
})
···
69
78
}
70
79
query += ` ORDER BY n.created_at DESC LIMIT ?`
71
80
72
-
var queryArgs []interface{}
81
+
var queryArgs []any
73
82
queryArgs = append(queryArgs, viewer)
74
83
if cursor > 0 {
75
84
queryArgs = append(queryArgs, cursor)
···
77
86
queryArgs = append(queryArgs, limit)
78
87
79
88
if err := db.Raw(query, queryArgs...).Scan(&rows).Error; err != nil {
80
-
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
89
+
return c.JSON(http.StatusInternalServerError, map[string]any{
81
90
"error": "InternalError",
82
91
"message": "failed to query notifications",
83
92
})
84
93
}
85
94
86
95
// Hydrate notifications
87
-
notifications := make([]interface{}, 0)
96
+
notifications := make([]*bsky.NotificationListNotifications_Notification, 0)
88
97
for _, row := range rows {
89
98
authorInfo, err := hydrator.HydrateActor(ctx, row.AuthorDid)
90
99
if err != nil {
91
100
continue
92
101
}
93
102
94
-
notif := map[string]interface{}{
95
-
"uri": row.Source,
96
-
"author": views.ProfileView(authorInfo),
97
-
"reason": mapNotifKind(row.Kind),
98
-
"record": nil, // Could hydrate the source record here
99
-
"isRead": false,
100
-
"indexedAt": row.CreatedAt,
101
-
"labels": []interface{}{},
103
+
// Skip notifications without CIDs as they're invalid
104
+
if row.SourceCid == "" {
105
+
continue
102
106
}
103
107
104
-
// Only include CID if we have one (required field)
105
-
if row.SourceCid != "" {
106
-
notif["cid"] = row.SourceCid
107
-
} else {
108
-
// Skip notifications without CIDs as they're invalid
108
+
// Fetch and decode the raw record
109
+
recordDecoder, err := fetchNotificationRecord(db, row.Source, row.Kind)
110
+
if err != nil {
109
111
continue
110
112
}
111
113
114
+
notif := &bsky.NotificationListNotifications_Notification{
115
+
Uri: row.Source,
116
+
Cid: row.SourceCid,
117
+
Author: views.ProfileView(authorInfo),
118
+
Reason: mapNotifKind(row.Kind),
119
+
Record: recordDecoder,
120
+
IsRead: false,
121
+
IndexedAt: row.CreatedAt,
122
+
}
123
+
112
124
notifications = append(notifications, notif)
113
125
}
114
126
115
127
// Generate next cursor
116
-
var nextCursor string
128
+
var cursorPtr *string
117
129
if len(rows) > 0 {
118
-
nextCursor = strconv.FormatUint(uint64(rows[len(rows)-1].ID), 10)
130
+
cursor := strconv.FormatUint(uint64(rows[len(rows)-1].ID), 10)
131
+
cursorPtr = &cursor
119
132
}
120
133
121
-
return c.JSON(http.StatusOK, map[string]interface{}{
122
-
"notifications": notifications,
123
-
"cursor": nextCursor,
124
-
})
134
+
var lastSeen time.Time
135
+
if err := db.Raw("SELECT seen_at FROM notification_seens WHERE repo = (select id from repos where did = ?)", viewer).Scan(&lastSeen).Error; err != nil {
136
+
return err
137
+
}
138
+
139
+
var lastSeenStr *string
140
+
if !lastSeen.IsZero() {
141
+
s := lastSeen.Format(time.RFC3339)
142
+
lastSeenStr = &s
143
+
}
144
+
145
+
output := &bsky.NotificationListNotifications_Output{
146
+
Notifications: notifications,
147
+
Cursor: cursorPtr,
148
+
SeenAt: lastSeenStr,
149
+
}
150
+
151
+
return c.JSON(http.StatusOK, output)
125
152
}
126
153
127
154
// HandleGetUnreadCount implements app.bsky.notification.getUnreadCount
128
155
func HandleGetUnreadCount(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
129
156
viewer := getUserDID(c)
130
157
if viewer == "" {
131
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
158
+
return c.JSON(http.StatusUnauthorized, map[string]any{
132
159
"error": "AuthenticationRequired",
133
160
"message": "authentication required",
134
161
})
135
162
}
136
163
137
-
// For now, return 0 - we'd need to track read state in the database
138
-
return c.JSON(http.StatusOK, map[string]interface{}{
139
-
"count": 0,
164
+
var repo models.Repo
165
+
if err := db.Find(&repo, "did = ?", viewer).Error; err != nil {
166
+
return err
167
+
}
168
+
169
+
var lastSeen time.Time
170
+
if err := db.Raw("SELECT seen_at FROM notification_seens WHERE repo = ?", repo.ID).Scan(&lastSeen).Error; err != nil {
171
+
return err
172
+
}
173
+
174
+
var count int
175
+
query := `SELECT count(*) FROM notifications WHERE created_at > ? AND for = ?`
176
+
if err := db.Raw(query, lastSeen, repo.ID).Scan(&count).Error; err != nil {
177
+
return c.JSON(http.StatusInternalServerError, map[string]any{
178
+
"error": "InternalError",
179
+
"message": "failed to count unread notifications",
180
+
})
181
+
}
182
+
183
+
return c.JSON(http.StatusOK, map[string]any{
184
+
"count": count,
140
185
})
141
186
}
142
187
···
144
189
func HandleUpdateSeen(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
145
190
viewer := getUserDID(c)
146
191
if viewer == "" {
147
-
return c.JSON(http.StatusUnauthorized, map[string]interface{}{
192
+
return c.JSON(http.StatusUnauthorized, map[string]any{
148
193
"error": "AuthenticationRequired",
149
194
"message": "authentication required",
150
195
})
151
196
}
152
197
153
-
// For now, just return success - we'd need to track seen timestamps in the database
154
-
return c.JSON(http.StatusOK, map[string]interface{}{})
198
+
var body bsky.NotificationUpdateSeen_Input
199
+
if err := c.Bind(&body); err != nil {
200
+
return c.JSON(http.StatusBadRequest, map[string]any{
201
+
"error": "InvalidRequest",
202
+
"message": "invalid request body",
203
+
})
204
+
}
205
+
206
+
// Parse the seenAt timestamp
207
+
seenAt, err := time.Parse(time.RFC3339, body.SeenAt)
208
+
if err != nil {
209
+
return c.JSON(http.StatusBadRequest, map[string]any{
210
+
"error": "InvalidRequest",
211
+
"message": "invalid seenAt timestamp",
212
+
})
213
+
}
214
+
215
+
// Get the viewer's repo ID
216
+
var repoID uint
217
+
if err := db.Raw("SELECT id FROM repos WHERE did = ?", viewer).Scan(&repoID).Error; err != nil {
218
+
return c.JSON(http.StatusInternalServerError, map[string]any{
219
+
"error": "InternalError",
220
+
"message": "failed to find viewer repo",
221
+
})
222
+
}
223
+
224
+
if repoID == 0 {
225
+
return c.JSON(http.StatusInternalServerError, map[string]any{
226
+
"error": "InternalError",
227
+
"message": "viewer repo not found",
228
+
})
229
+
}
230
+
231
+
// Upsert the NotificationSeen record
232
+
notifSeen := models.NotificationSeen{
233
+
Repo: repoID,
234
+
SeenAt: seenAt,
235
+
}
236
+
237
+
err = db.Clauses(clause.OnConflict{
238
+
Columns: []clause.Column{{Name: "repo"}},
239
+
DoUpdates: clause.AssignmentColumns([]string{"seen_at"}),
240
+
}).Create(¬ifSeen).Error
241
+
242
+
if err != nil {
243
+
return c.JSON(http.StatusInternalServerError, map[string]any{
244
+
"error": "InternalError",
245
+
"message": "failed to update seen timestamp",
246
+
})
247
+
}
248
+
249
+
return c.JSON(http.StatusOK, map[string]any{})
155
250
}
156
251
157
252
func getUserDID(c echo.Context) string {
···
175
270
return "repost"
176
271
case "mention":
177
272
return "mention"
273
+
case "follow":
274
+
return "follow"
178
275
default:
179
276
return kind
180
277
}
181
278
}
279
+
280
+
// fetchNotificationRecord fetches and decodes the raw record for a notification
281
+
func fetchNotificationRecord(db *gorm.DB, sourceURI string, kind string) (*util.LexiconTypeDecoder, error) {
282
+
// Parse the source URI to extract DID and rkey
283
+
// URI format: at://did:plc:xxx/collection/rkey
284
+
did := extractDIDFromURI(sourceURI)
285
+
rkey := extractRkeyFromURI(sourceURI)
286
+
287
+
if did == "" || rkey == "" {
288
+
return nil, fmt.Errorf("invalid source URI")
289
+
}
290
+
291
+
var raw []byte
292
+
var err error
293
+
294
+
// Fetch raw data based on notification kind
295
+
switch kind {
296
+
case "reply", "mention", "quote":
297
+
// These reference posts
298
+
err = db.Raw(`
299
+
SELECT p.raw
300
+
FROM posts p
301
+
JOIN repos r ON r.id = p.author
302
+
WHERE r.did = ? AND p.rkey = ?
303
+
`, did, rkey).Scan(&raw).Error
304
+
305
+
case "like":
306
+
// we don't store the raw like objects, so we just reconstruct it here...
307
+
// These reference like records
308
+
var like models.Like
309
+
err = db.Raw(`
310
+
SELECT *
311
+
FROM likes l
312
+
JOIN repos r ON r.id = l.author
313
+
WHERE r.did = ? AND l.rkey = ?
314
+
`, did, rkey).Scan(&like).Error
315
+
316
+
lk := bsky.FeedLike{
317
+
CreatedAt: like.Created.Format(time.RFC3339),
318
+
Subject: &atproto.RepoStrongRef{
319
+
Cid: "",
320
+
Uri: "",
321
+
},
322
+
}
323
+
buf := new(bytes.Buffer)
324
+
if err := lk.MarshalCBOR(buf); err != nil {
325
+
return nil, fmt.Errorf("failed to marshal reconstructed like: %w", err)
326
+
}
327
+
raw = buf.Bytes()
328
+
329
+
case "repost":
330
+
// These reference repost records
331
+
err = db.Raw(`
332
+
SELECT r.raw
333
+
FROM reposts r
334
+
JOIN repos repo ON repo.id = r.author
335
+
WHERE repo.did = ? AND r.rkey = ?
336
+
`, did, rkey).Scan(&raw).Error
337
+
338
+
case "follow":
339
+
// These reference follow records
340
+
err = db.Raw(`
341
+
SELECT f.raw
342
+
FROM follows f
343
+
JOIN repos r ON r.id = f.author
344
+
WHERE r.did = ? AND f.rkey = ?
345
+
`, did, rkey).Scan(&raw).Error
346
+
347
+
default:
348
+
return nil, fmt.Errorf("unknown notification kind: %s", kind)
349
+
}
350
+
351
+
if err != nil || len(raw) == 0 {
352
+
return nil, fmt.Errorf("failed to fetch record: %w", err)
353
+
}
354
+
355
+
// Decode the CBOR data
356
+
decoded, err := lexutil.CborDecodeValue(raw)
357
+
if err != nil {
358
+
return nil, fmt.Errorf("failed to decode CBOR: %w", err)
359
+
}
360
+
361
+
return &util.LexiconTypeDecoder{
362
+
Val: decoded,
363
+
}, nil
364
+
}
365
+
366
+
func extractDIDFromURI(uri string) string {
367
+
// URI format: at://did:plc:xxx/collection/rkey
368
+
if len(uri) < 5 || uri[:5] != "at://" {
369
+
return ""
370
+
}
371
+
parts := []rune(uri[5:])
372
+
for i, r := range parts {
373
+
if r == '/' {
374
+
return string(parts[:i])
375
+
}
376
+
}
377
+
return string(parts)
378
+
}
379
+
380
+
func extractRkeyFromURI(uri string) string {
381
+
// URI format: at://did:plc:xxx/collection/rkey
382
+
if len(uri) < 5 || uri[:5] != "at://" {
383
+
return ""
384
+
}
385
+
// Find last slash
386
+
for i := len(uri) - 1; i >= 5; i-- {
387
+
if uri[i] == '/' {
388
+
return uri[i+1:]
389
+
}
390
+
}
391
+
return ""
392
+
}
+23
-7
xrpc/server.go
+23
-7
xrpc/server.go
···
1
1
package xrpc
2
2
3
3
import (
4
+
"context"
4
5
"log/slog"
5
6
"net/http"
6
7
7
8
"github.com/bluesky-social/indigo/atproto/identity"
8
9
"github.com/labstack/echo/v4"
9
10
"github.com/labstack/echo/v4/middleware"
11
+
"github.com/whyrusleeping/konbini/backend"
10
12
"github.com/whyrusleeping/konbini/hydration"
13
+
"github.com/whyrusleeping/konbini/models"
11
14
"github.com/whyrusleeping/konbini/xrpc/actor"
12
15
"github.com/whyrusleeping/konbini/xrpc/feed"
13
16
"github.com/whyrusleeping/konbini/xrpc/graph"
···
31
34
type Backend interface {
32
35
// Add methods as needed for data access
33
36
34
-
TrackMissingActor(did string)
37
+
TrackMissingRecord(identifier string, wait bool)
38
+
GetOrCreateRepo(ctx context.Context, did string) (*models.Repo, error)
35
39
}
36
40
37
41
// NewServer creates a new XRPC server
38
-
func NewServer(db *gorm.DB, dir identity.Directory, backend Backend) *Server {
42
+
func NewServer(db *gorm.DB, dir identity.Directory, backend *backend.PostgresBackend) *Server {
39
43
e := echo.New()
40
44
e.HidePort = true
41
45
e.HideBanner = true
···
56
60
db: db,
57
61
dir: dir,
58
62
backend: backend,
59
-
hydrator: hydration.NewHydrator(db, dir),
63
+
hydrator: hydration.NewHydrator(db, dir, backend),
60
64
}
61
-
62
-
s.hydrator.SetMissingActorCallback(backend.TrackMissingActor)
63
65
64
66
// Register XRPC endpoints
65
67
s.registerEndpoints()
···
76
78
// registerEndpoints registers all XRPC endpoints
77
79
func (s *Server) registerEndpoints() {
78
80
// XRPC endpoints follow the pattern: /xrpc/<namespace>.<method>
81
+
82
+
s.e.GET("/.well-known/did.json", func(c echo.Context) error {
83
+
return c.File("did.json")
84
+
})
85
+
79
86
xrpcGroup := s.e.Group("/xrpc")
80
87
81
88
// com.atproto.identity.*
···
89
96
// app.bsky.actor.*
90
97
xrpcGroup.GET("/app.bsky.actor.getProfile", func(c echo.Context) error {
91
98
return actor.HandleGetProfile(c, s.hydrator)
92
-
})
99
+
}, s.optionalAuth)
93
100
xrpcGroup.GET("/app.bsky.actor.getProfiles", func(c echo.Context) error {
94
101
return actor.HandleGetProfiles(c, s.db, s.hydrator)
95
-
})
102
+
}, s.optionalAuth)
96
103
xrpcGroup.GET("/app.bsky.actor.getPreferences", func(c echo.Context) error {
97
104
return actor.HandleGetPreferences(c, s.db, s.hydrator)
98
105
}, s.requireAuth)
···
124
131
xrpcGroup.GET("/app.bsky.feed.getActorLikes", func(c echo.Context) error {
125
132
return feed.HandleGetActorLikes(c, s.db, s.hydrator)
126
133
}, s.requireAuth)
134
+
xrpcGroup.GET("/app.bsky.feed.getFeed", func(c echo.Context) error {
135
+
return feed.HandleGetFeed(c, s.db, s.hydrator, s.dir)
136
+
}, s.optionalAuth)
137
+
xrpcGroup.GET("/app.bsky.feed.getFeedGenerator", func(c echo.Context) error {
138
+
return feed.HandleGetFeedGenerator(c, s.db, s.hydrator, s.dir)
139
+
})
127
140
128
141
// app.bsky.graph.*
129
142
xrpcGroup.GET("/app.bsky.graph.getFollows", func(c echo.Context) error {
···
166
179
})
167
180
xrpcGroup.GET("/app.bsky.unspecced.getTrendingTopics", func(c echo.Context) error {
168
181
return unspecced.HandleGetTrendingTopics(c)
182
+
})
183
+
xrpcGroup.GET("/app.bsky.unspecced.getPostThreadV2", func(c echo.Context) error {
184
+
return unspecced.HandleGetPostThreadV2(c, s.db, s.hydrator)
169
185
})
170
186
}
171
187
+362
xrpc/unspecced/getPostThreadV2.go
+362
xrpc/unspecced/getPostThreadV2.go
···
1
+
package unspecced
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"log/slog"
8
+
"net/http"
9
+
"strconv"
10
+
"sync"
11
+
12
+
"github.com/bluesky-social/indigo/api/bsky"
13
+
"github.com/labstack/echo/v4"
14
+
"github.com/whyrusleeping/konbini/hydration"
15
+
"github.com/whyrusleeping/konbini/views"
16
+
"github.com/whyrusleeping/market/models"
17
+
"go.opentelemetry.io/otel"
18
+
"gorm.io/gorm"
19
+
)
20
+
21
+
var tracer = otel.Tracer("xrpc/unspecced")
22
+
23
+
// HandleGetPostThreadV2 implements app.bsky.unspecced.getPostThreadV2
24
+
func HandleGetPostThreadV2(c echo.Context, db *gorm.DB, hydrator *hydration.Hydrator) error {
25
+
ctx, span := tracer.Start(c.Request().Context(), "getPostThreadV2")
26
+
defer span.End()
27
+
ctx = context.WithValue(ctx, "auto-fetch", true)
28
+
29
+
// Parse parameters
30
+
anchorRaw := c.QueryParam("anchor")
31
+
if anchorRaw == "" {
32
+
return c.JSON(http.StatusBadRequest, map[string]interface{}{
33
+
"error": "InvalidRequest",
34
+
"message": "anchor parameter is required",
35
+
})
36
+
}
37
+
38
+
anchorUri, err := hydrator.NormalizeUri(ctx, anchorRaw)
39
+
if err != nil {
40
+
return err
41
+
}
42
+
43
+
// Parse optional parameters with defaults
44
+
above := c.QueryParam("above") != "false" // default true
45
+
46
+
below := int64(6) // default
47
+
if belowParam := c.QueryParam("below"); belowParam != "" {
48
+
if b, err := strconv.ParseInt(belowParam, 10, 64); err == nil && b >= 0 && b <= 20 {
49
+
below = b
50
+
}
51
+
}
52
+
53
+
branchingFactor := int64(10) // default
54
+
if bfParam := c.QueryParam("branchingFactor"); bfParam != "" {
55
+
if bf, err := strconv.ParseInt(bfParam, 10, 64); err == nil && bf > 0 {
56
+
branchingFactor = bf
57
+
}
58
+
}
59
+
60
+
_ = c.QueryParam("prioritizeFollowedUsers") == "true" // TODO: implement prioritization
61
+
62
+
sort := c.QueryParam("sort")
63
+
if sort == "" {
64
+
sort = "newest"
65
+
}
66
+
67
+
viewer := getUserDID(c)
68
+
69
+
// Hydrate the anchor post
70
+
anchorPostInfo, err := hydrator.HydratePost(ctx, anchorUri, viewer)
71
+
if err != nil {
72
+
slog.Error("failed to hydrate post", "error", err, "anchor", anchorUri)
73
+
return c.JSON(http.StatusNotFound, map[string]interface{}{
74
+
"error": "NotFound",
75
+
"message": "anchor post not found",
76
+
})
77
+
}
78
+
79
+
threadID := anchorPostInfo.InThread
80
+
if threadID == 0 {
81
+
threadID = anchorPostInfo.ID
82
+
}
83
+
84
+
var threadPosts []*models.Post
85
+
if err := db.Raw("SELECT * FROM posts WHERE in_thread = ? OR id = ?", threadID, anchorPostInfo.ID).Scan(&threadPosts).Error; err != nil {
86
+
return err
87
+
}
88
+
89
+
fmt.Println("GOT THREAD POSTS: ", len(threadPosts))
90
+
91
+
treeNodes, err := buildThreadTree(ctx, hydrator, db, threadPosts)
92
+
if err != nil {
93
+
return fmt.Errorf("failed to construct tree: %w", err)
94
+
}
95
+
96
+
anchor := treeNodes[anchorPostInfo.ID]
97
+
98
+
// Build flat thread items list
99
+
var threadItems []*bsky.UnspeccedGetPostThreadV2_ThreadItem
100
+
hasOtherReplies := false
101
+
102
+
// Add parents if requested
103
+
if above {
104
+
parent := anchor.parent
105
+
depth := int64(-1)
106
+
for parent != nil {
107
+
if parent.missing {
108
+
fmt.Println("Parent missing: ", depth)
109
+
item := &bsky.UnspeccedGetPostThreadV2_ThreadItem{
110
+
Depth: depth,
111
+
Uri: parent.uri,
112
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
113
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
114
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
115
+
},
116
+
},
117
+
}
118
+
119
+
threadItems = append(threadItems, item)
120
+
break
121
+
}
122
+
123
+
item := buildThreadItem(ctx, hydrator, parent, depth, viewer)
124
+
if item != nil {
125
+
threadItems = append(threadItems, item)
126
+
}
127
+
128
+
parent = parent.parent
129
+
depth--
130
+
}
131
+
}
132
+
133
+
// Add anchor post (depth 0)
134
+
anchorItem := buildThreadItem(ctx, hydrator, anchor, 0, viewer)
135
+
if anchorItem != nil {
136
+
threadItems = append(threadItems, anchorItem)
137
+
}
138
+
139
+
// Add replies below anchor
140
+
if below > 0 {
141
+
replies, err := collectReplies(ctx, hydrator, anchor, 0, below, branchingFactor, sort, viewer)
142
+
if err != nil {
143
+
return err
144
+
}
145
+
threadItems = append(threadItems, replies...)
146
+
//hasOtherReplies = hasMore
147
+
}
148
+
149
+
return c.JSON(http.StatusOK, &bsky.UnspeccedGetPostThreadV2_Output{
150
+
Thread: threadItems,
151
+
HasOtherReplies: hasOtherReplies,
152
+
})
153
+
}
154
+
155
+
func collectReplies(ctx context.Context, hydrator *hydration.Hydrator, curnode *threadTree, depth int64, below int64, branchingFactor int64, sort string, viewer string) ([]*bsky.UnspeccedGetPostThreadV2_ThreadItem, error) {
156
+
if below == 0 {
157
+
return nil, nil
158
+
}
159
+
160
+
type parThreadResults struct {
161
+
node *bsky.UnspeccedGetPostThreadV2_ThreadItem
162
+
children []*bsky.UnspeccedGetPostThreadV2_ThreadItem
163
+
}
164
+
165
+
results := make([]parThreadResults, len(curnode.children))
166
+
167
+
var wg sync.WaitGroup
168
+
for i := range curnode.children {
169
+
ix := i
170
+
wg.Go(func() {
171
+
child := curnode.children[ix]
172
+
173
+
results[ix].node = buildThreadItem(ctx, hydrator, child, depth+1, viewer)
174
+
if child.missing {
175
+
return
176
+
}
177
+
178
+
sub, err := collectReplies(ctx, hydrator, child, depth+1, below-1, branchingFactor, sort, viewer)
179
+
if err != nil {
180
+
slog.Error("failed to collect replies", "node", child.uri, "error", err)
181
+
return
182
+
}
183
+
184
+
results[ix].children = sub
185
+
})
186
+
}
187
+
188
+
wg.Wait()
189
+
190
+
var out []*bsky.UnspeccedGetPostThreadV2_ThreadItem
191
+
for _, res := range results {
192
+
out = append(out, res.node)
193
+
out = append(out, res.children...)
194
+
}
195
+
196
+
return out, nil
197
+
}
198
+
199
+
func buildThreadItem(ctx context.Context, hydrator *hydration.Hydrator, node *threadTree, depth int64, viewer string) *bsky.UnspeccedGetPostThreadV2_ThreadItem {
200
+
if node.missing {
201
+
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
202
+
Depth: depth,
203
+
Uri: node.uri,
204
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
205
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
206
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
207
+
},
208
+
},
209
+
}
210
+
}
211
+
212
+
// Hydrate the post
213
+
postInfo, err := hydrator.HydratePostDB(ctx, node.uri, node.val, viewer)
214
+
if err != nil {
215
+
slog.Error("failed to hydrate post in thread item", "uri", node.uri, "error", err)
216
+
// Return not found item
217
+
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
218
+
Depth: depth,
219
+
Uri: node.uri,
220
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
221
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
222
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
223
+
},
224
+
},
225
+
}
226
+
}
227
+
228
+
// Hydrate author
229
+
authorInfo, err := hydrator.HydrateActor(ctx, postInfo.Author)
230
+
if err != nil {
231
+
slog.Error("failed to hydrate actor in thread item", "author", postInfo.Author, "error", err)
232
+
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
233
+
Depth: depth,
234
+
Uri: node.uri,
235
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
236
+
UnspeccedDefs_ThreadItemNotFound: &bsky.UnspeccedDefs_ThreadItemNotFound{
237
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemNotFound",
238
+
},
239
+
},
240
+
}
241
+
}
242
+
243
+
// Build post view
244
+
postView := views.PostView(postInfo, authorInfo)
245
+
246
+
// Calculate moreReplies count
247
+
moreReplies := int64(0)
248
+
if len(node.children) > 0 {
249
+
// This is a simplified calculation - actual count would need more complex logic
250
+
moreReplies = int64(len(node.children))
251
+
}
252
+
253
+
return &bsky.UnspeccedGetPostThreadV2_ThreadItem{
254
+
Depth: depth,
255
+
Uri: node.uri,
256
+
Value: &bsky.UnspeccedGetPostThreadV2_ThreadItem_Value{
257
+
UnspeccedDefs_ThreadItemPost: &bsky.UnspeccedDefs_ThreadItemPost{
258
+
LexiconTypeID: "app.bsky.unspecced.defs#threadItemPost",
259
+
Post: postView,
260
+
HiddenByThreadgate: false,
261
+
MoreParents: false,
262
+
MoreReplies: moreReplies,
263
+
MutedByViewer: false,
264
+
OpThread: false, // TODO: Calculate this properly
265
+
},
266
+
},
267
+
}
268
+
}
269
+
270
+
func getUserDID(c echo.Context) string {
271
+
did := c.Get("viewer")
272
+
if did == nil {
273
+
return ""
274
+
}
275
+
if s, ok := did.(string); ok {
276
+
return s
277
+
}
278
+
return ""
279
+
}
280
+
281
+
func extractDIDFromURI(uri string) string {
282
+
// URI format: at://did:plc:xxx/collection/rkey
283
+
if len(uri) < 5 || uri[:5] != "at://" {
284
+
return ""
285
+
}
286
+
parts := []rune(uri[5:])
287
+
for i, r := range parts {
288
+
if r == '/' {
289
+
return string(parts[:i])
290
+
}
291
+
}
292
+
return string(parts)
293
+
}
294
+
295
+
type threadTree struct {
296
+
parent *threadTree
297
+
children []*threadTree
298
+
299
+
val *models.Post
300
+
301
+
missing bool
302
+
303
+
uri string
304
+
cid string
305
+
}
306
+
307
+
func buildThreadTree(ctx context.Context, hydrator *hydration.Hydrator, db *gorm.DB, posts []*models.Post) (map[uint]*threadTree, error) {
308
+
nodes := make(map[uint]*threadTree)
309
+
for _, p := range posts {
310
+
puri, err := hydrator.UriForPost(ctx, p)
311
+
if err != nil {
312
+
return nil, err
313
+
}
314
+
315
+
t := &threadTree{
316
+
val: p,
317
+
uri: puri,
318
+
}
319
+
320
+
nodes[p.ID] = t
321
+
}
322
+
323
+
missing := make(map[uint]*threadTree)
324
+
for _, node := range nodes {
325
+
if node.val.ReplyTo == 0 {
326
+
continue
327
+
}
328
+
329
+
pnode, ok := nodes[node.val.ReplyTo]
330
+
if !ok {
331
+
pnode = &threadTree{
332
+
missing: true,
333
+
}
334
+
missing[node.val.ReplyTo] = pnode
335
+
336
+
var bspost bsky.FeedPost
337
+
if err := bspost.UnmarshalCBOR(bytes.NewReader(node.val.Raw)); err != nil {
338
+
return nil, err
339
+
}
340
+
341
+
if bspost.Reply == nil || bspost.Reply.Parent == nil {
342
+
return nil, fmt.Errorf("node with parent had no parent in object")
343
+
}
344
+
345
+
pnode.uri = bspost.Reply.Parent.Uri
346
+
pnode.cid = bspost.Reply.Parent.Cid
347
+
348
+
/* Maybe we could force hydrate these?
349
+
hydrator.AddMissingRecord(puri, true)
350
+
*/
351
+
}
352
+
353
+
pnode.children = append(pnode.children, node)
354
+
node.parent = pnode
355
+
}
356
+
357
+
for k, v := range missing {
358
+
nodes[k] = v
359
+
}
360
+
361
+
return nodes, nil
362
+
}