commit 646c17e2a0b8a8a7f116a6d0972b4cdabdf01938 · edavis.dev/bsky-feeds

+2

.gitignore

··· 1 + bin/ 2 + db/

+7

Makefile

··· 1 + all: bin/mostliked bin/feedweb 2 + 3 + bin/mostliked: cmd/mostliked/*.go pkg/mostliked/*.go 4 + go build -o $@ ./cmd/mostliked 5 + 6 + bin/feedweb: cmd/feedweb/*.go pkg/*/*.go mostliked/*.go 7 + go build -o $@ ./cmd/feedweb

+61

cmd/feedweb/feedweb.go

··· 1 + package main 2 + 3 + import ( 4 + "net/http" 5 + 6 + "github.com/labstack/echo/v4" 7 + "github.com/labstack/echo/v4/middleware" 8 + "github.com/edavis/bsky-feeds/mostliked" 9 + ) 10 + 11 + type SkeletonRequest struct { 12 + Feed string `query:"feed"` 13 + Limit int64 `query:"limit"` 14 + Offset string `query:"offset"` 15 + } 16 + 17 + type SkeletonResponse struct { 18 + Cursor string `json:"cursor,omitempty"` 19 + Feed []Post `json:"feed"` 20 + } 21 + 22 + type Post struct { 23 + Uri string `json:"post"` 24 + } 25 + 26 + type SkeletonHeader struct { 27 + Langs []string `header:"Accept-Language"` 28 + } 29 + 30 + func getFeedSkeleton(c echo.Context) error { 31 + var req SkeletonRequest 32 + if err := c.Bind(&req); err != nil { 33 + return c.String(http.StatusBadRequest, "bad request") 34 + } 35 + var hdr SkeletonHeader 36 + if err := (&echo.DefaultBinder{}).BindHeaders(c, &hdr); err != nil { 37 + return c.String(http.StatusBadRequest, "bad request") 38 + } 39 + 40 + var posts []Post 41 + uris := mostliked.Feed(mostliked.FeedViewParams{ 42 + Limit: req.Limit, 43 + Offset: req.Offset, 44 + Langs: hdr.Langs, 45 + }) 46 + for _, uri := range uris { 47 + posts = append(posts, Post{uri}) 48 + } 49 + response := SkeletonResponse{ 50 + Feed: posts, 51 + } 52 + return c.JSON(http.StatusOK, response) 53 + } 54 + 55 + func main() { 56 + e := echo.New() 57 + e.Use(middleware.Logger()) 58 + e.Use(middleware.Recover()) 59 + e.GET("/xrpc/app.bsky.feed.getFeedSkeleton", getFeedSkeleton) 60 + e.Logger.Fatal(e.Start(":5000")) 61 + }

+8 -8

cmd/mostliked/mostliked.go

··· 12 12 13 13 appbsky "github.com/bluesky-social/indigo/api/bsky" 14 14 jetstream "github.com/bluesky-social/jetstream/pkg/models" 15 - "github.com/edavis/bsky-feeds/pkg/mostliked" 15 + db "github.com/edavis/bsky-feeds/pkg/mostliked" 16 16 "github.com/gorilla/websocket" 17 17 "github.com/karlseguin/ccache/v3" 18 18 _ "github.com/mattn/go-sqlite3" ··· 60 60 } 61 61 } 62 62 63 - func trimPostsTable(ctx context.Context, queries *mostliked.Queries) { 63 + func trimPostsTable(ctx context.Context, queries *db.Queries) { 64 64 ticker := time.NewTicker(1 * time.Minute) 65 65 defer ticker.Stop() 66 66 ··· 94 94 func processEvents(events <-chan []byte) { 95 95 ctx := context.Background() 96 96 97 - db, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on") 97 + dbCnx, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on") 98 98 if err != nil { 99 99 log.Fatal("error opening db") 100 100 } 101 - if _, err := db.ExecContext(ctx, ddl); err != nil { 101 + if _, err := dbCnx.ExecContext(ctx, ddl); err != nil { 102 102 log.Fatal("couldn't create tables") 103 103 } 104 - defer db.Close() 104 + defer dbCnx.Close() 105 105 106 - queries := mostliked.New(db) 106 + queries := db.New(dbCnx) 107 107 108 108 drafts := ccache.New(ccache.Configure[DraftPost]().MaxSize(50_000).GetsPerPromote(1)) 109 109 ··· 174 174 } 175 175 drafts.Delete(like.Subject.Uri) 176 176 log.Println("storing", like.Subject.Uri, "in database") 177 - err := queries.InsertPost(ctx, mostliked.InsertPostParams{ 177 + err := queries.InsertPost(ctx, db.InsertPostParams{ 178 178 Uri: like.Subject.Uri, 179 179 CreateTs: draftPost.Created, 180 180 Likes: draftPost.Likes, ··· 182 182 if err != nil { 183 183 log.Println("error inserting post") 184 184 } 185 - err = queries.InsertLang(ctx, mostliked.InsertLangParams{ 185 + err = queries.InsertLang(ctx, db.InsertLangParams{ 186 186 Uri: like.Subject.Uri, 187 187 Lang: strings.ToLower(draftPost.Language.IsoCode639_1().String()), 188 188 })

+7

cmd/mostliked/query.sql

··· 9 9 10 10 -- name: TrimPosts :exec 11 11 delete from posts where create_ts < unixepoch('now', '-24 hours'); 12 + 13 + -- name: ViewFeed :many 14 + select posts.uri, create_ts, likes, lang 15 + from posts 16 + left join langs on posts.uri = langs.uri 17 + order by likes desc 18 + limit ? offset ?;

+9

go.mod

··· 7 7 github.com/bluesky-social/jetstream v0.0.0-20240925210745-2cd1b6147279 8 8 github.com/gorilla/websocket v1.5.3 9 9 github.com/karlseguin/ccache/v3 v3.0.5 10 + github.com/labstack/echo/v4 v4.11.3 10 11 github.com/mattn/go-sqlite3 v1.14.22 11 12 github.com/pemistahl/lingua-go v1.4.0 12 13 ) ··· 18 19 github.com/go-logr/stdr v1.2.2 // indirect 19 20 github.com/goccy/go-json v0.10.2 // indirect 20 21 github.com/gogo/protobuf v1.3.2 // indirect 22 + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect 21 23 github.com/google/uuid v1.6.0 // indirect 22 24 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 23 25 github.com/hashicorp/go-retryablehttp v0.7.5 // indirect ··· 36 38 github.com/ipfs/go-metrics-interface v0.0.1 // indirect 37 39 github.com/jbenet/goprocess v0.1.4 // indirect 38 40 github.com/klauspost/cpuid/v2 v2.2.7 // indirect 41 + github.com/labstack/gommon v0.4.1 // indirect 42 + github.com/mattn/go-colorable v0.1.13 // indirect 39 43 github.com/mattn/go-isatty v0.0.20 // indirect 40 44 github.com/minio/sha256-simd v1.0.1 // indirect 41 45 github.com/mr-tron/base58 v1.2.0 // indirect ··· 48 52 github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect 49 53 github.com/shopspring/decimal v1.3.1 // indirect 50 54 github.com/spaolacci/murmur3 v1.1.0 // indirect 55 + github.com/valyala/bytebufferpool v1.0.0 // indirect 56 + github.com/valyala/fasttemplate v1.2.2 // indirect 51 57 github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c // indirect 52 58 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect 53 59 go.opentelemetry.io/otel v1.21.0 // indirect ··· 58 64 go.uber.org/zap v1.26.0 // indirect 59 65 golang.org/x/crypto v0.22.0 // indirect 60 66 golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect 67 + golang.org/x/net v0.24.0 // indirect 61 68 golang.org/x/sys v0.22.0 // indirect 69 + golang.org/x/text v0.16.0 // indirect 70 + golang.org/x/time v0.5.0 // indirect 62 71 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 63 72 google.golang.org/protobuf v1.34.2 // indirect 64 73 lukechampine.com/blake3 v1.2.1 // indirect

+20

go.sum

··· 22 22 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= 23 23 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 24 24 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 25 + github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= 26 + github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= 25 27 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 26 28 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 27 29 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= ··· 84 86 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 85 87 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 86 88 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 89 + github.com/labstack/echo/v4 v4.11.3 h1:Upyu3olaqSHkCjs1EJJwQ3WId8b8b1hxbogyommKktM= 90 + github.com/labstack/echo/v4 v4.11.3/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws= 91 + github.com/labstack/gommon v0.4.1 h1:gqEff0p/hTENGMABzezPoPSRtIh1Cvw0ueMOe0/dfOk= 92 + github.com/labstack/gommon v0.4.1/go.mod h1:TyTrpPqxR5KMk8LKVtLmfMjeQ5FEkBYdxLYPw/WfrOM= 93 + github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 94 + github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 87 95 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 96 + github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 88 97 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 89 98 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 90 99 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= ··· 133 142 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 134 143 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 135 144 github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 145 + github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 146 + github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 147 + github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 148 + github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 136 149 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= 137 150 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= 138 151 github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c h1:UsxJNcLPfyLyVaA4iusIrsLAqJn/xh36Qgb8emqtXzk= ··· 183 196 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 184 197 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 185 198 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 199 + golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= 200 + golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= 186 201 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 187 202 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 203 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= ··· 194 209 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 195 210 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 196 211 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 212 + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 197 213 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 198 214 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 199 215 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= ··· 201 217 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 202 218 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 203 219 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 220 + golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= 221 + golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= 222 + golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= 223 + golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 204 224 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 205 225 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 206 226 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=

+45

mostliked/feed.go

··· 1 + package mostliked 2 + 3 + import ( 4 + "context" 5 + "database/sql" 6 + "log" 7 + "strconv" 8 + 9 + db "github.com/edavis/bsky-feeds/pkg/mostliked" 10 + _ "github.com/mattn/go-sqlite3" 11 + ) 12 + 13 + type FeedViewParams struct { 14 + Limit int64 15 + Offset string 16 + Langs []string 17 + } 18 + 19 + func Feed(args FeedViewParams) []string { 20 + ctx := context.Background() 21 + dbCnx, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on") 22 + if err != nil { 23 + log.Fatal("error opening db") 24 + } 25 + defer dbCnx.Close() 26 + 27 + offset, err := strconv.Atoi(args.Offset) 28 + if err != nil { 29 + log.Println("error converting offset to integer") 30 + } 31 + 32 + queries := db.New(dbCnx) 33 + rows, err := queries.ViewFeed(ctx, db.ViewFeedParams{ 34 + Limit: args.Limit, 35 + Offset: int64(offset), 36 + }) 37 + if err != nil { 38 + log.Println("error fetching rows") 39 + } 40 + var uris []string 41 + for _, row := range rows { 42 + uris = append(uris, row.Uri) 43 + } 44 + return uris 45 + }

+49

pkg/mostliked/query.sql.go

··· 7 7 8 8 import ( 9 9 "context" 10 + "database/sql" 10 11 ) 11 12 12 13 const insertLang = `-- name: InsertLang :exec ··· 55 56 _, err := q.db.ExecContext(ctx, updateLikes, uri) 56 57 return err 57 58 } 59 + 60 + const viewFeed = `-- name: ViewFeed :many 61 + select posts.uri, create_ts, likes, lang 62 + from posts 63 + left join langs on posts.uri = langs.uri 64 + order by likes desc 65 + limit ? offset ? 66 + ` 67 + 68 + type ViewFeedParams struct { 69 + Limit int64 70 + Offset int64 71 + } 72 + 73 + type ViewFeedRow struct { 74 + Uri string 75 + CreateTs int64 76 + Likes int64 77 + Lang sql.NullString 78 + } 79 + 80 + func (q *Queries) ViewFeed(ctx context.Context, arg ViewFeedParams) ([]ViewFeedRow, error) { 81 + rows, err := q.db.QueryContext(ctx, viewFeed, arg.Limit, arg.Offset) 82 + if err != nil { 83 + return nil, err 84 + } 85 + defer rows.Close() 86 + var items []ViewFeedRow 87 + for rows.Next() { 88 + var i ViewFeedRow 89 + if err := rows.Scan( 90 + &i.Uri, 91 + &i.CreateTs, 92 + &i.Likes, 93 + &i.Lang, 94 + ); err != nil { 95 + return nil, err 96 + } 97 + items = append(items, i) 98 + } 99 + if err := rows.Close(); err != nil { 100 + return nil, err 101 + } 102 + if err := rows.Err(); err != nil { 103 + return nil, err 104 + } 105 + return items, nil 106 + }