+31

.github/workflows/sync-internal.yaml

··· 1 1 + name: Sync to internal repo 2 2 + 3 3 + on: 4 4 + push: 5 5 + branches: [main] 6 6 + 7 7 + jobs: 8 8 + sync: 9 9 + runs-on: ubuntu-latest 10 10 + if: github.repository == 'bluesky-social/indigo' 11 11 + steps: 12 12 + - name: Checkout public repo 13 13 + uses: actions/checkout@v4 14 14 + with: 15 15 + fetch-depth: 0 16 16 + - name: Generate GitHub App Token 17 17 + id: app-token 18 18 + uses: actions/create-github-app-token@v1 19 19 + with: 20 20 + app-id: ${{ vars.SYNC_INTERNAL_APP_ID }} 21 21 + private-key: ${{ secrets.SYNC_INTERNAL_PK }} 22 22 + repositories: indigo-internal 23 23 + - name: Push to internal repo 24 24 + env: 25 25 + TOKEN: ${{ steps.app-token.outputs.token }} 26 26 + run: | 27 27 + git config user.name "github-actions" 28 28 + git config user.email "test@users.noreply.github.com" 29 29 + git config --unset-all http.https://github.com/.extraheader 30 30 + git remote add internal https://x-access-token:${TOKEN}@github.com/bluesky-social/indigo-internal.git 31 31 + git push internal main --force

+2

atproto/label/label.go

··· 140 140 Cid: l.CID, 141 141 Cts: l.CreatedAt, 142 142 Exp: l.ExpiresAt, 143 143 + Neg: l.Negated, 143 144 Sig: []byte(l.Sig), 144 145 Src: l.SourceDID, 145 146 Uri: l.URI, ··· 157 158 CID: l.Cid, 158 159 CreatedAt: l.Cts, 159 160 ExpiresAt: l.Exp, 161 161 + Negated: l.Neg, 160 162 Sig: []byte(l.Sig), 161 163 SourceDID: l.Src, 162 164 URI: l.Uri,

+62

atproto/label/label_test.go

··· 4 4 "encoding/json" 5 5 "testing" 6 6 7 7 + comatproto "github.com/bluesky-social/indigo/api/atproto" 7 8 "github.com/bluesky-social/indigo/atproto/crypto" 8 9 9 10 "github.com/stretchr/testify/assert" ··· 89 90 assert.NoError(l.Sign(priv)) 90 91 assert.NoError(l.VerifySignature(pub)) 91 92 } 93 93 + 94 94 + func TestToLexicon(t *testing.T) { 95 95 + assert := assert.New(t) 96 96 + 97 97 + expiresAt := "2025-07-28T23:53:19.804Z" 98 98 + negated := true 99 99 + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" 100 100 + 101 101 + l := Label{ 102 102 + CID: &cid, 103 103 + CreatedAt: "2024-10-23T17:51:19.128Z", 104 104 + ExpiresAt: &expiresAt, 105 105 + Negated: &negated, 106 106 + SourceDID: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", 107 107 + URI: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", 108 108 + Val: "good", 109 109 + Version: ATPROTO_LABEL_VERSION, 110 110 + Sig: []byte("sig"), // invalid, but we only care about the conversion 111 111 + } 112 112 + 113 113 + lex := l.ToLexicon() 114 114 + assert.Equal(l.Version, *lex.Ver) 115 115 + assert.Equal(l.CreatedAt, lex.Cts) 116 116 + assert.Equal(l.URI, lex.Uri) 117 117 + assert.Equal(l.Val, lex.Val) 118 118 + assert.Equal(l.CID, lex.Cid) 119 119 + assert.Equal(l.ExpiresAt, lex.Exp) 120 120 + assert.Equal(l.Negated, lex.Neg) 121 121 + assert.Equal(l.SourceDID, lex.Src) 122 122 + } 123 123 + 124 124 + func TestFromLexicon(t *testing.T) { 125 125 + assert := assert.New(t) 126 126 + 127 127 + expiresAt := "2025-07-28T23:53:19.804Z" 128 128 + negated := true 129 129 + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" 130 130 + version := int64(1) 131 131 + 132 132 + lex := &comatproto.LabelDefs_Label{ 133 133 + Cid: &cid, 134 134 + Cts: "2024-10-23T17:51:19.128Z", 135 135 + Exp: &expiresAt, 136 136 + Neg: &negated, 137 137 + Src: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", 138 138 + Uri: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", 139 139 + Val: "good", 140 140 + Ver: &version, 141 141 + Sig: []byte("sig"), // invalid, but we only care about the conversion 142 142 + } 143 143 + 144 144 + l := FromLexicon(lex) 145 145 + assert.Equal(lex.Ver, &l.Version) 146 146 + assert.Equal(lex.Cts, l.CreatedAt) 147 147 + assert.Equal(lex.Uri, l.URI) 148 148 + assert.Equal(lex.Val, l.Val) 149 149 + assert.Equal(lex.Cid, l.CID) 150 150 + assert.Equal(lex.Exp, l.ExpiresAt) 151 151 + assert.Equal(lex.Neg, l.Negated) 152 152 + assert.Equal(lex.Src, l.SourceDID) 153 153 + }

+84 -1

cmd/goat/account.go

··· 8 8 "time" 9 9 10 10 comatproto "github.com/bluesky-social/indigo/api/atproto" 11 11 + "github.com/bluesky-social/indigo/atproto/auth" 12 12 + "github.com/bluesky-social/indigo/atproto/crypto" 11 13 "github.com/bluesky-social/indigo/atproto/syntax" 12 14 "github.com/bluesky-social/indigo/xrpc" 13 15 ··· 89 91 }, 90 92 &cli.Command{ 91 93 Name: "service-auth", 92 92 - Usage: "create service auth token", 94 94 + Usage: "ask the PDS to create a service auth token", 93 95 Flags: []cli.Flag{ 94 96 &cli.StringFlag{ 95 97 Name: "endpoint", ··· 109 111 }, 110 112 }, 111 113 Action: runAccountServiceAuth, 114 114 + }, 115 115 + &cli.Command{ 116 116 + Name: "service-auth-offline", 117 117 + Usage: "create service auth token via locally-held signing key", 118 118 + Flags: []cli.Flag{ 119 119 + &cli.StringFlag{ 120 120 + Name: "atproto-signing-key", 121 121 + Required: true, 122 122 + Usage: "private key used to sign the token (multibase syntax)", 123 123 + EnvVars: []string{"ATPROTO_SIGNING_KEY"}, 124 124 + }, 125 125 + &cli.StringFlag{ 126 126 + Name: "iss", 127 127 + Required: true, 128 128 + Usage: "the DID of the account issuing the token", 129 129 + }, 130 130 + &cli.StringFlag{ 131 131 + Name: "endpoint", 132 132 + Aliases: []string{"lxm"}, 133 133 + Usage: "restrict token to API endpoint (NSID, optional)", 134 134 + }, 135 135 + &cli.StringFlag{ 136 136 + Name: "audience", 137 137 + Aliases: []string{"aud"}, 138 138 + Required: true, 139 139 + Usage: "DID of service that will receive and validate token", 140 140 + }, 141 141 + &cli.IntFlag{ 142 142 + Name: "duration-sec", 143 143 + Value: 60, 144 144 + Usage: "validity time window of token (seconds)", 145 145 + }, 146 146 + }, 147 147 + Action: runAccountServiceAuthOffline, 112 148 }, 113 149 &cli.Command{ 114 150 Name: "create", ··· 365 401 } 366 402 367 403 fmt.Println(resp.Token) 404 404 + 405 405 + return nil 406 406 + } 407 407 + 408 408 + func runAccountServiceAuthOffline(cctx *cli.Context) error { 409 409 + privStr := cctx.String("atproto-signing-key") 410 410 + if privStr == "" { 411 411 + return fmt.Errorf("private key must be provided") 412 412 + } 413 413 + privkey, err := crypto.ParsePrivateMultibase(privStr) 414 414 + if err != nil { 415 415 + return fmt.Errorf("failed parsing private key: %w", err) 416 416 + } 417 417 + 418 418 + issString := cctx.String("iss") 419 419 + // TODO: support fragment identifiers 420 420 + iss, err := syntax.ParseDID(issString) 421 421 + if err != nil { 422 422 + return fmt.Errorf("iss argument must be a valid DID: %w", err) 423 423 + } 424 424 + 425 425 + lxmString := cctx.String("endpoint") 426 426 + var lxm *syntax.NSID = nil 427 427 + if lxmString != "" { 428 428 + lxmTmp, err := syntax.ParseNSID(lxmString) 429 429 + if err != nil { 430 430 + return fmt.Errorf("lxm argument must be a valid NSID: %w", err) 431 431 + } 432 432 + lxm = &lxmTmp 433 433 + } 434 434 + 435 435 + aud := cctx.String("audience") 436 436 + // TODO: can aud DID have a fragment? 437 437 + _, err = syntax.ParseDID(aud) 438 438 + if err != nil { 439 439 + return fmt.Errorf("aud argument must be a valid DID: %w", err) 440 440 + } 441 441 + 442 442 + durSec := cctx.Int("duration-sec") 443 443 + duration := time.Duration(durSec * int(time.Second)) 444 444 + 445 445 + token, err := auth.SignServiceAuth(iss, aud, duration, lxm, privkey) 446 446 + if err != nil { 447 447 + return fmt.Errorf("failed signing token: %w", err) 448 448 + } 449 449 + 450 450 + fmt.Println(token) 368 451 369 452 return nil 370 453 }

-26

docs/auth.md

··· 1 1 - # Auth 2 2 - 3 3 - The auth system uses two tokens, an access token and a refresh token. 4 4 - 5 5 - The access token is a jwt with the following values: 6 6 - ``` 7 7 - scope: "com.atproto.access" 8 8 - sub: <the users DID> 9 9 - iat: the current time, in unix epoch seconds 10 10 - exp: the expiry date, usually around an hour, but at least 15 minutes 11 11 - ``` 12 12 - 13 13 - The refresh token is a jwt with the following values: 14 14 - ``` 15 15 - scope: "com.atproto.refresh" 16 16 - sub: <the users DID> 17 17 - iat: the current time, in unix epoch seconds 18 18 - exp: the expiry date, usually around a week, must be significantly longer than the access token 19 19 - jti: a unique identifier for this token 20 20 - ``` 21 21 - 22 22 - The access token is what is used for all requests, however since it expires 23 23 - quickly, it must be refreshed periodically using the refresh token. 24 24 - When the refresh token is used, it must be marked as deleted, and the new token then replaces it. 25 25 - Note: The old access token is not necessarily disabled at that point of refreshing. 26 26 -

-37

docs/feed-proposal.md

··· 1 1 - # Feed Structuring Proposal 2 2 - 3 3 - Some thoughts on a new format for feeds. 4 4 - 5 5 - ## Motivation 6 6 - The interface for requesting and getting back feeds is something that I feel is really at the core of what bluesky offers. The user should be able to choose what feeds they subscribe to, feeds should be first class objects, they should be able to be efficiently generated and consumed, and they should be able to trustlessly come from anywhere. 7 7 - There are a lot of changes we *could* make to the current structure, but I don't want to stray too far from where we are at right now. 8 8 - 9 9 - 10 10 - ```go 11 11 - type Feed struct { 12 12 - Items []FeedItem 13 13 - Values map[Cid]Record 14 14 - ItemInfos map[Uri]ItemInfo 15 15 - ActorInfos map[Did]ActorInfo 16 16 - } 17 17 - 18 18 - type FeedItem struct { 19 19 - Uri string 20 20 - Replies []Uri 21 21 - ReplyTo Uri 22 22 - RepostedBy Did 23 23 - } 24 24 - 25 25 - type ItemInfo struct { 26 26 - Cid Cid 27 27 - Upvotes int 28 28 - Reposts int 29 29 - Replies int 30 30 - Author Did 31 31 - } 32 32 - ``` 33 33 - 34 34 - The main idea here is not repeating ourselves, while still providing all the information the client might need. 35 35 - With this structure too, the user could easily request *less* data, asking to 36 36 - skip the inclusion of records older than X, or saying they are okay with stale 37 37 - information in certain places for the sake of efficiency.

+1 -1

mst/mst_test.go

··· 532 532 for i := 0; i < 256; i++ { 533 533 f.Add([]byte{byte(i)}) 534 534 } 535 535 - rx := regexp.MustCompile("^[a-zA-Z0-9_:.-]+$") 535 535 + rx := regexp.MustCompile("^[a-zA-Z0-9_:.~-]+$") 536 536 f.Fuzz(func(t *testing.T, in []byte) { 537 537 s := string(in) 538 538 if a, b := rx.MatchString(s), keyHasAllValidChars(s); a != b {

+2 -2

mst/mst_util.go

··· 197 197 } 198 198 199 199 // keyHasAllValidChars reports whether s matches 200 200 - // the regexp /^[a-zA-Z0-9_:.-]+$/ without using regexp, 200 200 + // the regexp /^[a-zA-Z0-9_:.~-]+$/ without using regexp, 201 201 // which is slower. 202 202 func keyHasAllValidChars(s string) bool { 203 203 if len(s) == 0 { ··· 211 211 continue 212 212 } 213 213 switch b { 214 214 - case '_', ':', '.', '-': 214 214 + case '_', ':', '.', '~', '-': 215 215 continue 216 216 default: 217 217 return false

+135

pkg/robusthttp/client.go

··· 1 1 + package robusthttp 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "log/slog" 6 6 + "net/http" 7 7 + "time" 8 8 + 9 9 + "github.com/hashicorp/go-cleanhttp" 10 10 + "github.com/hashicorp/go-retryablehttp" 11 11 + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" 12 12 + ) 13 13 + 14 14 + type LeveledSlog struct { 15 15 + inner *slog.Logger 16 16 + } 17 17 + 18 18 + // re-writes HTTP client ERROR to WARN level (because of retries) 19 19 + func (l LeveledSlog) Error(msg string, keysAndValues ...any) { 20 20 + l.inner.Warn(msg, keysAndValues...) 21 21 + } 22 22 + 23 23 + func (l LeveledSlog) Warn(msg string, keysAndValues ...any) { 24 24 + l.inner.Warn(msg, keysAndValues...) 25 25 + } 26 26 + 27 27 + func (l LeveledSlog) Info(msg string, keysAndValues ...any) { 28 28 + l.inner.Info(msg, keysAndValues...) 29 29 + } 30 30 + 31 31 + func (l LeveledSlog) Debug(msg string, keysAndValues ...any) { 32 32 + l.inner.Debug(msg, keysAndValues...) 33 33 + } 34 34 + 35 35 + type Option func(*retryablehttp.Client) 36 36 + 37 37 + // WithMaxRetries sets the maximum number of retries for the HTTP client. 38 38 + func WithMaxRetries(maxRetries int) Option { 39 39 + return func(client *retryablehttp.Client) { 40 40 + client.RetryMax = maxRetries 41 41 + } 42 42 + } 43 43 + 44 44 + // WithRetryWaitMin sets the minimum wait time between retries. 45 45 + func WithRetryWaitMin(waitMin time.Duration) Option { 46 46 + return func(client *retryablehttp.Client) { 47 47 + client.RetryWaitMin = waitMin 48 48 + } 49 49 + } 50 50 + 51 51 + // WithRetryWaitMax sets the maximum wait time between retries. 52 52 + func WithRetryWaitMax(waitMax time.Duration) Option { 53 53 + return func(client *retryablehttp.Client) { 54 54 + client.RetryWaitMax = waitMax 55 55 + } 56 56 + } 57 57 + 58 58 + // WithLogger sets a custom logger for the HTTP client. 59 59 + func WithLogger(logger *slog.Logger) Option { 60 60 + return func(client *retryablehttp.Client) { 61 61 + client.Logger = retryablehttp.LeveledLogger(LeveledSlog{inner: logger}) 62 62 + } 63 63 + } 64 64 + 65 65 + // WithTransport sets a custom transport for the HTTP client. 66 66 + func WithTransport(transport http.RoundTripper) Option { 67 67 + return func(client *retryablehttp.Client) { 68 68 + client.HTTPClient.Transport = transport 69 69 + } 70 70 + } 71 71 + 72 72 + // WithRetryPolicy sets a custom retry policy for the HTTP client. 73 73 + func WithRetryPolicy(policy retryablehttp.CheckRetry) Option { 74 74 + return func(client *retryablehttp.Client) { 75 75 + client.CheckRetry = policy 76 76 + } 77 77 + } 78 78 + 79 79 + // Generates an HTTP client with decent general-purpose defaults around 80 80 + // timeouts and retries. The returned client has the stdlib http.Client 81 81 + // interface, but has Hashicorp retryablehttp logic internally. 82 82 + // 83 83 + // This client will retry on connection errors, 5xx status (except 501). 84 84 + // It will log intermediate failures with WARN level. This does not start from 85 85 + // http.DefaultClient. 86 86 + // 87 87 + // This should be usable for XRPC clients, and other general inter-service 88 88 + // client needs. CLI tools might want shorter timeouts and fewer retries by 89 89 + // default. 90 90 + func NewClient(options ...Option) *http.Client { 91 91 + logger := LeveledSlog{inner: slog.Default().With("subsystem", "RobustHTTPClient")} 92 92 + retryClient := retryablehttp.NewClient() 93 93 + retryClient.HTTPClient.Transport = otelhttp.NewTransport(cleanhttp.DefaultPooledTransport()) 94 94 + retryClient.RetryMax = 3 95 95 + retryClient.RetryWaitMin = 1 * time.Second 96 96 + retryClient.RetryWaitMax = 10 * time.Second 97 97 + retryClient.Logger = retryablehttp.LeveledLogger(logger) 98 98 + retryClient.CheckRetry = DefaultRetryPolicy 99 99 + 100 100 + for _, option := range options { 101 101 + option(retryClient) 102 102 + } 103 103 + 104 104 + client := retryClient.StandardClient() 105 105 + client.Timeout = 30 * time.Second 106 106 + return client 107 107 + } 108 108 + 109 109 + // For use in local integration tests. Short timeouts, no retries, etc 110 110 + func TestingHTTPClient() *http.Client { 111 111 + 112 112 + client := http.DefaultClient 113 113 + client.Timeout = 1 * time.Second 114 114 + return client 115 115 + } 116 116 + 117 117 + // DefaultRetryPolicy is a custom wrapper around retryablehttp.DefaultRetryPolicy. 118 118 + // It treats `429 Too Many Requests` as non-retryable, so the application can decide 119 119 + // how to deal with rate-limiting. 120 120 + func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { 121 121 + if err == nil && resp.StatusCode == http.StatusTooManyRequests { 122 122 + return false, nil 123 123 + } 124 124 + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) 125 125 + } 126 126 + 127 127 + func NoInternalServerErrorPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { 128 128 + if err == nil && resp.StatusCode == http.StatusTooManyRequests { 129 129 + return false, nil 130 130 + } 131 131 + if err == nil && resp.StatusCode == http.StatusInternalServerError { 132 132 + return false, nil 133 133 + } 134 134 + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) 135 135 + }