+31

.github/workflows/sync-internal.yaml

··· 1 1 + name: Sync to internal repo 2 2 + 3 3 + on: 4 4 + push: 5 5 + branches: [main] 6 6 + 7 7 + jobs: 8 8 + sync: 9 9 + runs-on: ubuntu-latest 10 10 + if: github.repository == 'bluesky-social/indigo' 11 11 + steps: 12 12 + - name: Checkout public repo 13 13 + uses: actions/checkout@v4 14 14 + with: 15 15 + fetch-depth: 0 16 16 + - name: Generate GitHub App Token 17 17 + id: app-token 18 18 + uses: actions/create-github-app-token@v1 19 19 + with: 20 20 + app-id: ${{ vars.SYNC_INTERNAL_APP_ID }} 21 21 + private-key: ${{ secrets.SYNC_INTERNAL_PK }} 22 22 + repositories: indigo-internal 23 23 + - name: Push to internal repo 24 24 + env: 25 25 + TOKEN: ${{ steps.app-token.outputs.token }} 26 26 + run: | 27 27 + git config user.name "github-actions" 28 28 + git config user.email "test@users.noreply.github.com" 29 29 + git config --unset-all http.https://github.com/.extraheader 30 30 + git remote add internal https://x-access-token:${TOKEN}@github.com/bluesky-social/indigo-internal.git 31 31 + git push internal main --force

+2

atproto/label/label.go

··· 140 140 Cid: l.CID, 141 141 Cts: l.CreatedAt, 142 142 Exp: l.ExpiresAt, 143 143 + Neg: l.Negated, 143 144 Sig: []byte(l.Sig), 144 145 Src: l.SourceDID, 145 146 Uri: l.URI, ··· 157 158 CID: l.Cid, 158 159 CreatedAt: l.Cts, 159 160 ExpiresAt: l.Exp, 161 161 + Negated: l.Neg, 160 162 Sig: []byte(l.Sig), 161 163 SourceDID: l.Src, 162 164 URI: l.Uri,

+62

atproto/label/label_test.go

··· 4 4 "encoding/json" 5 5 "testing" 6 6 7 7 + comatproto "github.com/bluesky-social/indigo/api/atproto" 7 8 "github.com/bluesky-social/indigo/atproto/crypto" 8 9 9 10 "github.com/stretchr/testify/assert" ··· 89 90 assert.NoError(l.Sign(priv)) 90 91 assert.NoError(l.VerifySignature(pub)) 91 92 } 93 93 + 94 94 + func TestToLexicon(t *testing.T) { 95 95 + assert := assert.New(t) 96 96 + 97 97 + expiresAt := "2025-07-28T23:53:19.804Z" 98 98 + negated := true 99 99 + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" 100 100 + 101 101 + l := Label{ 102 102 + CID: &cid, 103 103 + CreatedAt: "2024-10-23T17:51:19.128Z", 104 104 + ExpiresAt: &expiresAt, 105 105 + Negated: &negated, 106 106 + SourceDID: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", 107 107 + URI: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", 108 108 + Val: "good", 109 109 + Version: ATPROTO_LABEL_VERSION, 110 110 + Sig: []byte("sig"), // invalid, but we only care about the conversion 111 111 + } 112 112 + 113 113 + lex := l.ToLexicon() 114 114 + assert.Equal(l.Version, *lex.Ver) 115 115 + assert.Equal(l.CreatedAt, lex.Cts) 116 116 + assert.Equal(l.URI, lex.Uri) 117 117 + assert.Equal(l.Val, lex.Val) 118 118 + assert.Equal(l.CID, lex.Cid) 119 119 + assert.Equal(l.ExpiresAt, lex.Exp) 120 120 + assert.Equal(l.Negated, lex.Neg) 121 121 + assert.Equal(l.SourceDID, lex.Src) 122 122 + } 123 123 + 124 124 + func TestFromLexicon(t *testing.T) { 125 125 + assert := assert.New(t) 126 126 + 127 127 + expiresAt := "2025-07-28T23:53:19.804Z" 128 128 + negated := true 129 129 + cid := "bafyreifxykqhed72s26cr4i64rxvrtofeqrly3j4vjzbkvo3ckkjbxjqtq" 130 130 + version := int64(1) 131 131 + 132 132 + lex := &comatproto.LabelDefs_Label{ 133 133 + Cid: &cid, 134 134 + Cts: "2024-10-23T17:51:19.128Z", 135 135 + Exp: &expiresAt, 136 136 + Neg: &negated, 137 137 + Src: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", 138 138 + Uri: "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/app.bsky.actor.profile/self", 139 139 + Val: "good", 140 140 + Ver: &version, 141 141 + Sig: []byte("sig"), // invalid, but we only care about the conversion 142 142 + } 143 143 + 144 144 + l := FromLexicon(lex) 145 145 + assert.Equal(lex.Ver, &l.Version) 146 146 + assert.Equal(lex.Cts, l.CreatedAt) 147 147 + assert.Equal(lex.Uri, l.URI) 148 148 + assert.Equal(lex.Val, l.Val) 149 149 + assert.Equal(lex.Cid, l.CID) 150 150 + assert.Equal(lex.Exp, l.ExpiresAt) 151 151 + assert.Equal(lex.Neg, l.Negated) 152 152 + assert.Equal(lex.Src, l.SourceDID) 153 153 + }

+4 -3

cmd/goat/account.go

··· 113 113 Action: runAccountServiceAuth, 114 114 }, 115 115 &cli.Command{ 116 116 - Name: "service-auth-local", 116 116 + Name: "service-auth-offline", 117 117 Usage: "create service auth token via locally-held signing key", 118 118 Flags: []cli.Flag{ 119 119 &cli.StringFlag{ ··· 144 144 Usage: "validity time window of token (seconds)", 145 145 }, 146 146 }, 147 147 - Action: runAccountServiceAuthLocal, 147 147 + Action: runAccountServiceAuthOffline, 148 148 }, 149 149 &cli.Command{ 150 150 Name: "create", ··· 405 405 return nil 406 406 } 407 407 408 408 - func runAccountServiceAuthLocal(cctx *cli.Context) error { 408 408 + func runAccountServiceAuthOffline(cctx *cli.Context) error { 409 409 privStr := cctx.String("atproto-signing-key") 410 410 if privStr == "" { 411 411 return fmt.Errorf("private key must be provided") ··· 416 416 } 417 417 418 418 issString := cctx.String("iss") 419 419 + // TODO: support fragment identifiers 419 420 iss, err := syntax.ParseDID(issString) 420 421 if err != nil { 421 422 return fmt.Errorf("iss argument must be a valid DID: %w", err)

-26

docs/auth.md

··· 1 1 - # Auth 2 2 - 3 3 - The auth system uses two tokens, an access token and a refresh token. 4 4 - 5 5 - The access token is a jwt with the following values: 6 6 - ``` 7 7 - scope: "com.atproto.access" 8 8 - sub: <the users DID> 9 9 - iat: the current time, in unix epoch seconds 10 10 - exp: the expiry date, usually around an hour, but at least 15 minutes 11 11 - ``` 12 12 - 13 13 - The refresh token is a jwt with the following values: 14 14 - ``` 15 15 - scope: "com.atproto.refresh" 16 16 - sub: <the users DID> 17 17 - iat: the current time, in unix epoch seconds 18 18 - exp: the expiry date, usually around a week, must be significantly longer than the access token 19 19 - jti: a unique identifier for this token 20 20 - ``` 21 21 - 22 22 - The access token is what is used for all requests, however since it expires 23 23 - quickly, it must be refreshed periodically using the refresh token. 24 24 - When the refresh token is used, it must be marked as deleted, and the new token then replaces it. 25 25 - Note: The old access token is not necessarily disabled at that point of refreshing. 26 26 -

-37

docs/feed-proposal.md

··· 1 1 - # Feed Structuring Proposal 2 2 - 3 3 - Some thoughts on a new format for feeds. 4 4 - 5 5 - ## Motivation 6 6 - The interface for requesting and getting back feeds is something that I feel is really at the core of what bluesky offers. The user should be able to choose what feeds they subscribe to, feeds should be first class objects, they should be able to be efficiently generated and consumed, and they should be able to trustlessly come from anywhere. 7 7 - There are a lot of changes we *could* make to the current structure, but I don't want to stray too far from where we are at right now. 8 8 - 9 9 - 10 10 - ```go 11 11 - type Feed struct { 12 12 - Items []FeedItem 13 13 - Values map[Cid]Record 14 14 - ItemInfos map[Uri]ItemInfo 15 15 - ActorInfos map[Did]ActorInfo 16 16 - } 17 17 - 18 18 - type FeedItem struct { 19 19 - Uri string 20 20 - Replies []Uri 21 21 - ReplyTo Uri 22 22 - RepostedBy Did 23 23 - } 24 24 - 25 25 - type ItemInfo struct { 26 26 - Cid Cid 27 27 - Upvotes int 28 28 - Reposts int 29 29 - Replies int 30 30 - Author Did 31 31 - } 32 32 - ``` 33 33 - 34 34 - The main idea here is not repeating ourselves, while still providing all the information the client might need. 35 35 - With this structure too, the user could easily request *less* data, asking to 36 36 - skip the inclusion of records older than X, or saying they are okay with stale 37 37 - information in certain places for the sake of efficiency.

+1 -1

mst/mst_test.go

··· 532 532 for i := 0; i < 256; i++ { 533 533 f.Add([]byte{byte(i)}) 534 534 } 535 535 - rx := regexp.MustCompile("^[a-zA-Z0-9_:.-]+$") 535 535 + rx := regexp.MustCompile("^[a-zA-Z0-9_:.~-]+$") 536 536 f.Fuzz(func(t *testing.T, in []byte) { 537 537 s := string(in) 538 538 if a, b := rx.MatchString(s), keyHasAllValidChars(s); a != b {

+2 -2

mst/mst_util.go

··· 197 197 } 198 198 199 199 // keyHasAllValidChars reports whether s matches 200 200 - // the regexp /^[a-zA-Z0-9_:.-]+$/ without using regexp, 200 200 + // the regexp /^[a-zA-Z0-9_:.~-]+$/ without using regexp, 201 201 // which is slower. 202 202 func keyHasAllValidChars(s string) bool { 203 203 if len(s) == 0 { ··· 211 211 continue 212 212 } 213 213 switch b { 214 214 - case '_', ':', '.', '-': 214 214 + case '_', ':', '.', '~', '-': 215 215 continue 216 216 default: 217 217 return false

+135

pkg/robusthttp/client.go

··· 1 1 + package robusthttp 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "log/slog" 6 6 + "net/http" 7 7 + "time" 8 8 + 9 9 + "github.com/hashicorp/go-cleanhttp" 10 10 + "github.com/hashicorp/go-retryablehttp" 11 11 + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" 12 12 + ) 13 13 + 14 14 + type LeveledSlog struct { 15 15 + inner *slog.Logger 16 16 + } 17 17 + 18 18 + // re-writes HTTP client ERROR to WARN level (because of retries) 19 19 + func (l LeveledSlog) Error(msg string, keysAndValues ...any) { 20 20 + l.inner.Warn(msg, keysAndValues...) 21 21 + } 22 22 + 23 23 + func (l LeveledSlog) Warn(msg string, keysAndValues ...any) { 24 24 + l.inner.Warn(msg, keysAndValues...) 25 25 + } 26 26 + 27 27 + func (l LeveledSlog) Info(msg string, keysAndValues ...any) { 28 28 + l.inner.Info(msg, keysAndValues...) 29 29 + } 30 30 + 31 31 + func (l LeveledSlog) Debug(msg string, keysAndValues ...any) { 32 32 + l.inner.Debug(msg, keysAndValues...) 33 33 + } 34 34 + 35 35 + type Option func(*retryablehttp.Client) 36 36 + 37 37 + // WithMaxRetries sets the maximum number of retries for the HTTP client. 38 38 + func WithMaxRetries(maxRetries int) Option { 39 39 + return func(client *retryablehttp.Client) { 40 40 + client.RetryMax = maxRetries 41 41 + } 42 42 + } 43 43 + 44 44 + // WithRetryWaitMin sets the minimum wait time between retries. 45 45 + func WithRetryWaitMin(waitMin time.Duration) Option { 46 46 + return func(client *retryablehttp.Client) { 47 47 + client.RetryWaitMin = waitMin 48 48 + } 49 49 + } 50 50 + 51 51 + // WithRetryWaitMax sets the maximum wait time between retries. 52 52 + func WithRetryWaitMax(waitMax time.Duration) Option { 53 53 + return func(client *retryablehttp.Client) { 54 54 + client.RetryWaitMax = waitMax 55 55 + } 56 56 + } 57 57 + 58 58 + // WithLogger sets a custom logger for the HTTP client. 59 59 + func WithLogger(logger *slog.Logger) Option { 60 60 + return func(client *retryablehttp.Client) { 61 61 + client.Logger = retryablehttp.LeveledLogger(LeveledSlog{inner: logger}) 62 62 + } 63 63 + } 64 64 + 65 65 + // WithTransport sets a custom transport for the HTTP client. 66 66 + func WithTransport(transport http.RoundTripper) Option { 67 67 + return func(client *retryablehttp.Client) { 68 68 + client.HTTPClient.Transport = transport 69 69 + } 70 70 + } 71 71 + 72 72 + // WithRetryPolicy sets a custom retry policy for the HTTP client. 73 73 + func WithRetryPolicy(policy retryablehttp.CheckRetry) Option { 74 74 + return func(client *retryablehttp.Client) { 75 75 + client.CheckRetry = policy 76 76 + } 77 77 + } 78 78 + 79 79 + // Generates an HTTP client with decent general-purpose defaults around 80 80 + // timeouts and retries. The returned client has the stdlib http.Client 81 81 + // interface, but has Hashicorp retryablehttp logic internally. 82 82 + // 83 83 + // This client will retry on connection errors, 5xx status (except 501). 84 84 + // It will log intermediate failures with WARN level. This does not start from 85 85 + // http.DefaultClient. 86 86 + // 87 87 + // This should be usable for XRPC clients, and other general inter-service 88 88 + // client needs. CLI tools might want shorter timeouts and fewer retries by 89 89 + // default. 90 90 + func NewClient(options ...Option) *http.Client { 91 91 + logger := LeveledSlog{inner: slog.Default().With("subsystem", "RobustHTTPClient")} 92 92 + retryClient := retryablehttp.NewClient() 93 93 + retryClient.HTTPClient.Transport = otelhttp.NewTransport(cleanhttp.DefaultPooledTransport()) 94 94 + retryClient.RetryMax = 3 95 95 + retryClient.RetryWaitMin = 1 * time.Second 96 96 + retryClient.RetryWaitMax = 10 * time.Second 97 97 + retryClient.Logger = retryablehttp.LeveledLogger(logger) 98 98 + retryClient.CheckRetry = DefaultRetryPolicy 99 99 + 100 100 + for _, option := range options { 101 101 + option(retryClient) 102 102 + } 103 103 + 104 104 + client := retryClient.StandardClient() 105 105 + client.Timeout = 30 * time.Second 106 106 + return client 107 107 + } 108 108 + 109 109 + // For use in local integration tests. Short timeouts, no retries, etc 110 110 + func TestingHTTPClient() *http.Client { 111 111 + 112 112 + client := http.DefaultClient 113 113 + client.Timeout = 1 * time.Second 114 114 + return client 115 115 + } 116 116 + 117 117 + // DefaultRetryPolicy is a custom wrapper around retryablehttp.DefaultRetryPolicy. 118 118 + // It treats `429 Too Many Requests` as non-retryable, so the application can decide 119 119 + // how to deal with rate-limiting. 120 120 + func DefaultRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { 121 121 + if err == nil && resp.StatusCode == http.StatusTooManyRequests { 122 122 + return false, nil 123 123 + } 124 124 + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) 125 125 + } 126 126 + 127 127 + func NoInternalServerErrorPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { 128 128 + if err == nil && resp.StatusCode == http.StatusTooManyRequests { 129 129 + return false, nil 130 130 + } 131 131 + if err == nil && resp.StatusCode == http.StatusInternalServerError { 132 132 + return false, nil 133 133 + } 134 134 + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) 135 135 + }