desertthunder.dev
a love letter to tangled (android, iOS, and a search API)
1package backfill
2
3import (
4 "context"
5 "encoding/json"
6 "fmt"
7 "io"
8 "net/http"
9 "net/url"
10 "strings"
11 "time"
12)
13
14const listReposByCollectionNSID = "com.atproto.sync.listReposByCollection"
15
16type lightrailRepoLister interface {
17 ListReposByCollection(
18 ctx context.Context, baseURL string, collections []string, limit int,
19 ) ([]string, error)
20}
21
22type listReposByCollectionResponse struct {
23 Cursor string `json:"cursor"`
24 Repos []listReposByCollection `json:"repos"`
25}
26
27type listReposByCollection struct {
28 DID string `json:"did"`
29}
30
31type HTTPLightrailClient struct {
32 client *http.Client
33}
34
35func NewHTTPLightrailClient() *HTTPLightrailClient {
36 return &HTTPLightrailClient{
37 client: &http.Client{Timeout: 15 * time.Second},
38 }
39}
40
41func (c *HTTPLightrailClient) ListReposByCollection(
42 ctx context.Context, baseURL string, collections []string, limit int,
43) ([]string, error) {
44 baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
45 if baseURL == "" {
46 return nil, fmt.Errorf("lightrail url is required")
47 }
48 if limit <= 0 {
49 limit = DefaultPageLimit
50 }
51
52 seen := make(map[string]bool)
53 dids := make([]string, 0)
54 cursor := ""
55 for {
56 resp, err := c.listReposByCollectionPage(ctx, baseURL, collections, limit, cursor)
57 if err != nil {
58 return nil, err
59 }
60 for _, repo := range resp.Repos {
61 did := strings.TrimSpace(repo.DID)
62 if did == "" || seen[did] {
63 continue
64 }
65 seen[did] = true
66 dids = append(dids, did)
67 }
68 if resp.Cursor == "" {
69 return dids, nil
70 }
71 if resp.Cursor == cursor {
72 return nil, fmt.Errorf("listReposByCollection repeated cursor %q", cursor)
73 }
74 cursor = resp.Cursor
75 }
76}
77
78func (c *HTTPLightrailClient) listReposByCollectionPage(
79 ctx context.Context, baseURL string, collections []string, limit int, cursor string,
80) (*listReposByCollectionResponse, error) {
81 params := url.Values{}
82 for _, collection := range collections {
83 collection = strings.TrimSpace(collection)
84 if collection != "" {
85 params.Add("collection", collection)
86 }
87 }
88 if limit > 0 {
89 params.Set("limit", fmt.Sprintf("%d", limit))
90 }
91 if cursor != "" {
92 params.Set("cursor", cursor)
93 }
94
95 endpoint := baseURL + "/xrpc/" + listReposByCollectionNSID
96 if encoded := params.Encode(); encoded != "" {
97 endpoint += "?" + encoded
98 }
99 req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
100 if err != nil {
101 return nil, fmt.Errorf("build listReposByCollection request: %w", err)
102 }
103
104 resp, err := c.client.Do(req)
105 if err != nil {
106 return nil, fmt.Errorf("listReposByCollection request: %w", err)
107 }
108 defer resp.Body.Close()
109
110 if resp.StatusCode < 200 || resp.StatusCode >= 300 {
111 body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
112 return nil, fmt.Errorf(
113 "listReposByCollection failed: status %d: %s",
114 resp.StatusCode,
115 strings.TrimSpace(string(body)),
116 )
117 }
118
119 var payload listReposByCollectionResponse
120 if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
121 return nil, fmt.Errorf("decode listReposByCollection response: %w", err)
122 }
123 return &payload, nil
124}