oppi.li
fork of indigo with slightly nicer lexgen
bnewbold.net
1package main
2
3import (
4 "errors"
5 "fmt"
6
7 comatproto "github.com/bluesky-social/indigo/api/atproto"
8 "github.com/bluesky-social/indigo/atproto/identity"
9 "github.com/bluesky-social/indigo/cmd/relay/relay"
10 "github.com/bluesky-social/indigo/cmd/relay/relay/models"
11 "github.com/bluesky-social/indigo/util/cliutil"
12 "github.com/bluesky-social/indigo/xrpc"
13
14 "github.com/urfave/cli/v2"
15)
16
17var cmdPullHosts = &cli.Command{
18 Name: "pull-hosts",
19 Usage: "initializes or updates host list from an existing relay (public API)",
20 Action: runPullHosts,
21 Flags: []cli.Flag{
22 &cli.StringFlag{
23 Name: "relay-host",
24 Usage: "method, hostname, and port of relay to pull from",
25 Value: "https://bsky.network",
26 EnvVars: []string{"RELAY_HOST"},
27 },
28 &cli.StringFlag{
29 Name: "db-url",
30 Usage: "database connection string for relay database",
31 Value: "sqlite://data/relay/relay.sqlite",
32 EnvVars: []string{"DATABASE_URL"},
33 },
34 &cli.IntFlag{
35 Name: "default-account-limit",
36 Value: 100,
37 Usage: "max number of active accounts for new upstream hosts",
38 EnvVars: []string{"RELAY_DEFAULT_ACCOUNT_LIMIT", "RELAY_DEFAULT_REPO_LIMIT"},
39 },
40 &cli.IntFlag{
41 Name: "batch-size",
42 Value: 500,
43 Usage: "host many hosts to pull at a time",
44 EnvVars: []string{"RELAY_PULL_HOSTS_BATCH_SIZE"},
45 },
46 &cli.StringSliceFlag{
47 Name: "trusted-domains",
48 Usage: "domain names which mark trusted hosts; use wildcard prefix to match suffixes",
49 Value: cli.NewStringSlice("*.host.bsky.network"),
50 EnvVars: []string{"RELAY_TRUSTED_DOMAINS"},
51 },
52 &cli.BoolFlag{
53 Name: "skip-host-checks",
54 Usage: "don't run describeServer requests to see if host is a PDS before adding",
55 EnvVars: []string{"RELAY_SKIP_HOST_CHECKS"},
56 },
57 },
58}
59
60func runPullHosts(cctx *cli.Context) error {
61 ctx := cctx.Context
62
63 if cctx.Args().Len() > 0 {
64 return fmt.Errorf("unexpected arguments")
65 }
66
67 client := xrpc.Client{
68 Host: cctx.String("relay-host"),
69 }
70
71 skipHostChecks := cctx.Bool("skip-host-checks")
72
73 dir := identity.DefaultDirectory()
74
75 dburl := cctx.String("db-url")
76 db, err := cliutil.SetupDatabase(dburl, 10)
77 if err != nil {
78 return err
79 }
80
81 relayConfig := relay.DefaultRelayConfig()
82 relayConfig.DefaultRepoLimit = cctx.Int64("default-account-limit")
83 relayConfig.TrustedDomains = cctx.StringSlice("trusted-domains")
84
85 // NOTE: setting evtmgr to nil
86 r, err := relay.NewRelay(db, nil, dir, relayConfig)
87 if err != nil {
88 return err
89 }
90
91 checker := relay.NewHostClient(relayConfig.UserAgent)
92
93 cursor := ""
94 size := cctx.Int64("batch-size")
95 for {
96 resp, err := comatproto.SyncListHosts(ctx, &client, cursor, size)
97 if err != nil {
98 return err
99 }
100 for _, h := range resp.Hosts {
101 if h.Status == nil {
102 fmt.Printf("%s: status=unknown\n", h.Hostname)
103 continue
104 }
105 if !(models.HostStatus(*h.Status) == models.HostStatusActive || models.HostStatus(*h.Status) == models.HostStatusIdle) {
106 fmt.Printf("%s: status=%s\n", h.Hostname, *h.Status)
107 continue
108 }
109 if h.Seq == nil || *h.Seq <= 0 {
110 fmt.Printf("%s: no-cursor\n", h.Hostname)
111 continue
112 }
113 existing, err := r.GetHost(ctx, h.Hostname)
114 if err != nil && !errors.Is(err, relay.ErrHostNotFound) {
115 return err
116 }
117 if existing != nil {
118 fmt.Printf("%s: exists\n", h.Hostname)
119 continue
120 }
121 hostname, noSSL, err := relay.ParseHostname(h.Hostname)
122 if err != nil {
123 return fmt.Errorf("%w: %s", err, h.Hostname)
124 }
125 if noSSL {
126 // skip "localhost" and non-SSL hosts (this is for public PDS instances)
127 fmt.Printf("%s: non-public\n", h.Hostname)
128 continue
129 }
130
131 accountLimit := r.Config.DefaultRepoLimit
132 trusted := relay.IsTrustedHostname(hostname, r.Config.TrustedDomains)
133 if trusted {
134 accountLimit = r.Config.TrustedRepoLimit
135 }
136
137 if !skipHostChecks {
138 if err := checker.CheckHost(ctx, "https://"+hostname); err != nil {
139 fmt.Printf("%s: checking host: %s\n", h.Hostname, err)
140 continue
141 }
142 }
143
144 host := models.Host{
145 Hostname: hostname,
146 NoSSL: noSSL,
147 Status: models.HostStatusActive,
148 Trusted: trusted,
149 AccountLimit: accountLimit,
150 }
151 if err := db.Create(&host).Error; err != nil {
152 return err
153 }
154 fmt.Printf("%s: added\n", h.Hostname)
155 }
156 if resp.Cursor == nil || *resp.Cursor == "" {
157 break
158 }
159 cursor = *resp.Cursor
160 }
161 return nil
162}