Weighs the soul of incoming HTTP requests to stop AI crawlers
techaro.lol
1package lib
2
3import (
4 "context"
5 "crypto/ed25519"
6 "crypto/rand"
7 "errors"
8 "fmt"
9 "io"
10 "log/slog"
11 "net/http"
12 "os"
13 "strings"
14 "time"
15
16 "github.com/TecharoHQ/anubis"
17 "github.com/TecharoHQ/anubis/data"
18 "github.com/TecharoHQ/anubis/internal"
19 "github.com/TecharoHQ/anubis/internal/ogtags"
20 "github.com/TecharoHQ/anubis/lib/challenge"
21 "github.com/TecharoHQ/anubis/lib/localization"
22 "github.com/TecharoHQ/anubis/lib/policy"
23 "github.com/TecharoHQ/anubis/lib/policy/config"
24 "github.com/TecharoHQ/anubis/web"
25 "github.com/TecharoHQ/anubis/xess"
26 "github.com/a-h/templ"
27)
28
29type Options struct {
30 Next http.Handler
31 Policy *policy.ParsedConfig
32 Target string
33 CookieDynamicDomain bool
34 CookieDomain string
35 CookieExpiration time.Duration
36 CookiePartitioned bool
37 BasePrefix string
38 WebmasterEmail string
39 RedirectDomains []string
40 ED25519PrivateKey ed25519.PrivateKey
41 HS512Secret []byte
42 StripBasePrefix bool
43 OpenGraph config.OpenGraph
44 ServeRobotsTXT bool
45 CookieSecure bool
46}
47
48func LoadPoliciesOrDefault(ctx context.Context, fname string, defaultDifficulty int) (*policy.ParsedConfig, error) {
49 var fin io.ReadCloser
50 var err error
51
52 if fname != "" {
53 fin, err = os.Open(fname)
54 if err != nil {
55 return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
56 }
57 } else {
58 fname = "(data)/botPolicies.yaml"
59 fin, err = data.BotPolicies.Open("botPolicies.yaml")
60 if err != nil {
61 return nil, fmt.Errorf("[unexpected] can't parse builtin policy file %s: %w", fname, err)
62 }
63 }
64
65 defer func(fin io.ReadCloser) {
66 err := fin.Close()
67 if err != nil {
68 slog.Error("failed to close policy file", "file", fname, "err", err)
69 }
70 }(fin)
71
72 anubisPolicy, err := policy.ParseConfig(ctx, fin, fname, defaultDifficulty)
73 if err != nil {
74 return nil, fmt.Errorf("can't parse policy file %s: %w", fname, err)
75 }
76 var validationErrs []error
77
78 for _, b := range anubisPolicy.Bots {
79 if _, ok := challenge.Get(b.Challenge.Algorithm); !ok {
80 validationErrs = append(validationErrs, fmt.Errorf("%w %s", policy.ErrChallengeRuleHasWrongAlgorithm, b.Challenge.Algorithm))
81 }
82 }
83
84 if len(validationErrs) != 0 {
85 return nil, fmt.Errorf("can't do final validation of Anubis config: %w", errors.Join(validationErrs...))
86 }
87
88 return anubisPolicy, err
89}
90
91func New(opts Options) (*Server, error) {
92 if opts.ED25519PrivateKey == nil && opts.HS512Secret == nil {
93 slog.Debug("opts.PrivateKey not set, generating a new one")
94 _, priv, err := ed25519.GenerateKey(rand.Reader)
95 if err != nil {
96 return nil, fmt.Errorf("lib: can't generate private key: %v", err)
97 }
98 opts.ED25519PrivateKey = priv
99 }
100
101 anubis.BasePrefix = opts.BasePrefix
102
103 result := &Server{
104 next: opts.Next,
105 ed25519Priv: opts.ED25519PrivateKey,
106 hs512Secret: opts.HS512Secret,
107 policy: opts.Policy,
108 opts: opts,
109 OGTags: ogtags.NewOGTagCache(opts.Target, opts.Policy.OpenGraph, opts.Policy.Store),
110 store: opts.Policy.Store,
111 }
112
113 mux := http.NewServeMux()
114 xess.Mount(mux)
115
116 // Helper to add global prefix
117 registerWithPrefix := func(pattern string, handler http.Handler, method string) {
118 if method != "" {
119 method = method + " " // methods must end with a space to register with them
120 }
121
122 // Ensure there's no double slash when concatenating BasePrefix and pattern
123 basePrefix := strings.TrimSuffix(anubis.BasePrefix, "/")
124 prefix := method + basePrefix
125
126 // If pattern doesn't start with a slash, add one
127 if !strings.HasPrefix(pattern, "/") {
128 pattern = "/" + pattern
129 }
130
131 mux.Handle(prefix+pattern, handler)
132 }
133
134 // Ensure there's no double slash when concatenating BasePrefix and StaticPath
135 stripPrefix := strings.TrimSuffix(anubis.BasePrefix, "/") + anubis.StaticPath
136 registerWithPrefix(anubis.StaticPath, internal.UnchangingCache(internal.NoBrowsing(http.StripPrefix(stripPrefix, http.FileServerFS(web.Static)))), "")
137
138 if opts.ServeRobotsTXT {
139 registerWithPrefix("/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
140 http.ServeFileFS(w, r, web.Static, "static/robots.txt")
141 }), "GET")
142 registerWithPrefix("/.well-known/robots.txt", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
143 http.ServeFileFS(w, r, web.Static, "static/robots.txt")
144 }), "GET")
145 }
146
147 if opts.Policy.Impressum != nil {
148 registerWithPrefix(anubis.APIPrefix+"imprint", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
149 templ.Handler(
150 web.Base(opts.Policy.Impressum.Page.Title, opts.Policy.Impressum.Page, opts.Policy.Impressum, localization.GetLocalizer(r)),
151 ).ServeHTTP(w, r)
152 }), "GET")
153 }
154
155 registerWithPrefix(anubis.APIPrefix+"pass-challenge", http.HandlerFunc(result.PassChallenge), "GET")
156 registerWithPrefix(anubis.APIPrefix+"check", http.HandlerFunc(result.maybeReverseProxyHttpStatusOnly), "")
157 registerWithPrefix("/", http.HandlerFunc(result.maybeReverseProxyOrPage), "")
158
159 //goland:noinspection GoBoolExpressions
160 if anubis.Version == "devel" {
161 // make-challenge is only used in tests. Only enable while version is devel
162 registerWithPrefix(anubis.APIPrefix+"make-challenge", http.HandlerFunc(result.MakeChallenge), "POST")
163 }
164
165 for _, implKind := range challenge.Methods() {
166 impl, _ := challenge.Get(implKind)
167 impl.Setup(mux)
168 }
169
170 result.mux = mux
171
172 return result, nil
173}