knotmirror: add `knotBackoff` and reachability test #1168

+85 -2

1 changed file

Diff round #4

expand all

knotmirror

resyncer.go

+85 -2

knotmirror/resyncer.go

··· 7 7 "fmt" 8 8 "log/slog" 9 9 "math/rand" 10 + "net/http" 11 + "net/url" 10 12 "strings" 11 13 "sync" 12 14 "time" ··· 31 33 repoFetchTimeout time.Duration 32 34 manualResyncTimeout time.Duration 33 35 parallelism int 36 + 37 + knotBackoff map[string]time.Time 38 + knotBackoffMu sync.RWMutex 34 39 } 35 40 36 41 func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config.Config) *Resyncer { ··· 44 49 repoFetchTimeout: cfg.GitRepoFetchTimeout, 45 50 manualResyncTimeout: 30 * time.Minute, 46 51 parallelism: cfg.ResyncParallelism, 52 + 53 + knotBackoff: make(map[string]time.Time), 47 54 } 48 55 } 49 56 ··· 203 210 return false, nil 204 211 } 205 212 206 - // TODO: check if Knot is on backoff list. If so, return (false, nil) 207 - // TODO: detect rate limit error (http.StatusTooManyRequests) to put Knot in backoff list 213 + r.knotBackoffMu.RLock() 214 + backoffUntil, inBackoff := r.knotBackoff[repo.KnotDomain] 215 + r.knotBackoffMu.RUnlock() 216 + if inBackoff && time.Now().Before(backoffUntil) { 217 + return false, nil 218 + } 219 + 220 + // HACK: check knot reachability with short timeout before running actual fetch. 221 + // This is crucial as git-cli doesn't support http connection timeout. 222 + // `http.lowSpeedTime` is only applied _after_ the connection. 223 + if err := r.checkKnotReachability(ctx, repo); err != nil { 224 + if isRateLimitError(err) { 225 + r.knotBackoffMu.Lock() 226 + r.knotBackoff[repo.KnotDomain] = time.Now().Add(10 * time.Second) 227 + r.knotBackoffMu.Unlock() 228 + return false, nil 229 + } 230 + // TODO: suspend repo on 404. KnotStream updates will change the repo state back online 231 + return false, fmt.Errorf("knot unreachable: %w", err) 232 + } 208 233 209 234 timeout := r.repoFetchTimeout 210 235 if repo.RetryAfter == -1 { ··· 229 254 return true, nil 230 255 } 231 256 257 + type knotStatusError struct { 258 + StatusCode int 259 + } 260 + 261 + func (ke *knotStatusError) Error() string { 262 + return fmt.Sprintf("request failed with status code (HTTP %d)", ke.StatusCode) 263 + } 264 + 265 + func isRateLimitError(err error) bool { 266 + var knotErr *knotStatusError 267 + if errors.As(err, &knotErr) { 268 + return knotErr.StatusCode == http.StatusTooManyRequests 269 + } 270 + return false 271 + } 272 + 273 + // checkKnotReachability checks if Knot is reachable and is valid git remote server 274 + func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) error { 275 + repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), true) 276 + if err != nil { 277 + return err 278 + } 279 + 280 + repoUrl += "/info/refs?service=git-upload-pack" 281 + 282 + client := http.Client{ 283 + Timeout: 30 * time.Second, 284 + } 285 + req, err := http.NewRequestWithContext(ctx, "GET", repoUrl, nil) 286 + if err != nil { 287 + return err 288 + } 289 + req.Header.Set("User-Agent", "git/2.x") 290 + req.Header.Set("Accept", "*/*") 291 + 292 + resp, err := client.Do(req) 293 + if err != nil { 294 + var uerr *url.Error 295 + if errors.As(err, &uerr) { 296 + return fmt.Errorf("request failed: %w", uerr.Unwrap()) 297 + } 298 + return fmt.Errorf("request failed: %w", err) 299 + } 300 + defer resp.Body.Close() 301 + 302 + if resp.StatusCode != http.StatusOK { 303 + return &knotStatusError{resp.StatusCode} 304 + } 305 + 306 + // check if target is git server 307 + ct := resp.Header.Get("Content-Type") 308 + if !strings.Contains(ct, "application/x-git-upload-pack-advertisement") { 309 + return fmt.Errorf("unexpected content-type: %s", ct) 310 + } 311 + 312 + return nil 313 + } 314 + 232 315 func (r *Resyncer) handleResyncFailure(ctx context.Context, repoAt syntax.ATURI, err error) error { 233 316 r.logger.Debug("handleResyncFailure", "at_uri", repoAt, "err", err) 234 317 var state models.RepoState