knotmirror: add `knotBackoff` and reachability test #1168

+71 -6

1 changed file

Diff round #1

expand all

knotmirror

resyncer.go

+71 -6

knotmirror/resyncer.go

··· 7 7 "fmt" 8 8 "log/slog" 9 9 "math/rand" 10 + "net/http" 11 + "net/url" 10 12 "strings" 11 13 "sync" 12 14 "time" ··· 25 27 26 28 claimJobMu sync.Mutex 27 29 28 - runningJobs map[syntax.ATURI]context.CancelFunc 30 + runningJobs map[syntax.ATURI]context.CancelFunc 29 31 runningJobsMu sync.Mutex 30 32 31 33 repoFetchTimeout time.Duration 32 34 manualResyncTimeout time.Duration 35 + parallelism int 33 36 34 - parallelism int 37 + knotBackoff map[string]time.Time 38 + knotBackoffMu sync.RWMutex 35 39 } 36 40 37 41 func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config.Config) *Resyncer { ··· 42 46 43 47 runningJobs: make(map[syntax.ATURI]context.CancelFunc), 44 48 45 - repoFetchTimeout: cfg.GitRepoFetchTimeout, 46 - parallelism: cfg.ResyncParallelism, 47 - 49 + repoFetchTimeout: cfg.GitRepoFetchTimeout, 48 50 manualResyncTimeout: 30 * time.Minute, 51 + parallelism: cfg.ResyncParallelism, 52 + 53 + knotBackoff: make(map[string]time.Time), 49 54 } 50 55 } 51 56 ··· 147 152 select at_uri from repos 148 153 where state in ($2, $3, $4) 149 154 and (retry_after = -1 or retry_after = 0 or retry_after < $5) 155 + order by 156 + (retry_after = -1) desc, 157 + (retry_after = 0) desc, 158 + retry_after 150 159 limit 1 151 160 ) 152 161 returning at_uri ··· 201 210 return false, nil 202 211 } 203 212 204 - // TODO: check if Knot is on backoff list. If so, return (false, nil) 213 + r.knotBackoffMu.RLock() 214 + backoffUntil, inBackoff := r.knotBackoff[repo.KnotDomain] 215 + r.knotBackoffMu.RUnlock() 216 + if inBackoff && time.Now().Before(backoffUntil) { 217 + return false, nil 218 + } 219 + 220 + // HACK: check knot reachability with short timeout before running actual fetch. 221 + // This is crucial as git-cli doesn't support http connection timeout. 222 + // `http.lowSpeedTime` is only applied _after_ the connection. 223 + if err := r.checkKnotReachability(ctx, repo); err != nil { 224 + return false, fmt.Errorf("knot unreachable: %w", err) 225 + } 226 + 205 227 // TODO: detect rate limit error (http.StatusTooManyRequests) to put Knot in backoff list 228 + // we can use http statuscode for that. 206 229 207 230 timeout := r.repoFetchTimeout 208 231 if repo.RetryAfter == -1 { ··· 227 250 return true, nil 228 251 } 229 252 253 + // checkKnotReachability checks if Knot is reachable and is valid git remote server 254 + func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) error { 255 + repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), true) 256 + if err != nil { 257 + return err 258 + } 259 + 260 + repoUrl += "/info/refs?service=git-upload-pack" 261 + 262 + client := http.Client{ 263 + Timeout: 30 * time.Second, 264 + } 265 + req, err := http.NewRequestWithContext(ctx, "GET", repoUrl, nil) 266 + if err != nil { 267 + return err 268 + } 269 + req.Header.Set("User-Agent", "git/2.x") 270 + req.Header.Set("Accept", "*/*") 271 + 272 + resp, err := client.Do(req) 273 + if err != nil { 274 + var uerr *url.Error 275 + if errors.As(err, &uerr) { 276 + return fmt.Errorf("request failed: %w", uerr.Unwrap()) 277 + } 278 + return fmt.Errorf("request failed: %w", err) 279 + } 280 + defer resp.Body.Close() 281 + 282 + if resp.StatusCode != http.StatusOK { 283 + return fmt.Errorf("unexpected status: %s", resp.Status) 284 + } 285 + 286 + // check if target is git server 287 + ct := resp.Header.Get("Content-Type") 288 + if !strings.Contains(ct, "application/x-git-upload-pack-advertisement") { 289 + return fmt.Errorf("unexpected content-type: %s", ct) 290 + } 291 + 292 + return nil 293 + } 294 + 230 295 func (r *Resyncer) handleResyncFailure(ctx context.Context, repoAt syntax.ATURI, err error) error { 231 296 r.logger.Debug("handleResyncFailure", "at_uri", repoAt, "err", err) 232 297 var state models.RepoState