better handling of backfill errors on pds fetch (#964)

Have a fair amount of random missing data coming from my consumer. Some
of it i've found is that the initial repo fetch errorred for some reason
and then we just drop all future events (which kinda makes sense i
guess)

However its useful to distinguish why the fetch failed, distinguishing
between a rate limit and 'this repo does not exist' is important for
recovery situations.

backfill/backfill.go

···
 	err        error
 }
 
+type FetchRepoError struct {
+	StatusCode int
+	Status     string
+}
+
+func (e *FetchRepoError) Error() string {
+	reason := "unknown error"
+	if e.StatusCode == http.StatusBadRequest {
+		reason = "repo not found"
+	} else {
+		reason = e.Status
+	}
+	return fmt.Sprintf("failed to get repo: %s (%d)", reason, e.StatusCode)
+}
+
 // Fetches a repo CAR file over HTTP from the indicated host. If successful, parses the CAR and returns repo.Repo
 func (b *Backfiller) fetchRepo(ctx context.Context, did, since, host string) (*repo.Repo, error) {
 	url := fmt.Sprintf("%s/xrpc/com.atproto.sync.getRepo?did=%s", host, did)
···
 	}
 
 	if resp.StatusCode != http.StatusOK {
-		reason := "unknown error"
-		if resp.StatusCode == http.StatusBadRequest {
-			reason = "repo not found"
-		} else {
-			reason = resp.Status
+		return nil, &FetchRepoError{
+			StatusCode: resp.StatusCode,
+			Status:     resp.Status,
 		}
-		return nil, fmt.Errorf("failed to get repo: %s", reason)
 	}
 
 	instrumentedReader := instrumentedReader{
···
 		r, err = b.fetchRepo(ctx, repoDID, job.Rev(), pdsHost)
 		if err != nil {
 			slog.Warn("repo CAR fetch from PDS failed", "did", repoDID, "since", job.Rev(), "pdsHost", pdsHost, "err", err)
-			return "repo CAR fetch from PDS failed", err
+			rfe, ok := err.(*FetchRepoError)
+			if ok {
+				return fmt.Sprintf("failed to fetch repo CAR from PDS (http %d:%s)", rfe.StatusCode, rfe.Status), err
+			}
+			return "failed to fetch repo CAR from PDS", err
 		}
 	}