diff --git a/spindle/notify_admission_test.go b/spindle/notify_admission_test.go new file mode 100644 index 000000000..3e22b15a9 --- /dev/null +++ b/spindle/notify_admission_test.go @@ -0,0 +1,90 @@ +package spindle + +import ( + "context" + "fmt" + "hash/fnv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func admissionStripe(did syntax.DID) uint32 { + h := fnv.New32a() + h.Write([]byte(did)) + return h.Sum32() % repoLockStripes +} + +// not on the stripe of the served repo, so a held stranger can't hold its lock +func strangersAwayFrom(served syntax.DID, n int) []syntax.DID { + var out []syntax.DID + for i := 0; len(out) < n; i++ { + did := syntax.DID(fmt.Sprintf("did:plc:stranger%d", i)) + if admissionStripe(did) != admissionStripe(served) { + out = append(out, did) + } + } + return out +} + +func TestNotifyEventUnknownReposDontTakeKnownReposSlots(t *testing.T) { + e := newPushEnv(t) + release := e.w.holdManifest() + defer release(false) + + strangers := strangersAwayFrom(testRepoDid, 8) + var answered atomic.Int32 + var wg sync.WaitGroup + for _, did := range strangers { + wg.Go(func() { + e.n.s.NotifyEvent(context.Background(), e.w.knotRef(), did, faultPush("refs/heads/main", zeroSha, strings.Repeat("a", 40))) + answered.Add(1) + }) + } + eventually(t, 10*time.Second, "the strangers to be held on the knot or turned away", func() bool { + return int(e.w.heldReads.Load()+answered.Load()) == len(strangers) + }) + + uri, err := e.notify(e.n.s, e.firstPush(t)) + if err != nil || uri == "" { + t.Fatalf("the served repo's push answered %q, %v while strangers were held, want a pipeline", uri, err) + } + release(false) + wg.Wait() +} + +func TestNotifyEventRemembersReposItDoesNotServe(t *testing.T) { + e := newPushEnv(t) + stranger := syntax.DID("did:plc:stranger") + push := faultPush("refs/heads/main", zeroSha, strings.Repeat("a", 40)) + + if uri, err := e.n.s.NotifyEvent(context.Background(), e.w.knotRef(), stranger, push); err != nil || uri != "" { + t.Fatalf("first notify answered %q, %v, want an empty answer", uri, err) + } + before := e.w.traffic() + if uri, err := e.n.s.NotifyEvent(context.Background(), e.w.knotRef(), stranger, push); err != nil || uri != "" { + t.Fatalf("second notify answered %q, %v, want an empty answer", uri, err) + } + if got := e.w.traffic() - before; got != 0 { + t.Fatalf("second notify of an unknown repo made %d knot requests, want 0", got) + } +} + +func TestNotifyAdmissionStillRemembersAfterAFloodOfDistinctDids(t *testing.T) { + var a notifyAdmission + for i := range unknownRepoMemoryMax + 10 { + a.miss(syntax.DID(fmt.Sprintf("did:plc:flood%d", i))) + } + + last := syntax.DID("did:plc:flood" + fmt.Sprint(unknownRepoMemoryMax+9)) + if !a.missedRecently(last) { + t.Fatal("a miss after the memory filled was forgotten, so real misses cost a read again") + } + if a.missedRecently("did:plc:flood0") { + t.Fatal("the oldest miss outlived the cap") + } +} diff --git a/spindle/notify_event.go b/spindle/notify_event.go index 06262425d..55d88bc2e 100644 --- a/spindle/notify_event.go +++ b/spindle/notify_event.go @@ -12,6 +12,7 @@ import ( "time" "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/hashicorp/golang-lru/v2/expirable" "tangled.org/core/api/org_tangled" "tangled.org/core/api/tangled" "tangled.org/core/gitutil" @@ -33,6 +34,13 @@ const ( // a hostile knot can sit on its slots for the whole budget, so one knot never gets them all notifyMaxPerKnot = 4 + // repos nobody serves yet cost a knot read and a pds read before anything else, so they get a + // small pool of their own and can't crowd out the repos that are served + notifyMaxUnknown = 2 + notifyMaxUnknownPerKnot = 1 + unknownRepoMemory = time.Minute + unknownRepoMemoryMax = 4096 + changedFileOverheadBytes = 48 changedFilesMaxBytes = 524288 changedFilesMaxCount = 8192 @@ -75,6 +83,55 @@ func (k *knotSlots) release(host string) { } } +// the pool for repos the db doesn't know, plus a short memory of the ones that turned out not served here +type notifyAdmission struct { + mu sync.Mutex + held map[string]int + total int + + once sync.Once + missed *expirable.LRU[syntax.DID, struct{}] +} + +func (a *notifyAdmission) acquire(host string) bool { + a.mu.Lock() + defer a.mu.Unlock() + if a.total >= notifyMaxUnknown || a.held[host] >= notifyMaxUnknownPerKnot { + return false + } + if a.held == nil { + a.held = map[string]int{} + } + a.held[host]++ + a.total++ + return true +} + +func (a *notifyAdmission) release(host string) { + a.mu.Lock() + defer a.mu.Unlock() + a.total-- + if a.held[host]--; a.held[host] == 0 { + delete(a.held, host) + } +} + +func (a *notifyAdmission) memory() *expirable.LRU[syntax.DID, struct{}] { + a.once.Do(func() { + a.missed = expirable.NewLRU[syntax.DID, struct{}](unknownRepoMemoryMax, nil, unknownRepoMemory) + }) + return a.missed +} + +func (a *notifyAdmission) missedRecently(did syntax.DID) bool { + _, ok := a.memory().Get(did) + return ok +} + +func (a *notifyAdmission) miss(did syntax.DID) { + a.memory().Add(did, struct{}{}) +} + // an error makes the knot retry, an empty uri means nothing ran func (s *Spindle) NotifyEvent(ctx context.Context, k knotclient.Knot, repoDid syntax.DID, event *org_tangled.EventNotifyEvent_Input_Event) (syntax.ATURI, error) { l := log.FromContext(ctx).With("repo", repoDid) @@ -116,34 +173,53 @@ func (s *Spindle) acquireNotifySlot(host string) (release func(), ok bool) { }, true } -func (s *Spindle) runPush(ctx context.Context, k knotclient.Knot, repoDid syntax.DID, push *org_tangled.EventPush) (syntax.ATURI, error) { +func (s *Spindle) runPush(ctx context.Context, k knotclient.Knot, repoDid syntax.DID, push *org_tangled.EventPush) (pipeline syntax.ATURI, err error) { + defer func() { + if err != nil && !errors.Is(err, xrpc.ErrNotifyBusy) { + log.FromContext(ctx).Error("notify event failed", "repo", repoDid, "err", err) + } + }() + + repo, err := s.repoForPush(ctx, k, repoDid) + if err != nil || repo == nil { + return "", err + } release, ok := s.acquireNotifySlot(k.Host) if !ok { return "", xrpc.ErrNotifyBusy } defer release() - pipeline, err := s.notifyPush(ctx, k, repoDid, push) - if err != nil { - log.FromContext(ctx).Error("notify event failed", "repo", repoDid, "err", err) - } - return pipeline, err + defer s.lockRepo(repoDid)() + return s.handlePush(ctx, k, repo, push) } -// a repo in the db already passed the manifest check in ensureRepo -func (s *Spindle) notifyPush(ctx context.Context, k knotclient.Knot, repoDid syntax.DID, push *org_tangled.EventPush) (syntax.ATURI, error) { +// a repo in the db already passed the manifest check in ensureRepo, so only the others read the knot +func (s *Spindle) repoForPush(ctx context.Context, k knotclient.Knot, repoDid syntax.DID) (*db.Repo, error) { repo, err := s.db.GetRepoByDid(repoDid) - if errors.Is(err, sql.ErrNoRows) { - if repo, err = s.ensureRepo(ctx, k, repoDid); err == nil && repo == nil { - log.FromContext(ctx).Info("ignoring event for a repo this spindle does not serve", "repo", repoDid) - return "", nil - } + if err == nil { + return repo, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("looking up repo %s: %w", repoDid, err) + } + if s.notifyAdmission.missedRecently(repoDid) { + return nil, nil + } + if !s.notifyAdmission.acquire(k.Host) { + return nil, xrpc.ErrNotifyBusy } + defer s.notifyAdmission.release(k.Host) + + repo, err = s.ensureRepo(ctx, k, repoDid) if err != nil { - return "", fmt.Errorf("looking up repo %s: %w", repoDid, err) + return nil, fmt.Errorf("looking up repo %s: %w", repoDid, err) } - defer s.lockRepo(repoDid)() - return s.handlePush(ctx, k, repo, push) + if repo == nil { + s.notifyAdmission.miss(repoDid) + log.FromContext(ctx).Info("ignoring event for a repo this spindle does not serve", "repo", repoDid) + } + return repo, nil } func (s *Spindle) handlePush(ctx context.Context, k knotclient.Knot, repo *db.Repo, push *org_tangled.EventPush) (syntax.ATURI, error) { diff --git a/spindle/server.go b/spindle/server.go index 89144693d..42f3bd234 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -121,9 +121,10 @@ type Spindle struct { retries repoRetries knots *knotclient.Client - pushGroup singleflight.Group - notifyInflight atomic.Int32 - knotInflight knotSlots + pushGroup singleflight.Group + notifyInflight atomic.Int32 + knotInflight knotSlots + notifyAdmission notifyAdmission } func newCacheStore(ctx context.Context, cfg *config.Config) (storage.Storage, error) {