From eb4cc871f34bf519d44cb84239dd648f44abd2c2 Mon Sep 17 00:00:00 2001 From: Eric Davis Date: Thu, 12 Nov 2026 11:28:41 -0800 Subject: [PATCH] Add feed rules for OR/AND pattern combinations and targets A feed can now list rules instead of a single pattern. An event that satisfies any rule adds that rule's target; a rule is satisfied when all of its patterns match. A rule's target is a dot path to a {uri, cid} reference in the event, such as the reply parent or quoted post, and is added only when it names an app.bsky.feed.post by DID. Without a target, the rule adds the matched post. `pattern` remains as shorthand for a single one-pattern rule. Co-Authored-By: Claude Opus 5.5 --- README.md | 39 +++++- examples/feeds.toml | 24 ++++ examples/patterns/dudes-quote-media.json | 8 ++ examples/patterns/dudes-quote.json | 8 ++ examples/patterns/dudes-reply.json | 8 ++ examples/patterns/standard-document.json | 7 + examples/patterns/standard-publication.json | 7 + internal/config/feeds.go | 135 ++++++++++++++++---- internal/config/feeds_test.go | 55 +++++++- internal/ingest/ingest.go | 64 ++++++++-- internal/ingest/ingest_test.go | 119 ++++++++++++++++- internal/matcher/matcher.go | 61 +++++++-- internal/matcher/matcher_test.go | 86 +++++++++++-- 13 files changed, 553 insertions(+), 68 deletions(-) create mode 100644 examples/patterns/dudes-quote-media.json create mode 100644 examples/patterns/dudes-quote.json create mode 100644 examples/patterns/dudes-reply.json create mode 100644 examples/patterns/standard-document.json create mode 100644 examples/patterns/standard-publication.json diff --git a/README.md b/README.md index 24b86e5..d938ac5 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ sets a value, Updraft uses its built-in default. | Key | Built-in default | Meaning | |---|---|---| -| `pattern` | (none, required) | Path to the feed's Quamina pattern file, relative to `feeds.toml` | +| `pattern` | (none) | Path to the feed's Quamina pattern file, relative to `feeds.toml`. Set this or `rules` | | `page_size` | `30` | Posts per page when the request omits `limit` (1 to 100) | | `max_age` | `"48h"` | Posts older than this are purged (Go duration syntax) | | `algorithm` | `"chronological"` | `"chronological"` (newest first) or `"hot"` | @@ -83,6 +83,43 @@ See Quamina's full syntax (`prefix`, `shellstyle`, `wildcard`, `anything-but`, `equals-ignore-case`, `exists`, `regexp`). +### Rules + +When a single pattern can't describe a feed, list `rules` instead of +`pattern`. An event that satisfies any rule adds that rule's target to the +feed. A rule is satisfied when all of its patterns match. + +```toml +[feeds.dudes] +max_age = "24h" + +[[feeds.dudes.rules]] +patterns = ["patterns/dudes-reply.json"] +target = "commit.record.reply.parent" + +[[feeds.dudes.rules]] +patterns = ["patterns/dudes-quote.json"] +target = "commit.record.embed.record" + +[[feeds.standard.rules]] +patterns = ["patterns/standard-publication.json", "patterns/standard-document.json"] +``` + +| Key | Meaning | +|---|---| +| `patterns` | Paths to Quamina pattern files, relative to `feeds.toml`. All of them must match | +| `target` | Dot path, from the event root, to a `{uri, cid}` reference naming the post to add. Without it, the rule adds the matched post | + +`pattern = "p.json"` is shorthand for one rule with `patterns = ["p.json"]` and +no `target`. + +A target is skipped unless it is an `app.bsky.feed.post` URI, so a rule +targeting quotes ignores quoted lists, feeds and starter packs. A quote with +images or video nests the quoted post one level deeper, at +`commit.record.embed.record.record`, so it needs its own rule. If an event +satisfies several rules, every target is added. A post that is already in the +feed keeps the time it was first added. + ## Ranking - **chronological**: newest first, using the Jetstream event time. diff --git a/examples/feeds.toml b/examples/feeds.toml index 09c0ded..8b7d4dc 100644 --- a/examples/feeds.toml +++ b/examples/feeds.toml @@ -17,3 +17,27 @@ max_age = "72h" [feeds.atproto] pattern = "patterns/atproto.json" + +# A feed can list rules instead of a single pattern. An event that satisfies +# any rule adds that rule's target. A rule is satisfied when all of its +# patterns match. "target" is a dot path to a {uri, cid} reference in the +# event; without it, the rule adds the matched post. +[feeds.dudes] +max_age = "24h" + +[[feeds.dudes.rules]] +patterns = ["patterns/dudes-reply.json"] +target = "commit.record.reply.parent" + +[[feeds.dudes.rules]] +patterns = ["patterns/dudes-quote.json"] +target = "commit.record.embed.record" + +# A quote with images or video nests the quoted post one level deeper. +[[feeds.dudes.rules]] +patterns = ["patterns/dudes-quote-media.json"] +target = "commit.record.embed.record.record" + +# Both patterns must match, so a post needs a publication ref and a document ref. +[[feeds.standard.rules]] +patterns = ["patterns/standard-publication.json", "patterns/standard-document.json"] diff --git a/examples/patterns/dudes-quote-media.json b/examples/patterns/dudes-quote-media.json new file mode 100644 index 0000000..b165ce0 --- /dev/null +++ b/examples/patterns/dudes-quote-media.json @@ -0,0 +1,8 @@ +{ + "commit": { + "record": { + "text": [{"shellstyle": "*dudes rock*"}], + "embed": {"record": {"record": {"uri": [{"exists": true}]}}} + } + } +} diff --git a/examples/patterns/dudes-quote.json b/examples/patterns/dudes-quote.json new file mode 100644 index 0000000..568b222 --- /dev/null +++ b/examples/patterns/dudes-quote.json @@ -0,0 +1,8 @@ +{ + "commit": { + "record": { + "text": [{"shellstyle": "*dudes rock*"}], + "embed": {"record": {"uri": [{"exists": true}]}} + } + } +} diff --git a/examples/patterns/dudes-reply.json b/examples/patterns/dudes-reply.json new file mode 100644 index 0000000..2b4a5f6 --- /dev/null +++ b/examples/patterns/dudes-reply.json @@ -0,0 +1,8 @@ +{ + "commit": { + "record": { + "text": [{"shellstyle": "*dudes rock*"}], + "reply": {"parent": {"uri": [{"exists": true}]}} + } + } +} diff --git a/examples/patterns/standard-document.json b/examples/patterns/standard-document.json new file mode 100644 index 0000000..1d5770e --- /dev/null +++ b/examples/patterns/standard-document.json @@ -0,0 +1,7 @@ +{ + "commit": { + "record": { + "embed": {"external": {"associatedRefs": {"uri": [{"shellstyle": "*/site.standard.document/*"}]}}} + } + } +} diff --git a/examples/patterns/standard-publication.json b/examples/patterns/standard-publication.json new file mode 100644 index 0000000..f2998f7 --- /dev/null +++ b/examples/patterns/standard-publication.json @@ -0,0 +1,7 @@ +{ + "commit": { + "record": { + "embed": {"external": {"associatedRefs": {"uri": [{"shellstyle": "*/site.standard.publication/*"}]}}} + } + } +} diff --git a/internal/config/feeds.go b/internal/config/feeds.go index 85526d3..3e400b5 100644 --- a/internal/config/feeds.go +++ b/internal/config/feeds.go @@ -31,10 +31,9 @@ const MaxPageSize = 100 type Feed struct { // Name is the feed's record key in at:///app.bsky.feed.generator/. Name string - // PatternPath is the resolved path of the feed's Quamina pattern file. - PatternPath string - // Pattern is the Quamina pattern JSON read from PatternPath. - Pattern string + // Rules decide which posts the feed adds. An event that satisfies any rule + // adds that rule's target. + Rules []Rule // PageSize is the number of posts returned when a request does not specify a limit. PageSize int // MaxAge is how long matched posts are kept before being purged. @@ -47,15 +46,40 @@ type Feed struct { TrackLikes bool } +// Rule is a set of patterns that must all match an event for the feed to add the rule's target. +type Rule struct { + // Patterns are the Quamina patterns that must all match. + Patterns []Pattern + // Target is the dot path, from the event root, to a strongRef ({uri, cid}) + // naming the post to add. An empty Target adds the matched post itself. + Target string +} + +// Pattern is a Quamina pattern loaded from a file. +type Pattern struct { + // Path is the resolved path of the pattern file. + Path string + // JSON is the Quamina pattern read from Path. + JSON string +} + // feedSettings is one TOML table of feed settings. Pointer fields distinguish // "not set" from a zero value so a feed can override a default with zero/false. type feedSettings struct { - Pattern *string `toml:"pattern"` - PageSize *int `toml:"page_size"` - MaxAge *string `toml:"max_age"` - Algorithm *string `toml:"algorithm"` - Gravity *float64 `toml:"gravity"` - TrackLikes *bool `toml:"track_likes"` + // Pattern is shorthand for a single rule holding one pattern and no target. + Pattern *string `toml:"pattern"` + Rules []ruleSettings `toml:"rules"` + PageSize *int `toml:"page_size"` + MaxAge *string `toml:"max_age"` + Algorithm *string `toml:"algorithm"` + Gravity *float64 `toml:"gravity"` + TrackLikes *bool `toml:"track_likes"` +} + +// ruleSettings is one [[feeds..rules]] table. +type ruleSettings struct { + Patterns []string `toml:"patterns"` + Target string `toml:"target"` } type feedsFile struct { @@ -117,8 +141,11 @@ func LoadFeeds(path string) ([]Feed, error) { // withFallback returns s with every unset field taken from fallback. func (s feedSettings) withFallback(fallback feedSettings) feedSettings { - if s.Pattern == nil { + // pattern and rules are alternatives, so a feed that sets either one + // inherits neither. + if s.Pattern == nil && s.Rules == nil { s.Pattern = fallback.Pattern + s.Rules = fallback.Rules } if s.PageSize == nil { s.PageSize = fallback.PageSize @@ -179,25 +206,81 @@ func resolveFeed(name string, settings feedSettings, baseDir string) (Feed, erro errs = append(errs, fmt.Errorf("algorithm must be %q or %q", AlgorithmChronological, AlgorithmHot)) } - if settings.Pattern == nil || *settings.Pattern == "" { - errs = append(errs, errors.New("pattern is required")) - } else { - feed.PatternPath = *settings.Pattern - if !filepath.IsAbs(feed.PatternPath) { - feed.PatternPath = filepath.Join(baseDir, feed.PatternPath) + rules, err := resolveRules(settings, baseDir) + if err != nil { + errs = append(errs, err) + } + feed.Rules = rules + + return feed, errors.Join(errs...) +} + +// resolveRules validates the feed's rules and loads their pattern files. +// A pattern setting becomes a single rule holding that pattern and no target. +func resolveRules(settings feedSettings, baseDir string) ([]Rule, error) { + if settings.Pattern != nil && settings.Rules != nil { + return nil, errors.New("set pattern or rules, not both") + } + if settings.Pattern != nil { + pattern, err := loadPattern(*settings.Pattern, baseDir) + if err != nil { + return nil, err } - pattern, err := os.ReadFile(feed.PatternPath) - switch { - case err != nil: - errs = append(errs, fmt.Errorf("reading pattern: %w", err)) - case !json.Valid(pattern): - errs = append(errs, fmt.Errorf("pattern %s is not valid JSON", feed.PatternPath)) - default: - feed.Pattern = string(pattern) + return []Rule{{Patterns: []Pattern{pattern}}}, nil + } + if len(settings.Rules) == 0 { + return nil, errors.New("pattern or rules is required") + } + + rules := make([]Rule, len(settings.Rules)) + var errs []error + for i, ruleSettings := range settings.Rules { + rule, err := resolveRule(ruleSettings, baseDir) + if err != nil { + errs = append(errs, fmt.Errorf("rules[%d]: %w", i, err)) } + rules[i] = rule } + return rules, errors.Join(errs...) +} - return feed, errors.Join(errs...) +// resolveRule validates one rule and loads its pattern files. +func resolveRule(settings ruleSettings, baseDir string) (Rule, error) { + rule := Rule{Target: settings.Target} + var errs []error + if len(settings.Patterns) == 0 { + errs = append(errs, errors.New("patterns is required")) + } + if rule.Target != "" && slices.Contains(strings.Split(rule.Target, "."), "") { + errs = append(errs, fmt.Errorf("target %q is not a valid dot path", rule.Target)) + } + for _, path := range settings.Patterns { + pattern, err := loadPattern(path, baseDir) + if err != nil { + errs = append(errs, err) + continue + } + rule.Patterns = append(rule.Patterns, pattern) + } + return rule, errors.Join(errs...) +} + +// loadPattern reads the Quamina pattern file at path, resolved relative to baseDir. +func loadPattern(path, baseDir string) (Pattern, error) { + if path == "" { + return Pattern{}, errors.New("pattern path must not be empty") + } + if !filepath.IsAbs(path) { + path = filepath.Join(baseDir, path) + } + data, err := os.ReadFile(path) + switch { + case err != nil: + return Pattern{}, fmt.Errorf("reading pattern: %w", err) + case !json.Valid(data): + return Pattern{}, fmt.Errorf("pattern %s is not valid JSON", path) + } + return Pattern{Path: path, JSON: string(data)}, nil } func ptr[T any](value T) *T { diff --git a/internal/config/feeds_test.go b/internal/config/feeds_test.go index 8d80d2b..cb6bac9 100644 --- a/internal/config/feeds_test.go +++ b/internal/config/feeds_test.go @@ -63,11 +63,15 @@ track_likes = false plain.Algorithm != AlgorithmChronological || plain.Gravity != 0.1 || !plain.TrackLikes { t.Errorf("plain = %+v, want [default] values plus built-in defaults", plain) } - if plain.PatternPath != filepath.Join(filepath.Dir(path), "patterns", "p.json") { - t.Errorf("PatternPath = %q, want path relative to feeds.toml", plain.PatternPath) + if len(plain.Rules) != 1 || len(plain.Rules[0].Patterns) != 1 || plain.Rules[0].Target != "" { + t.Fatalf("Rules = %+v, want pattern shorthand as one single-pattern rule targeting the matched post", plain.Rules) } - if !strings.Contains(plain.Pattern, "shellstyle") { - t.Errorf("Pattern = %q, want pattern file contents", plain.Pattern) + pattern := plain.Rules[0].Patterns[0] + if pattern.Path != filepath.Join(filepath.Dir(path), "patterns", "p.json") { + t.Errorf("Path = %q, want path relative to feeds.toml", pattern.Path) + } + if !strings.Contains(pattern.JSON, "shellstyle") { + t.Errorf("JSON = %q, want pattern file contents", pattern.JSON) } if hot.Algorithm != AlgorithmHot || hot.Gravity != 0.5 || hot.MaxAge != 72*time.Hour || hot.PageSize != 25 { @@ -87,7 +91,12 @@ func TestLoadFeedsValidationErrors(t *testing.T) { }{ {"no feeds", `[default]`, "defines no feeds"}, {"unknown key", "[feeds.a]\npattern = \"patterns/p.json\"\npagesize = 3", "unknown keys"}, - {"missing pattern", "[feeds.a]\n", "pattern is required"}, + {"missing pattern", "[feeds.a]\n", "pattern or rules is required"}, + {"pattern and rules", "[feeds.a]\npattern = \"patterns/p.json\"\n[[feeds.a.rules]]\npatterns = [\"patterns/p.json\"]", "not both"}, + {"rule without patterns", "[[feeds.a.rules]]\ntarget = \"commit.record.reply.parent\"", "rules[0]: patterns is required"}, + {"rule pattern file missing", "[[feeds.a.rules]]\npatterns = [\"patterns/p.json\", \"nope.json\"]", "rules[0]: reading pattern"}, + {"bad target", "[[feeds.a.rules]]\npatterns = [\"patterns/p.json\"]\ntarget = \"commit..parent\"", "not a valid dot path"}, + {"unknown rule key", "[[feeds.a.rules]]\npatterns = [\"patterns/p.json\"]\ntargte = \"x\"", "unknown keys"}, {"pattern file missing", "[feeds.a]\npattern = \"nope.json\"", "reading pattern"}, {"bad page size", "[feeds.a]\npattern = \"patterns/p.json\"\npage_size = 101", "page_size"}, {"bad max age", "[feeds.a]\npattern = \"patterns/p.json\"\nmax_age = \"soon\"", "max_age"}, @@ -106,6 +115,42 @@ func TestLoadFeedsValidationErrors(t *testing.T) { } } +func TestLoadFeedsResolvesRules(t *testing.T) { + path := writeFeedsDir(t, ` +[feeds.a] +max_age = "1h" + +[[feeds.a.rules]] +patterns = ["patterns/p.json"] +target = "commit.record.reply.parent" + +[[feeds.a.rules]] +patterns = ["patterns/p.json", "patterns/p.json"] +`) + + feeds, err := LoadFeeds(path) + if err != nil { + t.Fatalf("LoadFeeds: %v", err) + } + rules := feeds[0].Rules + if len(rules) != 2 { + t.Fatalf("got %d rules, want 2", len(rules)) + } + if rules[0].Target != "commit.record.reply.parent" || len(rules[0].Patterns) != 1 { + t.Errorf("rules[0] = %+v, want one pattern with the reply parent target", rules[0]) + } + if rules[1].Target != "" || len(rules[1].Patterns) != 2 { + t.Errorf("rules[1] = %+v, want two patterns targeting the matched post", rules[1]) + } + for _, rule := range rules { + for _, pattern := range rule.Patterns { + if !strings.Contains(pattern.JSON, "shellstyle") { + t.Errorf("JSON = %q, want pattern file contents", pattern.JSON) + } + } + } +} + func TestLoadFeedsRejectsInvalidPatternJSON(t *testing.T) { path := writeFeedsDir(t, "[feeds.a]\npattern = \"bad.json\"") if err := os.WriteFile(filepath.Join(filepath.Dir(path), "bad.json"), []byte("{nope"), 0o644); err != nil { diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 9de7fbd..55ca7e4 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "time" "github.com/bluesky-social/jetstream" @@ -187,12 +188,30 @@ func (i *Ingester) processPostCreate(ctx context.Context, tx *store.Tx, event je return nil } - for _, name := range matches { - if err := tx.InsertPost(ctx, name, uri, event.Commit.CID, event.TimeUS); err != nil { + // fields is the event decoded generically, so rule targets can be looked + // up by dot path. It is decoded only when a matching rule has a target. + var fields map[string]any + for _, match := range matches { + targetURI, targetCID := uri, event.Commit.CID + if match.Target != "" { + if fields == nil { + if err := json.Unmarshal(data, &fields); err != nil { + i.log.Warn("skipping undecodable post", "uri", uri, "err", err) + return nil + } + } + var ok bool + targetURI, targetCID, ok = lookupStrongRef(fields, match.Target) + if !ok || !isPostURI(targetURI) { + i.log.Debug("skipping rule target that is not a post", "uri", uri, "feed", match.Feed, "target", match.Target) + continue + } + } + if err := tx.InsertPost(ctx, match.Feed, targetURI, targetCID, event.TimeUS); err != nil { return err } - if i.feeds[name].TrackLikes { - i.tracked[uri] = struct{}{} + if i.feeds[match.Feed].TrackLikes { + i.tracked[targetURI] = struct{}{} } } return nil @@ -238,12 +257,39 @@ func recordURI(did, collection, rkey string) string { return fmt.Sprintf("at://%s/%s/%s", did, collection, rkey) } -// likeSubjectURI extracts subject.uri from an app.bsky.feed.like record. -func likeSubjectURI(record map[string]any) (string, bool) { - subject, ok := record["subject"].(map[string]any) +// isPostURI reports whether uri is an at:// URI naming an app.bsky.feed.post +// record by DID, the form used by firehose deletes. +func isPostURI(uri string) bool { + rest, ok := strings.CutPrefix(uri, "at://") if !ok { - return "", false + return false } - uri, ok := subject["uri"].(string) + parts := strings.Split(rest, "/") + return len(parts) == 3 && strings.HasPrefix(parts[0], "did:") && parts[1] == postCollection && parts[2] != "" +} + +// likeSubjectURI extracts subject.uri from an app.bsky.feed.like record. +func likeSubjectURI(record map[string]any) (string, bool) { + uri, _, ok := lookupStrongRef(record, "subject") return uri, ok } + +// lookupStrongRef follows the dot path through fields to a strongRef object +// and returns its uri and cid. ok is false when the path or uri is missing. +func lookupStrongRef(fields map[string]any, path string) (uri, cid string, ok bool) { + var value any = fields + for _, key := range strings.Split(path, ".") { + object, isObject := value.(map[string]any) + if !isObject { + return "", "", false + } + value = object[key] + } + ref, isObject := value.(map[string]any) + if !isObject { + return "", "", false + } + uri, ok = ref["uri"].(string) + cid, _ = ref["cid"].(string) + return uri, cid, ok +} diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index e103e5d..4d4d79d 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -18,7 +18,7 @@ import ( var testFeeds = []config.Feed{ { Name: "golang", - Pattern: `{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`, + Rules: singlePatternRules(`{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`), MaxAge: time.Hour, Algorithm: config.AlgorithmHot, Gravity: 0.1, @@ -26,12 +26,17 @@ var testFeeds = []config.Feed{ }, { Name: "rust", - Pattern: `{"commit": {"record": {"text": [{"shellstyle": "*rust*"}]}}}`, + Rules: singlePatternRules(`{"commit": {"record": {"text": [{"shellstyle": "*rust*"}]}}}`), MaxAge: 2 * time.Hour, Algorithm: config.AlgorithmChronological, }, } +// singlePatternRules returns one rule holding pattern and targeting the matched post. +func singlePatternRules(pattern string) []config.Rule { + return []config.Rule{{Patterns: []config.Pattern{{JSON: pattern}}}} +} + func newTestIngester(t *testing.T, feeds []config.Feed) (*Ingester, *store.Store) { t.Helper() st, err := store.Open(filepath.Join(t.TempDir(), "test.db")) @@ -214,3 +219,113 @@ func TestCollectionsIncludesLikesOnlyWhenTracked(t *testing.T) { t.Errorf("collections = %v", got) } } + +// strongRef returns a {uri, cid} record reference. +func strongRef(uri string) map[string]any { + return map[string]any{"uri": uri, "cid": "cid-of-" + uri} +} + +// dudesFeed adds the parent of a matching reply and the post a matching post quotes. +var dudesFeed = config.Feed{ + Name: "dudes", + Rules: []config.Rule{ + { + Patterns: []config.Pattern{{JSON: `{"commit": {"record": {"text": [{"shellstyle": "*dudes rock*"}], "reply": {"parent": {"uri": [{"exists": true}]}}}}}`}}, + Target: "commit.record.reply.parent", + }, + { + Patterns: []config.Pattern{{JSON: `{"commit": {"record": {"text": [{"shellstyle": "*dudes rock*"}], "embed": {"record": {"uri": [{"exists": true}]}}}}}`}}, + Target: "commit.record.embed.record", + }, + }, + MaxAge: time.Hour, + Algorithm: config.AlgorithmHot, + Gravity: 0.1, + TrackLikes: true, +} + +func TestProcessBatchStoresRuleTargets(t *testing.T) { + i, st := newTestIngester(t, []config.Feed{dudesFeed}) + parentURI := "at://did:plc:p/app.bsky.feed.post/parent" + quotedURI := "at://did:plc:q/app.bsky.feed.post/quoted" + + reply := postCreate("did:plc:a", "reply", "dudes rock", 100) + reply.Commit.Record["reply"] = map[string]any{"root": strongRef(parentURI), "parent": strongRef(parentURI)} + + replyAndQuote := postCreate("did:plc:a", "both", "dudes rock", 100) + replyAndQuote.Commit.Record["reply"] = map[string]any{"root": strongRef(parentURI), "parent": strongRef(parentURI)} + replyAndQuote.Commit.Record["embed"] = map[string]any{"$type": "app.bsky.embed.record", "record": strongRef(quotedURI)} + + plain := postCreate("did:plc:a", "plain", "dudes rock", 100) + + quotesList := postCreate("did:plc:a", "list", "dudes rock", 100) + quotesList.Commit.Record["embed"] = map[string]any{"$type": "app.bsky.embed.record", "record": strongRef("at://did:plc:q/app.bsky.graph.list/l")} + + process(t, i, 1, reply, replyAndQuote, plain, quotesList) + + if got, want := feedURIs(t, st, "dudes"), []string{parentURI, quotedURI}; !slices.Equal(got, want) { + t.Errorf("dudes feed = %v, want %v", got, want) + } + if _, ok := i.tracked[parentURI]; !ok { + t.Errorf("tracked = %v, want the parent post tracked", i.tracked) + } + + process(t, i, 2, likeCreate(parentURI, 200)) + posts, _ := st.FeedPosts(context.Background(), "dudes") + for _, p := range posts { + if p.URI == parentURI && p.Score != 1 { + t.Errorf("parent score = %d, want 1", p.Score) + } + } +} + +func TestProcessBatchRequiresEveryPatternOfARule(t *testing.T) { + feed := config.Feed{ + Name: "standard", + Rules: []config.Rule{{Patterns: []config.Pattern{ + {JSON: `{"commit": {"record": {"embed": {"external": {"associatedRefs": {"uri": [{"shellstyle": "*/site.standard.publication/*"}]}}}}}}`}, + {JSON: `{"commit": {"record": {"embed": {"external": {"associatedRefs": {"uri": [{"shellstyle": "*/site.standard.document/*"}]}}}}}}`}, + }}}, + MaxAge: time.Hour, + Algorithm: config.AlgorithmChronological, + } + i, st := newTestIngester(t, []config.Feed{feed}) + + withRefs := func(rkey string, uris ...string) jetstream.Event { + event := postCreate("did:plc:a", rkey, "new post", 100) + var refs []any + for _, uri := range uris { + refs = append(refs, map[string]any{"uri": uri}) + } + event.Commit.Record["embed"] = map[string]any{"external": map[string]any{"associatedRefs": refs}} + return event + } + process(t, i, 1, + withRefs("both", "at://did:plc:a/site.standard.publication/p", "at://did:plc:a/site.standard.document/d"), + withRefs("pub", "at://did:plc:a/site.standard.publication/p"), + withRefs("doc", "at://did:plc:a/site.standard.document/d"), + ) + + if got, want := feedURIs(t, st, "standard"), []string{"at://did:plc:a/app.bsky.feed.post/both"}; !slices.Equal(got, want) { + t.Errorf("standard feed = %v, want %v", got, want) + } +} + +func TestIsPostURI(t *testing.T) { + tests := []struct { + uri string + want bool + }{ + {"at://did:plc:a/app.bsky.feed.post/3kabc", true}, + {"at://did:plc:a/app.bsky.graph.list/3kabc", false}, + {"at://alice.example.com/app.bsky.feed.post/3kabc", false}, + {"at://did:plc:a/app.bsky.feed.post", false}, + {"at://did:plc:a/app.bsky.feed.post/3kabc/extra", false}, + {"https://did:plc:a/app.bsky.feed.post/3kabc", false}, + } + for _, tt := range tests { + if got := isPostURI(tt.uri); got != tt.want { + t.Errorf("isPostURI(%q) = %v, want %v", tt.uri, got, tt.want) + } + } +} diff --git a/internal/matcher/matcher.go b/internal/matcher/matcher.go index 9da2a19..bbb2fa8 100644 --- a/internal/matcher/matcher.go +++ b/internal/matcher/matcher.go @@ -9,35 +9,74 @@ import ( "tangled.org/edavis.dev/updraft/internal/config" ) -// Matcher reports which feeds' patterns an event matches. +// Matcher reports which feed rules an event satisfies. // A Matcher is not safe for concurrent use. type Matcher struct { q *quamina.Quamina + // rules holds every feed rule by its ID. + rules map[ruleID]config.Rule } -// New builds a Matcher holding the pattern of every feed. +// Match is a feed rule whose patterns all matched an event. +type Match struct { + // Feed is the name of the feed the rule belongs to. + Feed string + // Target is the rule's target path. An empty Target means the matched post. + Target string +} + +// ruleID identifies one rule of one feed. +type ruleID struct { + feed string + rule int +} + +// patternID identifies one pattern of one feed rule. It is the value Quamina +// reports for each matching pattern. +type patternID struct { + ruleID + pattern int +} + +// New builds a Matcher holding the patterns of every feed rule. func New(feeds []config.Feed) (*Matcher, error) { q, err := quamina.New() if err != nil { return nil, fmt.Errorf("creating quamina: %w", err) } + rules := make(map[ruleID]config.Rule) for _, feed := range feeds { - if err := q.AddPattern(feed.Name, feed.Pattern); err != nil { - return nil, fmt.Errorf("feed %q: invalid pattern %s: %w", feed.Name, feed.PatternPath, err) + for r, rule := range feed.Rules { + id := ruleID{feed: feed.Name, rule: r} + rules[id] = rule + for p, pattern := range rule.Patterns { + if err := q.AddPattern(patternID{ruleID: id, pattern: p}, pattern.JSON); err != nil { + return nil, fmt.Errorf("feed %q: invalid pattern %s: %w", feed.Name, pattern.Path, err) + } + } } } - return &Matcher{q: q}, nil + return &Matcher{q: q, rules: rules}, nil } -// Match returns the names of the feeds whose patterns match the JSON event. -func (m *Matcher) Match(event []byte) ([]string, error) { +// Match returns the feed rules whose patterns all match the JSON event. +func (m *Matcher) Match(event []byte) ([]Match, error) { matches, err := m.q.MatchesForEvent(event) if err != nil { return nil, err } - names := make([]string, len(matches)) - for i, match := range matches { - names[i] = match.(string) + // Quamina reports each matching pattern once, so counting a rule's + // matching patterns tells whether all of them matched. + matchedPatterns := make(map[ruleID]int) + for _, match := range matches { + matchedPatterns[match.(patternID).ruleID]++ + } + var satisfied []Match + for id, count := range matchedPatterns { + rule := m.rules[id] + if count == len(rule.Patterns) { + satisfied = append(satisfied, Match{Feed: id.feed, Target: rule.Target}) + } } - return names, nil + return satisfied, nil } diff --git a/internal/matcher/matcher_test.go b/internal/matcher/matcher_test.go index 3654f57..5db14fc 100644 --- a/internal/matcher/matcher_test.go +++ b/internal/matcher/matcher_test.go @@ -39,12 +39,13 @@ func postEvent(t *testing.T, did, text string, langs ...string) []byte { return data } -func newMatcher(t *testing.T, patterns map[string]string) *Matcher { +// singlePatternFeed returns a feed whose one rule holds pattern and targets the matched post. +func singlePatternFeed(name, pattern string) config.Feed { + return config.Feed{Name: name, Rules: []config.Rule{{Patterns: []config.Pattern{{JSON: pattern}}}}} +} + +func newMatcher(t *testing.T, feeds ...config.Feed) *Matcher { t.Helper() - var feeds []config.Feed - for name, pattern := range patterns { - feeds = append(feeds, config.Feed{Name: name, Pattern: pattern}) - } m, err := New(feeds) if err != nil { t.Fatalf("New: %v", err) @@ -53,11 +54,11 @@ func newMatcher(t *testing.T, patterns map[string]string) *Matcher { } func TestMatch(t *testing.T) { - m := newMatcher(t, map[string]string{ - "golang": `{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`, - "english": `{"commit": {"record": {"langs": ["en"]}}}`, - "alice": `{"did": ["did:plc:alice"]}`, - }) + m := newMatcher(t, + singlePatternFeed("golang", `{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`), + singlePatternFeed("english", `{"commit": {"record": {"langs": ["en"]}}}`), + singlePatternFeed("alice", `{"did": ["did:plc:alice"]}`), + ) tests := []struct { name string @@ -75,17 +76,74 @@ func TestMatch(t *testing.T) { if err != nil { t.Fatal(err) } - slices.Sort(got) - if !slices.Equal(got, tt.want) { - t.Errorf("Match = %v, want %v", got, tt.want) + var feeds []string + for _, match := range got { + feeds = append(feeds, match.Feed) + } + slices.Sort(feeds) + if !slices.Equal(feeds, tt.want) { + t.Errorf("Match feeds = %v, want %v", feeds, tt.want) } }) } } func TestNewRejectsInvalidPattern(t *testing.T) { - _, err := New([]config.Feed{{Name: "bad", PatternPath: "bad.json", Pattern: `{"text": "not-an-array"}`}}) + _, err := New([]config.Feed{singlePatternFeed("bad", `{"text": "not-an-array"}`)}) if err == nil || !strings.Contains(err.Error(), `feed "bad"`) { t.Fatalf("err = %v, want invalid pattern error naming the feed", err) } } + +func TestMatchRequiresEveryPatternOfARule(t *testing.T) { + m := newMatcher(t, config.Feed{Name: "both", Rules: []config.Rule{{Patterns: []config.Pattern{ + {JSON: `{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`}, + {JSON: `{"commit": {"record": {"langs": ["en"]}}}`}, + }}}}) + + tests := []struct { + name string + event []byte + want int + }{ + {"neither", postEvent(t, "did:plc:bob", "hello", "fr"), 0}, + {"only first", postEvent(t, "did:plc:bob", "golang", "fr"), 0}, + {"only second", postEvent(t, "did:plc:bob", "hello", "en"), 0}, + {"both", postEvent(t, "did:plc:bob", "golang", "en"), 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := m.Match(tt.event) + if err != nil { + t.Fatal(err) + } + if len(got) != tt.want { + t.Errorf("Match = %v, want %d matches", got, tt.want) + } + }) + } +} + +func TestMatchReturnsEachSatisfiedRuleWithItsTarget(t *testing.T) { + m := newMatcher(t, config.Feed{Name: "f", Rules: []config.Rule{ + {Patterns: []config.Pattern{{JSON: `{"commit": {"record": {"text": [{"shellstyle": "*golang*"}]}}}`}}, Target: "golang.target"}, + {Patterns: []config.Pattern{{JSON: `{"commit": {"record": {"langs": ["en"]}}}`}}, Target: "english.target"}, + {Patterns: []config.Pattern{{JSON: `{"did": ["did:plc:alice"]}`}}, Target: "alice.target"}, + }}) + + got, err := m.Match(postEvent(t, "did:plc:bob", "golang", "en")) + if err != nil { + t.Fatal(err) + } + var targets []string + for _, match := range got { + if match.Feed != "f" { + t.Errorf("Feed = %q, want f", match.Feed) + } + targets = append(targets, match.Target) + } + slices.Sort(targets) + if want := []string{"english.target", "golang.target"}; !slices.Equal(targets, want) { + t.Errorf("targets = %v, want %v", targets, want) + } +} -- knotmirror 0.0.1