palomar: refactor query string parsing; updates to schemas and transform (#643)

TODO:

- [x] normalize URLs: maybe with `purell`, maybe stripping tracking
parameters
- [x] date-level queries
- [x] add end-to-end tests (similar to japanese queries)
- [x] implement "merge" of query params (parsed and actual params)
- [x] run lexgen and wire up actual query params
- [x] `from:me` post search behavior (server-side, using viewer param)
- [ ] alternate post "sort" order (top vs. latest)

Closes: https://github.com/bluesky-social/indigo/issues/191
Closes: https://github.com/bluesky-social/indigo/issues/593

api/bsky/actorsearchActorsTypeahead.go

···
 //
 // q: Search query prefix; not a full query string.
 // term: DEPRECATED: use 'q' instead.
-func ActorSearchActorsTypeahead(ctx context.Context, c *xrpc.Client, limit int64, q string, term string) (*ActorSearchActorsTypeahead_Output, error) {
+// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking.
+func ActorSearchActorsTypeahead(ctx context.Context, c *xrpc.Client, limit int64, q string, term string, viewer string) (*ActorSearchActorsTypeahead_Output, error) {
 	var out ActorSearchActorsTypeahead_Output
 
 	params := map[string]interface{}{
-		"limit": limit,
-		"q":     q,
-		"term":  term,
+		"limit":  limit,
+		"q":      q,
+		"term":   term,
+		"viewer": viewer,
 	}
 	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.actor.searchActorsTypeahead", params, nil, &out); err != nil {
 		return nil, err

api/bsky/feedsearchPosts.go

···
 
 // FeedSearchPosts calls the XRPC method "app.bsky.feed.searchPosts".
 //
+// author: Filter to posts by the given account. Handles are resolved to DID before query-time.
 // cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set.
+// domain: Filter to posts with URLs (facet links or embeds) linking to the given domain (hostname). Server may apply hostname normalization.
+// lang: Filter to posts in the given language. Expected to be based on post language field, though server may override language detection.
+// mentions: Filter to posts which mention the given account. Handles are resolved to DID before query-time. Only matches rich-text facet mentions.
 // q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.
-func FeedSearchPosts(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string) (*FeedSearchPosts_Output, error) {
+// since: Filter results for posts after the indicated datetime (inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYYY-MM-DD).
+// sort: Specifies the ranking order of results.
+// tag: Filter to posts with the given tag (hashtag), based on rich-text facet or tag field. Do not include the hash (#) prefix. Multiple tags can be specified, with 'AND' matching.
+// until: Filter results for posts before the indicated datetime (not inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYY-MM-DD).
+// url: Filter to posts with links (facet links or embeds) pointing to this URL. Server may apply URL normalization or fuzzy matching.
+func FeedSearchPosts(ctx context.Context, c *xrpc.Client, author string, cursor string, domain string, lang string, limit int64, mentions string, q string, since string, sort string, tag []string, until string, url string) (*FeedSearchPosts_Output, error) {
 	var out FeedSearchPosts_Output
 
 	params := map[string]interface{}{
-		"cursor": cursor,
-		"limit":  limit,
-		"q":      q,
+		"author":   author,
+		"cursor":   cursor,
+		"domain":   domain,
+		"lang":     lang,
+		"limit":    limit,
+		"mentions": mentions,
+		"q":        q,
+		"since":    since,
+		"sort":     sort,
+		"tag":      tag,
+		"until":    until,
+		"url":      url,
 	}
 	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.searchPosts", params, nil, &out); err != nil {
 		return nil, err

api/bsky/unspeccedsearchActorsSkeleton.go

···
 // cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set.
 // q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended. For typeahead search, only simple term match is supported, not full syntax.
 // typeahead: If true, acts as fast/simple 'typeahead' query.
-func UnspeccedSearchActorsSkeleton(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string, typeahead bool) (*UnspeccedSearchActorsSkeleton_Output, error) {
+// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used to boost followed accounts in ranking.
+func UnspeccedSearchActorsSkeleton(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string, typeahead bool, viewer string) (*UnspeccedSearchActorsSkeleton_Output, error) {
 	var out UnspeccedSearchActorsSkeleton_Output
 
 	params := map[string]interface{}{
···
 		"limit":     limit,
 		"q":         q,
 		"typeahead": typeahead,
+		"viewer":    viewer,
 	}
 	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.unspecced.searchActorsSkeleton", params, nil, &out); err != nil {
 		return nil, err

api/bsky/unspeccedsearchPostsSkeleton.go

···
 
 // UnspeccedSearchPostsSkeleton calls the XRPC method "app.bsky.unspecced.searchPostsSkeleton".
 //
+// author: Filter to posts by the given account. Handles are resolved to DID before query-time.
 // cursor: Optional pagination mechanism; may not necessarily allow scrolling through entire result set.
+// domain: Filter to posts with URLs (facet links or embeds) linking to the given domain (hostname). Server may apply hostname normalization.
+// lang: Filter to posts in the given language. Expected to be based on post language field, though server may override language detection.
+// mentions: Filter to posts which mention the given account. Handles are resolved to DID before query-time. Only matches rich-text facet mentions.
 // q: Search query string; syntax, phrase, boolean, and faceting is unspecified, but Lucene query syntax is recommended.
-func UnspeccedSearchPostsSkeleton(ctx context.Context, c *xrpc.Client, cursor string, limit int64, q string) (*UnspeccedSearchPostsSkeleton_Output, error) {
+// since: Filter results for posts after the indicated datetime (inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYYY-MM-DD).
+// sort: Specifies the ranking order of results.
+// tag: Filter to posts with the given tag (hashtag), based on rich-text facet or tag field. Do not include the hash (#) prefix. Multiple tags can be specified, with 'AND' matching.
+// until: Filter results for posts before the indicated datetime (not inclusive). Expected to use 'sortAt' timestamp, which may not match 'createdAt'. Can be a datetime, or just an ISO date (YYY-MM-DD).
+// url: Filter to posts with links (facet links or embeds) pointing to this URL. Server may apply URL normalization or fuzzy matching.
+// viewer: DID of the account making the request (not included for public/unauthenticated queries). Used for 'from:me' queries.
+func UnspeccedSearchPostsSkeleton(ctx context.Context, c *xrpc.Client, author string, cursor string, domain string, lang string, limit int64, mentions string, q string, since string, sort string, tag []string, until string, url string, viewer string) (*UnspeccedSearchPostsSkeleton_Output, error) {
 	var out UnspeccedSearchPostsSkeleton_Output
 
 	params := map[string]interface{}{
-		"cursor": cursor,
-		"limit":  limit,
-		"q":      q,
+		"author":   author,
+		"cursor":   cursor,
+		"domain":   domain,
+		"lang":     lang,
+		"limit":    limit,
+		"mentions": mentions,
+		"q":        q,
+		"since":    since,
+		"sort":     sort,
+		"tag":      tag,
+		"until":    until,
+		"url":      url,
+		"viewer":   viewer,
 	}
 	if err := c.Do(ctx, xrpc.Query, "", "app.bsky.unspecced.searchPostsSkeleton", params, nil, &out); err != nil {
 		return nil, err

cmd/palomar/main.go

···
 			identity.DefaultDirectory(), // TODO: parse PLC arg
 			escli,
 			cctx.String("es-post-index"),
-			strings.Join(cctx.Args().Slice(), " "),
-			0,
-			20,
+			&search.PostSearchParams{
+				Query:  strings.Join(cctx.Args().Slice(), " "),
+				Offset: 0,
+				Size:   20,
+			},
 		)
 		if err != nil {
 			return err
···
 				context.Background(),
 				escli,
 				cctx.String("es-profile-index"),
-				strings.Join(cctx.Args().Slice(), " "),
-				10,
+				&search.ActorSearchParams{
+					Query: strings.Join(cctx.Args().Slice(), " "),
+					Size:  10,
+				},
 			)
 			if err != nil {
 				return err
···
 				identity.DefaultDirectory(), // TODO: parse PLC arg
 				escli,
 				cctx.String("es-profile-index"),
-				strings.Join(cctx.Args().Slice(), " "),
-				0,
-				20,
+				&search.ActorSearchParams{
+					Query:  strings.Join(cctx.Args().Slice(), " "),
+					Offset: 0,
+					Size:   20,
+				},
 			)
 			if err != nil {
 				return err

go.mod

···
 )
 
 require (
+	github.com/PuerkitoBio/purell v1.2.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-redis/redis v6.15.9+incompatible // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect

go.sum

···
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/purell v1.2.1 h1:QsZ4TjvwiMpat6gBCBxEQI0rcS9ehtkKtSpiUnd9N28=
+github.com/PuerkitoBio/purell v1.2.1/go.mod h1:ZwHcC/82TOaovDi//J/804umJFFmbOHPngi8iYYv/Eo=
 github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU=
 github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=

search/handlers.go

···
 		})
 	}
 
+	params := PostSearchParams{
+		Query: q,
+		// TODO: parse/validate the sort options here?
+		Sort:   e.QueryParam("sort"),
+		Domain: e.QueryParam("domain"),
+		URL:    e.QueryParam("url"),
+	}
+
+	viewerStr := e.QueryParam("viewer")
+	if viewerStr != "" {
+		d, err := syntax.ParseDID(viewerStr)
+		if err != nil {
+			return e.JSON(400, map[string]any{
+				"error":   "BadRequest",
+				"message": fmt.Sprintf("invalid DID for 'viewer': %s", err),
+			})
+		}
+		params.Viewer = &d
+	}
+	authorStr := e.QueryParam("author")
+	if authorStr != "" {
+		atid, err := syntax.ParseAtIdentifier(authorStr)
+		if err != nil {
+			return &echo.HTTPError{
+				Code:    400,
+				Message: fmt.Sprintf("invalid DID for 'author': %s", err),
+			}
+		}
+		if atid.IsHandle() {
+			ident, err := s.dir.Lookup(context.TODO(), *atid)
+			if err != nil {
+				return e.JSON(400, map[string]any{
+					"error":   "BadRequest",
+					"message": fmt.Sprintf("invalid Handle for 'author': %s", err),
+				})
+			}
+			params.Author = &ident.DID
+		} else {
+			d, err := atid.AsDID()
+			if err != nil {
+				return err
+			}
+			params.Author = &d
+		}
+	}
+
+	mentionsStr := e.QueryParam("mentions")
+	if mentionsStr != "" {
+		atid, err := syntax.ParseAtIdentifier(mentionsStr)
+		if err != nil {
+			return &echo.HTTPError{
+				Code:    400,
+				Message: fmt.Sprintf("invalid DID for 'mentions': %s", err),
+			}
+		}
+		if atid.IsHandle() {
+			ident, err := s.dir.Lookup(context.TODO(), *atid)
+			if err != nil {
+				return e.JSON(400, map[string]any{
+					"error":   "BadRequest",
+					"message": fmt.Sprintf("invalid Handle for 'mentions': %s", err),
+				})
+			}
+			params.Mentions = &ident.DID
+		} else {
+			d, err := atid.AsDID()
+			if err != nil {
+				return err
+			}
+			params.Mentions = &d
+		}
+	}
+
+	sinceStr := e.QueryParam("since")
+	if sinceStr != "" {
+		dt, err := syntax.ParseDatetime(sinceStr)
+		if err != nil {
+			return e.JSON(400, map[string]any{
+				"error":   "BadRequest",
+				"message": fmt.Sprintf("invalid Datetime for 'since': %s", err),
+			})
+		}
+		params.Since = &dt
+	}
+
+	untilStr := e.QueryParam("until")
+	if untilStr != "" {
+		dt, err := syntax.ParseDatetime(untilStr)
+		if err != nil {
+			return e.JSON(400, map[string]any{
+				"error":   "BadRequest",
+				"message": fmt.Sprintf("invalid Datetime for 'until': %s", err),
+			})
+		}
+		params.Until = &dt
+	}
+
+	langStr := e.QueryParam("lang")
+	if langStr != "" {
+		l, err := syntax.ParseLanguage(langStr)
+		if err != nil {
+			return e.JSON(400, map[string]any{
+				"error":   "BadRequest",
+				"message": fmt.Sprintf("invalid Language for 'lang': %s", err),
+			})
+		}
+		params.Lang = &l
+	}
+	// TODO: could be multiple tag params; guess we should "bind"?
+	tags := e.Request().URL.Query()["tags"]
+	if len(tags) > 0 {
+		params.Tags = tags
+	}
+
 	offset, limit, err := parseCursorLimit(e)
 	if err != nil {
 		span.SetAttributes(attribute.String("error", fmt.Sprintf("invalid cursor/limit: %s", err)))
···
 		return err
 	}
 
+	params.Offset = offset
+	params.Size = limit
 	span.SetAttributes(attribute.Int("offset", offset), attribute.Int("limit", limit))
 
-	out, err := s.SearchPosts(ctx, q, offset, limit)
+	out, err := s.SearchPosts(ctx, &params)
 	if err != nil {
 		span.SetAttributes(attribute.String("error", fmt.Sprintf("failed to SearchPosts: %s", err)))
 		span.SetStatus(codes.Error, err.Error())
···
 	q := strings.TrimSpace(e.QueryParam("q"))
 	if q == "" {
 		return e.JSON(400, map[string]any{
-			"error": "must pass non-empty search query",
+			"error":   "BadRequest",
+			"message": "must pass non-empty search query",
 		})
 	}
 
···
 		typeahead = true
 	}
 
+	params := ActorSearchParams{
+		Query:     q,
+		Typeahead: typeahead,
+		Offset:    offset,
+		Size:      limit,
+	}
+
+	viewerStr := e.QueryParam("viewer")
+	if viewerStr != "" {
+		d, err := syntax.ParseDID(viewerStr)
+		if err != nil {
+			return e.JSON(400, map[string]any{
+				"error":   "BadRequest",
+				"message": fmt.Sprintf("invalid DID for 'viewer': %s", err),
+			})
+		}
+		params.Viewer = &d
+	}
+
 	span.SetAttributes(
 		attribute.Int("offset", offset),
 		attribute.Int("limit", limit),
 		attribute.Bool("typeahead", typeahead),
 	)
 
-	out, err := s.SearchProfiles(ctx, q, typeahead, offset, limit)
+	out, err := s.SearchProfiles(ctx, &params)
 	if err != nil {
 		span.SetAttributes(attribute.String("error", fmt.Sprintf("failed to SearchProfiles: %s", err)))
 		span.SetStatus(codes.Error, err.Error())
···
 	})
 }
 
-func (s *Server) SearchPosts(ctx context.Context, q string, offset, size int) (*appbsky.UnspeccedSearchPostsSkeleton_Output, error) {
+func (s *Server) SearchPosts(ctx context.Context, params *PostSearchParams) (*appbsky.UnspeccedSearchPostsSkeleton_Output, error) {
 	ctx, span := tracer.Start(ctx, "SearchPosts")
 	defer span.End()
 
-	resp, err := DoSearchPosts(ctx, s.dir, s.escli, s.postIndex, q, offset, size)
+	resp, err := DoSearchPosts(ctx, s.dir, s.escli, s.postIndex, params)
 	if err != nil {
 		return nil, err
 	}
···
 	}
 
 	out := appbsky.UnspeccedSearchPostsSkeleton_Output{Posts: posts}
-	if len(posts) == size && (offset+size) < 10000 {
-		s := fmt.Sprintf("%d", offset+size)
+	if len(posts) == params.Size && (params.Offset+params.Size) < 10000 {
+		s := fmt.Sprintf("%d", params.Offset+params.Size)
 		out.Cursor = &s
 	}
 	if resp.Hits.Total.Relation == "eq" {
···
 	return &out, nil
 }
 
-func (s *Server) SearchProfiles(ctx context.Context, q string, typeahead bool, offset, size int) (*appbsky.UnspeccedSearchActorsSkeleton_Output, error) {
+func (s *Server) SearchProfiles(ctx context.Context, params *ActorSearchParams) (*appbsky.UnspeccedSearchActorsSkeleton_Output, error) {
 	ctx, span := tracer.Start(ctx, "SearchProfiles")
 	defer span.End()
 
 	var resp *EsSearchResponse
 	var err error
-	if typeahead {
-		resp, err = DoSearchProfilesTypeahead(ctx, s.escli, s.profileIndex, q, size)
+	if params.Typeahead {
+		resp, err = DoSearchProfilesTypeahead(ctx, s.escli, s.profileIndex, params)
 	} else {
-		resp, err = DoSearchProfiles(ctx, s.dir, s.escli, s.profileIndex, q, offset, size)
+		resp, err = DoSearchProfiles(ctx, s.dir, s.escli, s.profileIndex, params)
 	}
 	if err != nil {
 		return nil, err
···
 	}
 
 	out := appbsky.UnspeccedSearchActorsSkeleton_Output{Actors: actors}
-	if len(actors) == size && (offset+size) < 10000 {
-		s := fmt.Sprintf("%d", offset+size)
+	if len(actors) == params.Size && (params.Offset+params.Size) < 10000 {
+		s := fmt.Sprintf("%d", params.Offset+params.Size)
 		out.Cursor = &s
 	}
 	if resp.Hits.Total.Relation == "eq" {

search/parse_query.go

···
 
 import (
 	"context"
-	"fmt"
 	"log/slog"
 	"strings"
+	"time"
 
 	"github.com/bluesky-social/indigo/atproto/identity"
 	"github.com/bluesky-social/indigo/atproto/syntax"
 )
 
 // ParseQuery takes a query string and pulls out some facet patterns ("from:handle.net") as filters
-func ParseQuery(ctx context.Context, dir identity.Directory, raw string) (string, []map[string]interface{}) {
-	var filters []map[string]interface{}
+func ParsePostQuery(ctx context.Context, dir identity.Directory, raw string, viewer *syntax.DID) PostSearchParams {
 	quoted := false
 	parts := strings.FieldsFunc(raw, func(r rune) bool {
 		if r == '"' {
···
 		return r == ' ' && !quoted
 	})
 
+	params := PostSearchParams{}
+
 	keep := make([]string, 0, len(parts))
 	for _, p := range parts {
-		p = strings.Trim(p, "\"")
+		// pass-through quoted, either phrase or single token
+		if strings.HasPrefix(p, "\"") {
+			keep = append(keep, p)
+			continue
+		}
 
+		// tags (array)
 		if strings.HasPrefix(p, "#") && len(p) > 1 {
-			filters = append(filters, map[string]interface{}{
-				"term": map[string]interface{}{
-					"tag": map[string]interface{}{
-						"value":            p[1:],
-						"case_insensitive": true,
-					},
-				},
-			})
+			params.Tags = append(params.Tags, p[1:])
+			continue
+		}
+
+		// handle (mention)
+		if strings.HasPrefix(p, "@") && len(p) > 1 {
+			handle, err := syntax.ParseHandle(p[1:])
+			if err != nil {
+				keep = append(keep, p)
+				continue
+			}
+			id, err := dir.LookupHandle(ctx, handle)
+			if err != nil {
+				if err != identity.ErrHandleNotFound {
+					slog.Error("failed to resolve handle", "err", err)
+				}
+				continue
+			}
+			params.Mentions = &id.DID
 			continue
 		}
-		if strings.HasPrefix(p, "did:") {
-			filters = append(filters, map[string]interface{}{
-				"term": map[string]interface{}{"did": p},
-			})
+
+		tokParts := strings.SplitN(p, ":", 2)
+		if len(tokParts) == 1 {
+			keep = append(keep, p)
 			continue
 		}
-		if strings.HasPrefix(p, "from:") && len(p) > 6 {
-			h := p[5:]
-			if h[0] == '@' {
-				h = h[1:]
+
+		switch tokParts[0] {
+		case "did":
+			// TODO: not really clear what to do here; treating like a mention doesn't really make sense?
+		case "from", "to", "mentions":
+			raw := tokParts[1]
+			if raw == "me" {
+				if viewer != nil && tokParts[0] == "from" {
+					params.Author = viewer
+				} else if viewer != nil {
+					params.Mentions = viewer
+				}
+				continue
+			}
+			if strings.HasPrefix(raw, "@") && len(raw) > 1 {
+				raw = raw[1:]
 			}
-			handle, err := syntax.ParseHandle(h)
+			handle, err := syntax.ParseHandle(raw)
 			if err != nil {
-				keep = append(keep, p)
 				continue
 			}
 			id, err := dir.LookupHandle(ctx, handle)
···
 				}
 				continue
 			}
-			filters = append(filters, map[string]interface{}{
-				"term": map[string]interface{}{"did": id.DID.String()},
-			})
+			if tokParts[0] == "from" {
+				params.Author = &id.DID
+			} else {
+				params.Mentions = &id.DID
+			}
+			continue
+		case "http", "https":
+			params.URL = p
+			continue
+		case "domain":
+			params.Domain = tokParts[1]
+			continue
+		case "lang":
+			lang, err := syntax.ParseLanguage(tokParts[1])
+			if nil == err {
+				params.Lang = &lang
+			}
+			continue
+		case "since", "until":
+			var dt syntax.Datetime
+			// first try just date
+			date, err := time.Parse(time.DateOnly, tokParts[1])
+			if nil == err {
+				dt = syntax.Datetime(date.Format(syntax.AtprotoDatetimeLayout))
+			} else {
+				// fallback to formal atproto datetime format
+				dt, err = syntax.ParseDatetimeLenient(tokParts[1])
+				if err != nil {
+					continue
+				}
+			}
+			if tokParts[0] == "since" {
+				params.Since = &dt
+			} else {
+				params.Until = &dt
+			}
 			continue
 		}
 
···
 
 	out := ""
 	for _, p := range keep {
-		if strings.ContainsRune(p, ' ') {
-			if out == "" {
-				out = fmt.Sprintf(`"%s"`, p)
-			} else {
-				out += " " + fmt.Sprintf(`"%s"`, p)
-			}
+		if out == "" {
+			out = p
 		} else {
-			if out == "" {
-				out = p
-			} else {
-				out += " " + p
-			}
+			out += " " + p
 		}
 	}
-	if out == "" && len(filters) >= 1 {
+	if out == "" {
 		out = "*"
 	}
-	return out, filters
+	params.Query = out
+	return params
 }

search/parse_query_test.go

···
 	ctx := context.Background()
 	assert := assert.New(t)
 	dir := identity.NewMockDirectory()
-	dir.Insert(identity.Identity{
+	ident := identity.Identity{
 		Handle: syntax.Handle("known.example.com"),
 		DID:    syntax.DID("did:plc:abc222"),
-	})
+	}
+	dir.Insert(ident)
+
+	var p PostSearchParams
+
+	p = ParsePostQuery(ctx, &dir, "", nil)
+	assert.Equal("*", p.Query)
+	assert.Empty(p.Filters())
 
-	var q string
-	var f []map[string]interface{}
+	q1 := "some +test \"with phrase\" -ok"
+	p = ParsePostQuery(ctx, &dir, q1, nil)
+	assert.Equal(q1, p.Query)
+	assert.Empty(p.Filters())
 
-	q, f = ParseQuery(ctx, &dir, "")
-	assert.Equal("", q)
-	assert.Empty(f)
+	q2 := "missing from:missing.example.com"
+	p = ParsePostQuery(ctx, &dir, q2, nil)
+	assert.Equal("missing", p.Query)
+	assert.Empty(p.Filters())
 
-	p1 := "some +test \"with phrase\" -ok"
-	q, f = ParseQuery(ctx, &dir, p1)
-	assert.Equal(p1, q)
-	assert.Empty(f)
+	q3 := "known from:known.example.com"
+	p = ParsePostQuery(ctx, &dir, q3, nil)
+	assert.Equal("known", p.Query)
+	assert.NotNil(p.Author)
+	if p.Author != nil {
+		assert.Equal("did:plc:abc222", p.Author.String())
+	}
 
-	p2 := "missing from:missing.example.com"
-	q, f = ParseQuery(ctx, &dir, p2)
-	assert.Equal("missing", q)
-	assert.Empty(f)
+	q4 := "from:known.example.com"
+	p = ParsePostQuery(ctx, &dir, q4, nil)
+	assert.Equal("*", p.Query)
+	assert.Equal(1, len(p.Filters()))
 
-	p3 := "known from:known.example.com"
-	q, f = ParseQuery(ctx, &dir, p3)
-	assert.Equal("known", q)
-	assert.Equal(1, len(f))
+	q5 := `from:known.example.com "multi word phrase" coolio blorg`
+	p = ParsePostQuery(ctx, &dir, q5, nil)
+	assert.Equal(`"multi word phrase" coolio blorg`, p.Query)
+	assert.NotNil(p.Author)
+	if p.Author != nil {
+		assert.Equal("did:plc:abc222", p.Author.String())
+	}
+	assert.Equal(1, len(p.Filters()))
 
-	p4 := "from:known.example.com"
-	q, f = ParseQuery(ctx, &dir, p4)
-	assert.Equal("*", q)
-	assert.Equal(1, len(f))
+	q6 := `from:known.example.com #cool_tag some other stuff`
+	p = ParsePostQuery(ctx, &dir, q6, nil)
+	assert.Equal(`some other stuff`, p.Query)
+	assert.NotNil(p.Author)
+	if p.Author != nil {
+		assert.Equal("did:plc:abc222", p.Author.String())
+	}
+	assert.Equal([]string{"cool_tag"}, p.Tags)
+	assert.Equal(2, len(p.Filters()))
 
-	p5 := `from:known.example.com "multi word phrase" coolio blorg`
-	q, f = ParseQuery(ctx, &dir, p5)
-	assert.Equal(`"multi word phrase" coolio blorg`, q)
-	assert.Equal(1, len(f))
+	q7 := "known from:@known.example.com"
+	p = ParsePostQuery(ctx, &dir, q7, nil)
+	assert.Equal("known", p.Query)
+	assert.NotNil(p.Author)
+	if p.Author != nil {
+		assert.Equal("did:plc:abc222", p.Author.String())
+	}
+	assert.Equal(1, len(p.Filters()))
 
-	p6 := `from:known.example.com #cool_tag some other stuff`
-	q, f = ParseQuery(ctx, &dir, p6)
-	assert.Equal(`some other stuff`, q)
-	assert.Equal(2, len(f))
+	q8 := "known from:me"
+	p = ParsePostQuery(ctx, &dir, q8, &ident.DID)
+	assert.Equal("known", p.Query)
+	assert.NotNil(p.Author)
+	if p.Author != nil {
+		assert.Equal("did:plc:abc222", p.Author.String())
+	}
+	assert.Equal(1, len(p.Filters()))
 
-	p7 := "known from:@known.example.com"
-	q, f = ParseQuery(ctx, &dir, p7)
-	assert.Equal("known", q)
-	assert.Equal(1, len(f))
+	// TODO: more parsing tests: bare handles, to:, since:, until:, URL, domain:, lang
 }

search/post_schema.json

···
         "lang_code":      { "type": "keyword", "normalizer": "default" },
         "lang_code_iso2": { "type": "keyword", "normalizer": "default" },
         "mention_did":    { "type": "keyword", "normalizer": "default" },
-        "link_url":       { "type": "keyword", "normalizer": "default" },
-        "embed_url":      { "type": "keyword", "normalizer": "default" },
         "embed_aturi":    { "type": "keyword", "normalizer": "default" },
         "reply_root_aturi": { "type": "keyword", "normalizer": "default" },
         "embed_img_count": { "type": "integer" },
···
         "embed_img_alt_text_ja": { "type": "text", "analyzer": "textJapanese", "search_analyzer": "textJapaneseSearch", "copy_to": "everything_ja" },
         "self_label":     { "type": "keyword", "normalizer": "default" },
 
+        "url":            { "type": "keyword", "normalizer": "default" },
+        "domain":         { "type": "keyword", "normalizer": "default" },
         "tag":            { "type": "keyword", "normalizer": "default" },
         "emoji":          { "type": "keyword", "normalizer": "caseSensitive" },
+
+        "likesFuzzy":     { "type": "integer" },
 
         "everything":     { "type": "text", "analyzer": "textIcu", "search_analyzer": "textIcuSearch" },
         "everything_ja":  { "type": "text", "analyzer": "textJapanese", "search_analyzer": "textJapaneseSearch" },

search/profile_schema.json

···
         "img_alt_text":   { "type": "text", "analyzer": "textIcu", "search_analyzer": "textIcuSearch", "copy_to": "everything" },
         "self_label":     { "type": "keyword", "normalizer": "default" },
 
+        "url":            { "type": "keyword", "normalizer": "default" },
+        "domain":         { "type": "keyword", "normalizer": "default" },
         "tag":            { "type": "keyword", "normalizer": "default" },
         "emoji":          { "type": "keyword", "normalizer": "caseSensitive" },
 
         "has_avatar":     { "type": "boolean" },
         "has_banner":     { "type": "boolean" },
+
+        "pagerank":       { "type": "float" },
+        "followersFuzzy": { "type": "integer" },
 
         "typeahead":      { "type": "search_as_you_type", "analyzer": "textIcu", "search_analyzer": "textIcuSearch" },
         "everything":     { "type": "text", "analyzer": "textIcu", "search_analyzer": "textIcuSearch" }

search/query.go

···
 	Post any        `json:"post"`
 }
 
+type PostSearchParams struct {
+	Query    string           `json:"q"`
+	Sort     string           `json:"sort"`
+	Author   *syntax.DID      `json:"author"`
+	Since    *syntax.Datetime `json:"since"`
+	Until    *syntax.Datetime `json:"until"`
+	Mentions *syntax.DID      `json:"mentions"`
+	Lang     *syntax.Language `json:"lang"`
+	Domain   string           `json:"domain"`
+	URL      string           `json:"url"`
+	Tags     []string         `json:"tag"`
+	Viewer   *syntax.DID      `json:"viewer"`
+	Offset   int              `json:"offset"`
+	Size     int              `json:"size"`
+}
+
+type ActorSearchParams struct {
+	Query     string       `json:"q"`
+	Typeahead bool         `json:"typeahead"`
+	Follows   []syntax.DID `json:"follows"`
+	Viewer    *syntax.DID  `json:"viewer"`
+	Offset    int          `json:"offset"`
+	Size      int          `json:"size"`
+}
+
+// Merges params from another param object in to this one. Intended to meld parsed query with HTTP query params, so not all functionality is supported, and priority is with the "current" object
+func (p *PostSearchParams) Update(other *PostSearchParams) {
+	p.Query = other.Query
+	if p.Author == nil {
+		p.Author = other.Author
+	}
+	if p.Since == nil {
+		p.Since = other.Since
+	}
+	if p.Until == nil {
+		p.Until = other.Until
+	}
+	if p.Mentions == nil {
+		p.Mentions = other.Mentions
+	}
+	if p.Lang == nil {
+		p.Lang = other.Lang
+	}
+	if p.Domain == "" {
+		p.Domain = other.Domain
+	}
+	if p.URL == "" {
+		p.URL = other.URL
+	}
+	if len(p.Tags) == 0 {
+		p.Tags = other.Tags
+	}
+}
+
+// turns search params in to actual elasticsearch/opensearch filter DSL
+func (p *PostSearchParams) Filters() []map[string]interface{} {
+	var filters []map[string]interface{}
+
+	if p.Author != nil {
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{"did": map[string]interface{}{
+				"value":            p.Author.String(),
+				"case_insensitive": true,
+			}},
+		})
+	}
+
+	if p.Mentions != nil {
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{"mention_did": map[string]interface{}{
+				"value":            p.Mentions.String(),
+				"case_insensitive": true,
+			}},
+		})
+	}
+
+	if p.Lang != nil {
+		// TODO: extracting just the 2-char code would be good
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{"lang_code_iso2": map[string]interface{}{
+				"value":            p.Lang.String(),
+				"case_insensitive": true,
+			}},
+		})
+	}
+
+	if p.Since != nil {
+		filters = append(filters, map[string]interface{}{
+			"range": map[string]interface{}{
+				"created_at": map[string]interface{}{
+					"gte": p.Since.String(),
+				},
+			},
+		})
+	}
+
+	if p.Until != nil {
+		filters = append(filters, map[string]interface{}{
+			"range": map[string]interface{}{
+				"created_at": map[string]interface{}{
+					"lt": p.Until.String(),
+				},
+			},
+		})
+	}
+
+	if p.URL != "" {
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{"url": map[string]interface{}{
+				"value":            NormalizeLossyURL(p.URL),
+				"case_insensitive": true,
+			}},
+		})
+	}
+
+	if p.Domain != "" {
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{"domain": map[string]interface{}{
+				"value":            p.Domain,
+				"case_insensitive": true,
+			}},
+		})
+	}
+
+	for _, tag := range p.Tags {
+		filters = append(filters, map[string]interface{}{
+			"term": map[string]interface{}{
+				"tag": map[string]interface{}{
+					"value":            tag,
+					"case_insensitive": true,
+				},
+			},
+		})
+	}
+
+	return filters
+}
+
 func checkParams(offset, size int) error {
 	if offset+size > 10000 || size > 250 || offset > 10000 || offset < 0 || size < 0 {
 		return fmt.Errorf("disallowed size/offset parameters")
···
 	return nil
 }
 
-func DoSearchPosts(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
+func DoSearchPosts(ctx context.Context, dir identity.Directory, escli *es.Client, index string, params *PostSearchParams) (*EsSearchResponse, error) {
 	ctx, span := tracer.Start(ctx, "DoSearchPosts")
 	defer span.End()
 
-	if err := checkParams(offset, size); err != nil {
+	if err := checkParams(params.Offset, params.Size); err != nil {
 		return nil, err
 	}
-	queryStr, filters := ParseQuery(ctx, dir, q)
+	queryStringParams := ParsePostQuery(ctx, dir, params.Query, params.Viewer)
+	params.Update(&queryStringParams)
 	idx := "everything"
-	if containsJapanese(queryStr) {
+	if containsJapanese(params.Query) {
 		idx = "everything_ja"
 	}
 	basic := map[string]interface{}{
 		"simple_query_string": map[string]interface{}{
-			"query":            queryStr,
+			"query":            params.Query,
 			"fields":           []string{idx},
 			"flags":            "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE",
 			"default_operator": "and",
···
 			"analyze_wildcard": false,
 		},
 	}
+	filters := params.Filters()
 	// filter out future posts (TODO: temporary hack)
 	now := syntax.DatetimeNow()
 	filters = append(filters, map[string]interface{}{
···
 				"order": "desc",
 			},
 		},
-		"size": size,
-		"from": offset,
+		"size": params.Size,
+		"from": params.Offset,
 	}
 
 	return doSearch(ctx, escli, index, query)
 }
 
-func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) {
+func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Client, index string, params *ActorSearchParams) (*EsSearchResponse, error) {
 	ctx, span := tracer.Start(ctx, "DoSearchProfiles")
 	defer span.End()
 
-	if err := checkParams(offset, size); err != nil {
+	if err := checkParams(params.Offset, params.Size); err != nil {
 		return nil, err
 	}
 
-	queryStr, filters := ParseQuery(ctx, dir, q)
 	basic := map[string]interface{}{
 		"simple_query_string": map[string]interface{}{
-			"query":            queryStr,
+			"query":            params.Query,
 			"fields":           []string{"everything"},
 			"flags":            "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE",
 			"default_operator": "and",
···
 					map[string]interface{}{"term": map[string]interface{}{"has_banner": true}},
 				},
 				"minimum_should_match": 0,
-				"filter":               filters,
 				"boost":                0.5,
 			},
 		},
-		"size": size,
-		"from": offset,
+		"size": params.Size,
+		"from": params.Offset,
 	}
 
 	return doSearch(ctx, escli, index, query)
 }
 
-func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q string, size int) (*EsSearchResponse, error) {
+func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index string, params *ActorSearchParams) (*EsSearchResponse, error) {
 	ctx, span := tracer.Start(ctx, "DoSearchProfilesTypeahead")
 	defer span.End()
 
-	if err := checkParams(0, size); err != nil {
+	if err := checkParams(0, params.Size); err != nil {
 		return nil, err
 	}
 
 	query := map[string]interface{}{
 		"query": map[string]interface{}{
 			"multi_match": map[string]interface{}{
-				"query":    q,
+				"query":    params.Query,
 				"type":     "bool_prefix",
 				"operator": "and",
 				"fields": []string{
···
 				},
 			},
 		},
-		"size": size,
+		"size": params.Size,
 	}
 
 	return doSearch(ctx, escli, index, query)

search/query_test.go

···
 	}
 	assert.Equal(1, len(res.Hits.Hits))
 }
+
+func TestParsedQuery(t *testing.T) {
+	assert := assert.New(t)
+	ctx := context.Background()
+	escli := testEsClient(t)
+	dir := identity.NewMockDirectory()
+	srv := testServer(ctx, t, escli, &dir)
+	ident := identity.Identity{
+		DID:    syntax.DID("did:plc:abc111"),
+		Handle: syntax.Handle("handle.example.com"),
+	}
+	other := identity.Identity{
+		DID:    syntax.DID("did:plc:abc222"),
+		Handle: syntax.Handle("other.example.com"),
+	}
+	dir.Insert(ident)
+	dir.Insert(other)
+
+	res, err := DoSearchPosts(ctx, &dir, escli, testPostIndex, "english", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+
+	p1 := appbsky.FeedPost{Text: "basic english post", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p1, "app.bsky.feed.post/3kpnillluoh2y", cid.Undef))
+	p2 := appbsky.FeedPost{Text: "another english post", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p2, "app.bsky.feed.post/3kpnilllu2222", cid.Undef))
+	p3 := appbsky.FeedPost{
+		Text:      "#cat post with hashtag",
+		CreatedAt: "2024-01-02T03:04:05.006Z",
+		Facets: []*appbsky.RichtextFacet{
+			&appbsky.RichtextFacet{
+				Features: []*appbsky.RichtextFacet_Features_Elem{
+					&appbsky.RichtextFacet_Features_Elem{
+						RichtextFacet_Tag: &appbsky.RichtextFacet_Tag{
+							Tag: "trick",
+						},
+					},
+				},
+				Index: &appbsky.RichtextFacet_ByteSlice{
+					ByteStart: 0,
+					ByteEnd:   4,
+				},
+			},
+		},
+	}
+	assert.NoError(srv.indexPost(ctx, &ident, &p3, "app.bsky.feed.post/3kpnilllu3333", cid.Undef))
+	p4 := appbsky.FeedPost{
+		Text:      "@other.example.com post with mention",
+		CreatedAt: "2024-01-02T03:04:05.006Z",
+		Facets: []*appbsky.RichtextFacet{
+			&appbsky.RichtextFacet{
+				Features: []*appbsky.RichtextFacet_Features_Elem{
+					&appbsky.RichtextFacet_Features_Elem{
+						RichtextFacet_Mention: &appbsky.RichtextFacet_Mention{
+							Did: "did:plc:abc222",
+						},
+					},
+				},
+				Index: &appbsky.RichtextFacet_ByteSlice{
+					ByteStart: 0,
+					ByteEnd:   18,
+				},
+			},
+		},
+	}
+	assert.NoError(srv.indexPost(ctx, &ident, &p4, "app.bsky.feed.post/3kpnilllu4444", cid.Undef))
+	p5 := appbsky.FeedPost{
+		Text:      "https://bsky.app... post with hashtag #cat",
+		CreatedAt: "2024-01-02T03:04:05.006Z",
+		Facets: []*appbsky.RichtextFacet{
+			&appbsky.RichtextFacet{
+				Features: []*appbsky.RichtextFacet_Features_Elem{
+					&appbsky.RichtextFacet_Features_Elem{
+						RichtextFacet_Link: &appbsky.RichtextFacet_Link{
+							Uri: "htTPS://www.en.wikipedia.org/wiki/CBOR?q=3&a=1&utm_campaign=123",
+						},
+					},
+				},
+				Index: &appbsky.RichtextFacet_ByteSlice{
+					ByteStart: 0,
+					ByteEnd:   19,
+				},
+			},
+		},
+	}
+	assert.NoError(srv.indexPost(ctx, &ident, &p5, "app.bsky.feed.post/3kpnilllu5555", cid.Undef))
+	p6 := appbsky.FeedPost{
+		Text:      "post with lang (deutsch)",
+		CreatedAt: "2024-01-02T03:04:05.006Z",
+		Langs:     []string{"ja", "de-DE"},
+	}
+	assert.NoError(srv.indexPost(ctx, &ident, &p6, "app.bsky.feed.post/3kpnilllu6666", cid.Undef))
+	p7 := appbsky.FeedPost{Text: "post with old date", CreatedAt: "2020-05-03T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p7, "app.bsky.feed.post/3kpnilllu7777", cid.Undef))
+
+	_, err = srv.escli.Indices.Refresh()
+	assert.NoError(err)
+
+	// expect all to be indexed
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "*", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(7, len(res.Hits.Hits))
+
+	// check that english matches both
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "english", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(2, len(res.Hits.Hits))
+
+	// phrase only matches one
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "\"basic english\"", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// posts-by
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "from:handle.example.com", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(7, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "from:@handle.example.com", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(7, len(res.Hits.Hits))
+
+	// hashtag query
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "post #trick", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "post #Trick", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "post #trick #allMustMatch", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+
+	// mention query
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "@other.example.com", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// URL and domain queries
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "https://en.wikipedia.org/wiki/CBOR?a=1&q=3", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "\"https://en.wikipedia.org/wiki/CBOR?a=1&q=3\"", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "https://en.wikipedia.org/wiki/CBOR", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "domain:en.wikipedia.org", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// lang filter
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "lang:de", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// date range filters
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "since:2023-01-01T00:00:00Z", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(6, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "since:2023-01-01", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(6, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "until:2023-01-01", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "until:asdf", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(7, len(res.Hits.Hits))
+}

search/testdata/transform-post-fixtures.json

···
   			"embed": {
     			"$type": "app.bsky.embed.external",
     			"external": {
-      			"uri": "https://bsky.app",
+      			"uri": "https://www.bsky.app:443/index.html",
       			"title": "Bluesky Social",
       			"description": "See what's next.",
       			"thumb": {
···
   			"record_cid": "bafyreibjifzpqj6o6wcq3hejh7y4z4z2vmiklkvykc57tw3pcbx3kxifpm",
   			"created_at": "2023-08-07T05:46:14.423045Z",
   			"text": "post which embeds an external URL as a card",
-  			"embed_url": "https://bsky.app",
+  			"url": ["https://bsky.app"],
+  			"domain": ["bsky.app"],
   			"embed_img_count": 0
 		}
 	},
···
       			"features": [
         			{
           			"$type": "app.bsky.richtext.facet#link",
-          			"uri": "https://en.wikipedia.org/wiki/CBOR"
+          			"uri": "https://en.wikipedia.org/wiki/CBOR?utm_campaign=123"
         			}
       			]
     			},
···
   			"created_at": "2023-08-07T05:46:14.423045Z",
   			"text": "longer example with #some #hashtags, emoji \u2620 \ud83d\ude42 \ud83c\udf85\ud83c\udfff, flags \ud83c\uddf8\ud83c\udde8 ",
   			"reply_root_aturi": "at://did:plc:u5cwb2mwiv2bfq53cjufe6yn/app.bsky.feed.post/3k43tv4rft22g",
-  			"link_url": [ "https://en.wikipedia.org/wiki/CBOR" ],
   			"mention_did": [ "did:plc:ewvi7nxzyoun6zhxrhs64oiz" ],
   			"embed_aturi": "at://did:plc:u5cwb2mwiv2bfq53cjufe6yn/app.bsky.feed.post/3k44deefqdk2g",
   			"lang_code": ["th", "en-US"],
   			"lang_code_iso2": ["th", "en"],
             "self_label": ["nudity"],
+  			"url": [ "https://en.wikipedia.org/wiki/CBOR" ],
+  			"domain": [ "en.wikipedia.org" ],
   			"tag": ["some", "thing"],
   			"emoji": ["\u2620", "\ud83d\ude42", "\ud83c\udf85\ud83c\udfff", "\ud83c\uddf8\ud83c\udde8"],
   			"embed_img_count": 0

search/transform.go

···
 
 import (
 	"log/slog"
+	"net/url"
 	"strings"
 	"time"
 
···
 	Description *string  `json:"description,omitempty"`
 	ImgAltText  []string `json:"img_alt_text,omitempty"`
 	SelfLabel   []string `json:"self_label,omitempty"`
+	URL         []string `json:"url,omitempty"`
+	Domain      []string `json:"domain,omitempty"`
 	Tag         []string `json:"tag,omitempty"`
 	Emoji       []string `json:"emoji,omitempty"`
 	HasAvatar   bool     `json:"has_avatar"`
···
 	LangCode          []string `json:"lang_code,omitempty"`
 	LangCodeIso2      []string `json:"lang_code_iso2,omitempty"`
 	MentionDID        []string `json:"mention_did,omitempty"`
-	LinkURL           []string `json:"link_url,omitempty"`
-	EmbedURL          *string  `json:"embed_url,omitempty"`
 	EmbedATURI        *string  `json:"embed_aturi,omitempty"`
 	ReplyRootATURI    *string  `json:"reply_root_aturi,omitempty"`
 	EmbedImgCount     int      `json:"embed_img_count"`
 	EmbedImgAltText   []string `json:"embed_img_alt_text,omitempty"`
 	EmbedImgAltTextJA []string `json:"embed_img_alt_text_ja,omitempty"`
 	SelfLabel         []string `json:"self_label,omitempty"`
+	URL               []string `json:"url,omitempty"`
+	Domain            []string `json:"domain,omitempty"`
 	Tag               []string `json:"tag,omitempty"`
 	Emoji             []string `json:"emoji,omitempty"`
 }
···
 		}
 	}
 	var mentionDIDs []string
-	var linkURLs []string
+	var urls []string
 	for _, facet := range post.Facets {
 		for _, feat := range facet.Features {
 			if feat.RichtextFacet_Mention != nil {
 				mentionDIDs = append(mentionDIDs, feat.RichtextFacet_Mention.Did)
 			}
 			if feat.RichtextFacet_Link != nil {
-				linkURLs = append(linkURLs, feat.RichtextFacet_Link.Uri)
+				urls = append(urls, feat.RichtextFacet_Link.Uri)
 			}
 		}
 	}
···
 	if post.Reply != nil {
 		replyRootATURI = &(post.Reply.Root.Uri)
 	}
-	var embedURL *string
 	if post.Embed != nil && post.Embed.EmbedExternal != nil {
-		embedURL = &post.Embed.EmbedExternal.External.Uri
+		urls = append(urls, post.Embed.EmbedExternal.External.Uri)
 	}
 	var embedATURI *string
 	if post.Embed != nil && post.Embed.EmbedRecord != nil {
···
 		}
 	}
 
+	var domains []string
+	for i, raw := range urls {
+		clean := NormalizeLossyURL(raw)
+		urls[i] = clean
+		u, err := url.Parse(clean)
+		if nil == err {
+			domains = append(domains, u.Hostname())
+		}
+	}
+
 	doc := PostDoc{
 		DocIndexTs:        syntax.DatetimeNow().String(),
 		DID:               ident.DID.String(),
···
 		LangCode:          post.Langs,
 		LangCodeIso2:      langCodeIso2,
 		MentionDID:        mentionDIDs,
-		LinkURL:           linkURLs,
-		EmbedURL:          embedURL,
 		EmbedATURI:        embedATURI,
 		ReplyRootATURI:    replyRootATURI,
 		EmbedImgCount:     embedImgCount,
 		EmbedImgAltText:   embedImgAltText,
 		EmbedImgAltTextJA: embedImgAltTextJA,
 		SelfLabel:         selfLabels,
+		URL:               urls,
+		Domain:            domains,
 		Tag:               parsePostTags(post),
 		Emoji:             parseEmojis(post.Text),
 	}

search/url.go

···
+package search
+
+import (
+	"net/url"
+
+	"github.com/PuerkitoBio/purell"
+)
+
+var trackingParams = []string{
+	"__s",
+	"_ga",
+	"campaign_id",
+	"ceid",
+	"emci",
+	"emdi",
+	"fbclid",
+	"gclid",
+	"hootPostID",
+	"mc_eid",
+	"mkclid",
+	"mkt_tok",
+	"msclkid",
+	"pk_campaign",
+	"pk_kwd",
+	"sessionid",
+	"sourceid",
+	"utm_campaign",
+	"utm_content",
+	"utm_id",
+	"utm_medium",
+	"utm_source",
+	"utm_term",
+	"xpid",
+}
+
+// aggressively normalizes URL, for search indexing and matching. it is possible the URL won't be directly functional after this normalization
+func NormalizeLossyURL(raw string) string {
+	clean, err := purell.NormalizeURLString(raw, purell.FlagsUsuallySafeGreedy|purell.FlagRemoveDirectoryIndex|purell.FlagRemoveFragment|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveWWW|purell.FlagSortQuery)
+	if err != nil {
+		return raw
+	}
+
+	// remove tracking params
+	u, err := url.Parse(clean)
+	if err != nil {
+		return clean
+	}
+	if u.RawQuery == "" {
+		return clean
+	}
+	params := u.Query()
+
+	// there is probably a more efficient way to do this
+	for _, p := range trackingParams {
+		if params.Has(p) {
+			params.Del(p)
+		}
+	}
+	u.RawQuery = params.Encode()
+	return u.String()
+}

search/url_test.go

···
+package search
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeLossyURL(t *testing.T) {
+	assert := assert.New(t)
+
+	fixtures := []struct {
+		orig  string
+		clean string
+	}{
+		{orig: "", clean: ""},
+		{orig: "asdf", clean: "asdf"},
+		{orig: "HTTP://bSky.app:80/index.html", clean: "http://bsky.app"},
+		{orig: "https://example.com/thing?c=123&utm_campaign=blah&a=first", clean: "https://example.com/thing?a=first&c=123"},
+		{orig: "https://example.com/thing?c=123&utm_campaign=blah&a=first", clean: "https://example.com/thing?a=first&c=123"},
+		{orig: "http://example.com/foo//bar.html", clean: "http://example.com/foo/bar.html"},
+		{orig: "http://example.com/bar.html#section1", clean: "http://example.com/bar.html"},
+		{orig: "http://example.com/foo/", clean: "http://example.com/foo"},
+		{orig: "http://example.com/", clean: "http://example.com"},
+		{orig: "http://example.com/%7Efoo", clean: "http://example.com/~foo"},
+		{orig: "http://example.com/foo/./bar/baz/../qux", clean: "http://example.com/foo/bar/qux"},
+		{orig: "http://www.example.com/", clean: "http://example.com"},
+	}
+
+	for _, fix := range fixtures {
+		assert.Equal(fix.clean, NormalizeLossyURL(fix.orig))
+	}
+}