search: normalize URLs for indexing and query

search/query.go

···
 
 	if p.URL != "" {
 		filters = append(filters, map[string]interface{}{
-			"term": map[string]interface{}{"url": p.URL},
+			"term": map[string]interface{}{"url": NormalizeLossyURL(p.URL)},
 		})
 	}
 

search/testdata/transform-post-fixtures.json

···
   			"embed": {
     			"$type": "app.bsky.embed.external",
     			"external": {
-      			"uri": "https://bsky.app",
+      			"uri": "https://www.bsky.app:443/index.html",
       			"title": "Bluesky Social",
       			"description": "See what's next.",
       			"thumb": {
···
       			"features": [
         			{
           			"$type": "app.bsky.richtext.facet#link",
-          			"uri": "https://en.wikipedia.org/wiki/CBOR"
+          			"uri": "https://en.wikipedia.org/wiki/CBOR?utm_campaign=123"
         			}
       			]
     			},

search/transform.go

···
 	}
 
 	var domains []string
-	for _, raw := range urls {
-		u, err := url.Parse(raw)
+	for i, raw := range urls {
+		clean := NormalizeLossyURL(raw)
+		urls[i] = clean
+		u, err := url.Parse(clean)
 		if nil == err {
 			domains = append(domains, u.Hostname())
 		}

search/url.go

···
+package search
+
+import (
+	"net/url"
+
+	"github.com/PuerkitoBio/purell"
+)
+
+var trackingParams = []string{
+	"__s",
+	"_ga",
+	"campaign_id",
+	"ceid",
+	"emci",
+	"emdi",
+	"fbclid",
+	"gclid",
+	"hootPostID",
+	"mc_eid",
+	"mkclid",
+	"mkt_tok",
+	"msclkid",
+	"pk_campaign",
+	"pk_kwd",
+	"sessionid",
+	"sourceid",
+	"utm_campaign",
+	"utm_content",
+	"utm_id",
+	"utm_medium",
+	"utm_source",
+	"utm_term",
+	"xpid",
+}
+
+// aggressively normalizes URL, for search indexing and matching. it is possible the URL won't be directly functional after this normalization
+func NormalizeLossyURL(raw string) string {
+	clean, err := purell.NormalizeURLString(raw, purell.FlagsUsuallySafeGreedy|purell.FlagRemoveDirectoryIndex|purell.FlagRemoveFragment|purell.FlagRemoveDuplicateSlashes|purell.FlagRemoveWWW|purell.FlagSortQuery)
+	if err != nil {
+		return raw
+	}
+
+	// remove tracking params
+	u, err := url.Parse(clean)
+	if u.RawQuery == "" {
+		return clean
+	}
+	params := u.Query()
+
+	// there is probably a more efficient way to do this
+	for _, p := range trackingParams {
+		if params.Has(p) {
+			params.Del(p)
+		}
+	}
+	u.RawQuery = params.Encode()
+	return u.String()
+}

search/url_test.go

···
+package search
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeLossyURL(t *testing.T) {
+	assert := assert.New(t)
+
+	fixtures := []struct {
+		orig  string
+		clean string
+	}{
+		{orig: "", clean: ""},
+		{orig: "asdf", clean: "asdf"},
+		{orig: "HTTP://bSky.app:80/index.html", clean: "http://bsky.app"},
+		{orig: "https://example.com/thing?c=123&utm_campaign=blah&a=first", clean: "https://example.com/thing?a=first&c=123"},
+		{orig: "https://example.com/thing?c=123&utm_campaign=blah&a=first", clean: "https://example.com/thing?a=first&c=123"},
+		{orig: "http://example.com/foo//bar.html", clean: "http://example.com/foo/bar.html"},
+		{orig: "http://example.com/bar.html#section1", clean: "http://example.com/bar.html"},
+		{orig: "http://example.com/foo/", clean: "http://example.com/foo"},
+		{orig: "http://example.com/", clean: "http://example.com"},
+		{orig: "http://example.com/%7Efoo", clean: "http://example.com/~foo"},
+		{orig: "http://example.com/foo/./bar/baz/../qux", clean: "http://example.com/foo/bar/qux"},
+		{orig: "http://www.example.com/", clean: "http://example.com"},
+	}
+
+	for _, fix := range fixtures {
+		assert.Equal(fix.clean, NormalizeLossyURL(fix.orig))
+	}
+}