first pass at japanese-specific post search

This is only for posts (not profiles).

search/japanese.go

···
+package search
+
+import (
+	"regexp"
+)
+
+// U+3040 - U+30FF: hiragana and katakana (Japanese only)
+// U+FF66 - U+FF9F: half-width katakana (Japanese only)
+var japaneseRegex = regexp.MustCompile(`[\x{3040}-\x{30ff}\x{ff66}-\x{ff9f}]`)
+
+// helper to check if an input string contains any Japanese-specific characters (hiragana or katakana). will not trigger on CJK characters which are not specific to Japanese
+func containsJapanese(text string) bool {
+	return japaneseRegex.MatchString(text)
+}

search/japanese_test.go

···
+package search
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJapaneseDetection(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.False(containsJapanese(""))
+	assert.False(containsJapanese("basic english"))
+	assert.False(containsJapanese("basic english"))
+
+	assert.True(containsJapanese("学校から帰って熱いお風呂に入ったら力一杯がんばる"))
+	assert.True(containsJapanese("パリ"))
+	assert.True(containsJapanese("ハリー・ポッター"))
+	assert.True(containsJapanese("some japanese パリ and some english"))
+
+	// CJK, but not japanese-specific
+	assert.False(containsJapanese("熱力学"))
+}

search/query.go

···
 		return nil, err
 	}
 	queryStr, filters := ParseQuery(ctx, dir, q)
+	idx := "everything"
+	if containsJapanese(queryStr) {
+		idx = "everything_ja"
+	}
 	basic := map[string]interface{}{
 		"simple_query_string": map[string]interface{}{
 			"query":            queryStr,
-			"fields":           []string{"everything"},
+			"fields":           []string{idx},
 			"flags":            "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE",
 			"default_operator": "and",
 			"lenient":          true,

search/query_test.go

···
+package search
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"log/slog"
+	"net/http"
+	"testing"
+
+	appbsky "github.com/bluesky-social/indigo/api/bsky"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+
+	"github.com/ipfs/go-cid"
+	es "github.com/opensearch-project/opensearch-go/v2"
+	"github.com/stretchr/testify/assert"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+var (
+	testPostIndex    = "palomar_test_post"
+	testProfileIndex = "palomar_test_profile"
+)
+
+func testEsClient(t *testing.T) *es.Client {
+	cfg := es.Config{
+		Addresses: []string{"http://localhost:9200"},
+		Username:  "admin",
+		Password:  "0penSearch-Pal0mar",
+		CACert:    nil,
+		Transport: &http.Transport{
+			MaxIdleConnsPerHost: 5,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	escli, err := es.NewClient(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, err := escli.Info()
+	if err != nil {
+		t.Fatal(err)
+	}
+	info.Body.Close()
+	return escli
+
+}
+
+func testServer(ctx context.Context, t *testing.T, escli *es.Client, dir identity.Directory) *Server {
+	db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srv, err := NewServer(
+		db,
+		escli,
+		dir,
+		Config{
+			RelayHost:           "wss://relay.invalid",
+			PostIndex:           testPostIndex,
+			ProfileIndex:        testProfileIndex,
+			Logger:              slog.Default(),
+			RelaySyncRateLimit:  1,
+			IndexMaxConcurrency: 1,
+		},
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// NOTE: skipping errors
+	resp, _ := srv.escli.Indices.Delete([]string{testPostIndex, testProfileIndex})
+	defer resp.Body.Close()
+	io.ReadAll(resp.Body)
+
+	if err := srv.EnsureIndices(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	return srv
+}
+
+func TestJapaneseRegressions(t *testing.T) {
+	assert := assert.New(t)
+	ctx := context.Background()
+	escli := testEsClient(t)
+	dir := identity.NewMockDirectory()
+	srv := testServer(ctx, t, escli, &dir)
+	ident := identity.Identity{
+		DID:    syntax.DID("did:plc:abc111"),
+		Handle: syntax.Handle("handle.example.com"),
+	}
+
+	res, err := DoSearchPosts(ctx, &dir, escli, testPostIndex, "english", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+
+	p1 := appbsky.FeedPost{Text: "basic english post", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p1, "app.bsky.feed.post/3kpnillluoh2y", cid.Undef))
+
+	// https://github.com/bluesky-social/indigo/issues/302
+	p2 := appbsky.FeedPost{Text: "学校から帰って熱いお風呂に入ったら力一杯がんばる", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p2, "app.bsky.feed.post/3kpnillluo222", cid.Undef))
+	p3 := appbsky.FeedPost{Text: "熱力学", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p3, "app.bsky.feed.post/3kpnillluo333", cid.Undef))
+	p4 := appbsky.FeedPost{Text: "東京都", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p4, "app.bsky.feed.post/3kpnillluo444", cid.Undef))
+	p5 := appbsky.FeedPost{Text: "京都", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p5, "app.bsky.feed.post/3kpnillluo555", cid.Undef))
+	p6 := appbsky.FeedPost{Text: "パリ", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p6, "app.bsky.feed.post/3kpnillluo666", cid.Undef))
+	p7 := appbsky.FeedPost{Text: "ハリー・ポッター", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p7, "app.bsky.feed.post/3kpnillluo777", cid.Undef))
+	p8 := appbsky.FeedPost{Text: "ハリ", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p8, "app.bsky.feed.post/3kpnillluo223", cid.Undef))
+	p9 := appbsky.FeedPost{Text: "multilingual 多言語", CreatedAt: "2024-01-02T03:04:05.006Z"}
+	assert.NoError(srv.indexPost(ctx, &ident, &p9, "app.bsky.feed.post/3kpnillluo224", cid.Undef))
+
+	_, err = srv.escli.Indices.Refresh()
+	assert.NoError(err)
+
+	// expect all to be indexed
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "*", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(9, len(res.Hits.Hits))
+
+	// check that english matches (single post)
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "english", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// "thermodynamics"; should return only one match
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "熱力学", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// "Kyoto"; should return only one match
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "京都", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// "Paris"; should return only one match
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "パリ", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// should return only one match
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "ハリー", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	// part of a word; should match none
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "ハ", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(0, len(res.Hits.Hits))
+
+	// should match both ways, and together
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "multilingual", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "多言語", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "multilingual 多言語", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+	res, err = DoSearchPosts(ctx, &dir, escli, testPostIndex, "\"multilingual 多言語\"", 0, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(1, len(res.Hits.Hits))
+}

search/testdata/transform-post-fixtures.json

···
   			],
   			"embed_img_count": 2
 		}
+	},
+	{
+		"did": "did:plc:u5cwb2mwiv2bfq53cjufe6yn",
+  		"handle": "handle.example.com",
+		"rkey": "3k4duaz5vfs2b",
+		"cid": "bafyreibjifzpqj6o6wcq3hejh7y4z4z2vmiklkvykc57tw3pcbx3kxifpm",
+		"PostRecord": {
+  			"$type": "app.bsky.feed.post",
+  			"text": "学校から帰って熱いお風呂に入ったら力一杯がんばる",
+  			"createdAt": "2023-08-07T05:46:14.423045Z",
+  			"embed": {
+    			"$type": "app.bsky.embed.images",
+    			"images": [
+      			{
+        			"alt": "brief alt text description of the first image ハリー・ポッター",
+        			"image": {
+          			"$type": "blob",
+          			"ref": {
+            			"$link": "bafkreibabalobzn6cd366ukcsjycp4yymjymgfxcv6xczmlgpemzkz3cfa"
+          			},
+          			"mimeType": "image/webp",
+          			"size": 760898
+        			}
+      			},
+      			{
+        			"alt": "brief alt text description of the second image",
+        			"image": {
+          			"$type": "blob",
+          			"ref": {
+            			"$link": "bafkreif3fouono2i3fmm5moqypwskh3yjtp7snd5hfq5pr453oggygyrte"
+          			},
+          			"mimeType": "image/png",
+          			"size": 13208
+        			}
+      			}
+    			]
+  			}
+		},
+		"doc_id": "did:plc:u5cwb2mwiv2bfq53cjufe6yn_3k4duaz5vfs2b",
+		"PostDoc": {
+            "doc_index_ts": "2006-01-02T15:04:05.000Z",
+  			"did": "did:plc:u5cwb2mwiv2bfq53cjufe6yn",
+  			"handle": "handle.example.com",
+  			"record_rkey": "3k4duaz5vfs2b",
+  			"record_cid": "bafyreibjifzpqj6o6wcq3hejh7y4z4z2vmiklkvykc57tw3pcbx3kxifpm",
+  			"created_at": "2023-08-07T05:46:14.423045Z",
+  			"text": "学校から帰って熱いお風呂に入ったら力一杯がんばる",
+  			"text_ja": "学校から帰って熱いお風呂に入ったら力一杯がんばる",
+  			"embed_img_alt_text": [
+  				"brief alt text description of the first image ハリー・ポッター",
+  				"brief alt text description of the second image"
+  			],
+  			"embed_img_alt_text_ja": [
+  				"brief alt text description of the first image ハリー・ポッター"
+  			],
+  			"embed_img_count": 2
+		}
 	}
 ]

search/transform.go

···
 }
 
 type PostDoc struct {
-	DocIndexTs      string   `json:"doc_index_ts"`
-	DID             string   `json:"did"`
-	RecordRkey      string   `json:"record_rkey"`
-	RecordCID       string   `json:"record_cid"`
-	CreatedAt       *string  `json:"created_at,omitempty"`
-	Text            string   `json:"text"`
-	LangCode        []string `json:"lang_code,omitempty"`
-	LangCodeIso2    []string `json:"lang_code_iso2,omitempty"`
-	MentionDID      []string `json:"mention_did,omitempty"`
-	LinkURL         []string `json:"link_url,omitempty"`
-	EmbedURL        *string  `json:"embed_url,omitempty"`
-	EmbedATURI      *string  `json:"embed_aturi,omitempty"`
-	ReplyRootATURI  *string  `json:"reply_root_aturi,omitempty"`
-	EmbedImgCount   int      `json:"embed_img_count"`
-	EmbedImgAltText []string `json:"embed_img_alt_text,omitempty"`
-	SelfLabel       []string `json:"self_label,omitempty"`
-	Tag             []string `json:"tag,omitempty"`
-	Emoji           []string `json:"emoji,omitempty"`
+	DocIndexTs        string   `json:"doc_index_ts"`
+	DID               string   `json:"did"`
+	RecordRkey        string   `json:"record_rkey"`
+	RecordCID         string   `json:"record_cid"`
+	CreatedAt         *string  `json:"created_at,omitempty"`
+	Text              string   `json:"text"`
+	TextJA            string   `json:"text_ja,omitempty"`
+	LangCode          []string `json:"lang_code,omitempty"`
+	LangCodeIso2      []string `json:"lang_code_iso2,omitempty"`
+	MentionDID        []string `json:"mention_did,omitempty"`
+	LinkURL           []string `json:"link_url,omitempty"`
+	EmbedURL          *string  `json:"embed_url,omitempty"`
+	EmbedATURI        *string  `json:"embed_aturi,omitempty"`
+	ReplyRootATURI    *string  `json:"reply_root_aturi,omitempty"`
+	EmbedImgCount     int      `json:"embed_img_count"`
+	EmbedImgAltText   []string `json:"embed_img_alt_text,omitempty"`
+	EmbedImgAltTextJA []string `json:"embed_img_alt_text_ja,omitempty"`
+	SelfLabel         []string `json:"self_label,omitempty"`
+	Tag               []string `json:"tag,omitempty"`
+	Emoji             []string `json:"emoji,omitempty"`
 }
 
 // Returns the search index document ID (`_id`) for this document.
···
 	}
 	var embedImgCount int = 0
 	var embedImgAltText []string
+	var embedImgAltTextJA []string
 	if post.Embed != nil && post.Embed.EmbedImages != nil {
 		embedImgCount = len(post.Embed.EmbedImages.Images)
 		for _, img := range post.Embed.EmbedImages.Images {
 			if img.Alt != "" {
 				embedImgAltText = append(embedImgAltText, img.Alt)
+				if containsJapanese(img.Alt) {
+					embedImgAltTextJA = append(embedImgAltTextJA, img.Alt)
+				}
 			}
 		}
 	}
···
 	}
 
 	doc := PostDoc{
-		DocIndexTs:      syntax.DatetimeNow().String(),
-		DID:             ident.DID.String(),
-		RecordRkey:      rkey,
-		RecordCID:       cid,
-		Text:            post.Text,
-		LangCode:        post.Langs,
-		LangCodeIso2:    langCodeIso2,
-		MentionDID:      mentionDIDs,
-		LinkURL:         linkURLs,
-		EmbedURL:        embedURL,
-		EmbedATURI:      embedATURI,
-		ReplyRootATURI:  replyRootATURI,
-		EmbedImgCount:   embedImgCount,
-		EmbedImgAltText: embedImgAltText,
-		SelfLabel:       selfLabels,
-		Tag:             parsePostTags(post),
-		Emoji:           parseEmojis(post.Text),
+		DocIndexTs:        syntax.DatetimeNow().String(),
+		DID:               ident.DID.String(),
+		RecordRkey:        rkey,
+		RecordCID:         cid,
+		Text:              post.Text,
+		LangCode:          post.Langs,
+		LangCodeIso2:      langCodeIso2,
+		MentionDID:        mentionDIDs,
+		LinkURL:           linkURLs,
+		EmbedURL:          embedURL,
+		EmbedATURI:        embedATURI,
+		ReplyRootATURI:    replyRootATURI,
+		EmbedImgCount:     embedImgCount,
+		EmbedImgAltText:   embedImgAltText,
+		EmbedImgAltTextJA: embedImgAltTextJA,
+		SelfLabel:         selfLabels,
+		Tag:               parsePostTags(post),
+		Emoji:             parseEmojis(post.Text),
+	}
+
+	if containsJapanese(post.Text) {
+		doc.TextJA = post.Text
 	}
 
 	if post.CreatedAt != "" {