appview/indexer: add indexer for repos #1213

+4

appview/indexer/indexer.go

··· 7 7 "tangled.org/core/appview/db" 8 8 issues_indexer "tangled.org/core/appview/indexer/issues" 9 9 pulls_indexer "tangled.org/core/appview/indexer/pulls" 10 10 + repos_indexer "tangled.org/core/appview/indexer/repos" 10 11 "tangled.org/core/appview/notify" 11 12 tlog "tangled.org/core/log" 12 13 ) ··· 14 15 type Indexer struct { 15 16 Issues *issues_indexer.Indexer 16 17 Pulls *pulls_indexer.Indexer 18 18 + Repos *repos_indexer.Indexer 17 19 logger *slog.Logger 18 20 notify.BaseNotifier 19 21 } ··· 22 24 return &Indexer{ 23 25 issues_indexer.NewIndexer("indexes/issues.bleve"), 24 26 pulls_indexer.NewIndexer("indexes/pulls.bleve"), 27 27 + repos_indexer.NewIndexer("indexes/repos.bleve"), 25 28 logger, 26 29 notify.BaseNotifier{}, 27 30 } ··· 32 35 ctx = tlog.IntoContext(ctx, ix.logger) 33 36 ix.Issues.Init(ctx, db) 34 37 ix.Pulls.Init(ctx, db) 38 38 + ix.Repos.Init(ctx, db) 35 39 return nil 36 40 }

+9

appview/indexer/notifier.go

··· 73 73 l.Error("failed to index a pr", "err", err) 74 74 } 75 75 } 76 76 + 77 77 + func (ix *Indexer) NewRepo(ctx context.Context, repo *models.Repo) { 78 78 + l := log.FromContext(ctx).With("notifier", "indexer", "repo", repo) 79 79 + l.Debug("indexing new repo") 80 80 + err := ix.Repos.Index(ctx, *repo) 81 81 + if err != nil { 82 82 + l.Error("failed to index a repo", "err", err) 83 83 + } 84 84 + }

+372

appview/indexer/repos/indexer.go

··· 1 1 + // heavily inspired by gitea's model (basically copy-pasted) 2 2 + package repos_indexer 3 3 + 4 4 + import ( 5 5 + "context" 6 6 + "errors" 7 7 + "log" 8 8 + "os" 9 9 + 10 10 + "github.com/blevesearch/bleve/v2" 11 11 + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" 12 12 + "github.com/blevesearch/bleve/v2/analysis/token/camelcase" 13 13 + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" 14 14 + "github.com/blevesearch/bleve/v2/analysis/token/ngram" 15 15 + "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" 16 16 + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" 17 17 + "github.com/blevesearch/bleve/v2/index/upsidedown" 18 18 + "github.com/blevesearch/bleve/v2/mapping" 19 19 + "github.com/blevesearch/bleve/v2/search/query" 20 20 + "tangled.org/core/appview/db" 21 21 + "tangled.org/core/appview/indexer/base36" 22 22 + bleveutil "tangled.org/core/appview/indexer/bleve" 23 23 + "tangled.org/core/appview/models" 24 24 + "tangled.org/core/appview/pagination" 25 25 + tlog "tangled.org/core/log" 26 26 + ) 27 27 + 28 28 + const ( 29 29 + repoIndexerAnalyzer = "repoIndexer" 30 30 + repoIndexerDocType = "repoIndexerDocType" 31 31 + 32 32 + unicodeNormalizeName = "unicodeNormalize" 33 33 + 34 34 + // Bump this when the index mapping changes to trigger a rebuild. 35 35 + repoIndexerVersion = 5 36 36 + ) 37 37 + 38 38 + type Indexer struct { 39 39 + indexer bleve.Index 40 40 + path string 41 41 + } 42 42 + 43 43 + func NewIndexer(indexDir string) *Indexer { 44 44 + return &Indexer{ 45 45 + path: indexDir, 46 46 + } 47 47 + } 48 48 + 49 49 + // Init initializes the indexer 50 50 + func (ix *Indexer) Init(ctx context.Context, e db.Execer) { 51 51 + l := tlog.FromContext(ctx) 52 52 + existed, err := ix.intialize(ctx) 53 53 + if err != nil { 54 54 + log.Fatalln("failed to initialize repo indexer", err) 55 55 + } 56 56 + if !existed { 57 57 + l.Debug("Populating the repo indexer") 58 58 + err := PopulateIndexer(ctx, ix, e) 59 59 + if err != nil { 60 60 + log.Fatalln("failed to populate repo indexer", err) 61 61 + } 62 62 + } 63 63 + 64 64 + count, _ := ix.indexer.DocCount() 65 65 + l.Info("Initialized the repo indexer", "docCount", count) 66 66 + } 67 67 + 68 68 + func generateRepoIndexMapping() (mapping.IndexMapping, error) { 69 69 + mapping := bleve.NewIndexMapping() 70 70 + docMapping := bleve.NewDocumentMapping() 71 71 + 72 72 + textFieldMapping := bleve.NewTextFieldMapping() 73 73 + textFieldMapping.Store = false 74 74 + textFieldMapping.IncludeInAll = false 75 75 + 76 76 + keywordFieldMapping := bleve.NewKeywordFieldMapping() 77 77 + keywordFieldMapping.Store = false 78 78 + keywordFieldMapping.IncludeInAll = false 79 79 + 80 80 + // case-insensitive keyword field for language and topics 81 81 + caseInsensitiveKeywordMapping := bleve.NewTextFieldMapping() 82 82 + caseInsensitiveKeywordMapping.Store = false 83 83 + caseInsensitiveKeywordMapping.IncludeInAll = false 84 84 + caseInsensitiveKeywordMapping.Analyzer = "keyword_lowercase" 85 85 + 86 86 + // trigram field for partial repo name matching 87 87 + trigramFieldMapping := bleve.NewTextFieldMapping() 88 88 + trigramFieldMapping.Store = false 89 89 + trigramFieldMapping.IncludeInAll = false 90 90 + trigramFieldMapping.Analyzer = "trigram" 91 91 + 92 92 + // text fields 93 93 + docMapping.AddFieldMappingsAt("name", textFieldMapping) 94 94 + docMapping.AddFieldMappingsAt("name_trigram", trigramFieldMapping) 95 95 + docMapping.AddFieldMappingsAt("description", textFieldMapping) 96 96 + docMapping.AddFieldMappingsAt("website", textFieldMapping) 97 97 + docMapping.AddFieldMappingsAt("topics", textFieldMapping) 98 98 + 99 99 + // keyword fields 100 100 + docMapping.AddFieldMappingsAt("language", caseInsensitiveKeywordMapping) 101 101 + docMapping.AddFieldMappingsAt("topics_exact", caseInsensitiveKeywordMapping) 102 102 + docMapping.AddFieldMappingsAt("did", keywordFieldMapping) 103 103 + docMapping.AddFieldMappingsAt("knot", keywordFieldMapping) 104 104 + docMapping.AddFieldMappingsAt("repo_at", keywordFieldMapping) 105 105 + 106 106 + err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ 107 107 + "type": unicodenorm.Name, 108 108 + "form": unicodenorm.NFC, 109 109 + }) 110 110 + if err != nil { 111 111 + return nil, err 112 112 + } 113 113 + 114 114 + err = mapping.AddCustomTokenFilter("edgeNgram3", map[string]any{ 115 115 + "type": ngram.Name, 116 116 + "min": 2.0, 117 117 + "max": 3.0, 118 118 + }) 119 119 + if err != nil { 120 120 + return nil, err 121 121 + } 122 122 + 123 123 + err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{ 124 124 + "type": custom.Name, 125 125 + "char_filters": []string{}, 126 126 + "tokenizer": unicode.Name, 127 127 + "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, 128 128 + }) 129 129 + if err != nil { 130 130 + return nil, err 131 131 + } 132 132 + 133 133 + err = mapping.AddCustomAnalyzer("keyword_lowercase", map[string]any{ 134 134 + "type": custom.Name, 135 135 + "char_filters": []string{}, 136 136 + "tokenizer": "single", 137 137 + "token_filters": []string{lowercase.Name}, 138 138 + }) 139 139 + if err != nil { 140 140 + return nil, err 141 141 + } 142 142 + 143 143 + err = mapping.AddCustomAnalyzer("trigram", map[string]any{ 144 144 + "type": custom.Name, 145 145 + "char_filters": []string{}, 146 146 + "tokenizer": "single", 147 147 + "token_filters": []string{lowercase.Name, "edgeNgram3"}, 148 148 + }) 149 149 + if err != nil { 150 150 + return nil, err 151 151 + } 152 152 + 153 153 + mapping.DefaultAnalyzer = repoIndexerAnalyzer 154 154 + mapping.AddDocumentMapping(repoIndexerDocType, docMapping) 155 155 + mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping()) 156 156 + mapping.DefaultMapping = bleve.NewDocumentDisabledMapping() 157 157 + 158 158 + return mapping, nil 159 159 + } 160 160 + 161 161 + func (ix *Indexer) intialize(ctx context.Context) (bool, error) { 162 162 + if ix.indexer != nil { 163 163 + return false, errors.New("indexer is already initialized") 164 164 + } 165 165 + 166 166 + indexer, err := openIndexer(ctx, ix.path, repoIndexerVersion) 167 167 + if err != nil { 168 168 + return false, err 169 169 + } 170 170 + if indexer != nil { 171 171 + ix.indexer = indexer 172 172 + return true, nil 173 173 + } 174 174 + 175 175 + mapping, err := generateRepoIndexMapping() 176 176 + if err != nil { 177 177 + return false, err 178 178 + } 179 179 + indexer, err = bleve.New(ix.path, mapping) 180 180 + if err != nil { 181 181 + return false, err 182 182 + } 183 183 + indexer.SetInternal([]byte("mapping_version"), []byte{byte(repoIndexerVersion)}) 184 184 + 185 185 + ix.indexer = indexer 186 186 + 187 187 + return false, nil 188 188 + } 189 189 + 190 190 + func openIndexer(ctx context.Context, path string, version int) (bleve.Index, error) { 191 191 + l := tlog.FromContext(ctx) 192 192 + indexer, err := bleve.Open(path) 193 193 + if err != nil { 194 194 + if errors.Is(err, upsidedown.IncompatibleVersion) { 195 195 + l.Info("Indexer was built with a previous version of bleve, deleting and rebuilding") 196 196 + return nil, os.RemoveAll(path) 197 197 + } 198 198 + return nil, nil 199 199 + } 200 200 + 201 201 + storedVersion, _ := indexer.GetInternal([]byte("mapping_version")) 202 202 + if storedVersion == nil || int(storedVersion[0]) != version { 203 203 + l.Info("Indexer mapping version changed, deleting and rebuilding") 204 204 + indexer.Close() 205 205 + return nil, os.RemoveAll(path) 206 206 + } 207 207 + 208 208 + return indexer, nil 209 209 + } 210 210 + 211 211 + func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error { 212 212 + l := tlog.FromContext(ctx) 213 213 + count := 0 214 214 + 215 215 + err := pagination.IterateAll( 216 216 + func(page pagination.Page) ([]models.Repo, error) { 217 217 + return db.GetReposPaginated(e, page) 218 218 + }, 219 219 + func(repos []models.Repo) error { 220 220 + count += len(repos) 221 221 + return ix.Index(ctx, repos...) 222 222 + }, 223 223 + ) 224 224 + 225 225 + l.Info("repos indexed", "count", count) 226 226 + return err 227 227 + } 228 228 + 229 229 + type repoData struct { 230 230 + ID int64 `json:"id"` 231 231 + RepoAt string `json:"repo_at"` 232 232 + Did string `json:"did"` 233 233 + Name string `json:"name"` 234 234 + NameTrigram string `json:"name_trigram"` 235 235 + Description string `json:"description"` 236 236 + Website string `json:"website"` 237 237 + Topics []string `json:"topics"` 238 238 + TopicsExact []string `json:"topics_exact"` 239 239 + Knot string `json:"knot"` 240 240 + Language string `json:"language"` 241 241 + } 242 242 + 243 243 + func makeRepoData(repo *models.Repo) *repoData { 244 244 + return &repoData{ 245 245 + ID: repo.Id, 246 246 + RepoAt: repo.RepoAt().String(), 247 247 + Did: repo.Did, 248 248 + Name: repo.Name, 249 249 + NameTrigram: repo.Name, 250 250 + Description: repo.Description, 251 251 + Website: repo.Website, 252 252 + Topics: repo.Topics, 253 253 + TopicsExact: repo.Topics, 254 254 + Knot: repo.Knot, 255 255 + Language: repo.RepoStats.Language, 256 256 + } 257 257 + } 258 258 + 259 259 + // Type returns the document type, for bleve's mapping.Classifier interface. 260 260 + func (r *repoData) Type() string { 261 261 + return repoIndexerDocType 262 262 + } 263 263 + 264 264 + type SearchResult struct { 265 265 + Hits []int64 266 266 + Total uint64 267 267 + } 268 268 + 269 269 + const maxBatchSize = 20 270 270 + 271 271 + func (ix *Indexer) Index(ctx context.Context, repos ...models.Repo) error { 272 272 + batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize) 273 273 + for _, repo := range repos { 274 274 + repoData := makeRepoData(&repo) 275 275 + if err := batch.Index(base36.Encode(repo.Id), repoData); err != nil { 276 276 + return err 277 277 + } 278 278 + } 279 279 + return batch.Flush() 280 280 + } 281 281 + 282 282 + func (ix *Indexer) Delete(ctx context.Context, repoID int64) error { 283 283 + return ix.indexer.Delete(base36.Encode(repoID)) 284 284 + } 285 285 + 286 286 + func (ix *Indexer) Search(ctx context.Context, opts models.RepoSearchOptions) (*SearchResult, error) { 287 287 + var musts []query.Query 288 288 + var mustNots []query.Query 289 289 + 290 290 + for _, keyword := range opts.Keywords { 291 291 + musts = append(musts, bleve.NewDisjunctionQuery( 292 292 + bleveutil.MatchAndQuery("name", keyword, repoIndexerAnalyzer, 0), 293 293 + bleveutil.MatchAndQuery("name_trigram", keyword, "trigram", 0), 294 294 + bleveutil.MatchAndQuery("description", keyword, repoIndexerAnalyzer, 0), 295 295 + bleveutil.MatchAndQuery("website", keyword, repoIndexerAnalyzer, 0), 296 296 + bleveutil.MatchAndQuery("topics", keyword, repoIndexerAnalyzer, 0), 297 297 + )) 298 298 + } 299 299 + 300 300 + for _, phrase := range opts.Phrases { 301 301 + musts = append(musts, bleve.NewDisjunctionQuery( 302 302 + bleveutil.MatchPhraseQuery("name", phrase, repoIndexerAnalyzer), 303 303 + bleveutil.MatchPhraseQuery("description", phrase, repoIndexerAnalyzer), 304 304 + bleveutil.MatchPhraseQuery("website", phrase, repoIndexerAnalyzer), 305 305 + bleveutil.MatchPhraseQuery("topics", phrase, repoIndexerAnalyzer), 306 306 + )) 307 307 + } 308 308 + 309 309 + for _, keyword := range opts.NegatedKeywords { 310 310 + mustNots = append(mustNots, bleve.NewDisjunctionQuery( 311 311 + bleveutil.MatchAndQuery("name", keyword, repoIndexerAnalyzer, 0), 312 312 + bleveutil.MatchAndQuery("name_trigram", keyword, "trigram", 0), 313 313 + bleveutil.MatchAndQuery("description", keyword, repoIndexerAnalyzer, 0), 314 314 + bleveutil.MatchAndQuery("website", keyword, repoIndexerAnalyzer, 0), 315 315 + bleveutil.MatchAndQuery("topics", keyword, repoIndexerAnalyzer, 0), 316 316 + )) 317 317 + } 318 318 + 319 319 + for _, phrase := range opts.NegatedPhrases { 320 320 + mustNots = append(mustNots, bleve.NewDisjunctionQuery( 321 321 + bleveutil.MatchPhraseQuery("name", phrase, repoIndexerAnalyzer), 322 322 + bleveutil.MatchPhraseQuery("description", phrase, repoIndexerAnalyzer), 323 323 + bleveutil.MatchPhraseQuery("website", phrase, repoIndexerAnalyzer), 324 324 + bleveutil.MatchPhraseQuery("topics", phrase, repoIndexerAnalyzer), 325 325 + )) 326 326 + } 327 327 + 328 328 + // keyword filters 329 329 + if opts.Language != "" { 330 330 + musts = append(musts, bleveutil.MatchAndQuery("language", opts.Language, "keyword_lowercase", 0)) 331 331 + } 332 332 + 333 333 + if opts.Knot != "" { 334 334 + musts = append(musts, bleveutil.KeywordFieldQuery("knot", opts.Knot)) 335 335 + } 336 336 + 337 337 + if opts.Did != "" { 338 338 + musts = append(musts, bleveutil.KeywordFieldQuery("did", opts.Did)) 339 339 + } 340 340 + 341 341 + for _, topic := range opts.Topics { 342 342 + musts = append(musts, bleveutil.MatchAndQuery("topics_exact", topic, "keyword_lowercase", 0)) 343 343 + } 344 344 + 345 345 + for _, topic := range opts.NegatedTopics { 346 346 + mustNots = append(mustNots, bleveutil.MatchAndQuery("topics_exact", topic, "keyword_lowercase", 0)) 347 347 + } 348 348 + 349 349 + indexerQuery := bleve.NewBooleanQuery() 350 350 + if len(musts) == 0 { 351 351 + musts = append(musts, bleve.NewMatchAllQuery()) 352 352 + } 353 353 + indexerQuery.AddMust(musts...) 354 354 + indexerQuery.AddMustNot(mustNots...) 355 355 + searchReq := bleve.NewSearchRequestOptions(indexerQuery, opts.Page.Limit, opts.Page.Offset, false) 356 356 + res, err := ix.indexer.SearchInContext(ctx, searchReq) 357 357 + if err != nil { 358 358 + return nil, nil 359 359 + } 360 360 + ret := &SearchResult{ 361 361 + Total: res.Total, 362 362 + Hits: make([]int64, len(res.Hits)), 363 363 + } 364 364 + for i, hit := range res.Hits { 365 365 + id, err := base36.Decode(hit.ID) 366 366 + if err != nil { 367 367 + return nil, err 368 368 + } 369 369 + ret.Hits[i] = id 370 370 + } 371 371 + return ret, nil 372 372 + }

+639

appview/indexer/repos/indexer_test.go

··· 1 1 + package repos_indexer 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "os" 6 6 + "testing" 7 7 + 8 8 + "github.com/blevesearch/bleve/v2" 9 9 + "github.com/stretchr/testify/assert" 10 10 + "github.com/stretchr/testify/require" 11 11 + "tangled.org/core/appview/models" 12 12 + "tangled.org/core/appview/pagination" 13 13 + ) 14 14 + 15 15 + func setupTestIndexer(t *testing.T) (*Indexer, func()) { 16 16 + t.Helper() 17 17 + 18 18 + tmpDir, err := os.MkdirTemp("", "repo_indexer_test") 19 19 + require.NoError(t, err) 20 20 + 21 21 + ix := NewIndexer(tmpDir) 22 22 + 23 23 + mapping, err := generateRepoIndexMapping() 24 24 + require.NoError(t, err) 25 25 + 26 26 + indexer, err := bleve.New(tmpDir, mapping) 27 27 + require.NoError(t, err) 28 28 + ix.indexer = indexer 29 29 + 30 30 + cleanup := func() { 31 31 + ix.indexer.Close() 32 32 + os.RemoveAll(tmpDir) 33 33 + } 34 34 + 35 35 + return ix, cleanup 36 36 + } 37 37 + 38 38 + func TestBasicIndexingAndSearch(t *testing.T) { 39 39 + ix, cleanup := setupTestIndexer(t) 40 40 + defer cleanup() 41 41 + 42 42 + ctx := context.Background() 43 43 + 44 44 + err := ix.Index(ctx, 45 45 + models.Repo{ 46 46 + Id: 1, 47 47 + Did: "did:plc:alice", 48 48 + Name: "web-framework", 49 49 + Knot: "example.com", 50 50 + Description: "A modern web framework for Go", 51 51 + Website: "https://example.com/web-framework", 52 52 + Topics: []string{"web", "framework", "golang"}, 53 53 + RepoStats: &models.RepoStats{Language: "Go"}, 54 54 + }, 55 55 + models.Repo{ 56 56 + Id: 2, 57 57 + Did: "did:plc:bob", 58 58 + Name: "cli-tool", 59 59 + Knot: "example.com", 60 60 + Description: "Command line utility for developers", 61 61 + Website: "", 62 62 + Topics: []string{"cli", "tool"}, 63 63 + RepoStats: &models.RepoStats{Language: "Rust"}, 64 64 + }, 65 65 + models.Repo{ 66 66 + Id: 3, 67 67 + Did: "did:plc:alice", 68 68 + Name: "javascript-parser", 69 69 + Knot: "example.com", 70 70 + Description: "Fast JavaScript parser", 71 71 + Website: "", 72 72 + Topics: []string{"javascript", "parser"}, 73 73 + RepoStats: &models.RepoStats{Language: "JavaScript"}, 74 74 + }, 75 75 + ) 76 76 + require.NoError(t, err) 77 77 + 78 78 + // search by name 79 79 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 80 80 + Keywords: []string{"framework"}, 81 81 + Page: pagination.Page{Limit: 10}, 82 82 + }) 83 83 + require.NoError(t, err) 84 84 + assert.Equal(t, uint64(1), result.Total) 85 85 + assert.Contains(t, result.Hits, int64(1)) 86 86 + 87 87 + // search by description 88 88 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 89 89 + Keywords: []string{"utility"}, 90 90 + Page: pagination.Page{Limit: 10}, 91 91 + }) 92 92 + require.NoError(t, err) 93 93 + assert.Equal(t, uint64(1), result.Total) 94 94 + assert.Contains(t, result.Hits, int64(2)) 95 95 + 96 96 + // search by website 97 97 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 98 98 + Keywords: []string{"example.com/web-framework"}, 99 99 + Page: pagination.Page{Limit: 10}, 100 100 + }) 101 101 + require.NoError(t, err) 102 102 + assert.Equal(t, uint64(1), result.Total) 103 103 + assert.Contains(t, result.Hits, int64(1)) 104 104 + } 105 105 + 106 106 + func TestLanguageFiltering(t *testing.T) { 107 107 + ix, cleanup := setupTestIndexer(t) 108 108 + defer cleanup() 109 109 + 110 110 + ctx := context.Background() 111 111 + 112 112 + err := ix.Index(ctx, 113 113 + models.Repo{ 114 114 + Id: 1, 115 115 + Did: "did:plc:alice", 116 116 + Name: "go-project", 117 117 + RepoStats: &models.RepoStats{Language: "Go"}, 118 118 + }, 119 119 + models.Repo{ 120 120 + Id: 2, 121 121 + Did: "did:plc:bob", 122 122 + Name: "rust-project", 123 123 + RepoStats: &models.RepoStats{Language: "Rust"}, 124 124 + }, 125 125 + models.Repo{ 126 126 + Id: 3, 127 127 + Did: "did:plc:alice", 128 128 + Name: "another-go-project", 129 129 + RepoStats: &models.RepoStats{Language: "Go"}, 130 130 + }, 131 131 + ) 132 132 + require.NoError(t, err) 133 133 + 134 134 + // filter by go language 135 135 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 136 136 + Language: "Go", 137 137 + Page: pagination.Page{Limit: 10}, 138 138 + }) 139 139 + require.NoError(t, err) 140 140 + assert.Equal(t, uint64(2), result.Total) 141 141 + assert.Contains(t, result.Hits, int64(1)) 142 142 + assert.Contains(t, result.Hits, int64(3)) 143 143 + 144 144 + // filter by rust language 145 145 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 146 146 + Language: "Rust", 147 147 + Page: pagination.Page{Limit: 10}, 148 148 + }) 149 149 + require.NoError(t, err) 150 150 + assert.Equal(t, uint64(1), result.Total) 151 151 + assert.Contains(t, result.Hits, int64(2)) 152 152 + } 153 153 + 154 154 + func TestTopicExactMatching(t *testing.T) { 155 155 + ix, cleanup := setupTestIndexer(t) 156 156 + defer cleanup() 157 157 + 158 158 + ctx := context.Background() 159 159 + 160 160 + err := ix.Index(ctx, 161 161 + models.Repo{ 162 162 + Id: 1, 163 163 + Did: "did:plc:alice", 164 164 + Name: "js-tool", 165 165 + Topics: []string{"javascript", "tool"}, 166 166 + RepoStats: &models.RepoStats{}, 167 167 + }, 168 168 + models.Repo{ 169 169 + Id: 2, 170 170 + Did: "did:plc:bob", 171 171 + Name: "java-app", 172 172 + Topics: []string{"java", "application"}, 173 173 + RepoStats: &models.RepoStats{}, 174 174 + }, 175 175 + models.Repo{ 176 176 + Id: 3, 177 177 + Did: "did:plc:alice", 178 178 + Name: "cli-tool", 179 179 + Topics: []string{"cli", "tool"}, 180 180 + RepoStats: &models.RepoStats{}, 181 181 + }, 182 182 + ) 183 183 + require.NoError(t, err) 184 184 + 185 185 + // exact match for "javascript" topic 186 186 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 187 187 + Topics: []string{"javascript"}, 188 188 + Page: pagination.Page{Limit: 10}, 189 189 + }) 190 190 + require.NoError(t, err) 191 191 + assert.Equal(t, uint64(1), result.Total) 192 192 + assert.Contains(t, result.Hits, int64(1)) 193 193 + 194 194 + // exact match for "tool" topic (should match repos 1 and 3) 195 195 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 196 196 + Topics: []string{"tool"}, 197 197 + Page: pagination.Page{Limit: 10}, 198 198 + }) 199 199 + require.NoError(t, err) 200 200 + assert.Equal(t, uint64(2), result.Total) 201 201 + assert.Contains(t, result.Hits, int64(1)) 202 202 + assert.Contains(t, result.Hits, int64(3)) 203 203 + } 204 204 + 205 205 + func TestTopicTextSearch(t *testing.T) { 206 206 + ix, cleanup := setupTestIndexer(t) 207 207 + defer cleanup() 208 208 + 209 209 + ctx := context.Background() 210 210 + 211 211 + err := ix.Index(ctx, 212 212 + models.Repo{ 213 213 + Id: 1, 214 214 + Did: "did:plc:alice", 215 215 + Name: "js-tool", 216 216 + Topics: []string{"JavaScript"}, 217 217 + RepoStats: &models.RepoStats{}, 218 218 + }, 219 219 + models.Repo{ 220 220 + Id: 2, 221 221 + Did: "did:plc:bob", 222 222 + Name: "java-app", 223 223 + Topics: []string{"Java"}, 224 224 + RepoStats: &models.RepoStats{}, 225 225 + }, 226 226 + ) 227 227 + require.NoError(t, err) 228 228 + 229 229 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 230 230 + Keywords: []string{"Java"}, 231 231 + Page: pagination.Page{Limit: 10}, 232 232 + }) 233 233 + require.NoError(t, err) 234 234 + assert.Equal(t, uint64(2), result.Total) 235 235 + assert.Contains(t, result.Hits, int64(1)) 236 236 + assert.Contains(t, result.Hits, int64(2)) 237 237 + } 238 238 + 239 239 + func TestNegatedFilters(t *testing.T) { 240 240 + ix, cleanup := setupTestIndexer(t) 241 241 + defer cleanup() 242 242 + 243 243 + ctx := context.Background() 244 244 + 245 245 + err := ix.Index(ctx, 246 246 + models.Repo{ 247 247 + Id: 1, 248 248 + Did: "did:plc:alice", 249 249 + Name: "active-project", 250 250 + Description: "An active development project", 251 251 + Topics: []string{"active"}, 252 252 + RepoStats: &models.RepoStats{Language: "Go"}, 253 253 + }, 254 254 + models.Repo{ 255 255 + Id: 2, 256 256 + Did: "did:plc:bob", 257 257 + Name: "archived-project", 258 258 + Description: "An archived project", 259 259 + Topics: []string{"archived"}, 260 260 + RepoStats: &models.RepoStats{Language: "Python"}, 261 261 + }, 262 262 + models.Repo{ 263 263 + Id: 3, 264 264 + Did: "did:plc:alice", 265 265 + Name: "another-project", 266 266 + Description: "Another active project", 267 267 + Topics: []string{"active"}, 268 268 + RepoStats: &models.RepoStats{Language: "Go"}, 269 269 + }, 270 270 + ) 271 271 + require.NoError(t, err) 272 272 + 273 273 + // exclude archived topic 274 274 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 275 275 + NegatedTopics: []string{"archived"}, 276 276 + Page: pagination.Page{Limit: 10}, 277 277 + }) 278 278 + require.NoError(t, err) 279 279 + assert.Equal(t, uint64(2), result.Total) 280 280 + assert.Contains(t, result.Hits, int64(1)) 281 281 + assert.Contains(t, result.Hits, int64(3)) 282 282 + 283 283 + // exclude keyword "archived" 284 284 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 285 285 + NegatedKeywords: []string{"archived"}, 286 286 + Page: pagination.Page{Limit: 10}, 287 287 + }) 288 288 + require.NoError(t, err) 289 289 + assert.Equal(t, uint64(2), result.Total) 290 290 + assert.Contains(t, result.Hits, int64(1)) 291 291 + assert.Contains(t, result.Hits, int64(3)) 292 292 + 293 293 + // exclude phrase 294 294 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 295 295 + NegatedPhrases: []string{"archived project"}, 296 296 + Page: pagination.Page{Limit: 10}, 297 297 + }) 298 298 + require.NoError(t, err) 299 299 + assert.Equal(t, uint64(2), result.Total) 300 300 + assert.Contains(t, result.Hits, int64(1)) 301 301 + assert.Contains(t, result.Hits, int64(3)) 302 302 + } 303 303 + 304 304 + func TestPagination(t *testing.T) { 305 305 + ix, cleanup := setupTestIndexer(t) 306 306 + defer cleanup() 307 307 + 308 308 + ctx := context.Background() 309 309 + 310 310 + // index multiple repos 311 311 + var repos []models.Repo 312 312 + for i := 1; i <= 25; i++ { 313 313 + repos = append(repos, models.Repo{ 314 314 + Id: int64(i), 315 315 + Did: "did:plc:alice", 316 316 + Name: "project", 317 317 + Topics: []string{"test"}, 318 318 + RepoStats: &models.RepoStats{}, 319 319 + }) 320 320 + } 321 321 + err := ix.Index(ctx, repos...) 322 322 + require.NoError(t, err) 323 323 + 324 324 + // first page 325 325 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 326 326 + Topics: []string{"test"}, 327 327 + Page: pagination.Page{Limit: 10, Offset: 0}, 328 328 + }) 329 329 + require.NoError(t, err) 330 330 + assert.Equal(t, uint64(25), result.Total) 331 331 + assert.Len(t, result.Hits, 10) 332 332 + 333 333 + // second page 334 334 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 335 335 + Topics: []string{"test"}, 336 336 + Page: pagination.Page{Limit: 10, Offset: 10}, 337 337 + }) 338 338 + require.NoError(t, err) 339 339 + assert.Equal(t, uint64(25), result.Total) 340 340 + assert.Len(t, result.Hits, 10) 341 341 + 342 342 + // third page - 5 items 343 343 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 344 344 + Topics: []string{"test"}, 345 345 + Page: pagination.Page{Limit: 10, Offset: 20}, 346 346 + }) 347 347 + require.NoError(t, err) 348 348 + assert.Equal(t, uint64(25), result.Total) 349 349 + assert.Len(t, result.Hits, 5) 350 350 + } 351 351 + 352 352 + func TestUpdateReindex(t *testing.T) { 353 353 + ix, cleanup := setupTestIndexer(t) 354 354 + defer cleanup() 355 355 + 356 356 + ctx := context.Background() 357 357 + 358 358 + // initial index 359 359 + err := ix.Index(ctx, models.Repo{ 360 360 + Id: 1, 361 361 + Did: "did:plc:alice", 362 362 + Name: "my-project", 363 363 + Description: "Initial description", 364 364 + Topics: []string{"initial"}, 365 365 + RepoStats: &models.RepoStats{Language: "Go"}, 366 366 + }) 367 367 + require.NoError(t, err) 368 368 + 369 369 + // search for initial state 370 370 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 371 371 + Keywords: []string{"Initial"}, 372 372 + Page: pagination.Page{Limit: 10}, 373 373 + }) 374 374 + require.NoError(t, err) 375 375 + assert.Equal(t, uint64(1), result.Total) 376 376 + 377 377 + // update the repo 378 378 + err = ix.Index(ctx, models.Repo{ 379 379 + Id: 1, 380 380 + Did: "did:plc:alice", 381 381 + Name: "my-project", 382 382 + Description: "Updated description", 383 383 + Topics: []string{"updated"}, 384 384 + RepoStats: &models.RepoStats{Language: "Rust"}, 385 385 + }) 386 386 + require.NoError(t, err) 387 387 + 388 388 + // search for old description should return nothing 389 389 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 390 390 + Keywords: []string{"Initial"}, 391 391 + Page: pagination.Page{Limit: 10}, 392 392 + }) 393 393 + require.NoError(t, err) 394 394 + assert.Equal(t, uint64(0), result.Total) 395 395 + 396 396 + // search for new description should work 397 397 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 398 398 + Keywords: []string{"Updated"}, 399 399 + Page: pagination.Page{Limit: 10}, 400 400 + }) 401 401 + require.NoError(t, err) 402 402 + assert.Equal(t, uint64(1), result.Total) 403 403 + 404 404 + // language should be updated 405 405 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 406 406 + Language: "Rust", 407 407 + Page: pagination.Page{Limit: 10}, 408 408 + }) 409 409 + require.NoError(t, err) 410 410 + assert.Equal(t, uint64(1), result.Total) 411 411 + } 412 412 + 413 413 + func TestEmptyResults(t *testing.T) { 414 414 + ix, cleanup := setupTestIndexer(t) 415 415 + defer cleanup() 416 416 + 417 417 + ctx := context.Background() 418 418 + 419 419 + err := ix.Index(ctx, models.Repo{ 420 420 + Id: 1, 421 421 + Did: "did:plc:alice", 422 422 + Name: "my-project", 423 423 + RepoStats: &models.RepoStats{}, 424 424 + }) 425 425 + require.NoError(t, err) 426 426 + 427 427 + // search for non-existent keyword 428 428 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 429 429 + Keywords: []string{"nonexistent"}, 430 430 + Page: pagination.Page{Limit: 10}, 431 431 + }) 432 432 + require.NoError(t, err) 433 433 + assert.Equal(t, uint64(0), result.Total) 434 434 + assert.Empty(t, result.Hits) 435 435 + 436 436 + // search for non-existent language 437 437 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 438 438 + Language: "NonexistentLanguage", 439 439 + Page: pagination.Page{Limit: 10}, 440 440 + }) 441 441 + require.NoError(t, err) 442 442 + assert.Equal(t, uint64(0), result.Total) 443 443 + assert.Empty(t, result.Hits) 444 444 + } 445 445 + 446 446 + func TestCombinedFilters(t *testing.T) { 447 447 + ix, cleanup := setupTestIndexer(t) 448 448 + defer cleanup() 449 449 + 450 450 + ctx := context.Background() 451 451 + 452 452 + err := ix.Index(ctx, 453 453 + models.Repo{ 454 454 + Id: 1, 455 455 + Did: "did:plc:alice", 456 456 + Name: "web-server", 457 457 + Knot: "example.com", 458 458 + Description: "A web server in Go", 459 459 + Topics: []string{"web", "server"}, 460 460 + RepoStats: &models.RepoStats{Language: "Go"}, 461 461 + }, 462 462 + models.Repo{ 463 463 + Id: 2, 464 464 + Did: "did:plc:bob", 465 465 + Name: "web-client", 466 466 + Knot: "example.org", 467 467 + Description: "A web client in Rust", 468 468 + Topics: []string{"web", "client"}, 469 469 + RepoStats: &models.RepoStats{Language: "Rust"}, 470 470 + }, 471 471 + models.Repo{ 472 472 + Id: 3, 473 473 + Did: "did:plc:alice", 474 474 + Name: "cli-tool", 475 475 + Knot: "example.com", 476 476 + Description: "A CLI tool in Go", 477 477 + Topics: []string{"cli", "tool"}, 478 478 + RepoStats: &models.RepoStats{Language: "Go"}, 479 479 + }, 480 480 + ) 481 481 + require.NoError(t, err) 482 482 + 483 483 + // combine language + topic + keyword 484 484 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 485 485 + Language: "Go", 486 486 + Topics: []string{"web"}, 487 487 + Keywords: []string{"server"}, 488 488 + Page: pagination.Page{Limit: 10}, 489 489 + }) 490 490 + require.NoError(t, err) 491 491 + assert.Equal(t, uint64(1), result.Total) 492 492 + assert.Contains(t, result.Hits, int64(1)) 493 493 + 494 494 + // combine did + language 495 495 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 496 496 + Did: "did:plc:alice", 497 497 + Language: "Go", 498 498 + Page: pagination.Page{Limit: 10}, 499 499 + }) 500 500 + require.NoError(t, err) 501 501 + assert.Equal(t, uint64(2), result.Total) 502 502 + assert.Contains(t, result.Hits, int64(1)) 503 503 + assert.Contains(t, result.Hits, int64(3)) 504 504 + 505 505 + // combine knot + language 506 506 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 507 507 + Knot: "example.com", 508 508 + Language: "Go", 509 509 + Page: pagination.Page{Limit: 10}, 510 510 + }) 511 511 + require.NoError(t, err) 512 512 + assert.Equal(t, uint64(2), result.Total) 513 513 + assert.Contains(t, result.Hits, int64(1)) 514 514 + assert.Contains(t, result.Hits, int64(3)) 515 515 + } 516 516 + 517 517 + func TestRepoWithoutLanguage(t *testing.T) { 518 518 + ix, cleanup := setupTestIndexer(t) 519 519 + defer cleanup() 520 520 + 521 521 + ctx := context.Background() 522 522 + 523 523 + err := ix.Index(ctx, 524 524 + models.Repo{ 525 525 + Id: 1, 526 526 + Did: "did:plc:alice", 527 527 + Name: "project-with-language", 528 528 + RepoStats: &models.RepoStats{Language: "Go"}, 529 529 + }, 530 530 + models.Repo{ 531 531 + Id: 2, 532 532 + Did: "did:plc:bob", 533 533 + Name: "project-without-language", 534 534 + RepoStats: &models.RepoStats{Language: ""}, 535 535 + }, 536 536 + ) 537 537 + require.NoError(t, err) 538 538 + 539 539 + // search without language filter should return both 540 540 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 541 541 + Keywords: []string{"project"}, 542 542 + Page: pagination.Page{Limit: 10}, 543 543 + }) 544 544 + require.NoError(t, err) 545 545 + assert.Equal(t, uint64(2), result.Total) 546 546 + 547 547 + // language filter should only return repo with language 548 548 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 549 549 + Language: "Go", 550 550 + Page: pagination.Page{Limit: 10}, 551 551 + }) 552 552 + require.NoError(t, err) 553 553 + assert.Equal(t, uint64(1), result.Total) 554 554 + assert.Contains(t, result.Hits, int64(1)) 555 555 + } 556 556 + 557 557 + func TestRepoWithoutTopics(t *testing.T) { 558 558 + ix, cleanup := setupTestIndexer(t) 559 559 + defer cleanup() 560 560 + 561 561 + ctx := context.Background() 562 562 + 563 563 + err := ix.Index(ctx, 564 564 + models.Repo{ 565 565 + Id: 1, 566 566 + Did: "did:plc:alice", 567 567 + Name: "project-with-topics", 568 568 + Topics: []string{"cli", "tool"}, 569 569 + RepoStats: &models.RepoStats{}, 570 570 + }, 571 571 + models.Repo{ 572 572 + Id: 2, 573 573 + Did: "did:plc:bob", 574 574 + Name: "project-without-topics", 575 575 + Topics: []string{}, 576 576 + RepoStats: &models.RepoStats{}, 577 577 + }, 578 578 + ) 579 579 + require.NoError(t, err) 580 580 + 581 581 + // topic filter should only return repo with topics 582 582 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 583 583 + Topics: []string{"cli"}, 584 584 + Page: pagination.Page{Limit: 10}, 585 585 + }) 586 586 + require.NoError(t, err) 587 587 + assert.Equal(t, uint64(1), result.Total) 588 588 + assert.Contains(t, result.Hits, int64(1)) 589 589 + 590 590 + // general search should return both 591 591 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 592 592 + Keywords: []string{"project"}, 593 593 + Page: pagination.Page{Limit: 10}, 594 594 + }) 595 595 + require.NoError(t, err) 596 596 + assert.Equal(t, uint64(2), result.Total) 597 597 + } 598 598 + 599 599 + func TestDelete(t *testing.T) { 600 600 + ix, cleanup := setupTestIndexer(t) 601 601 + defer cleanup() 602 602 + 603 603 + ctx := context.Background() 604 604 + 605 605 + err := ix.Index(ctx, 606 606 + models.Repo{ 607 607 + Id: 1, 608 608 + Did: "did:plc:alice", 609 609 + Name: "to-delete", 610 610 + RepoStats: &models.RepoStats{}, 611 611 + }, 612 612 + models.Repo{ 613 613 + Id: 2, 614 614 + Did: "did:plc:bob", 615 615 + Name: "to-keep", 616 616 + RepoStats: &models.RepoStats{}, 617 617 + }, 618 618 + ) 619 619 + require.NoError(t, err) 620 620 + 621 621 + // verify both exist 622 622 + result, err := ix.Search(ctx, models.RepoSearchOptions{ 623 623 + Page: pagination.Page{Limit: 10}, 624 624 + }) 625 625 + require.NoError(t, err) 626 626 + assert.Equal(t, uint64(2), result.Total) 627 627 + 628 628 + // delete repo 1 629 629 + err = ix.Delete(ctx, 1) 630 630 + require.NoError(t, err) 631 631 + 632 632 + // verify only one remains 633 633 + result, err = ix.Search(ctx, models.RepoSearchOptions{ 634 634 + Page: pagination.Page{Limit: 10}, 635 635 + }) 636 636 + require.NoError(t, err) 637 637 + assert.Equal(t, uint64(1), result.Total) 638 638 + assert.Contains(t, result.Hits, int64(2)) 639 639 + }

+23

appview/models/search.go

··· 53 53 len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || 54 54 len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 55 55 } 56 56 + 57 57 + type RepoSearchOptions struct { 58 58 + Keywords []string // text search across name, description, website, topics 59 59 + Phrases []string // phrase search 60 60 + 61 61 + Language string // exact match on primary language 62 62 + Knot string // filter by knot domain 63 63 + Did string // filter by owner DID 64 64 + Topics []string // exact topic matches 65 65 + 66 66 + NegatedKeywords []string 67 67 + NegatedPhrases []string 68 68 + NegatedTopics []string 69 69 + 70 70 + Page pagination.Page 71 71 + } 72 72 + 73 73 + func (o *RepoSearchOptions) HasSearchFilters() bool { 74 74 + return len(o.Keywords) > 0 || len(o.Phrases) > 0 || 75 75 + o.Language != "" || o.Did != "" || 76 76 + len(o.Topics) > 0 || len(o.NegatedTopics) > 0 || 77 77 + len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 78 78 + }