appview/db: introduce GetReposPaginated #1212

+1 -1

appview/db/collaborators.go

··· 59 59 return nil, nil 60 60 } 61 61 62 62 - return GetRepos(e, 0, orm.FilterIn("at_uri", repoAts)) 62 62 + return GetRepos(e, orm.FilterIn("at_uri", repoAts)) 63 63 } 64 64 65 65 func GetCollaborators(e Execer, filters ...orm.Filter) ([]models.Collaborator, error) {

+1 -1

appview/db/issues.go

··· 206 206 repoAts = append(repoAts, string(issue.RepoAt)) 207 207 } 208 208 209 209 - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", repoAts)) 209 209 + repos, err := GetRepos(e, orm.FilterIn("at_uri", repoAts)) 210 210 if err != nil { 211 211 return nil, fmt.Errorf("failed to build repo mappings: %w", err) 212 212 }

+2 -2

appview/db/profile.go

··· 66 66 *items = append(*items, &issue) 67 67 } 68 68 69 69 - repos, err := GetRepos(e, 0, orm.FilterEq("did", forDid)) 69 69 + repos, err := GetRepos(e, orm.FilterEq("did", forDid)) 70 70 if err != nil { 71 71 return nil, fmt.Errorf("error getting all repos by did: %w", err) 72 72 } ··· 489 489 } 490 490 491 491 // ensure all pinned repos are either own repos or collaborating repos 492 492 - repos, err := GetRepos(e, 0, orm.FilterEq("did", profile.Did)) 492 492 + repos, err := GetRepos(e, orm.FilterEq("did", profile.Did)) 493 493 if err != nil { 494 494 log.Printf("getting repos for %s: %s", profile.Did, err) 495 495 }

+1 -1

appview/db/pulls.go

··· 264 264 sourceAts = append(sourceAts, *p.PullSource.RepoAt) 265 265 } 266 266 } 267 267 - sourceRepos, err := GetRepos(e, 0, orm.FilterIn("at_uri", sourceAts)) 267 267 + sourceRepos, err := GetRepos(e, orm.FilterIn("at_uri", sourceAts)) 268 268 if err != nil && !errors.Is(err, sql.ErrNoRows) { 269 269 return nil, fmt.Errorf("failed to get source repos: %w", err) 270 270 }

+72 -67

appview/db/repos.go

··· 11 11 12 12 "github.com/bluesky-social/indigo/atproto/syntax" 13 13 "tangled.org/core/appview/models" 14 14 + "tangled.org/core/appview/pagination" 14 15 "tangled.org/core/orm" 15 16 ) 16 17 17 17 - func GetRepos(e Execer, limit int, filters ...orm.Filter) ([]models.Repo, error) { 18 18 - repoMap := make(map[syntax.ATURI]*models.Repo) 18 18 + func GetRepos(e Execer, filters ...orm.Filter) ([]models.Repo, error) { 19 19 + return GetReposPaginated(e, pagination.Page{}, filters...) 20 20 + } 19 21 22 22 + func GetReposPaginated(e Execer, page pagination.Page, filters ...orm.Filter) ([]models.Repo, error) { 20 23 var conditions []string 21 24 var args []any 22 25 for _, filter := range filters { ··· 29 32 whereClause = " where " + strings.Join(conditions, " and ") 30 33 } 31 34 32 32 - limitClause := "" 33 33 - if limit != 0 { 34 34 - limitClause = fmt.Sprintf(" limit %d", limit) 35 35 + pageClause := "" 36 36 + if page.Limit != 0 { 37 37 + pageClause = fmt.Sprintf(" limit %d offset %d", page.Limit, page.Offset) 35 38 } 36 39 37 37 - repoQuery := fmt.Sprintf( 38 38 - `select 40 40 + // main query to get repos with pagination 41 41 + query := fmt.Sprintf(` 42 42 + select 39 43 id, 40 44 did, 41 45 name, ··· 47 51 topics, 48 52 source, 49 53 spindle 50 50 - from 51 51 - repos r 54 54 + from repos 52 55 %s 53 56 order by created desc 54 54 - %s`, 55 55 - whereClause, 56 56 - limitClause, 57 57 - ) 58 58 - rows, err := e.Query(repoQuery, args...) 57 57 + %s 58 58 + `, whereClause, pageClause) 59 59 + 60 60 + rows, err := e.Query(query, args...) 59 61 if err != nil { 60 60 - return nil, fmt.Errorf("failed to execute repo query: %w ", err) 62 62 + return nil, err 61 63 } 62 64 defer rows.Close() 63 65 66 66 + repoMap := make(map[syntax.ATURI]*models.Repo) 64 67 for rows.Next() { 65 68 var repo models.Repo 66 69 var createdAt string ··· 80 83 &spindle, 81 84 ) 82 85 if err != nil { 83 83 - return nil, fmt.Errorf("failed to execute repo query: %w ", err) 86 86 + return nil, err 84 87 } 85 88 89 89 + // parse created timestamp 86 90 if t, err := time.Parse(time.RFC3339, createdAt); err == nil { 87 91 repo.Created = t 88 92 } 93 93 + 94 94 + // handle nullable fields 89 95 if description.Valid { 90 96 repo.Description = description.String 91 97 } ··· 107 113 } 108 114 109 115 if err = rows.Err(); err != nil { 110 110 - return nil, fmt.Errorf("failed to execute repo query: %w ", err) 116 116 + return nil, err 111 117 } 112 118 119 119 + // if no repos, return early 120 120 + if len(repoMap) == 0 { 121 121 + return nil, nil 122 122 + } 123 123 + 124 124 + // build IN clause for related queries 113 125 inClause := strings.TrimSuffix(strings.Repeat("?, ", len(repoMap)), ", ") 114 126 args = make([]any, len(repoMap)) 115 115 - 116 127 i := 0 117 128 for _, r := range repoMap { 118 129 args[i] = r.RepoAt() 119 130 i++ 120 131 } 121 132 122 122 - // Get labels for all repos 133 133 + // get labels for all repos 123 134 labelsQuery := fmt.Sprintf( 124 135 `select repo_at, label_at from repo_labels where repo_at in (%s)`, 125 136 inClause, 126 137 ) 138 138 + 127 139 rows, err = e.Query(labelsQuery, args...) 128 140 if err != nil { 129 129 - return nil, fmt.Errorf("failed to execute labels query: %w ", err) 141 141 + return nil, err 130 142 } 131 143 defer rows.Close() 132 144 133 145 for rows.Next() { 134 146 var repoat, labelat string 135 147 if err := rows.Scan(&repoat, &labelat); err != nil { 136 136 - log.Println("err", "err", err) 137 148 continue 138 149 } 139 150 if r, ok := repoMap[syntax.ATURI(repoat)]; ok { 140 151 r.Labels = append(r.Labels, labelat) 141 152 } 142 153 } 143 143 - if err = rows.Err(); err != nil { 144 144 - return nil, fmt.Errorf("failed to execute labels query: %w ", err) 145 145 - } 146 154 147 147 - languageQuery := fmt.Sprintf( 148 148 - ` 155 155 + // get primary language for all repos 156 156 + languageQuery := fmt.Sprintf(` 149 157 select repo_at, language 150 158 from ( 151 159 select 152 152 - repo_at, 153 153 - language, 154 154 - row_number() over ( 155 155 - partition by repo_at 156 156 - order by bytes desc 157 157 - ) as rn 160 160 + repo_at, language, 161 161 + row_number() over ( 162 162 + partition by repo_at 163 163 + order by bytes desc 164 164 + ) as rn 158 165 from repo_languages 159 166 where repo_at in (%s) 160 160 - and is_default_ref = 1 161 161 - and language <> '' 167 167 + and is_default_ref = 1 168 168 + and language <> '' 162 169 ) 163 170 where rn = 1 164 164 - `, 165 165 - inClause, 166 166 - ) 171 171 + `, inClause) 172 172 + 167 173 rows, err = e.Query(languageQuery, args...) 168 174 if err != nil { 169 169 - return nil, fmt.Errorf("failed to execute lang query: %w ", err) 175 175 + return nil, fmt.Errorf("failed to execute lang query: %w", err) 170 176 } 171 177 defer rows.Close() 172 178 ··· 181 187 } 182 188 } 183 189 if err = rows.Err(); err != nil { 184 184 - return nil, fmt.Errorf("failed to execute lang query: %w ", err) 190 190 + return nil, fmt.Errorf("failed to execute lang query: %w", err) 185 191 } 186 192 193 193 + // get star counts 187 194 starCountQuery := fmt.Sprintf( 188 188 - `select 189 189 - subject_at, count(1) 190 190 - from stars 191 191 - where subject_at in (%s) 192 192 - group by subject_at`, 195 195 + `select subject_at, count(1) from stars where subject_at in (%s) group by subject_at`, 193 196 inClause, 194 197 ) 198 198 + 195 199 rows, err = e.Query(starCountQuery, args...) 196 200 if err != nil { 197 197 - return nil, fmt.Errorf("failed to execute star-count query: %w ", err) 201 201 + return nil, fmt.Errorf("failed to execute star-count query: %w", err) 198 202 } 199 203 defer rows.Close() 200 204 ··· 210 214 } 211 215 } 212 216 if err = rows.Err(); err != nil { 213 213 - return nil, fmt.Errorf("failed to execute star-count query: %w ", err) 217 217 + return nil, fmt.Errorf("failed to execute star-count query: %w", err) 214 218 } 215 219 216 216 - issueCountQuery := fmt.Sprintf( 217 217 - `select 220 220 + // get issue counts 221 221 + issueCountQuery := fmt.Sprintf(` 222 222 + select 218 223 repo_at, 219 224 count(case when open = 1 then 1 end) as open_count, 220 225 count(case when open = 0 then 1 end) as closed_count 221 226 from issues 222 227 where repo_at in (%s) 223 223 - group by repo_at`, 224 224 - inClause, 225 225 - ) 228 228 + group by repo_at 229 229 + `, inClause) 230 230 + 226 231 rows, err = e.Query(issueCountQuery, args...) 227 232 if err != nil { 228 228 - return nil, fmt.Errorf("failed to execute issue-count query: %w ", err) 233 233 + return nil, fmt.Errorf("failed to execute issue-count query: %w", err) 229 234 } 230 235 defer rows.Close() 231 236 ··· 242 247 } 243 248 } 244 249 if err = rows.Err(); err != nil { 245 245 - return nil, fmt.Errorf("failed to execute issue-count query: %w ", err) 250 250 + return nil, fmt.Errorf("failed to execute issue-count query: %w", err) 246 251 } 247 252 248 248 - pullCountQuery := fmt.Sprintf( 249 249 - `select 253 253 + // get pull counts 254 254 + pullCountQuery := fmt.Sprintf(` 255 255 + select 250 256 repo_at, 251 257 count(case when state = ? then 1 end) as open_count, 252 258 count(case when state = ? then 1 end) as merged_count, ··· 254 260 count(case when state = ? then 1 end) as deleted_count 255 261 from pulls 256 262 where repo_at in (%s) 257 257 - group by repo_at`, 258 258 - inClause, 259 259 - ) 260 260 - args = append([]any{ 263 263 + group by repo_at 264 264 + `, inClause) 265 265 + 266 266 + pullArgs := append([]any{ 261 267 models.PullOpen, 262 268 models.PullMerged, 263 269 models.PullClosed, 264 270 models.PullDeleted, 265 271 }, args...) 266 266 - rows, err = e.Query( 267 267 - pullCountQuery, 268 268 - args..., 269 269 - ) 272 272 + 273 273 + rows, err = e.Query(pullCountQuery, pullArgs...) 270 274 if err != nil { 271 271 - return nil, fmt.Errorf("failed to execute pulls-count query: %w ", err) 275 275 + return nil, fmt.Errorf("failed to execute pulls-count query: %w", err) 272 276 } 273 277 defer rows.Close() 274 278 ··· 287 291 } 288 292 } 289 293 if err = rows.Err(); err != nil { 290 290 - return nil, fmt.Errorf("failed to execute pulls-count query: %w ", err) 294 294 + return nil, fmt.Errorf("failed to execute pulls-count query: %w", err) 291 295 } 292 296 293 297 var repos []models.Repo ··· 295 299 repos = append(repos, *r) 296 300 } 297 301 302 302 + // sort by created timestamp (desc) 298 303 slices.SortFunc(repos, func(a, b models.Repo) int { 299 304 if a.Created.After(b.Created) { 300 305 return -1 ··· 307 312 308 313 // helper to get exactly one repo 309 314 func GetRepo(e Execer, filters ...orm.Filter) (*models.Repo, error) { 310 310 - repos, err := GetRepos(e, 0, filters...) 315 315 + repos, err := GetReposPaginated(e, pagination.Page{Limit: 1}, filters...) 311 316 if err != nil { 312 317 return nil, err 313 318 } ··· 317 322 } 318 323 319 324 if len(repos) != 1 { 320 320 - return nil, fmt.Errorf("too many rows returned") 325 325 + return nil, fmt.Errorf("too few rows returned") 321 326 } 322 327 323 328 return &repos[0], nil

+2 -2

appview/db/star.go

··· 197 197 return nil, nil 198 198 } 199 199 200 200 - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", args)) 200 200 + repos, err := GetRepos(e, orm.FilterIn("at_uri", args)) 201 201 if err != nil { 202 202 return nil, err 203 203 } ··· 300 300 } 301 301 302 302 // get full repo data 303 303 - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", repoUris)) 303 303 + repos, err := GetRepos(e, orm.FilterIn("at_uri", repoUris)) 304 304 if err != nil { 305 305 return nil, err 306 306 }

+3 -2

appview/db/timeline.go

··· 5 5 6 6 "github.com/bluesky-social/indigo/atproto/syntax" 7 7 "tangled.org/core/appview/models" 8 8 + "tangled.org/core/appview/pagination" 8 9 "tangled.org/core/orm" 9 10 ) 10 11 ··· 90 91 filters = append(filters, orm.FilterIn("did", userIsFollowing)) 91 92 } 92 93 93 93 - repos, err := GetRepos(e, limit, filters...) 94 94 + repos, err := GetReposPaginated(e, pagination.Page{Limit: limit}, filters...) 94 95 if err != nil { 95 96 return nil, err 96 97 } ··· 105 106 106 107 var origRepos []models.Repo 107 108 if args != nil { 108 108 - origRepos, err = GetRepos(e, 0, orm.FilterIn("at_uri", args)) 109 109 + origRepos, err = GetRepos(e, orm.FilterIn("at_uri", args)) 109 110 } 110 111 if err != nil { 111 112 return nil, err

-1

appview/knots/knots.go

··· 112 112 113 113 repos, err := db.GetRepos( 114 114 k.Db, 115 115 - 0, 116 115 orm.FilterEq("knot", domain), 117 116 ) 118 117 if err != nil {

-1

appview/spindles/spindles.go

··· 109 109 110 110 repos, err := db.GetRepos( 111 111 s.Db, 112 112 - 0, 113 112 orm.FilterEq("spindle", instance), 114 113 ) 115 114 if err != nil {

-4

appview/state/knotstream.go

··· 107 107 var errWebhook error 108 108 repos, err := db.GetRepos( 109 109 d, 110 110 - 0, 111 110 orm.FilterEq("did", record.RepoDid), 112 111 orm.FilterEq("name", record.RepoName), 113 112 ) ··· 149 148 150 149 repos, err := db.GetRepos( 151 150 d, 152 152 - 0, 153 151 orm.FilterEq("did", record.RepoDid), 154 152 orm.FilterEq("name", record.RepoName), 155 153 ) ··· 241 239 242 240 repos, err := db.GetRepos( 243 241 d, 244 244 - 0, 245 242 orm.FilterEq("did", record.RepoDid), 246 243 orm.FilterEq("name", record.RepoName), 247 244 ) ··· 307 304 // does this repo have a spindle configured? 308 305 repos, err := db.GetRepos( 309 306 d, 310 310 - 0, 311 307 orm.FilterEq("did", record.TriggerMetadata.Repo.Did), 312 308 orm.FilterEq("name", record.TriggerMetadata.Repo.Repo), 313 309 )

+1 -3

appview/state/profile.go

··· 139 139 140 140 repos, err := db.GetRepos( 141 141 s.db, 142 142 - 0, 143 142 orm.FilterEq("did", profile.UserDid), 144 143 ) 145 144 if err != nil { ··· 231 230 232 231 repos, err := db.GetRepos( 233 232 s.db, 234 234 - 0, 235 233 orm.FilterEq("did", profile.UserDid), 236 234 ) 237 235 if err != nil { ··· 749 747 profile = &models.Profile{Did: user.Active.Did} 750 748 } 751 749 752 752 - repos, err := db.GetRepos(s.db, 0, orm.FilterEq("did", user.Active.Did)) 750 750 + repos, err := db.GetRepos(s.db, orm.FilterEq("did", user.Active.Did)) 753 751 if err != nil { 754 752 log.Printf("getting repos for %s: %s", user.Active.Did, err) 755 753 }