Write on the margins of the internet. Powered by the AT Protocol. margin.at
extension web atproto comments

Implement full PDS record synchronization for annotations, highlights, bookmarks, and collections, including upserting and deleting stale records.

+1374 -84
+3
backend/cmd/server/main.go
··· 41 41 } 42 42 43 43 ingester := firehose.NewIngester(database) 44 + firehose.RelayURL = getEnv("BLOCK_RELAY_URL", "wss://bsky.network/xrpc/com.atproto.sync.subscribeRepos") 45 + log.Printf("Firehose URL: %s", firehose.RelayURL) 46 + 44 47 go func() { 45 48 if err := ingester.Start(context.Background()); err != nil { 46 49 log.Printf("Firehose ingester error: %v", err)
+83 -25
backend/internal/api/annotations.go
··· 22 22 } 23 23 24 24 type CreateAnnotationRequest struct { 25 - URL string `json:"url"` 26 - Text string `json:"text"` 27 - Selector interface{} `json:"selector,omitempty"` 28 - Title string `json:"title,omitempty"` 29 - Tags []string `json:"tags,omitempty"` 25 + URL string `json:"url"` 26 + Text string `json:"text"` 27 + Selector json.RawMessage `json:"selector,omitempty"` 28 + Title string `json:"title,omitempty"` 29 + Tags []string `json:"tags,omitempty"` 30 30 } 31 31 32 32 type CreateAnnotationResponse struct { ··· 77 77 } 78 78 79 79 var result *xrpc.CreateRecordOutput 80 + 81 + if existing, err := s.checkDuplicateAnnotation(session.DID, req.URL, req.Text); err == nil && existing != nil { 82 + w.Header().Set("Content-Type", "application/json") 83 + json.NewEncoder(w).Encode(CreateAnnotationResponse{ 84 + URI: existing.URI, 85 + CID: *existing.CID, 86 + }) 87 + return 88 + } 89 + 80 90 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 81 91 var createErr error 82 92 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) ··· 237 247 return fmt.Errorf("failed to fetch existing record: %w", getErr) 238 248 } 239 249 240 - var record map[string]interface{} 250 + var record xrpc.AnnotationRecord 241 251 if err := json.Unmarshal(existing.Value, &record); err != nil { 242 252 return fmt.Errorf("failed to parse existing record: %w", err) 243 253 } 244 254 245 - record["text"] = req.Text 246 - if req.Tags != nil { 247 - record["tags"] = req.Tags 255 + record.Body = &xrpc.AnnotationBody{ 256 + Value: req.Text, 257 + Format: "text/plain", 258 + } 259 + if len(req.Tags) > 0 { 260 + record.Tags = req.Tags 248 261 } else { 249 - delete(record, "tags") 262 + record.Tags = nil 263 + } 264 + 265 + if err := record.Validate(); err != nil { 266 + return fmt.Errorf("validation failed: %w", err) 250 267 } 251 268 252 269 var updateErr error ··· 309 326 310 327 record := xrpc.NewLikeRecord(req.SubjectURI, req.SubjectCID) 311 328 329 + if err := record.Validate(); err != nil { 330 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 331 + return 332 + } 333 + 312 334 var result *xrpc.CreateRecordOutput 313 335 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 314 336 var createErr error ··· 402 424 } 403 425 404 426 record := xrpc.NewReplyRecord(req.ParentURI, req.ParentCID, req.RootURI, req.RootCID, req.Text) 427 + 428 + if err := record.Validate(); err != nil { 429 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 430 + return 431 + } 405 432 406 433 var result *xrpc.CreateRecordOutput 407 434 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { ··· 509 536 } 510 537 511 538 type CreateHighlightRequest struct { 512 - URL string `json:"url"` 513 - Title string `json:"title,omitempty"` 514 - Selector interface{} `json:"selector"` 515 - Color string `json:"color,omitempty"` 516 - Tags []string `json:"tags,omitempty"` 539 + URL string `json:"url"` 540 + Title string `json:"title,omitempty"` 541 + Selector json.RawMessage `json:"selector"` 542 + Color string `json:"color,omitempty"` 543 + Tags []string `json:"tags,omitempty"` 517 544 } 518 545 519 546 func (s *AnnotationService) CreateHighlight(w http.ResponseWriter, r *http.Request) { ··· 537 564 urlHash := db.HashURL(req.URL) 538 565 record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, req.Color, req.Tags) 539 566 567 + if err := record.Validate(); err != nil { 568 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 569 + return 570 + } 571 + 540 572 var result *xrpc.CreateRecordOutput 573 + 574 + if existing, err := s.checkDuplicateHighlight(session.DID, req.URL, req.Selector); err == nil && existing != nil { 575 + w.Header().Set("Content-Type", "application/json") 576 + json.NewEncoder(w).Encode(map[string]string{"uri": existing.URI, "cid": *existing.CID}) 577 + return 578 + } 579 + 541 580 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 542 581 var createErr error 543 582 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionHighlight, record) ··· 549 588 } 550 589 551 590 var selectorJSONPtr *string 552 - if req.Selector != nil { 553 - selectorBytes, _ := json.Marshal(req.Selector) 554 - selectorStr := string(selectorBytes) 591 + if len(record.Target.Selector) > 0 { 592 + selectorStr := string(record.Target.Selector) 555 593 selectorJSONPtr = &selectorStr 556 594 } 557 595 ··· 622 660 urlHash := db.HashURL(req.URL) 623 661 record := xrpc.NewBookmarkRecord(req.URL, urlHash, req.Title, req.Description) 624 662 663 + if err := record.Validate(); err != nil { 664 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 665 + return 666 + } 667 + 625 668 var result *xrpc.CreateRecordOutput 669 + 670 + if existing, err := s.checkDuplicateBookmark(session.DID, req.URL); err == nil && existing != nil { 671 + w.Header().Set("Content-Type", "application/json") 672 + json.NewEncoder(w).Encode(map[string]string{"uri": existing.URI, "cid": *existing.CID}) 673 + return 674 + } 675 + 626 676 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 627 677 var createErr error 628 678 result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionBookmark, record) ··· 759 809 return fmt.Errorf("failed to fetch record: %w", getErr) 760 810 } 761 811 762 - var record map[string]interface{} 812 + var record xrpc.HighlightRecord 763 813 json.Unmarshal(existing.Value, &record) 764 814 765 815 if req.Color != "" { 766 - record["color"] = req.Color 816 + record.Color = req.Color 767 817 } 768 818 if req.Tags != nil { 769 - record["tags"] = req.Tags 819 + record.Tags = req.Tags 820 + } 821 + 822 + if err := record.Validate(); err != nil { 823 + return fmt.Errorf("validation failed: %w", err) 770 824 } 771 825 772 826 var updateErr error ··· 839 893 return fmt.Errorf("failed to fetch record: %w", getErr) 840 894 } 841 895 842 - var record map[string]interface{} 896 + var record xrpc.BookmarkRecord 843 897 json.Unmarshal(existing.Value, &record) 844 898 845 899 if req.Title != "" { 846 - record["title"] = req.Title 900 + record.Title = req.Title 847 901 } 848 902 if req.Description != "" { 849 - record["description"] = req.Description 903 + record.Description = req.Description 850 904 } 851 905 if req.Tags != nil { 852 - record["tags"] = req.Tags 906 + record.Tags = req.Tags 907 + } 908 + 909 + if err := record.Validate(); err != nil { 910 + return fmt.Errorf("validation failed: %w", err) 853 911 } 854 912 855 913 var updateErr error
+59
backend/internal/api/annotations_helpers.go
··· 1 + package api 2 + 3 + import ( 4 + "encoding/json" 5 + "time" 6 + 7 + "margin.at/internal/db" 8 + ) 9 + 10 + func (s *AnnotationService) checkDuplicateAnnotation(did, url, text string) (*db.Annotation, error) { 11 + recentAnnos, err := s.db.GetAnnotationsByAuthor(did, 5, 0) 12 + if err != nil { 13 + return nil, err 14 + } 15 + for _, a := range recentAnnos { 16 + if a.TargetSource == url && 17 + ((a.BodyValue == nil && text == "") || (a.BodyValue != nil && *a.BodyValue == text)) && 18 + time.Since(a.CreatedAt) < 10*time.Second { 19 + return &a, nil 20 + } 21 + } 22 + return nil, nil 23 + } 24 + 25 + func (s *AnnotationService) checkDuplicateHighlight(did, url string, selector json.RawMessage) (*db.Highlight, error) { 26 + recentHighs, err := s.db.GetHighlightsByAuthor(did, 5, 0) 27 + if err != nil { 28 + return nil, err 29 + } 30 + for _, h := range recentHighs { 31 + matchSelector := false 32 + if h.SelectorJSON == nil && selector == nil { 33 + matchSelector = true 34 + } else if h.SelectorJSON != nil && selector != nil { 35 + selectorBytes, _ := json.Marshal(selector) 36 + if *h.SelectorJSON == string(selectorBytes) { 37 + matchSelector = true 38 + } 39 + } 40 + 41 + if h.TargetSource == url && matchSelector && time.Since(h.CreatedAt) < 10*time.Second { 42 + return &h, nil 43 + } 44 + } 45 + return nil, nil 46 + } 47 + 48 + func (s *AnnotationService) checkDuplicateBookmark(did, url string) (*db.Bookmark, error) { 49 + recentBooks, err := s.db.GetBookmarksByAuthor(did, 5, 0) 50 + if err != nil { 51 + return nil, err 52 + } 53 + for _, b := range recentBooks { 54 + if b.Source == url && time.Since(b.CreatedAt) < 10*time.Second { 55 + return &b, nil 56 + } 57 + } 58 + return nil, nil 59 + }
+114 -31
backend/internal/api/apikey.go
··· 157 157 urlHash := db.HashURL(req.URL) 158 158 record := xrpc.NewBookmarkRecord(req.URL, urlHash, req.Title, req.Description) 159 159 160 + if err := record.Validate(); err != nil { 161 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 162 + return 163 + } 164 + 160 165 var result *xrpc.CreateRecordOutput 161 166 err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 162 167 var createErr error ··· 200 205 }) 201 206 } 202 207 203 - type QuickAnnotationRequest struct { 204 - URL string `json:"url"` 205 - Text string `json:"text"` 208 + type QuickSaveRequest struct { 209 + URL string `json:"url"` 210 + Text string `json:"text,omitempty"` 211 + Selector json.RawMessage `json:"selector,omitempty"` 212 + Color string `json:"color,omitempty"` 206 213 } 207 214 208 - func (h *APIKeyHandler) QuickAnnotation(w http.ResponseWriter, r *http.Request) { 215 + func (h *APIKeyHandler) QuickSave(w http.ResponseWriter, r *http.Request) { 209 216 apiKey, err := h.authenticateAPIKey(r) 210 217 if err != nil { 211 218 http.Error(w, err.Error(), http.StatusUnauthorized) 212 219 return 213 220 } 214 221 215 - var req QuickAnnotationRequest 222 + var req QuickSaveRequest 216 223 if err := json.NewDecoder(r.Body).Decode(&req); err != nil { 217 224 http.Error(w, "Invalid request body", http.StatusBadRequest) 218 225 return 219 226 } 220 227 221 - if req.URL == "" || req.Text == "" { 222 - http.Error(w, "URL and text are required", http.StatusBadRequest) 228 + if req.URL == "" { 229 + http.Error(w, "URL is required", http.StatusBadRequest) 223 230 return 224 231 } 225 232 ··· 230 237 } 231 238 232 239 urlHash := db.HashURL(req.URL) 233 - record := xrpc.NewAnnotationRecord(req.URL, urlHash, req.Text, nil, "") 240 + 241 + var isHighlight bool 242 + if req.Selector != nil && req.Text == "" { 243 + isHighlight = true 244 + } 234 245 235 246 var result *xrpc.CreateRecordOutput 236 - err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 237 - var createErr error 238 - result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) 239 - return createErr 240 - }) 241 - if err != nil { 242 - http.Error(w, "Failed to create annotation: "+err.Error(), http.StatusInternalServerError) 243 - return 244 - } 247 + var createErr error 248 + 249 + if isHighlight { 250 + color := req.Color 251 + if color == "" { 252 + color = "yellow" 253 + } 254 + record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, color, nil) 255 + 256 + if err := record.Validate(); err != nil { 257 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 258 + return 259 + } 260 + 261 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 262 + result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionHighlight, record) 263 + return createErr 264 + }) 265 + if err == nil { 266 + h.db.UpdateAPIKeyLastUsed(apiKey.ID) 267 + selectorJSON, _ := json.Marshal(req.Selector) 268 + selectorStr := string(selectorJSON) 269 + colorPtr := &color 270 + 271 + highlight := &db.Highlight{ 272 + URI: result.URI, 273 + AuthorDID: apiKey.OwnerDID, 274 + TargetSource: req.URL, 275 + TargetHash: urlHash, 276 + SelectorJSON: &selectorStr, 277 + Color: colorPtr, 278 + CreatedAt: time.Now(), 279 + IndexedAt: time.Now(), 280 + CID: &result.CID, 281 + } 282 + go func() { 283 + if err := h.db.CreateHighlight(highlight); err != nil { 284 + fmt.Printf("Warning: failed to index highlight in local DB: %v\n", err) 285 + } 286 + }() 287 + } 288 + 289 + } else { 290 + record := xrpc.NewAnnotationRecord(req.URL, urlHash, req.Text, req.Selector, "") 291 + 292 + if err := record.Validate(); err != nil { 293 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 294 + return 295 + } 245 296 246 - h.db.UpdateAPIKeyLastUsed(apiKey.ID) 297 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 298 + result, createErr = client.CreateRecord(r.Context(), did, xrpc.CollectionAnnotation, record) 299 + return createErr 300 + }) 301 + if err == nil { 302 + h.db.UpdateAPIKeyLastUsed(apiKey.ID) 247 303 248 - bodyValue := req.Text 249 - annotation := &db.Annotation{ 250 - URI: result.URI, 251 - AuthorDID: apiKey.OwnerDID, 252 - Motivation: "commenting", 253 - BodyValue: &bodyValue, 254 - TargetSource: req.URL, 255 - TargetHash: urlHash, 256 - CreatedAt: time.Now(), 257 - IndexedAt: time.Now(), 258 - CID: &result.CID, 304 + var selectorStrPtr *string 305 + if req.Selector != nil { 306 + b, _ := json.Marshal(req.Selector) 307 + s := string(b) 308 + selectorStrPtr = &s 309 + } 310 + 311 + bodyValue := req.Text 312 + var bodyValuePtr *string 313 + if bodyValue != "" { 314 + bodyValuePtr = &bodyValue 315 + } 316 + 317 + annotation := &db.Annotation{ 318 + URI: result.URI, 319 + AuthorDID: apiKey.OwnerDID, 320 + Motivation: "commenting", 321 + BodyValue: bodyValuePtr, 322 + TargetSource: req.URL, 323 + TargetHash: urlHash, 324 + SelectorJSON: selectorStrPtr, 325 + CreatedAt: time.Now(), 326 + IndexedAt: time.Now(), 327 + CID: &result.CID, 328 + } 329 + go func() { 330 + h.db.CreateAnnotation(annotation) 331 + }() 332 + } 259 333 } 260 - h.db.CreateAnnotation(annotation) 334 + 335 + if err != nil { 336 + http.Error(w, "Failed to create record: "+err.Error(), http.StatusInternalServerError) 337 + return 338 + } 261 339 262 340 w.Header().Set("Content-Type", "application/json") 263 341 json.NewEncoder(w).Encode(map[string]string{ 264 342 "uri": result.URI, 265 343 "cid": result.CID, 266 - "message": "Annotation created successfully", 344 + "message": "Saved successfully", 267 345 }) 268 346 } 269 347 ··· 304 382 } 305 383 306 384 record := xrpc.NewHighlightRecord(req.URL, urlHash, req.Selector, color, nil) 385 + 386 + if err := record.Validate(); err != nil { 387 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 388 + return 389 + } 307 390 308 391 var result *xrpc.CreateRecordOutput 309 392 err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error {
+16
backend/internal/api/collections.go
··· 54 54 55 55 record := xrpc.NewCollectionRecord(req.Name, req.Description, req.Icon) 56 56 57 + if err := record.Validate(); err != nil { 58 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 59 + return 60 + } 61 + 57 62 var result *xrpc.CreateRecordOutput 58 63 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 59 64 var createErr error ··· 115 120 } 116 121 117 122 record := xrpc.NewCollectionItemRecord(collectionURI, req.AnnotationURI, req.Position) 123 + 124 + if err := record.Validate(); err != nil { 125 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 126 + return 127 + } 118 128 119 129 var result *xrpc.CreateRecordOutput 120 130 err = s.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { ··· 368 378 } 369 379 370 380 record := xrpc.NewCollectionRecord(req.Name, req.Description, req.Icon) 381 + 382 + if err := record.Validate(); err != nil { 383 + http.Error(w, "Validation error: "+err.Error(), http.StatusBadRequest) 384 + return 385 + } 386 + 371 387 parts := strings.Split(uri, "/") 372 388 rkey := parts[len(parts)-1] 373 389
+221 -12
backend/internal/api/handler.go
··· 8 8 "net/url" 9 9 "strconv" 10 10 "strings" 11 + "sync" 11 12 "time" 12 13 13 14 "github.com/go-chi/chi/v5" 14 15 15 16 "margin.at/internal/db" 17 + "margin.at/internal/xrpc" 16 18 ) 17 19 18 20 type Handler struct { ··· 57 59 r.Get("/collections/{collection}/items", collectionService.GetCollectionItems) 58 60 r.Delete("/collections/items", collectionService.RemoveCollectionItem) 59 61 r.Get("/collections/containing", collectionService.GetAnnotationCollections) 62 + r.Post("/sync", h.SyncAll) 60 63 61 64 r.Get("/targets", h.GetByTarget) 62 65 ··· 77 80 r.Delete("/keys/{id}", h.apiKeys.DeleteKey) 78 81 79 82 r.Post("/quick/bookmark", h.apiKeys.QuickBookmark) 80 - r.Post("/quick/annotation", h.apiKeys.QuickAnnotation) 81 - r.Post("/quick/highlight", h.apiKeys.QuickHighlight) 83 + r.Post("/quick/save", h.apiKeys.QuickSave) 82 84 }) 83 85 } 84 86 ··· 132 134 limit := parseIntParam(r, "limit", 50) 133 135 tag := r.URL.Query().Get("tag") 134 136 creator := r.URL.Query().Get("creator") 137 + 138 + viewerDID := h.getViewerDID(r) 139 + 140 + if viewerDID != "" && (creator == viewerDID || (creator == "" && tag == "")) { 141 + if creator == viewerDID { 142 + h.serveUserFeedFromPDS(w, r, viewerDID, tag, limit) 143 + return 144 + } 145 + } 135 146 136 147 var annotations []db.Annotation 137 148 var highlights []db.Highlight ··· 166 177 } 167 178 } 168 179 169 - viewerDID := h.getViewerDID(r) 170 180 authAnnos, _ := hydrateAnnotations(h.db, annotations, viewerDID) 171 181 authHighs, _ := hydrateHighlights(h.db, highlights, viewerDID) 172 182 authBooks, _ := hydrateBookmarks(h.db, bookmarks, viewerDID) ··· 187 197 feed = append(feed, ci) 188 198 } 189 199 190 - for i := 0; i < len(feed); i++ { 191 - for j := i + 1; j < len(feed); j++ { 192 - t1 := getCreatedAt(feed[i]) 193 - t2 := getCreatedAt(feed[j]) 194 - if t1.Before(t2) { 195 - feed[i], feed[j] = feed[j], feed[i] 200 + sortFeed(feed) 201 + 202 + if len(feed) > limit { 203 + feed = feed[:limit] 204 + } 205 + 206 + w.Header().Set("Content-Type", "application/json") 207 + json.NewEncoder(w).Encode(map[string]interface{}{ 208 + "@context": "http://www.w3.org/ns/anno.jsonld", 209 + "type": "Collection", 210 + "items": feed, 211 + "totalItems": len(feed), 212 + }) 213 + } 214 + 215 + func (h *Handler) serveUserFeedFromPDS(w http.ResponseWriter, r *http.Request, did, tag string, limit int) { 216 + var wg sync.WaitGroup 217 + var rawAnnos, rawHighs, rawBooks []interface{} 218 + var errAnnos, errHighs, errBooks error 219 + 220 + fetchLimit := limit * 2 221 + if fetchLimit < 50 { 222 + fetchLimit = 50 223 + } 224 + 225 + wg.Add(3) 226 + go func() { 227 + defer wg.Done() 228 + rawAnnos, errAnnos = h.FetchLatestUserRecords(r, did, xrpc.CollectionAnnotation, fetchLimit) 229 + }() 230 + go func() { 231 + defer wg.Done() 232 + rawHighs, errHighs = h.FetchLatestUserRecords(r, did, xrpc.CollectionHighlight, fetchLimit) 233 + }() 234 + go func() { 235 + defer wg.Done() 236 + rawBooks, errBooks = h.FetchLatestUserRecords(r, did, xrpc.CollectionBookmark, fetchLimit) 237 + }() 238 + wg.Wait() 239 + 240 + if errAnnos != nil { 241 + log.Printf("PDS Fetch Error (Annos): %v", errAnnos) 242 + } 243 + if errHighs != nil { 244 + log.Printf("PDS Fetch Error (Highs): %v", errHighs) 245 + } 246 + if errBooks != nil { 247 + log.Printf("PDS Fetch Error (Books): %v", errBooks) 248 + } 249 + 250 + var annotations []db.Annotation 251 + var highlights []db.Highlight 252 + var bookmarks []db.Bookmark 253 + 254 + for _, r := range rawAnnos { 255 + if a, ok := r.(*db.Annotation); ok { 256 + if tag == "" || containsTag(a.TagsJSON, tag) { 257 + annotations = append(annotations, *a) 258 + } 259 + } 260 + } 261 + for _, r := range rawHighs { 262 + if h, ok := r.(*db.Highlight); ok { 263 + if tag == "" || containsTag(h.TagsJSON, tag) { 264 + highlights = append(highlights, *h) 265 + } 266 + } 267 + } 268 + for _, r := range rawBooks { 269 + if b, ok := r.(*db.Bookmark); ok { 270 + if tag == "" || containsTag(b.TagsJSON, tag) { 271 + bookmarks = append(bookmarks, *b) 196 272 } 197 273 } 198 274 } 199 275 276 + go func() { 277 + for _, a := range annotations { 278 + h.db.CreateAnnotation(&a) 279 + } 280 + for _, hi := range highlights { 281 + h.db.CreateHighlight(&hi) 282 + } 283 + for _, b := range bookmarks { 284 + h.db.CreateBookmark(&b) 285 + } 286 + }() 287 + 288 + authAnnos, _ := hydrateAnnotations(h.db, annotations, did) 289 + authHighs, _ := hydrateHighlights(h.db, highlights, did) 290 + authBooks, _ := hydrateBookmarks(h.db, bookmarks, did) 291 + 292 + var feed []interface{} 293 + for _, a := range authAnnos { 294 + feed = append(feed, a) 295 + } 296 + for _, h := range authHighs { 297 + feed = append(feed, h) 298 + } 299 + for _, b := range authBooks { 300 + feed = append(feed, b) 301 + } 302 + 303 + sortFeed(feed) 304 + 200 305 if len(feed) > limit { 201 306 feed = feed[:limit] 202 307 } ··· 208 313 "items": feed, 209 314 "totalItems": len(feed), 210 315 }) 316 + 317 + } 318 + 319 + func containsTag(tagsJSON *string, tag string) bool { 320 + if tagsJSON == nil || *tagsJSON == "" { 321 + return false 322 + } 323 + var tags []string 324 + if err := json.Unmarshal([]byte(*tagsJSON), &tags); err != nil { 325 + return false 326 + } 327 + for _, t := range tags { 328 + if t == tag { 329 + return true 330 + } 331 + } 332 + return false 333 + } 334 + 335 + func sortFeed(feed []interface{}) { 336 + for i := 0; i < len(feed); i++ { 337 + for j := i + 1; j < len(feed); j++ { 338 + t1 := getCreatedAt(feed[i]) 339 + t2 := getCreatedAt(feed[j]) 340 + if t1.Before(t2) { 341 + feed[i], feed[j] = feed[j], feed[i] 342 + } 343 + } 344 + } 211 345 } 212 346 213 347 func getCreatedAt(item interface{}) time.Time { ··· 386 520 limit := parseIntParam(r, "limit", 50) 387 521 offset := parseIntParam(r, "offset", 0) 388 522 389 - annotations, err := h.db.GetAnnotationsByAuthor(did, limit, offset) 523 + var annotations []db.Annotation 524 + var err error 525 + 526 + viewerDID := h.getViewerDID(r) 527 + 528 + if offset == 0 && viewerDID != "" && did == viewerDID { 529 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionAnnotation, limit) 530 + if err == nil { 531 + for _, r := range raw { 532 + if a, ok := r.(*db.Annotation); ok { 533 + annotations = append(annotations, *a) 534 + } 535 + } 536 + go func() { 537 + for _, a := range annotations { 538 + h.db.CreateAnnotation(&a) 539 + } 540 + }() 541 + } else { 542 + log.Printf("PDS Fetch Error (User Annos): %v", err) 543 + annotations, err = h.db.GetAnnotationsByAuthor(did, limit, offset) 544 + } 545 + } else { 546 + annotations, err = h.db.GetAnnotationsByAuthor(did, limit, offset) 547 + } 548 + 390 549 if err != nil { 391 550 http.Error(w, err.Error(), http.StatusInternalServerError) 392 551 return ··· 412 571 limit := parseIntParam(r, "limit", 50) 413 572 offset := parseIntParam(r, "offset", 0) 414 573 415 - highlights, err := h.db.GetHighlightsByAuthor(did, limit, offset) 574 + var highlights []db.Highlight 575 + var err error 576 + 577 + viewerDID := h.getViewerDID(r) 578 + 579 + if offset == 0 && viewerDID != "" && did == viewerDID { 580 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionHighlight, limit) 581 + if err == nil { 582 + for _, r := range raw { 583 + if hi, ok := r.(*db.Highlight); ok { 584 + highlights = append(highlights, *hi) 585 + } 586 + } 587 + go func() { 588 + for _, hi := range highlights { 589 + h.db.CreateHighlight(&hi) 590 + } 591 + }() 592 + } else { 593 + log.Printf("PDS Fetch Error (User Highs): %v", err) 594 + highlights, err = h.db.GetHighlightsByAuthor(did, limit, offset) 595 + } 596 + } else { 597 + highlights, err = h.db.GetHighlightsByAuthor(did, limit, offset) 598 + } 599 + 416 600 if err != nil { 417 601 http.Error(w, err.Error(), http.StatusInternalServerError) 418 602 return ··· 438 622 limit := parseIntParam(r, "limit", 50) 439 623 offset := parseIntParam(r, "offset", 0) 440 624 441 - bookmarks, err := h.db.GetBookmarksByAuthor(did, limit, offset) 625 + var bookmarks []db.Bookmark 626 + var err error 627 + 628 + viewerDID := h.getViewerDID(r) 629 + 630 + if offset == 0 && viewerDID != "" && did == viewerDID { 631 + raw, err := h.FetchLatestUserRecords(r, did, xrpc.CollectionBookmark, limit) 632 + if err == nil { 633 + for _, r := range raw { 634 + if b, ok := r.(*db.Bookmark); ok { 635 + bookmarks = append(bookmarks, *b) 636 + } 637 + } 638 + go func() { 639 + for _, b := range bookmarks { 640 + h.db.CreateBookmark(&b) 641 + } 642 + }() 643 + } else { 644 + log.Printf("PDS Fetch Error (User Books): %v", err) 645 + bookmarks, err = h.db.GetBookmarksByAuthor(did, limit, offset) 646 + } 647 + } else { 648 + bookmarks, err = h.db.GetBookmarksByAuthor(did, limit, offset) 649 + } 650 + 442 651 if err != nil { 443 652 http.Error(w, err.Error(), http.StatusInternalServerError) 444 653 return
+226
backend/internal/api/pds.go
··· 1 + package api 2 + 3 + import ( 4 + "encoding/json" 5 + "fmt" 6 + "net/http" 7 + "time" 8 + 9 + "margin.at/internal/db" 10 + "margin.at/internal/xrpc" 11 + ) 12 + 13 + func (h *Handler) FetchLatestUserRecords(r *http.Request, did string, collection string, limit int) ([]interface{}, error) { 14 + session, err := h.refresher.GetSessionWithAutoRefresh(r) 15 + if err != nil { 16 + return nil, err 17 + } 18 + 19 + var results []interface{} 20 + 21 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, _ string) error { 22 + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s&limit=%d", client.PDS, did, collection, limit) 23 + 24 + req, _ := http.NewRequestWithContext(r.Context(), "GET", url, nil) 25 + req.Header.Set("Authorization", "Bearer "+client.AccessToken) 26 + 27 + resp, err := http.DefaultClient.Do(req) 28 + if err != nil { 29 + return fmt.Errorf("failed to fetch %s: %w", collection, err) 30 + } 31 + defer resp.Body.Close() 32 + 33 + if resp.StatusCode != 200 { 34 + return fmt.Errorf("XRPC error %d", resp.StatusCode) 35 + } 36 + 37 + var output struct { 38 + Records []struct { 39 + URI string `json:"uri"` 40 + CID string `json:"cid"` 41 + Value json.RawMessage `json:"value"` 42 + } `json:"records"` 43 + Cursor string `json:"cursor"` 44 + } 45 + 46 + if err := json.NewDecoder(resp.Body).Decode(&output); err != nil { 47 + return err 48 + } 49 + 50 + for _, rec := range output.Records { 51 + parsed, err := parseRecord(did, collection, rec.URI, rec.CID, rec.Value) 52 + if err == nil && parsed != nil { 53 + results = append(results, parsed) 54 + } 55 + } 56 + return nil 57 + }) 58 + 59 + if err != nil { 60 + return nil, err 61 + } 62 + 63 + return results, nil 64 + } 65 + 66 + func parseRecord(did, collection, uri, cid string, value json.RawMessage) (interface{}, error) { 67 + cidPtr := &cid 68 + 69 + switch collection { 70 + case xrpc.CollectionAnnotation: 71 + var record xrpc.AnnotationRecord 72 + if err := json.Unmarshal(value, &record); err != nil { 73 + return nil, err 74 + } 75 + 76 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 77 + 78 + targetSource := record.Target.Source 79 + 80 + targetHash := record.Target.SourceHash 81 + if targetHash == "" && targetSource != "" { 82 + targetHash = db.HashURL(targetSource) 83 + } 84 + 85 + motivation := record.Motivation 86 + if motivation == "" { 87 + motivation = "commenting" 88 + } 89 + 90 + var bodyValuePtr, bodyFormatPtr, bodyURIPtr *string 91 + if record.Body != nil { 92 + if record.Body.Value != "" { 93 + val := record.Body.Value 94 + bodyValuePtr = &val 95 + } 96 + if record.Body.Format != "" { 97 + fmt := record.Body.Format 98 + bodyFormatPtr = &fmt 99 + } 100 + } 101 + 102 + var targetTitlePtr, selectorJSONPtr, tagsJSONPtr *string 103 + if record.Target.Title != "" { 104 + t := record.Target.Title 105 + targetTitlePtr = &t 106 + } 107 + if len(record.Target.Selector) > 0 { 108 + selectorStr := string(record.Target.Selector) 109 + selectorJSONPtr = &selectorStr 110 + } 111 + if len(record.Tags) > 0 { 112 + tagsBytes, _ := json.Marshal(record.Tags) 113 + tagsStr := string(tagsBytes) 114 + tagsJSONPtr = &tagsStr 115 + } 116 + 117 + return &db.Annotation{ 118 + URI: uri, 119 + AuthorDID: did, 120 + Motivation: motivation, 121 + BodyValue: bodyValuePtr, 122 + BodyFormat: bodyFormatPtr, 123 + BodyURI: bodyURIPtr, 124 + TargetSource: targetSource, 125 + TargetHash: targetHash, 126 + TargetTitle: targetTitlePtr, 127 + SelectorJSON: selectorJSONPtr, 128 + TagsJSON: tagsJSONPtr, 129 + CreatedAt: createdAt, 130 + IndexedAt: time.Now(), 131 + CID: cidPtr, 132 + }, nil 133 + 134 + case xrpc.CollectionHighlight: 135 + var record xrpc.HighlightRecord 136 + if err := json.Unmarshal(value, &record); err != nil { 137 + return nil, err 138 + } 139 + 140 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 141 + if createdAt.IsZero() { 142 + createdAt = time.Now() 143 + } 144 + 145 + targetHash := record.Target.SourceHash 146 + if targetHash == "" && record.Target.Source != "" { 147 + targetHash = db.HashURL(record.Target.Source) 148 + } 149 + 150 + var titlePtr, selectorJSONPtr, colorPtr, tagsJSONPtr *string 151 + if record.Target.Title != "" { 152 + t := record.Target.Title 153 + titlePtr = &t 154 + } 155 + if len(record.Target.Selector) > 0 { 156 + selectorStr := string(record.Target.Selector) 157 + selectorJSONPtr = &selectorStr 158 + } 159 + if record.Color != "" { 160 + c := record.Color 161 + colorPtr = &c 162 + } 163 + if len(record.Tags) > 0 { 164 + tagsBytes, _ := json.Marshal(record.Tags) 165 + tagsStr := string(tagsBytes) 166 + tagsJSONPtr = &tagsStr 167 + } 168 + 169 + return &db.Highlight{ 170 + URI: uri, 171 + AuthorDID: did, 172 + TargetSource: record.Target.Source, 173 + TargetHash: targetHash, 174 + TargetTitle: titlePtr, 175 + SelectorJSON: selectorJSONPtr, 176 + Color: colorPtr, 177 + TagsJSON: tagsJSONPtr, 178 + CreatedAt: createdAt, 179 + IndexedAt: time.Now(), 180 + CID: cidPtr, 181 + }, nil 182 + 183 + case xrpc.CollectionBookmark: 184 + var record xrpc.BookmarkRecord 185 + if err := json.Unmarshal(value, &record); err != nil { 186 + return nil, err 187 + } 188 + 189 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 190 + 191 + sourceHash := record.SourceHash 192 + if sourceHash == "" && record.Source != "" { 193 + sourceHash = db.HashURL(record.Source) 194 + } 195 + 196 + var titlePtr, descPtr, tagsJSONPtr *string 197 + if record.Title != "" { 198 + t := record.Title 199 + titlePtr = &t 200 + } 201 + if record.Description != "" { 202 + d := record.Description 203 + descPtr = &d 204 + } 205 + if len(record.Tags) > 0 { 206 + tagsBytes, _ := json.Marshal(record.Tags) 207 + tagsStr := string(tagsBytes) 208 + tagsJSONPtr = &tagsStr 209 + } 210 + 211 + return &db.Bookmark{ 212 + URI: uri, 213 + AuthorDID: did, 214 + Source: record.Source, 215 + SourceHash: sourceHash, 216 + Title: titlePtr, 217 + Description: descPtr, 218 + TagsJSON: tagsJSONPtr, 219 + CreatedAt: createdAt, 220 + IndexedAt: time.Now(), 221 + CID: cidPtr, 222 + }, nil 223 + } 224 + 225 + return nil, nil 226 + }
+351
backend/internal/api/sync.go
··· 1 + package api 2 + 3 + import ( 4 + "encoding/json" 5 + "fmt" 6 + "io" 7 + "net/http" 8 + "time" 9 + 10 + "margin.at/internal/db" 11 + "margin.at/internal/xrpc" 12 + ) 13 + 14 + func (h *Handler) SyncAll(w http.ResponseWriter, r *http.Request) { 15 + session, err := h.refresher.GetSessionWithAutoRefresh(r) 16 + if err != nil { 17 + http.Error(w, err.Error(), http.StatusUnauthorized) 18 + return 19 + } 20 + 21 + collections := []string{ 22 + xrpc.CollectionAnnotation, 23 + xrpc.CollectionHighlight, 24 + xrpc.CollectionBookmark, 25 + xrpc.CollectionReply, 26 + xrpc.CollectionLike, 27 + xrpc.CollectionCollection, 28 + xrpc.CollectionCollectionItem, 29 + } 30 + 31 + results := make(map[string]string) 32 + 33 + err = h.refresher.ExecuteWithAutoRefresh(r, session, func(client *xrpc.Client, did string) error { 34 + for _, collectionNSID := range collections { 35 + count := 0 36 + cursor := "" 37 + fetchedURIs := make(map[string]bool) 38 + 39 + for { 40 + url := fmt.Sprintf("%s/xrpc/com.atproto.repo.listRecords?repo=%s&collection=%s&limit=100", client.PDS, did, collectionNSID) 41 + if cursor != "" { 42 + url += "&cursor=" + cursor 43 + } 44 + 45 + req, _ := http.NewRequestWithContext(r.Context(), "GET", url, nil) 46 + req.Header.Set("Authorization", "Bearer "+client.AccessToken) 47 + 48 + resp, err := http.DefaultClient.Do(req) 49 + if err != nil { 50 + return fmt.Errorf("failed to fetch %s: %w", collectionNSID, err) 51 + } 52 + defer resp.Body.Close() 53 + 54 + if resp.StatusCode != 200 { 55 + body, _ := io.ReadAll(resp.Body) 56 + results[collectionNSID] = fmt.Sprintf("error: %s", string(body)) 57 + break 58 + } 59 + 60 + var output struct { 61 + Records []struct { 62 + URI string `json:"uri"` 63 + CID string `json:"cid"` 64 + Value json.RawMessage `json:"value"` 65 + } `json:"records"` 66 + Cursor string `json:"cursor"` 67 + } 68 + 69 + if err := json.NewDecoder(resp.Body).Decode(&output); err != nil { 70 + return err 71 + } 72 + 73 + for _, rec := range output.Records { 74 + err := h.upsertRecord(did, collectionNSID, rec.URI, rec.CID, rec.Value) 75 + if err != nil { 76 + fmt.Printf("Error upserting %s: %v\n", rec.URI, err) 77 + } else { 78 + count++ 79 + fetchedURIs[rec.URI] = true 80 + } 81 + } 82 + 83 + if output.Cursor == "" { 84 + break 85 + } 86 + cursor = output.Cursor 87 + } 88 + 89 + deletedCount := 0 90 + if results[collectionNSID] == "" { 91 + var localURIs []string 92 + var err error 93 + 94 + switch collectionNSID { 95 + case xrpc.CollectionAnnotation: 96 + localURIs, err = h.db.GetAnnotationURIs(did) 97 + case xrpc.CollectionHighlight: 98 + localURIs, err = h.db.GetHighlightURIs(did) 99 + case xrpc.CollectionBookmark: 100 + localURIs, err = h.db.GetBookmarkURIs(did) 101 + } 102 + 103 + if err == nil { 104 + for _, uri := range localURIs { 105 + if !fetchedURIs[uri] { 106 + switch collectionNSID { 107 + case xrpc.CollectionAnnotation: 108 + _ = h.db.DeleteAnnotation(uri) 109 + case xrpc.CollectionHighlight: 110 + _ = h.db.DeleteHighlight(uri) 111 + case xrpc.CollectionBookmark: 112 + _ = h.db.DeleteBookmark(uri) 113 + } 114 + deletedCount++ 115 + } 116 + } 117 + } 118 + } 119 + 120 + if results[collectionNSID] == "" { 121 + results[collectionNSID] = fmt.Sprintf("synced %d records, deleted %d stale", count, deletedCount) 122 + } 123 + } 124 + return nil 125 + }) 126 + 127 + if err != nil { 128 + http.Error(w, "Sync failed: "+err.Error(), http.StatusInternalServerError) 129 + return 130 + } 131 + 132 + w.WriteHeader(http.StatusOK) 133 + json.NewEncoder(w).Encode(results) 134 + } 135 + 136 + func strPtr(s string) *string { 137 + if s == "" { 138 + return nil 139 + } 140 + return &s 141 + } 142 + 143 + func (h *Handler) upsertRecord(did, collection, uri, cid string, value json.RawMessage) error { 144 + cidPtr := strPtr(cid) 145 + switch collection { 146 + case xrpc.CollectionAnnotation: 147 + var record xrpc.AnnotationRecord 148 + if err := json.Unmarshal(value, &record); err != nil { 149 + return err 150 + } 151 + 152 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 153 + 154 + targetSource := record.Target.Source 155 + if targetSource == "" { 156 + 157 + } 158 + 159 + targetHash := record.Target.SourceHash 160 + if targetHash == "" && targetSource != "" { 161 + targetHash = db.HashURL(targetSource) 162 + } 163 + 164 + motivation := record.Motivation 165 + if motivation == "" { 166 + motivation = "commenting" 167 + } 168 + 169 + var bodyValuePtr, bodyFormatPtr, bodyURIPtr, targetTitlePtr, selectorJSONPtr, tagsJSONPtr *string 170 + if record.Body != nil { 171 + if record.Body.Value != "" { 172 + val := record.Body.Value 173 + bodyValuePtr = &val 174 + } 175 + if record.Body.Format != "" { 176 + fmt := record.Body.Format 177 + bodyFormatPtr = &fmt 178 + } 179 + } 180 + if record.Target.Title != "" { 181 + t := record.Target.Title 182 + targetTitlePtr = &t 183 + } 184 + if len(record.Target.Selector) > 0 { 185 + selectorStr := string(record.Target.Selector) 186 + selectorJSONPtr = &selectorStr 187 + } 188 + if len(record.Tags) > 0 { 189 + tagsBytes, _ := json.Marshal(record.Tags) 190 + tagsStr := string(tagsBytes) 191 + tagsJSONPtr = &tagsStr 192 + } 193 + 194 + return h.db.CreateAnnotation(&db.Annotation{ 195 + URI: uri, 196 + AuthorDID: did, 197 + Motivation: motivation, 198 + BodyValue: bodyValuePtr, 199 + BodyFormat: bodyFormatPtr, 200 + BodyURI: bodyURIPtr, 201 + TargetSource: targetSource, 202 + TargetHash: targetHash, 203 + TargetTitle: targetTitlePtr, 204 + SelectorJSON: selectorJSONPtr, 205 + TagsJSON: tagsJSONPtr, 206 + CreatedAt: createdAt, 207 + IndexedAt: time.Now(), 208 + CID: cidPtr, 209 + }) 210 + 211 + case xrpc.CollectionHighlight: 212 + var record xrpc.HighlightRecord 213 + if err := json.Unmarshal(value, &record); err != nil { 214 + return err 215 + } 216 + 217 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 218 + if createdAt.IsZero() { 219 + createdAt = time.Now() 220 + } 221 + 222 + targetHash := record.Target.SourceHash 223 + if targetHash == "" && record.Target.Source != "" { 224 + targetHash = db.HashURL(record.Target.Source) 225 + } 226 + 227 + var titlePtr, selectorJSONPtr, colorPtr, tagsJSONPtr *string 228 + if record.Target.Title != "" { 229 + t := record.Target.Title 230 + titlePtr = &t 231 + } 232 + if len(record.Target.Selector) > 0 { 233 + selectorStr := string(record.Target.Selector) 234 + selectorJSONPtr = &selectorStr 235 + } 236 + if record.Color != "" { 237 + c := record.Color 238 + colorPtr = &c 239 + } 240 + if len(record.Tags) > 0 { 241 + tagsBytes, _ := json.Marshal(record.Tags) 242 + tagsStr := string(tagsBytes) 243 + tagsJSONPtr = &tagsStr 244 + } 245 + 246 + return h.db.CreateHighlight(&db.Highlight{ 247 + URI: uri, 248 + AuthorDID: did, 249 + TargetSource: record.Target.Source, 250 + TargetHash: targetHash, 251 + TargetTitle: titlePtr, 252 + SelectorJSON: selectorJSONPtr, 253 + Color: colorPtr, 254 + TagsJSON: tagsJSONPtr, 255 + CreatedAt: createdAt, 256 + IndexedAt: time.Now(), 257 + CID: cidPtr, 258 + }) 259 + 260 + case xrpc.CollectionBookmark: 261 + var record xrpc.BookmarkRecord 262 + if err := json.Unmarshal(value, &record); err != nil { 263 + return err 264 + } 265 + 266 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 267 + 268 + sourceHash := record.SourceHash 269 + if sourceHash == "" && record.Source != "" { 270 + sourceHash = db.HashURL(record.Source) 271 + } 272 + 273 + var titlePtr, descPtr, tagsJSONPtr *string 274 + if record.Title != "" { 275 + t := record.Title 276 + titlePtr = &t 277 + } 278 + if record.Description != "" { 279 + d := record.Description 280 + descPtr = &d 281 + } 282 + if len(record.Tags) > 0 { 283 + tagsBytes, _ := json.Marshal(record.Tags) 284 + tagsStr := string(tagsBytes) 285 + tagsJSONPtr = &tagsStr 286 + } 287 + 288 + return h.db.CreateBookmark(&db.Bookmark{ 289 + URI: uri, 290 + AuthorDID: did, 291 + Source: record.Source, 292 + SourceHash: sourceHash, 293 + Title: titlePtr, 294 + Description: descPtr, 295 + TagsJSON: tagsJSONPtr, 296 + CreatedAt: createdAt, 297 + IndexedAt: time.Now(), 298 + CID: cidPtr, 299 + }) 300 + 301 + case xrpc.CollectionCollection: 302 + var record xrpc.CollectionRecord 303 + if err := json.Unmarshal(value, &record); err != nil { 304 + return err 305 + } 306 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 307 + 308 + var descPtr, iconPtr *string 309 + if record.Description != "" { 310 + d := record.Description 311 + descPtr = &d 312 + } 313 + if record.Icon != "" { 314 + i := record.Icon 315 + iconPtr = &i 316 + } 317 + 318 + return h.db.CreateCollection(&db.Collection{ 319 + URI: uri, 320 + AuthorDID: did, 321 + Name: record.Name, 322 + Description: descPtr, 323 + Icon: iconPtr, 324 + CreatedAt: createdAt, 325 + IndexedAt: time.Now(), 326 + }) 327 + 328 + case xrpc.CollectionCollectionItem: 329 + var record xrpc.CollectionItemRecord 330 + if err := json.Unmarshal(value, &record); err != nil { 331 + return err 332 + } 333 + createdAt, _ := time.Parse(time.RFC3339, record.CreatedAt) 334 + 335 + return h.db.AddToCollection(&db.CollectionItem{ 336 + URI: uri, 337 + AuthorDID: did, 338 + CollectionURI: record.Collection, 339 + AnnotationURI: record.Annotation, 340 + Position: record.Position, 341 + CreatedAt: createdAt, 342 + IndexedAt: time.Now(), 343 + }) 344 + 345 + case xrpc.CollectionReply: 346 + return nil 347 + case xrpc.CollectionLike: 348 + return nil 349 + } 350 + return nil 351 + }
+20
backend/internal/db/queries_annotations.go
··· 170 170 171 171 return scanAnnotations(rows) 172 172 } 173 + 174 + func (db *DB) GetAnnotationURIs(authorDID string) ([]string, error) { 175 + rows, err := db.Query(db.Rebind(` 176 + SELECT uri FROM annotations WHERE author_did = ? 177 + `), authorDID) 178 + if err != nil { 179 + return nil, err 180 + } 181 + defer rows.Close() 182 + 183 + var uris []string 184 + for rows.Next() { 185 + var uri string 186 + if err := rows.Scan(&uri); err != nil { 187 + return nil, err 188 + } 189 + uris = append(uris, uri) 190 + } 191 + return uris, nil 192 + }
+20
backend/internal/db/queries_bookmarks.go
··· 174 174 } 175 175 return bookmarks, nil 176 176 } 177 + 178 + func (db *DB) GetBookmarkURIs(authorDID string) ([]string, error) { 179 + rows, err := db.Query(db.Rebind(` 180 + SELECT uri FROM bookmarks WHERE author_did = ? 181 + `), authorDID) 182 + if err != nil { 183 + return nil, err 184 + } 185 + defer rows.Close() 186 + 187 + var uris []string 188 + for rows.Next() { 189 + var uri string 190 + if err := rows.Scan(&uri); err != nil { 191 + return nil, err 192 + } 193 + uris = append(uris, uri) 194 + } 195 + return uris, nil 196 + }
+23
backend/internal/db/queries_collections.go
··· 118 118 return items, nil 119 119 } 120 120 121 + func (db *DB) GetCollectionItemsByAuthor(authorDID string) ([]CollectionItem, error) { 122 + rows, err := db.Query(db.Rebind(` 123 + SELECT uri, author_did, collection_uri, annotation_uri, position, created_at, indexed_at 124 + FROM collection_items 125 + WHERE author_did = ? 126 + ORDER BY created_at DESC 127 + `), authorDID) 128 + if err != nil { 129 + return nil, err 130 + } 131 + defer rows.Close() 132 + 133 + var items []CollectionItem 134 + for rows.Next() { 135 + var item CollectionItem 136 + if err := rows.Scan(&item.URI, &item.AuthorDID, &item.CollectionURI, &item.AnnotationURI, &item.Position, &item.CreatedAt, &item.IndexedAt); err != nil { 137 + return nil, err 138 + } 139 + items = append(items, item) 140 + } 141 + return items, nil 142 + } 143 + 121 144 func (db *DB) GetCollectionURIsForAnnotation(annotationURI string) ([]string, error) { 122 145 rows, err := db.Query(db.Rebind(` 123 146 SELECT collection_uri FROM collection_items WHERE annotation_uri = ?
+20
backend/internal/db/queries_highlights.go
··· 199 199 } 200 200 return highlights, nil 201 201 } 202 + 203 + func (db *DB) GetHighlightURIs(authorDID string) ([]string, error) { 204 + rows, err := db.Query(db.Rebind(` 205 + SELECT uri FROM highlights WHERE author_did = ? 206 + `), authorDID) 207 + if err != nil { 208 + return nil, err 209 + } 210 + defer rows.Close() 211 + 212 + var uris []string 213 + for rows.Next() { 214 + var uri string 215 + if err := rows.Scan(&uri); err != nil { 216 + return nil, err 217 + } 218 + uris = append(uris, uri) 219 + } 220 + return uris, nil 221 + }
+23
backend/internal/db/queries_likes.go
··· 14 14 return err 15 15 } 16 16 17 + func (db *DB) GetLikesByAuthor(authorDID string) ([]Like, error) { 18 + rows, err := db.Query(db.Rebind(` 19 + SELECT uri, author_did, subject_uri, created_at, indexed_at 20 + FROM likes 21 + WHERE author_did = ? 22 + ORDER BY created_at DESC 23 + `), authorDID) 24 + if err != nil { 25 + return nil, err 26 + } 27 + defer rows.Close() 28 + 29 + var likes []Like 30 + for rows.Next() { 31 + var l Like 32 + if err := rows.Scan(&l.URI, &l.AuthorDID, &l.SubjectURI, &l.CreatedAt, &l.IndexedAt); err != nil { 33 + return nil, err 34 + } 35 + likes = append(likes, l) 36 + } 37 + return likes, nil 38 + } 39 + 17 40 func (db *DB) GetLikeCount(subjectURI string) (int, error) { 18 41 var count int 19 42 err := db.QueryRow(db.Rebind(`SELECT COUNT(*) FROM likes WHERE subject_uri = ?`), subjectURI).Scan(&count)
+185 -15
backend/internal/xrpc/records.go
··· 1 1 package xrpc 2 2 3 - import "time" 3 + import ( 4 + "encoding/json" 5 + "fmt" 6 + "time" 7 + "unicode/utf8" 8 + ) 4 9 5 10 const ( 6 11 CollectionAnnotation = "at.margin.annotation" ··· 12 17 CollectionCollectionItem = "at.margin.collectionItem" 13 18 ) 14 19 20 + const ( 21 + SelectorTypeQuote = "TextQuoteSelector" 22 + SelectorTypePosition = "TextPositionSelector" 23 + ) 24 + 25 + type Selector struct { 26 + Type string `json:"type"` 27 + } 28 + 29 + type TextQuoteSelector struct { 30 + Type string `json:"type"` 31 + Exact string `json:"exact"` 32 + Prefix string `json:"prefix,omitempty"` 33 + Suffix string `json:"suffix,omitempty"` 34 + } 35 + 36 + func (s *TextQuoteSelector) Validate() error { 37 + if s.Type != SelectorTypeQuote { 38 + return fmt.Errorf("invalid selector type: %s", s.Type) 39 + } 40 + if len(s.Exact) > 5000 { 41 + return fmt.Errorf("exact text too long: %d > 5000", len(s.Exact)) 42 + } 43 + if len(s.Prefix) > 500 { 44 + return fmt.Errorf("prefix too long: %d > 500", len(s.Prefix)) 45 + } 46 + if len(s.Suffix) > 500 { 47 + return fmt.Errorf("suffix too long: %d > 500", len(s.Suffix)) 48 + } 49 + return nil 50 + } 51 + 52 + type TextPositionSelector struct { 53 + Type string `json:"type"` 54 + Start int `json:"start"` 55 + End int `json:"end"` 56 + } 57 + 58 + func (s *TextPositionSelector) Validate() error { 59 + if s.Type != SelectorTypePosition { 60 + return fmt.Errorf("invalid selector type: %s", s.Type) 61 + } 62 + if s.Start < 0 { 63 + return fmt.Errorf("start position cannot be negative") 64 + } 65 + if s.End < s.Start { 66 + return fmt.Errorf("end position cannot be before start") 67 + } 68 + return nil 69 + } 70 + 15 71 type AnnotationRecord struct { 16 72 Type string `json:"$type"` 17 73 Motivation string `json:"motivation,omitempty"` ··· 27 83 } 28 84 29 85 type AnnotationTarget struct { 30 - Source string `json:"source"` 31 - SourceHash string `json:"sourceHash"` 32 - Title string `json:"title,omitempty"` 33 - Selector interface{} `json:"selector,omitempty"` 86 + Source string `json:"source"` 87 + SourceHash string `json:"sourceHash"` 88 + Title string `json:"title,omitempty"` 89 + Selector json.RawMessage `json:"selector,omitempty"` 34 90 } 35 91 36 - type TextQuoteSelector struct { 37 - Type string `json:"type"` 38 - Exact string `json:"exact"` 39 - Prefix string `json:"prefix,omitempty"` 40 - Suffix string `json:"suffix,omitempty"` 92 + func (r *AnnotationRecord) Validate() error { 93 + if r.Target.Source == "" { 94 + return fmt.Errorf("target source is required") 95 + } 96 + if r.Body != nil { 97 + if len(r.Body.Value) > 10000 { 98 + return fmt.Errorf("body too long: %d > 10000", len(r.Body.Value)) 99 + } 100 + if utf8.RuneCountInString(r.Body.Value) > 3000 { 101 + return fmt.Errorf("body too long (graphemes): %d > 3000", utf8.RuneCountInString(r.Body.Value)) 102 + } 103 + } 104 + if len(r.Tags) > 10 { 105 + return fmt.Errorf("too many tags: %d > 10", len(r.Tags)) 106 + } 107 + for _, tag := range r.Tags { 108 + if len(tag) > 64 { 109 + return fmt.Errorf("tag too long: %s", tag) 110 + } 111 + } 112 + 113 + if len(r.Target.Selector) > 0 { 114 + var typeCheck Selector 115 + if err := json.Unmarshal(r.Target.Selector, &typeCheck); err != nil { 116 + return fmt.Errorf("invalid selector format") 117 + } 118 + 119 + switch typeCheck.Type { 120 + case SelectorTypeQuote: 121 + var s TextQuoteSelector 122 + if err := json.Unmarshal(r.Target.Selector, &s); err != nil { 123 + return err 124 + } 125 + return s.Validate() 126 + case SelectorTypePosition: 127 + var s TextPositionSelector 128 + if err := json.Unmarshal(r.Target.Selector, &s); err != nil { 129 + return err 130 + } 131 + return s.Validate() 132 + } 133 + } 134 + 135 + return nil 41 136 } 42 137 43 138 func NewAnnotationRecord(url, urlHash, text string, selector interface{}, title string) *AnnotationRecord { ··· 45 140 } 46 141 47 142 func NewAnnotationRecordWithMotivation(url, urlHash, text string, selector interface{}, title string, motivation string) *AnnotationRecord { 143 + var selectorJSON json.RawMessage 144 + if selector != nil { 145 + b, _ := json.Marshal(selector) 146 + selectorJSON = b 147 + } 148 + 48 149 record := &AnnotationRecord{ 49 150 Type: CollectionAnnotation, 50 151 Motivation: motivation, ··· 52 153 Source: url, 53 154 SourceHash: urlHash, 54 155 Title: title, 156 + Selector: selectorJSON, 55 157 }, 56 158 CreatedAt: time.Now().UTC().Format(time.RFC3339), 57 159 } ··· 63 165 } 64 166 } 65 167 66 - if selector != nil { 67 - record.Target.Selector = selector 68 - } 69 - 70 168 return record 71 169 } 72 170 ··· 78 176 CreatedAt string `json:"createdAt"` 79 177 } 80 178 179 + func (r *HighlightRecord) Validate() error { 180 + if r.Target.Source == "" { 181 + return fmt.Errorf("target source is required") 182 + } 183 + if len(r.Tags) > 10 { 184 + return fmt.Errorf("too many tags: %d", len(r.Tags)) 185 + } 186 + if len(r.Color) > 20 { 187 + return fmt.Errorf("color too long") 188 + } 189 + return nil 190 + } 191 + 81 192 func NewHighlightRecord(url, urlHash string, selector interface{}, color string, tags []string) *HighlightRecord { 193 + var selectorJSON json.RawMessage 194 + if selector != nil { 195 + b, _ := json.Marshal(selector) 196 + selectorJSON = b 197 + } 198 + 82 199 return &HighlightRecord{ 83 200 Type: CollectionHighlight, 84 201 Target: AnnotationTarget{ 85 202 Source: url, 86 203 SourceHash: urlHash, 87 - Selector: selector, 204 + Selector: selectorJSON, 88 205 }, 89 206 Color: color, 90 207 Tags: tags, ··· 106 223 CreatedAt string `json:"createdAt"` 107 224 } 108 225 226 + func (r *ReplyRecord) Validate() error { 227 + if r.Text == "" { 228 + return fmt.Errorf("text is required") 229 + } 230 + if len(r.Text) > 2000 { 231 + return fmt.Errorf("reply text too long") 232 + } 233 + return nil 234 + } 235 + 109 236 func NewReplyRecord(parentURI, parentCID, rootURI, rootCID, text string) *ReplyRecord { 110 237 return &ReplyRecord{ 111 238 Type: CollectionReply, ··· 128 255 CreatedAt string `json:"createdAt"` 129 256 } 130 257 258 + func (r *LikeRecord) Validate() error { 259 + if r.Subject.URI == "" || r.Subject.CID == "" { 260 + return fmt.Errorf("invalid subject") 261 + } 262 + return nil 263 + } 264 + 131 265 func NewLikeRecord(subjectURI, subjectCID string) *LikeRecord { 132 266 return &LikeRecord{ 133 267 Type: CollectionLike, ··· 146 280 CreatedAt string `json:"createdAt"` 147 281 } 148 282 283 + func (r *BookmarkRecord) Validate() error { 284 + if r.Source == "" { 285 + return fmt.Errorf("source is required") 286 + } 287 + if len(r.Title) > 500 { 288 + return fmt.Errorf("title too long") 289 + } 290 + if len(r.Description) > 1000 { 291 + return fmt.Errorf("description too long") 292 + } 293 + if len(r.Tags) > 10 { 294 + return fmt.Errorf("too many tags") 295 + } 296 + return nil 297 + } 298 + 149 299 func NewBookmarkRecord(url, urlHash, title, description string) *BookmarkRecord { 150 300 return &BookmarkRecord{ 151 301 Type: CollectionBookmark, ··· 165 315 CreatedAt string `json:"createdAt"` 166 316 } 167 317 318 + func (r *CollectionRecord) Validate() error { 319 + if r.Name == "" { 320 + return fmt.Errorf("name is required") 321 + } 322 + if len(r.Name) > 100 { 323 + return fmt.Errorf("name too long") 324 + } 325 + if len(r.Description) > 500 { 326 + return fmt.Errorf("description too long") 327 + } 328 + return nil 329 + } 330 + 168 331 func NewCollectionRecord(name, description, icon string) *CollectionRecord { 169 332 return &CollectionRecord{ 170 333 Type: CollectionCollection, ··· 181 344 Annotation string `json:"annotation"` 182 345 Position int `json:"position,omitempty"` 183 346 CreatedAt string `json:"createdAt"` 347 + } 348 + 349 + func (r *CollectionItemRecord) Validate() error { 350 + if r.Collection == "" || r.Annotation == "" { 351 + return fmt.Errorf("collection and annotation URIs required") 352 + } 353 + return nil 184 354 } 185 355 186 356 func NewCollectionItemRecord(collection, annotation string, position int) *CollectionItemRecord {
+10 -1
web/src/App.jsx
··· 1 1 import { Routes, Route } from "react-router-dom"; 2 - import { AuthProvider } from "./context/AuthContext"; 2 + import { useEffect } from "react"; 3 + import { AuthProvider, useAuth } from "./context/AuthContext"; 3 4 import Sidebar from "./components/Sidebar"; 4 5 import RightSidebar from "./components/RightSidebar"; 5 6 import MobileNav from "./components/MobileNav"; ··· 19 20 import ScrollToTop from "./components/ScrollToTop"; 20 21 21 22 function AppContent() { 23 + const { user } = useAuth(); 24 + 25 + useEffect(() => { 26 + if (user) { 27 + fetch("/api/sync", { method: "POST" }).catch(console.error); 28 + } 29 + }, [user]); 30 + 22 31 return ( 23 32 <div className="layout"> 24 33 <ScrollToTop />