threadgrapher basic concurrency for backlink fetching

cmd/appview/main.go

···
 			// V2V2 still doesnt work. should probably make the handler from scratch to fully use the thread grapher.
 			// also the thread grapher is still sequental. pls fix that
 			//appbskyunspeccedgetpostthreadv2.HandleGetPostThreadV2V2(c, sl, cs, BSKYIMAGECDN_URL)
-			appbskyunspeccedgetpostthreadv2.HandleGetPostThreadV2V3(c, sl, cs, BSKYIMAGECDN_URL)
+
+			var existingGraph *appbskyunspeccedgetpostthreadv2.ThreadGraph
+			// var kvkey string
+			// threadAnchorURIraw := c.Query("anchor")
+			// if threadAnchorURIraw != "" {
+			// 	threadAnchorURI, err := syntax.ParseATURI(threadAnchorURIraw)
+			// 	if err == nil {
+			// 		kvkey = "ThreadGraph" + threadAnchorURI.String()
+			// 		val, ok := kv.Get(kvkey)
+			// 		if ok {
+			// 			parsed, err := appbskyunspeccedgetpostthreadv2.ThreadGraphFromBytes(val)
+			// 			if err != nil {
+			// 				existingGraph = parsed
+			// 			}
+			// 		}
+			// 	}
+			// }
+
+			returnedGraph := appbskyunspeccedgetpostthreadv2.HandleGetPostThreadV2V3(c, sl, cs, BSKYIMAGECDN_URL, existingGraph)
+			_ = returnedGraph
+			// bytes, err := returnedGraph.ToBytes()
+			// if err == nil && kvkey != "" {
+			// 	kv.Set(kvkey, bytes, 1*time.Minute)
+			// }
 		})
 
 	// weird stuff
···
 				}
 			}(clientUUID)
 		}
-		c.String(http.StatusOK, `
-    ____  __________     ____ _       _____    ____  ______   _____ __________ _    ____________ 
-   / __ \/ ____/ __ \   / __ \ |     / /   |  / __ \/ ____/  / ___// ____/ __ \ |  / / ____/ __ \
-  / /_/ / __/ / / / /  / / / / | /| / / /| | / /_/ / /_      \__ \/ __/ / /_/ / | / / __/ / /_/ /
- / _, _/ /___/ /_/ /  / /_/ /| |/ |/ / ___ |/ _, _/ __/     ___/ / /___/ _, _/| |/ / /___/ _, _/ 
-/_/ |_/_____/_____/  /_____/ |__/|__/_/  |_/_/ |_/_/       /____/_____/_/ |_| |___/_____/_/ |_|  
-
+		c.String(http.StatusOK, `    ____  __________     ____ _       _____    ____  ______
+   / __ \/ ____/ __ \   / __ \ |     / /   |  / __ \/ ____/
+  / /_/ / __/ / / / /  / / / / | /| / / /| | / /_/ / /_    
+ / _, _/ /___/ /_/ /  / /_/ /| |/ |/ / ___ |/ _, _/ __/    
+/_/ |_/_____/_____/  /_____/ |__/|__/_/  |_/_/ |_/_/       
+   _____ __________ _    ____________ 
+  / ___// ____/ __ \ |  / / ____/ __ \
+  \__ \/ __/ / /_/ / | / / __/ / /_/ /
+ ___/ / /___/ _, _/| |/ / /___/ _, _/ 
+/____/_____/_/ |_| |___/_____/_/ |_|  
 
 This is an AT Protocol Application View (AppView) for any application that supports app.bsky.* xrpc methods.
 

shims/lex/app/bsky/feed/defs/postview.go

···
 				subj, ok := like[".subject.uri"]
 				if ok {
 					likeCount = int64(subj.Records)
+				} else {
+					likeCount = int64(0)
 				}
+			} else {
+				likeCount = int64(0)
 			}
 		}
 		if links != nil &&
···
 				subj, ok := like[".subject.uri"]
 				if ok {
 					repostCount = int64(subj.Records)
+				} else {
+					repostCount = int64(0)
 				}
+			} else {
+				repostCount = int64(0)
 			}
 		}
 		if links != nil &&
···
 				subj, ok := like[".reply.parent.uri"]
 				if ok {
 					replyCount = int64(subj.Records)
+				} else {
+					replyCount = int64(0)
 				}
+			} else {
+				replyCount = int64(0)
 			}
 		}
 		if links != nil &&
···
 				subj, ok := like[".embed.record.uri"]
 				if ok {
 					quoteCount_noEmbed = int64(subj.Records)
+				} else {
+					quoteCount_noEmbed = int64(0)
 				}
+			} else {
+				quoteCount_noEmbed = int64(0)
 			}
 		}
 		if links != nil &&
···
 				subj, ok := like[".embed.record.record.uri"]
 				if ok {
 					quoteCount_withEmbed = int64(subj.Records)
+				} else {
+					quoteCount_withEmbed = int64(0)
 				}
+			} else {
+				quoteCount_withEmbed = int64(0)
 			}
 		}
 		quoteCount = quoteCount_noEmbed + quoteCount_withEmbed

shims/lex/app/bsky/unspecced/getpostthreadv2/query.go

···
 	depth int
 }
 
-func HandleGetPostThreadV2V3(c *gin.Context, sl *microcosm.MicrocosmClient, cs *microcosm.MicrocosmClient, imgcdn string) {
+func HandleGetPostThreadV2V3(c *gin.Context, sl *microcosm.MicrocosmClient, cs *microcosm.MicrocosmClient, imgcdn string, existingGraph *ThreadGraph) *ThreadGraph {
 	ctx := c.Request.Context()
 
 	rawdid := c.GetString("user_did")
···
 	threadAnchorURIraw := c.Query("anchor")
 	if threadAnchorURIraw == "" {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Missing feed param"})
-		return
+		return existingGraph
 	}
 
 	// "Whether to include parents above the anchor.
···
 
 	threadAnchorURI, err := syntax.ParseATURI(threadAnchorURIraw)
 	if err != nil {
-		return
+		return existingGraph
 	}
 
-	threadGraph, err := ThreadGrapher(ctx, cs, sl, threadAnchorURI)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, gin.H{"error": ("failed to graph the thread: " + err.Error())})
-		return
+	var workingGraph *ThreadGraph
+
+	if existingGraph != nil {
+		workingGraph = existingGraph
+		// update the existing graph to fit our needs in our subtree
+		workingGraph.UpdateGraphTo(threadAnchorURI)
+	} else {
+		newGraph, err := ThreadGrapher(ctx, cs, sl, threadAnchorURI)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": ("failed to graph the thread: " + err.Error())})
+			return nil
+		}
+		workingGraph = newGraph
 	}
+
 	var skeletonposts []SkeletonPost
 
 	// Parent Chain
···
 				// root = current
 				break
 			}
-			parent, ok := threadGraph.ParentsMap[current]
+			parent, ok := workingGraph.ParentsMap[current]
 			if !ok {
 				// root = current
 				break
···
 
 	// Tree Replies (with OP thread priority)
 	// should probably be recursive
-	recursiveHandleV2V3TreeReplies(threadGraph, &skeletonposts, threadAnchorURI, &below, &branchingFactor, &sort, 0, 0)
+	recursiveHandleV2V3TreeReplies(workingGraph, &skeletonposts, threadAnchorURI, &below, &branchingFactor, &sort, 0, 0)
 
 	//maplen := len(parentsMap)
 	concurrentResults := MapConcurrent(
···
 		HasOtherReplies: false,
 	}
 	c.JSON(http.StatusOK, resp)
+
+	return workingGraph
 }
 
 func flipArray[T any](s *[]T) {

shims/lex/app/bsky/unspecced/getpostthreadv2/threadgrapher.go

···
 	}
 
 	processingQueue := append([]syntax.ATURI{rootURI}, allRepliesATURI...)
-	for _, aturi := range processingQueue {
-		//tg.Nodes = append(tg.Nodes, aturi)
-		// graphinger
-		emptystrarray := &[]string{}
-		var localRepliesATURI []syntax.ATURI
-		limit := 100
-		var cursor *string
-		shouldContinue := true
-		for shouldContinue {
-			results, err := constellation.GetBacklinks(ctx, cs, aturi.String(), "app.bsky.feed.post:reply.parent.uri", *emptystrarray, &limit, cursor)
-			if err != nil {
-				log.Println("[ThreadGrapher] [parent graphing] exit by no replies")
-				return nil, fmt.Errorf("failed to get backlinks: %w", err)
-			}
-			if results.Records != nil {
-				for _, record := range results.Records {
-					aturi, err := syntax.ParseATURI("at://" + record.Did + "/" + record.Collection + "/" + record.Rkey)
-					if err == nil {
-						localRepliesATURI = append(localRepliesATURI, aturi)
+	// for _, aturi := range processingQueue {
+	// 	//tg.Nodes = append(tg.Nodes, aturi)
+	// 	// graphinger
+	// 	emptystrarray := &[]string{}
+	// 	var localRepliesATURI []syntax.ATURI
+	// 	limit := 100
+	// 	var cursor *string
+	// 	shouldContinue := true
+	// 	for shouldContinue {
+	// 		results, err := constellation.GetBacklinks(ctx, cs, aturi.String(), "app.bsky.feed.post:reply.parent.uri", *emptystrarray, &limit, cursor)
+	// 		if err != nil {
+	// 			log.Println("[ThreadGrapher] [parent graphing] exit by no replies")
+	// 			return nil, fmt.Errorf("failed to get backlinks: %w", err)
+	// 		}
+	// 		if results.Records != nil {
+	// 			for _, record := range results.Records {
+	// 				aturi, err := syntax.ParseATURI("at://" + record.Did + "/" + record.Collection + "/" + record.Rkey)
+	// 				if err == nil {
+	// 					localRepliesATURI = append(localRepliesATURI, aturi)
+	// 				}
+	// 			}
+	// 		}
+	// 		if results.Cursor != nil {
+	// 			cursor = results.Cursor
+	// 		} else {
+	// 			shouldContinue = false
+	// 		}
+	// 	}
+	// 	for _, reply := range localRepliesATURI {
+	// 		tg.ParentsMap[reply] = aturi
+	// 		tg.ChildrenMap[aturi] = append(tg.ChildrenMap[aturi], reply)
+	// 	}
+	// }
+
+	type concurrentStruct struct {
+		aturi   syntax.ATURI
+		replies []syntax.ATURI
+	}
+
+	localRepliesATURIConcurrent := MapConcurrent(
+		ctx,
+		processingQueue,
+		50,
+		func(ctx context.Context, aturi syntax.ATURI, idx int) (*concurrentStruct, error) {
+			//tg.Nodes = append(tg.Nodes, aturi)
+			// graphinger
+			emptystrarray := &[]string{}
+			var localRepliesATURI []syntax.ATURI
+			limit := 100
+			var cursor *string
+			shouldContinue := true
+			for shouldContinue {
+				results, err := constellation.GetBacklinks(ctx, cs, aturi.String(), "app.bsky.feed.post:reply.parent.uri", *emptystrarray, &limit, cursor)
+				if err != nil {
+					log.Println("[ThreadGrapher] [parent graphing] exit by no replies")
+					return nil, fmt.Errorf("failed to get backlinks: %w", err)
+				}
+				if results.Records != nil {
+					for _, record := range results.Records {
+						aturi, err := syntax.ParseATURI("at://" + record.Did + "/" + record.Collection + "/" + record.Rkey)
+						if err == nil {
+							localRepliesATURI = append(localRepliesATURI, aturi)
+						}
 					}
 				}
+				if results.Cursor != nil {
+					cursor = results.Cursor
+				} else {
+					shouldContinue = false
+				}
 			}
-			if results.Cursor != nil {
-				cursor = results.Cursor
-			} else {
-				shouldContinue = false
-			}
+			return &concurrentStruct{
+				aturi:   aturi,
+				replies: localRepliesATURI,
+			}, nil
+		},
+	)
+
+	localRepliesATURI := make([]*concurrentStruct, 0, len(localRepliesATURIConcurrent))
+	for _, r := range localRepliesATURIConcurrent {
+		if /*r != nil &&*/ r.Err == nil && r.Value != nil /*&& r.Value.aturi != nil*/ && r.Value.replies != nil {
+			localRepliesATURI = append(localRepliesATURI, r.Value)
 		}
-		for _, reply := range localRepliesATURI {
+	}
+	for _, replyStruct := range localRepliesATURI {
+		aturi := replyStruct.aturi
+		for _, reply := range replyStruct.replies {
 			tg.ParentsMap[reply] = aturi
 			tg.ChildrenMap[aturi] = append(tg.ChildrenMap[aturi], reply)
 		}
···
 
 	return tg, nil
 }
+
+// ToBytes serializes the ThreadGraph into a JSON byte slice.
+// It acquires a Read Lock to ensure thread safety during serialization.
+func (g *ThreadGraph) ToBytes() ([]byte, error) {
+	g.mu.RLock()
+	defer g.mu.RUnlock()
+
+	// sync.RWMutex is unexported, so json.Marshal will automatically skip it.
+	// syntax.ATURI implements TextMarshaler, so it works as a map key automatically.
+	return json.Marshal(g)
+}
+
+// ThreadGraphFromBytes deserializes a byte slice back into a ThreadGraph.
+// It ensures maps are initialized even if the JSON data was empty.
+func ThreadGraphFromBytes(data []byte) (*ThreadGraph, error) {
+	// Initialize with NewThreadGraph to ensure maps are allocated
+	tg := NewThreadGraph()
+
+	if err := json.Unmarshal(data, tg); err != nil {
+		return nil, fmt.Errorf("failed to deserialize ThreadGraph: %w", err)
+	}
+
+	// Safety check: specific to Go's JSON unmarshal behavior.
+	// If the JSON contained "null" for the maps, they might be nil again.
+	// We re-initialize them to avoid panics during concurrent writes later.
+	if tg.ParentsMap == nil {
+		tg.ParentsMap = make(map[syntax.ATURI]syntax.ATURI)
+	}
+	if tg.ChildrenMap == nil {
+		tg.ChildrenMap = make(map[syntax.ATURI][]syntax.ATURI)
+	}
+
+	// The Mutex (tg.mu) is zero-valued (unlocked) by default, which is exactly what we want.
+	return tg, nil
+}
+
+func (g *ThreadGraph) UpdateGraphTo(anchor syntax.ATURI) {
+	// path from anchor to root never needs to be updated
+	// all we need is to update all subtrees of the anchor
+	// so we should first do a
+	// recursiveHandleUpdateGraphTo(g, anchor)
+	// i dont think we should do a recursive thing
+	// it cant be optimized well, constellation queries will be sequential
+	// how about, grab the entire tree again, prune branches not part of the tree
+	// you will get a list of posts that are either new or a part of the subtree
+	//
+}