the big feed refactor

parakeet/src/hydration/posts.rs

···
 use lexica::app_bsky::actor::ProfileViewBasic;
 use lexica::app_bsky::embed::Embed;
 use lexica::app_bsky::feed::{
-    BlockedAuthor, FeedViewPost, PostView, PostViewerState, ReplyRef, ReplyRefPost, ThreadgateView,
+    BlockedAuthor, FeedReasonRepost, FeedViewPost, FeedViewPostReason, PostView, PostViewerState,
+    ReplyRef, ReplyRefPost, ThreadgateView,
 };
 use lexica::app_bsky::graph::ListViewBasic;
 use lexica::app_bsky::RecordStats;
···
     }
 }
 
+type HydratePostsRet = (
+    models::Post,
+    ProfileViewBasic,
+    Vec<models::Label>,
+    Option<Embed>,
+    Option<ThreadgateView>,
+    Option<PostViewerState>,
+    Option<PostStats>,
+);
+
 fn build_postview(
-    post: models::Post,
-    author: ProfileViewBasic,
-    labels: Vec<models::Label>,
-    embed: Option<Embed>,
-    threadgate: Option<ThreadgateView>,
-    viewer: Option<PostViewerState>,
-    stats: Option<PostStats>,
+    (post, author, labels, embed, threadgate, viewer, stats): HydratePostsRet,
 ) -> PostView {
     let stats = stats
         .map(|stats| RecordStats {
···
         let threadgate = self.hydrate_threadgate(threadgate).await;
         let labels = self.get_label(&post.at_uri).await;
 
-        Some(build_postview(
+        Some(build_postview((
             post, author, labels, embed, threadgate, viewer, stats,
-        ))
+        )))
     }
 
-    pub async fn hydrate_posts(&self, posts: Vec<String>) -> HashMap<String, PostView> {
+    async fn hydrate_posts_inner(&self, posts: Vec<String>) -> HashMap<String, HydratePostsRet> {
         let stats = self.loaders.post_stats.load_many(posts.clone()).await;
         let posts = self.loaders.posts.load_many(posts).await;
 
···
             .unzip::<_, _, Vec<_>, Vec<_>>();
         let authors = self.hydrate_profiles_basic(authors).await;
 
-        let post_labels = self.get_label_many(&post_uris).await;
-        let viewer_data = self.get_post_viewer_states(&post_uris).await;
+        let mut post_labels = self.get_label_many(&post_uris).await;
+        let mut viewer_data = self.get_post_viewer_states(&post_uris).await;
 
         let threadgates = posts
             .values()
···
             .collect();
         let threadgates = self.hydrate_threadgates(threadgates).await;
 
-        let embeds = self.hydrate_embeds(post_uris).await;
+        let mut embeds = self.hydrate_embeds(post_uris).await;
 
         posts
             .into_iter()
             .filter_map(|(uri, (post, threadgate))| {
-                let author = authors.get(&post.did)?;
-                let embed = embeds.get(&uri).cloned();
+                let author = authors.get(&post.did)?.clone();
+                let embed = embeds.remove(&uri);
                 let threadgate = threadgate.and_then(|tg| threadgates.get(&tg.at_uri).cloned());
-                let labels = post_labels.get(&uri).cloned().unwrap_or_default();
+                let labels = post_labels.remove(&uri).unwrap_or_default();
                 let stats = stats.get(&uri).cloned();
-                let viewer = viewer_data.get(&uri).cloned();
+                let viewer = viewer_data.remove(&uri);
 
                 Some((
                     uri,
-                    build_postview(
-                        post,
-                        author.to_owned(),
-                        labels,
-                        embed,
-                        threadgate,
-                        viewer,
-                        stats,
-                    ),
+                    (post, author, labels, embed, threadgate, viewer, stats),
                 ))
             })
             .collect()
     }
 
+    pub async fn hydrate_posts(&self, posts: Vec<String>) -> HashMap<String, PostView> {
+        self.hydrate_posts_inner(posts)
+            .await
+            .into_iter()
+            .map(|(uri, data)| (uri, build_postview(data)))
+            .collect()
+    }
+
     pub async fn hydrate_feed_posts(
         &self,
-        posts: Vec<String>,
+        posts: Vec<RawFeedItem>,
         author_threads_only: bool,
-    ) -> HashMap<String, FeedViewPost> {
-        let stats = self.loaders.post_stats.load_many(posts.clone()).await;
-        let posts = self.loaders.posts.load_many(posts).await;
-
-        let (authors, post_uris) = posts
-            .values()
-            .map(|(post, _)| (post.did.clone(), post.at_uri.clone()))
-            .unzip::<_, _, Vec<_>, Vec<_>>();
-        let authors = self.hydrate_profiles_basic(authors).await;
-
-        let post_labels = self.get_label_many(&post_uris).await;
-        let viewer_data = self.get_post_viewer_states(&post_uris).await;
-        let embeds = self.hydrate_embeds(post_uris.clone()).await;
+    ) -> Vec<FeedViewPost> {
+        let post_uris = posts
+            .iter()
+            .map(|item| item.post_uri().to_string())
+            .collect::<Vec<_>>();
+        let mut posts_hyd = self.hydrate_posts_inner(post_uris).await;
 
         // we shouldn't show the parent when the post violates a threadgate.
-        let reply_refs = posts
+        let reply_refs = posts_hyd
             .values()
-            .filter(|(post, _)| !post.violates_threadgate)
-            .flat_map(|(post, _)| [post.parent_uri.clone(), post.root_uri.clone()])
+            .filter(|(post, ..)| !post.violates_threadgate)
+            .flat_map(|(post, ..)| [post.parent_uri.clone(), post.root_uri.clone()])
             .flatten()
             .collect::<Vec<_>>();
-
         let reply_posts = self.hydrate_posts(reply_refs).await;
 
-        // hydrate all the posts.
-        let mut posts = posts
+        let repost_profiles = posts
+            .iter()
+            .filter_map(|item| item.repost_by())
+            .collect::<Vec<_>>();
+        let profiles_hydrated = self.hydrate_profiles_basic(repost_profiles).await;
+
+        posts
             .into_iter()
-            .filter_map(|(post_uri, (raw, _))| {
-                let root = raw.root_uri.clone();
-                let parent = raw.parent_uri.clone();
+            .filter_map(|item| {
+                let post = posts_hyd.remove(item.post_uri())?;
+                let context = item.context();
 
-                let author = authors.get(&raw.did)?;
-                let embed = embeds.get(&post_uri).cloned();
-                let labels = post_labels.get(&post_uri).cloned().unwrap_or_default();
-                let stats = stats.get(&post_uri).cloned();
-                let viewer = viewer_data.get(&post_uri).cloned();
-                let post =
-                    build_postview(raw, author.to_owned(), labels, embed, None, viewer, stats);
+                let reply = if let RawFeedItem::Post { .. } = item {
+                    let root_uri = post.0.root_uri.as_ref();
+                    let parent_uri = post.0.parent_uri.as_ref();
 
-                Some((post_uri, (post, root, parent)))
-            })
-            .collect::<HashMap<_, _>>();
+                    let (root, parent) = if author_threads_only {
+                        if root_uri.is_some() && parent_uri.is_some() {
+                            let root = root_uri.and_then(|uri| posts_hyd.get(uri))?;
+                            let parent = parent_uri.and_then(|uri| posts_hyd.get(uri))?;
 
-        post_uris
-            .into_iter()
-            .filter_map(|post_uri| {
-                let item = if author_threads_only {
-                    compile_feed_authors_threads_only(&post_uri, &mut posts)?
+                            let root = build_postview(root.clone());
+                            let parent = build_postview(parent.clone());
+
+                            (Some(root), Some(parent))
+                        } else {
+                            (None, None)
+                        }
+                    } else {
+                        let root = root_uri.and_then(|uri| reply_posts.get(uri)).cloned();
+                        let parent = parent_uri.and_then(|uri| reply_posts.get(uri)).cloned();
+
+                        (root, parent)
+                    };
+
+                    if root_uri.is_some() || parent_uri.is_some() {
+                        Some(ReplyRef {
+                            root: root.map(postview_to_replyref).unwrap_or(
+                                ReplyRefPost::NotFound {
+                                    uri: root_uri.unwrap().to_owned(),
+                                    not_found: true,
+                                },
+                            ),
+                            parent: parent.map(postview_to_replyref).unwrap_or(
+                                ReplyRefPost::NotFound {
+                                    uri: parent_uri.unwrap().to_owned(),
+                                    not_found: true,
+                                },
+                            ),
+                            grandparent_author: None,
+                        })
+                    } else {
+                        None
+                    }
                 } else {
-                    compile_feed(&post_uri, &mut posts, &reply_posts)?
+                    None
+                };
+
+                let reason = match item {
+                    RawFeedItem::Repost { uri, by, at, .. } => {
+                        Some(FeedViewPostReason::Repost(FeedReasonRepost {
+                            by: profiles_hydrated.get(&by).cloned()?,
+                            uri: Some(uri),
+                            cid: None,
+                            indexed_at: at,
+                        }))
+                    }
+                    RawFeedItem::Pin { .. } => Some(FeedViewPostReason::Pin),
+                    _ => None,
                 };
 
-                Some((post_uri, item))
+                let post = build_postview(post);
+
+                Some(FeedViewPost {
+                    post,
+                    reply,
+                    reason,
+                    feed_context: context,
+                })
             })
             .collect()
     }
···
     }
 }
 
-type FeedViewPartData = (PostView, Option<String>, Option<String>);
-
-// this is the 'normal' one that runs in most places
-fn compile_feed(
-    uri: &String,
-    posts: &mut HashMap<String, FeedViewPartData>,
-    reply_posts: &HashMap<String, PostView>,
-) -> Option<FeedViewPost> {
-    let (post, root_uri, parent_uri) = posts.remove(uri)?;
-
-    let root = root_uri.as_ref().and_then(|uri| reply_posts.get(uri));
-    let parent = parent_uri.as_ref().and_then(|uri| reply_posts.get(uri));
-
-    let reply = if parent_uri.is_some() && root_uri.is_some() {
-        Some(ReplyRef {
-            root: root
-                .cloned()
-                .map(postview_to_replyref)
-                .unwrap_or(ReplyRefPost::NotFound {
-                    uri: root_uri.as_ref().unwrap().clone(),
-                    not_found: true,
-                }),
-            parent: parent
-                .cloned()
-                .map(postview_to_replyref)
-                .unwrap_or(ReplyRefPost::NotFound {
-                    uri: parent_uri.as_ref().unwrap().clone(),
-                    not_found: true,
-                }),
-            grandparent_author: None,
-        })
-    } else {
-        None
-    };
-
-    Some(FeedViewPost {
-        post,
-        reply,
-        reason: None,
-        feed_context: None,
-    })
+#[derive(Debug)]
+pub enum RawFeedItem {
+    Pin {
+        uri: String,
+        context: Option<String>,
+    },
+    Post {
+        uri: String,
+        context: Option<String>,
+    },
+    Repost {
+        uri: String,
+        post: String,
+        by: String,
+        at: chrono::DateTime<chrono::Utc>,
+        context: Option<String>,
+    },
 }
 
-// and this one runs in getAuthorFeed when filter=PostsAndAuthorThreads
-fn compile_feed_authors_threads_only(
-    uri: &String,
-    posts: &mut HashMap<String, FeedViewPartData>,
-) -> Option<FeedViewPost> {
-    let (post, root_uri, parent_uri) = posts.get(uri)?.clone();
-
-    let root = root_uri.as_ref().and_then(|root| posts.get(root));
-    let parent = parent_uri.as_ref().and_then(|parent| posts.get(parent));
+impl RawFeedItem {
+    fn post_uri(&self) -> &str {
+        match self {
+            RawFeedItem::Pin { uri, .. } => uri,
+            RawFeedItem::Post { uri, .. } => uri,
+            RawFeedItem::Repost { post, .. } => post,
+        }
+    }
 
-    let reply = if parent_uri.is_some() && root_uri.is_some() {
-        Some(ReplyRef {
-            root: root
-                .cloned()
-                .map(|(post, _, _)| postview_to_replyref(post))?,
-            parent: parent
-                .cloned()
-                .map(|(post, _, _)| postview_to_replyref(post))?,
-            grandparent_author: None,
-        })
-    } else {
-        None
-    };
+    fn repost_by(&self) -> Option<String> {
+        match self {
+            RawFeedItem::Repost { by, .. } => Some(by.clone()),
+            _ => None,
+        }
+    }
 
-    Some(FeedViewPost {
-        post,
-        reply,
-        reason: None,
-        feed_context: None,
-    })
+    fn context(&self) -> Option<String> {
+        match self {
+            RawFeedItem::Pin { context, .. } => context.clone(),
+            RawFeedItem::Post { context, .. } => context.clone(),
+            RawFeedItem::Repost { context, .. } => context.clone(),
+        }
+    }
 }

parakeet/src/xrpc/app_bsky/feed/likes.rs

···
+use crate::hydration::posts::RawFeedItem;
 use crate::hydration::StatefulHydrator;
 use crate::xrpc::error::{Error, XrpcResult};
 use crate::xrpc::extract::{AtpAcceptLabelers, AtpAuth};
···
         .last()
         .map(|(last, _)| last.timestamp_millis().to_string());
 
-    let at_uris = results
+    let raw_feed = results
         .iter()
-        .map(|(_, uri)| uri.clone())
+        .map(|(_, uri)| RawFeedItem::Post {
+            uri: uri.clone(),
+            context: None,
+        })
         .collect::<Vec<_>>();
 
-    let mut posts = hyd.hydrate_feed_posts(at_uris, false).await;
-
-    let feed: Vec<_> = results
-        .into_iter()
-        .filter_map(|(_, uri)| posts.remove(&uri))
-        .collect();
+    let feed = hyd.hydrate_feed_posts(raw_feed, false).await;
 
     Ok(Json(FeedRes { cursor, feed }))
 }

parakeet/src/xrpc/app_bsky/feed/posts.rs

···
+use crate::hydration::posts::RawFeedItem;
 use crate::hydration::StatefulHydrator;
 use crate::xrpc::app_bsky::graph::lists::ListWithCursorQuery;
 use crate::xrpc::error::{Error, XrpcResult};
···
 use diesel_async::{AsyncPgConnection, RunQueryDsl};
 use lexica::app_bsky::actor::ProfileView;
 use lexica::app_bsky::feed::{
-    BlockedAuthor, FeedReasonRepost, FeedSkeletonResponse, FeedViewPost, FeedViewPostReason,
-    PostView, SkeletonReason, ThreadViewPost, ThreadViewPostType, ThreadgateView,
+    BlockedAuthor, FeedSkeletonResponse, FeedViewPost, PostView, SkeletonReason, ThreadViewPost,
+    ThreadViewPostType, ThreadgateView,
 };
 use parakeet_db::{models, schema};
 use reqwest::Url;
···
 
     let hyd = StatefulHydrator::new(&state.dataloaders, &state.cdn, &labelers, maybe_auth);
 
-    let at_uris = skeleton.feed.iter().map(|v| v.post.clone()).collect();
     let repost_skeleton = skeleton
         .feed
         .iter()
···
             _ => None,
         })
         .collect::<Vec<_>>();
+    let mut repost_data = get_skeleton_repost_data(&mut conn, repost_skeleton).await;
 
-    let mut posts = hyd.hydrate_feed_posts(at_uris, false).await;
-    let mut repost_data = get_skeleton_repost_data(&mut conn, &hyd, repost_skeleton).await;
-
-    let feed = skeleton
+    let raw_feed = skeleton
         .feed
         .into_iter()
-        .filter_map(|item| {
-            let mut post = posts.remove(&item.post)?;
-            let reason = match item.reason {
-                Some(SkeletonReason::Repost { repost }) => {
-                    repost_data.remove(&repost).map(FeedViewPostReason::Repost)
-                }
-                Some(SkeletonReason::Pin {}) => Some(FeedViewPostReason::Pin),
-                _ => None,
-            };
-
-            post.reason = reason;
-            post.feed_context = item.feed_context;
-
-            Some(post)
+        .filter_map(|v| match v.reason {
+            Some(SkeletonReason::Repost { repost }) => {
+                repost_data
+                    .remove_entry(&repost)
+                    .map(|(uri, (by, at))| RawFeedItem::Repost {
+                        uri,
+                        post: v.post,
+                        by,
+                        at: at.and_utc(),
+                        context: v.feed_context,
+                    })
+            }
+            Some(SkeletonReason::Pin {}) => Some(RawFeedItem::Pin {
+                uri: v.post,
+                context: v.feed_context,
+            }),
+            None => Some(RawFeedItem::Post {
+                uri: v.post,
+                context: v.feed_context,
+            }),
         })
         .collect();
+
+    let feed = hyd.hydrate_feed_posts(raw_feed, false).await;
 
     Ok(Json(FeedRes {
         cursor: skeleton.cursor,
···
 
     let pin = match query.include_pins && query.cursor.is_none() {
         false => None,
-        true => match crate::db::get_pinned_post_uri(&mut conn, &did).await? {
-            Some(post) => hyd.hydrate_post(post).await,
-            None => None,
-        },
+        true => crate::db::get_pinned_post_uri(&mut conn, &did).await?,
     };
 
     let limit = query.limit.unwrap_or(50).clamp(1, 100);
···
         .last()
         .map(|item| item.sort_at.timestamp_millis().to_string());
 
-    let at_uris = results
-        .iter()
-        .map(|item| item.post.clone())
-        .collect::<Vec<_>>();
-
-    // get the actor for if we have reposted
-    let profile = hyd
-        .hydrate_profile_basic(did)
-        .await
-        .ok_or(Error::server_error(None))?;
-
-    let mut posts = hyd.hydrate_feed_posts(at_uris, author_threads_only).await;
-
-    let mut feed: Vec<_> = results
+    let mut raw_feed = results
         .into_iter()
-        .filter_map(|item| {
-            posts.remove(&item.post).map(|mut fvp| {
-                if item.typ == "repost" {
-                    fvp.reason = Some(FeedViewPostReason::Repost(FeedReasonRepost {
-                        by: profile.clone(),
-                        uri: Some(item.uri),
-                        cid: Some(item.cid),
-                        indexed_at: Default::default(),
-                    }))
-                }
-                fvp
-            })
+        .filter_map(|item| match &*item.typ {
+            "post" => Some(RawFeedItem::Post {
+                uri: item.post,
+                context: None,
+            }),
+            "repost" => Some(RawFeedItem::Repost {
+                uri: item.uri,
+                post: item.post,
+                by: item.did,
+                at: item.sort_at,
+                context: None,
+            }),
+            _ => None,
         })
-        .collect();
+        .collect::<Vec<_>>();
 
     if let Some(post) = pin {
-        feed.insert(
+        raw_feed.insert(
             0,
-            FeedViewPost {
-                post,
-                reply: None,
-                reason: Some(FeedViewPostReason::Pin),
-                feed_context: None,
+            RawFeedItem::Pin {
+                uri: post,
+                context: None,
             },
         );
     }
+
+    let feed = hyd.hydrate_feed_posts(raw_feed, author_threads_only).await;
 
     Ok(Json(FeedRes { cursor, feed }))
 }
···
         .last()
         .map(|(last, _)| last.timestamp_millis().to_string());
 
-    let at_uris = results
+    let raw_feed = results
         .iter()
-        .map(|(_, uri)| uri.clone())
+        .map(|(_, uri)| RawFeedItem::Post {
+            uri: uri.clone(),
+            context: None,
+        })
         .collect::<Vec<_>>();
 
-    let mut posts = hyd.hydrate_feed_posts(at_uris, false).await;
-
-    let feed = results
-        .into_iter()
-        .filter_map(|(_, uri)| posts.remove(&uri))
-        .collect();
+    let feed = hyd.hydrate_feed_posts(raw_feed, false).await;
 
     Ok(Json(FeedRes { cursor, feed }))
 }
···
     }
 }
 
-async fn get_skeleton_repost_data<'a>(
+async fn get_skeleton_repost_data(
     conn: &mut AsyncPgConnection,
-    hyd: &StatefulHydrator<'a>,
     reposts: Vec<String>,
-) -> HashMap<String, FeedReasonRepost> {
+) -> HashMap<String, (String, NaiveDateTime)> {
     let Ok(repost_data) = schema::records::table
         .select((
             schema::records::at_uri,
···
         return HashMap::new();
     };
 
-    let profiles = repost_data.iter().map(|(_, did, _)| did.clone()).collect();
-    let profiles = hyd.hydrate_profiles_basic(profiles).await;
-
     repost_data
         .into_iter()
-        .filter_map(|(uri, did, indexed_at)| {
-            let by = profiles.get(&did).cloned()?;
-
-            let repost = FeedReasonRepost {
-                by,
-                uri: Some(uri.clone()),
-                cid: None, // okay, we do have this, but the app doesn't seem to be bothered about not setting it.
-                indexed_at: indexed_at.and_utc(),
-            };
-
-            Some((uri, repost))
-        })
+        .map(|(uri, did, at)| (uri, (did, at)))
         .collect()
 }