import { Op, QueryTypes } from "sequelize"; import { Ask, Blocks, Emoji, EmojiReaction, FederatedHost, Media, Post, PostEmojiRelations, PostMentionsUserRelation, PostTag, QuestionPoll, QuestionPollAnswer, QuestionPollQuestion, Quotes, sequelize, ServerBlock, User, UserBookmarkedPosts, UserEmojiRelation, UserLikesPostRelations, } from "../models/index.js"; import getPosstGroupDetails from "./getPostGroupDetails.js"; import getFollowedsIds from "./cacheGetters/getFollowedsIds.js"; import { Queue } from "bullmq"; import { completeEnvironment } from "./backendOptions.js"; import { InteractionControl, InteractionControlType, Privacy, } from "../models/post.js"; import { getAllLocalUserIds } from "./cacheGetters/getAllLocalUserIds.js"; import { checkBskyLabelersNSFW } from "./atproto/checkBskyLabelerNSFW.js"; import { isAdult } from "./isAdult.js"; import { logger } from "./logger.js"; const updateMediaDataQueue = new Queue("processRemoteMediaData", { connection: completeEnvironment.bullmqConnection, defaultJobOptions: { removeOnComplete: true, attempts: 3, backoff: { type: "exponential", delay: 1000, }, removeOnFail: true, }, }); async function getQuotes(postIds: string[]): Promise { return await Quotes.findAll({ where: { quoterPostId: { [Op.in]: postIds, }, }, }); } async function getMedias(postIds: string[]) { const medias = await Media.findAll({ attributes: [ "id", "NSFW", "description", "url", "external", "mediaOrder", "mediaType", "postId", "blurhash", "width", "height", ], where: { postId: { [Op.in]: postIds, }, }, }); let mediasToProcess = medias.filter( (elem: any) => !elem.mediaType || (elem.mediaType?.startsWith("image") && !elem.width) ); if (mediasToProcess && mediasToProcess.length > 0) { updateMediaDataQueue.addBulk( mediasToProcess.map((media: any) => { return { name: `getMediaData${media.id}`, data: { mediaId: media.id }, }; }) ); } return medias; } async function getMentionedUserIds( postIds: string[] ): Promise<{ usersMentioned: string[]; postMentionRelation: any[] }> { const mentions = await PostMentionsUserRelation.findAll({ attributes: ["userId", "postId"], where: { postId: { [Op.in]: postIds, }, }, }); const usersMentioned = mentions.map((elem: any) => elem.userId); const postMentionRelation = mentions.map((elem: any) => { return { userMentioned: elem.userId, post: elem.postId }; }); return { usersMentioned, postMentionRelation }; } async function getTags(postIds: string[]) { return await PostTag.findAll({ attributes: ["postId", "tagName"], where: { postId: { [Op.in]: postIds, }, }, }); } async function getLikes(postIds: string[]) { return await UserLikesPostRelations.findAll({ attributes: ["userId", "postId"], where: { postId: { [Op.in]: postIds, }, }, }); } async function getBookmarks(postIds: string[], userId: string) { return await UserBookmarkedPosts.findAll({ attributes: ["userId", "postId"], where: { userId: userId, postId: { [Op.in]: postIds, }, }, }); } async function getEmojis(input: { userIds: string[]; postIds: string[]; }): Promise<{ userEmojiRelation: UserEmojiRelation[]; postEmojiRelation: PostEmojiRelations[]; postEmojiReactions: EmojiReaction[]; emojis: Emoji[]; }> { let postEmojisIdsPromise = PostEmojiRelations.findAll({ attributes: ["emojiId", "postId"], where: { postId: { [Op.in]: input.postIds, }, }, }); let postEmojiReactionsPromise = EmojiReaction.findAll({ attributes: ["emojiId", "postId", "userId", "content"], where: { postId: { [Op.in]: input.postIds, }, }, }); let userEmojiIdPromise = UserEmojiRelation.findAll({ attributes: ["emojiId", "userId"], where: { userId: { [Op.in]: input.userIds, }, }, }); await Promise.all([ postEmojisIdsPromise, userEmojiIdPromise, postEmojiReactionsPromise, ]); let postEmojisIds = await postEmojisIdsPromise; let userEmojiId = await userEmojiIdPromise; let postEmojiReactions = await postEmojiReactionsPromise; const emojiIds: string[] = ([] as string[]) .concat(postEmojisIds.map((elem: any) => elem.emojiId)) .concat(userEmojiId.map((elem: any) => elem.emojiId)) .concat(postEmojiReactions.map((reaction: any) => reaction.emojiId)); return { userEmojiRelation: userEmojiId, postEmojiRelation: postEmojisIds, postEmojiReactions: postEmojiReactions, emojis: await Emoji.findAll({ attributes: ["id", "url", "external", "name", "uuid"], where: { id: { [Op.in]: emojiIds, }, }, }), }; } // TODO optimization: make more promise all and less await dothing await dothing async function getUnjointedPosts( postIdsInput: string[], posterId: string, doNotFullyHide = false ) { let user = await User.scope("full").findByPk(posterId); // we need a list of all the userId we just got from the post let userIds: string[] = []; let postIds: string[] = []; if (completeEnvironment.enableBsky) { // DETECT BSKY NSFW const bskyPosts = await Post.findAll({ where: { id: { [Op.in]: postIdsInput, }, userId: { [Op.notIn]: await getAllLocalUserIds(), }, bskyUri: { [Op.ne]: null, }, }, }); if (bskyPosts && bskyPosts.length) { await checkBskyLabelersNSFW( bskyPosts.filter((elem) => !elem.content_warning && elem.bskyUri) ); } // END DETECT BSKY NSFW } const posts = await Post.findAll({ include: [ { model: Post, as: "ancestors", required: false, where: { isDeleted: false, }, }, ], where: { id: { [Op.in]: postIdsInput, }, isDeleted: false, }, }); posts.forEach((post: any) => { userIds.push(post.userId); postIds.push(post.id); post.ancestors?.forEach((ancestor: any) => { userIds.push(ancestor.userId); postIds.push(ancestor.id); }); }); const quotes = await getQuotes(postIds); const quotedPostsIds = quotes.map((quote) => quote.quotedPostId); postIds = postIds.concat(quotedPostsIds); const quotedPosts = await Post.findAll({ where: { id: { [Op.in]: quotedPostsIds, }, }, }); const asks = await Ask.findAll({ attributes: [ "question", "apObject", "createdAt", "updatedAt", "postId", "userAsked", "userAsker", ], where: { postId: { [Op.in]: postIds, }, }, }); const rewootedPosts = await Post.findAll({ attributes: ["id", "parentId"], where: { isReblog: true, userId: posterId, parentId: { [Op.in]: postIds, }, }, }); const rewootIds = rewootedPosts.map((r: any) => r.id); userIds = userIds .concat(quotedPosts.map((q: any) => q.userId)) .concat(asks.map((elem: any) => elem.userAsked)) .concat(asks.map((elem: any) => elem.userAsker)); const emojis = getEmojis({ userIds, postIds, }); const mentions = await getMentionedUserIds(postIds); userIds = userIds.concat(mentions.usersMentioned); userIds = userIds.concat( (await emojis).postEmojiReactions.map((react: any) => react.userId) ); const polls = QuestionPoll.findAll({ where: { postId: { [Op.in]: postIds, }, }, include: [ { model: QuestionPollQuestion, include: [ { model: QuestionPollAnswer, required: false, where: { userId: posterId, }, }, ], }, ], }); let medias = getMedias([...postIds, ...rewootIds]); let tags = getTags([...postIds, ...rewootIds]); const likes = await getLikes(postIds); const bookmarks = await getBookmarks(postIds, posterId); userIds = userIds.concat(likes.map((like: any) => like.userId)); const users = User.findAll({ attributes: [ "url", "avatar", "id", "name", "remoteId", "banned", "bskyDid", "federatedHostId", "isBot" ], where: { id: { [Op.in]: userIds, }, }, }); const usersMap: Map = new Map(); for (const usr of await users) { usersMap.set(usr.id, usr); } const postWithNotes = getPosstGroupDetails(posts); await Promise.all([emojis, users, polls, medias, tags, postWithNotes]); const hostsIds = (await users) .filter((elem) => elem.federatedHostId) .map((elem) => elem.federatedHostId); const blockedHosts = await FederatedHost.findAll({ where: { id: { [Op.in]: hostsIds as string[], }, blocked: true, }, }); const blockedHostsIds = blockedHosts.map((elem) => elem.id); let blockedUsersSet: Set = new Set(); const blockedUsersQuery = await Blocks.findAll({ where: { [Op.or]: [ { blockerId: posterId, }, { blockedId: posterId, }, ], }, }); for (const block of blockedUsersQuery) { blockedUsersSet.add(block.blockedId); blockedUsersSet.add(block.blockerId); } blockedUsersSet.delete(posterId); const bannedUserIds = (await users) .filter( (elem) => elem.banned || (elem.federatedHostId && blockedHostsIds.includes(elem.federatedHostId)) ) .map((elem) => elem.id); let usersFollowedByPoster: string[] | Promise = getFollowedsIds(posterId); let usersFollowingPoster: string[] | Promise = getFollowedsIds( posterId, false, { getFollowersInstead: true, } ); await Promise.all([ usersFollowedByPoster, usersFollowingPoster, tags, medias, ]); usersFollowedByPoster = await usersFollowedByPoster; usersFollowingPoster = await usersFollowingPoster; const tagsAwaited = await tags; const mediasAwaited = await medias; const invalidRewoots = [] as string[]; for (const id of rewootIds) { const hasMedia = mediasAwaited.some((media: any) => media.postId === id); const hasTags = tagsAwaited.some((tag: any) => tag.postId === id); if (hasMedia || hasTags) { invalidRewoots.push(id); } } const finalRewootIds = rewootedPosts .filter((r: any) => !invalidRewoots.includes(r.id)) .map((r: any) => r.parentId); const blockedServers = ( await ServerBlock.findAll({ where: { userBlockerId: posterId } }) ).map((elem) => elem.blockedServerId); const postsMentioningUser: string[] = mentions.postMentionRelation .filter((mention: any) => mention.userMentioned === posterId) .map((mention: any) => mention.post); const allPosts = (await postWithNotes) .concat((await postWithNotes).flatMap((elem: any) => elem.ancestors)) .concat(await quotedPosts) .map((elem: any) => (elem.dataValues ? elem.dataValues : elem)); const postsToFullySend = allPosts.filter((post: any) => { const postIsPostedByUser = post.userId === posterId; const isReblog = post.content === "" && !tagsAwaited.some((tag: any) => tag.postId === post.id) && !mediasAwaited.some((media: any) => media.postId === post.id); const validPrivacy = [ Privacy.Public, Privacy.LocalOnly, Privacy.Unlisted, Privacy.LinkOnly, ].includes(post.privacy); const userFollowsPoster = usersFollowedByPoster.includes(post.userId) && post.privacy === Privacy.FollowersOnly; const userIsMentioned = postsMentioningUser.includes(post.id); const posterIsInBlockedServer = blockedServers.includes( usersMap.get(post.userId)?.federatedHostId as string ); return ( !bannedUserIds.includes(post.userId) && !posterIsInBlockedServer && (postIsPostedByUser || validPrivacy || userFollowsPoster || userIsMentioned || isReblog) ); }); const postIdsToFullySend: string[] = postsToFullySend .filter((elem) => !blockedUsersSet.has(elem.userId)) .map((post: any) => post.id); const postsToSendFiltered = (await postWithNotes) .map((post: any) => filterPost(post, postIdsToFullySend, doNotFullyHide)) .filter((elem: any) => !!elem); let mediasToSend = (await medias).filter((elem: any) => { return postIdsToFullySend.includes(elem.postId); }); const tagsFiltered = (await tags).filter((tag: any) => postIdsToFullySend.includes(tag.postId) ); const quotesFiltered = quotes.filter((quote: any) => postIdsToFullySend.includes(quote.quoterPostId) ); const pollsFiltered = (await polls).filter((poll: any) => postIdsToFullySend.includes(poll.postId) ); // we edit posts so we add the interactionPolicies let postsToSend = postsToSendFiltered .filter((elem) => !!elem) .map(async (elem) => addPostCanInteract( posterId, elem, usersFollowingPoster, usersFollowedByPoster, mentions ) ); let finalPostsToSend = await Promise.all(postsToSend); const userIsAdult = isAdult(user?.birthDate); if (!userIsAdult && user?.role !== 10) { finalPostsToSend = finalPostsToSend.filter((x) => { const cwToFilter = (x.content_warning || "").toLowerCase(); return ( !cwToFilter.includes("nsfw") && !cwToFilter.includes("lewd") && !cwToFilter.includes("sexual") && !cwToFilter.includes("nudity") && !cwToFilter.includes("porn") ); }); mediasToSend = mediasToSend.filter((x) => !x.NSFW); } return { rewootIds: finalRewootIds.filter((elem) => !!elem), posts: finalPostsToSend, emojiRelations: await emojis, mentions: mentions.postMentionRelation.filter((elem) => !!elem), users: (await users).filter((elem) => !!elem), polls: pollsFiltered.filter((elem) => !!elem), medias: mediasToSend.filter((elem) => !!elem), tags: tagsFiltered.filter((elem) => !!elem), likes: likes.filter((elem) => !!elem), bookmarks: bookmarks, quotes: quotesFiltered.filter((elem) => !!elem), quotedPosts: (await quotedPosts) .map((elem: any) => filterPost(elem, postIdsToFullySend, doNotFullyHide)) .filter((elem) => !!elem), asks: asks.filter((elem) => !!elem), }; } function filterPost( postToBeFilter: any, postIdsToFullySend: string[], donotHide = false ): any { let res = postToBeFilter; if (!postIdsToFullySend.includes(res.id)) { res = undefined; } if (res) { const ancestorsLength = res.ancestors ? res.ancestors.length : 0; res.ancestors = res.ancestors ? res.ancestors .map((elem: any) => filterPost(elem, postIdsToFullySend, donotHide)) .filter((elem: any) => !!elem) : []; res.ancestors = res.ancestors.filter((elem: any) => !(elem == undefined)); if (ancestorsLength != res.ancestors.length && !donotHide) { res = undefined; } } return res; } // we are gona do this for likes, quotes, replies and rewoots... and we may will this function too when user interacts with a post! async function canInteract( level: InteractionControlType, userId: string, postId: string, userFollowersInput?: string[], userFollowingInput?: string[], mentionsInput?: { usersMentioned: string[]; postMentionRelation: any[] } ): Promise { if (level == InteractionControl.Anyone) { return true; } let usersFollowing = userFollowingInput ? userFollowingInput : getFollowedsIds(userId); let userFollowers = userFollowersInput ? userFollowersInput : getFollowedsIds(userId, false, { getFollowersInstead: true, }); let mentions = mentionsInput ? mentionsInput : getMentionedUserIds([postId]); let post: Promise | Post | null = Post.findByPk(postId); await Promise.all([usersFollowing, userFollowers, mentions, post]); usersFollowing = await usersFollowing; userFollowers = await userFollowers; mentions = await mentions; post = await post; // TMP hack let res = false; if (post) { if (post.userId == userId) { return true; } switch (level) { case InteractionControl.Anyone: { res = false; break; } case InteractionControl.Followers: { res = usersFollowing.includes(post.userId); break; } case InteractionControl.Following: { // post creator follows you res = userFollowers.includes(post.userId); break; } case InteractionControl.FollowersAndMentioned: { // post creator follows you res = usersFollowing.includes(post.userId) || mentions.postMentionRelation.find( (elem) => elem.postId == postId && elem.userId == userId ); break; } case InteractionControl.FollowingAndMentioned: { // post creator follows you res = userFollowers.includes(post.userId) || mentions.postMentionRelation.find( (elem) => elem.postId == postId && elem.userId == userId ); break; } case InteractionControl.FollowersFollowersAndMentioned: { res = userFollowers.includes(post.userId) || userFollowingInput?.includes(post.userId) || mentions.postMentionRelation.find( (elem) => elem.postId == postId && elem.userId == userId ); break; } case InteractionControl.MentionedUsersOnly: { // post creator follows you res = mentions.postMentionRelation.find( (elem) => elem.postId == postId && elem.userId == userId ); break; } case InteractionControl.NoOne: { // we already check if user is from poster himself. This is a special one for bsky res = false; break; } case InteractionControl.SameAsOp: { // special one for bsky too // ok we need to check for the initial post and to the calculations with it. // we look for op post const parentsIds = ( await sequelize.query( `SELECT DISTINCT "ancestorId" FROM "postsancestors" where "postsId" = '${post.id}'`, { type: QueryTypes.SELECT, } ) ).map((elem: any) => elem.ancestorId as string); const originalPost = await Post.findOne({ where: { hierarchyLevel: 1, id: { [Op.in]: parentsIds, }, }, }); if (!originalPost || originalPost?.id === post.id) { return res; } else { // this will only be used for REPLIES res = await canInteract( originalPost.replyControl, userId, originalPost.id, userFollowersInput, userFollowingInput, mentionsInput ); } } } } return res; } async function addPostCanInteract( userId: string, postInput: any, userFollowersInput?: string[], userFollowingInput?: string[], mentionsInput?: { usersMentioned: string[]; postMentionRelation: any[] } ): Promise< Post & { canReply: boolean; canLike: boolean; canReblog: boolean; canQuote: boolean; } > { let post: any = { ...postInput }; let canReply = canInteract( post.replyControl, userId, post.id, userFollowersInput, userFollowingInput, mentionsInput ); let canLike = canInteract( post.likeControl, userId, post.id, userFollowersInput, userFollowingInput, mentionsInput ); let canReblog = canInteract( post.reblogControl, userId, post.id, userFollowersInput, userFollowingInput, mentionsInput ); let canQuote = canInteract( post.quoteControl, userId, post.id, userFollowersInput, userFollowingInput, mentionsInput ); await Promise.all([canReblog, canReply, canQuote, canLike]); post.canReply = await canReply; post.canLike = await canLike; post.canReblog = await canReblog; post.canQuote = await canQuote; if (post.ancestors) { post.ancestors = await Promise.all( post.ancestors.map((elem: Post) => addPostCanInteract( userId, elem.dataValues, userFollowersInput, userFollowingInput, mentionsInput ) ) ); } return post; } export { getUnjointedPosts, getMedias, getQuotes, getMentionedUserIds, getTags, getLikes, getBookmarks, getEmojis, addPostCanInteract, };