import { Op } from "sequelize"; import { Blocks, Emoji, FederatedHost, Media, Post, PostMentionsUserRelation, ServerBlock, PostTag, User, sequelize, Ask, Notification, EmojiReaction, PostAncestor, PostReport, QuestionPoll, Quotes, RemoteUserPostView, SilencedPost, UserBitesPostRelation, UserBookmarkedPosts, UserLikesPostRelations, } from "../../models/index.js"; import { completeEnvironment } from "../backendOptions.js"; import { logger } from "../logger.js"; import { getRemoteActor } from "./getRemoteActor.js"; import { getPetitionSigned } from "./getPetitionSigned.js"; import { fediverseTag } from "../../interfaces/fediverse/tags.js"; import { loadPoll } from "./loadPollFromPost.js"; import { getApObjectPrivacy } from "./getPrivacy.js"; import dompurify from "isomorphic-dompurify"; import { Queue } from "bullmq"; import { bulkCreateNotifications } from "../pushNotifications.js"; import { getDeletedUser } from "../cacheGetters/getDeletedUser.js"; import { Privacy } from "../../models/post.js"; import { getAtProtoThread, getPostThreadSafe, processSinglePost, } from "../../atproto/utils/getAtProtoThread.js"; import * as cheerio from "cheerio"; import { PostView, ThreadViewPost, } from "@atproto/api/dist/client/types/app/bsky/feed/defs.js"; import { getAdminUser } from "../getAdminAndDeletedUser.js"; const updateMediaDataQueue = new Queue("processRemoteMediaData", { connection: completeEnvironment.bullmqConnection, defaultJobOptions: { removeOnComplete: true, attempts: 3, backoff: { type: "exponential", delay: 1000, }, removeOnFail: true, }, }); async function getPostThreadRecursive( user: any, remotePostId: string | null, remotePostObject?: any, localPostToForceUpdate?: string, options?: any ) { if (remotePostId === null) return; const deletedUser = getDeletedUser(); try { remotePostId.startsWith( `${completeEnvironment.frontendUrl}/fediverse/post/` ); } catch (error) { logger.info({ message: "Error with url on post", object: remotePostId, stack: new Error().stack, }); return; } if ( remotePostId.startsWith( `${completeEnvironment.frontendUrl}/fediverse/post/` ) ) { // we are looking at a local post const partToRemove = `${completeEnvironment.frontendUrl}/fediverse/post/`; const postId = remotePostId.substring(partToRemove.length); return await Post.findOne({ where: { id: postId, }, }); } if (completeEnvironment.enableBsky && remotePostId.startsWith("at://")) { // Bluesky post. Likely coming from an import const postInDatabase = await Post.findOne({ where: { bskyUri: remotePostId, }, }); if (postInDatabase) { return postInDatabase; } else if (!remotePostObject) { const postId = await getAtProtoThread(remotePostId); return await Post.findByPk(postId); } } const postInDatabase = await Post.findOne({ where: { remotePostId: remotePostId, }, }); if (postInDatabase && !localPostToForceUpdate) { if (postInDatabase.remotePostId) { const parentPostPetition = await getPetitionSigned( user, postInDatabase.remotePostId ); if (parentPostPetition) { await loadPoll(parentPostPetition, postInDatabase, user); } } return postInDatabase; } else { try { const postPetition = remotePostObject ? remotePostObject : await getPetitionSigned(user, remotePostId); if (postPetition && !localPostToForceUpdate) { const remotePostInDatabase = await Post.findOne({ where: { remotePostId: postPetition.id, }, }); if (remotePostInDatabase) { if (remotePostInDatabase.remotePostId) { const parentPostPetition = await getPetitionSigned( user, remotePostInDatabase.remotePostId ); if (parentPostPetition) { await loadPoll(parentPostPetition, remotePostInDatabase, user); } } return remotePostInDatabase; } } // peertube: what the fuck let actorUrl = postPetition.attributedTo; if (Array.isArray(actorUrl)) { actorUrl = actorUrl[0].id; } const remoteUser = await getRemoteActor(actorUrl, user); if (remoteUser) { const remoteHost = (await FederatedHost.findByPk( remoteUser.federatedHostId as string )) as FederatedHost; const remoteUserServerBaned = remoteHost?.blocked ? remoteHost.blocked : false; // HACK: some implementations (GTS IM LOOKING AT YOU) may send a single element instead of an array // I should had used a funciton instead of this dirty thing, BUT you see, its late. Im eepy // also this code is CRITICAL. A failure here is a big problem. So this hack it is postPetition.tag = !Array.isArray(postPetition.tag) ? [postPetition.tag].filter((elem) => elem) : postPetition.tag; const medias: any[] = []; const fediTags: fediverseTag[] = [ ...new Set( postPetition.tag ?.filter((elem: fediverseTag) => [ postPetition.tag.some( (tag: fediverseTag) => tag.type == "WafrnHashtag" ) ? "WafrnHashtag" : "Hashtag", ].includes(elem.type) ) .map((elem: fediverseTag) => { return { href: elem.href, type: elem.type, name: elem.name }; }) ), ]; let fediMentions: fediverseTag[] = postPetition.tag?.filter( (elem: fediverseTag) => elem.type === "Mention" ); if (fediMentions == undefined) { fediMentions = postPetition.to.map((elem: string) => { return { href: elem }; }); } const fediEmojis: any[] = postPetition.tag?.filter( (elem: fediverseTag) => elem.type === "Emoji" ); const privacy = getApObjectPrivacy(postPetition, remoteUser); let postTextContent = `${ postPetition.content ? postPetition.content : "" }`; // Fix for bridgy giving this as undefined if (postPetition.type == "Video") { // peertube federation. We just add a link to the video, federating this is HELL postTextContent = postTextContent + ` ${postPetition.id}`; } if ( postPetition.tag && postPetition.tag.some( (tag: fediverseTag) => tag.type === "WafrnHashtag" ) ) { // Ok we have wafrn hashtags with us, we are probably talking with another wafrn! Crazy, I know const dom = cheerio.load(postTextContent); const tags = dom("a.hashtag").html(""); postTextContent = dom.html(); } if ( postPetition.attachment && postPetition.attachment.length > 0 && (!remoteUser.banned || options?.allowMediaFromBanned) ) { for await (const remoteFile of postPetition.attachment) { if (remoteFile.type !== "Link") { const wafrnMedia = await Media.create({ url: remoteFile.url, NSFW: postPetition?.sensitive, userId: remoteUserServerBaned || remoteUser.banned ? ( await deletedUser )?.id : remoteUser.id, description: remoteFile.name, ipUpload: "IMAGE_FROM_OTHER_FEDIVERSE_INSTANCE", mediaOrder: postPetition.attachment.indexOf(remoteFile), // could be non consecutive but its ok external: true, mediaType: remoteFile.mediaType ? remoteFile.mediaType : "", blurhash: remoteFile.blurhash ? remoteFile.blurhash : null, height: remoteFile.height ? remoteFile.height : null, width: remoteFile.width ? remoteFile.width : null, }); if ( !wafrnMedia.mediaType || (wafrnMedia.mediaType?.startsWith("image") && !wafrnMedia.width) ) { await updateMediaDataQueue.add(`updateMedia:${wafrnMedia.id}`, { mediaId: wafrnMedia.id, }); } medias.push(wafrnMedia); } else { postTextContent = "" + postTextContent + `${remoteFile.href}`; } } } const lemmyName = postPetition.name ? postPetition.name : ""; postTextContent = postTextContent ? postTextContent : `

${lemmyName}

`; let createdAt = new Date(postPetition.published); if (createdAt.getTime() > new Date().getTime()) { createdAt = new Date(); } let bskyUri: string | undefined, bskyCid: string | undefined; let existingBskyPost: Post | undefined; // check if it's a bridgy post or a post from a wafrn by checking a valid FEP-fffd if (postPetition.url && Array.isArray(postPetition.url)) { const url = postPetition.url as Array< string | { type: string; href: string } >; const firstFffd = url.find((x) => typeof x !== "string"); // check if it starts at at:// then its a bridged post, we do not touch it if it's not if (firstFffd && firstFffd.href.startsWith("at://")) { // get it's bsky counterparts first, we need the cid const thread = await getPostThreadSafe({ uri: firstFffd.href, }); if (thread && thread.success) { try { const threadView = thread.data.thread as ThreadViewPost; bskyCid = threadView.post.cid; bskyUri = threadView.post.uri; // check if it cames from wafrn if ( !( threadView.post.record as { fediverseId: string | undefined; } ).fediverseId ) { // this is a bridgy fed post, assume main post is on bsky, use bsky user const postId = await processSinglePost(threadView.post); if (postId) { const post = await Post.findByPk(postId); if (post) { post.remotePostId = postPetition.id; await post.save(); return post; } } } else { // now this is a wafrn post, where we do a thing little bit different // first we going to check if the post is already on db because this can break everything let existingPost = await Post.findOne({ where: { bskyCid: threadView.post.cid, remotePostId: null, }, }); if (existingPost) { existingBskyPost = existingPost; // do not attempt to merge it right now, this will crash backend bskyCid = undefined; bskyUri = undefined; } } } catch {} } } } const postToCreate: any = { content: "" + postTextContent, content_warning: postPetition.summary ? postPetition.summary : remoteUser.NSFW ? "User is marked as NSFW by this instance staff. Possible NSFW without tagging" : "", createdAt: createdAt, updatedAt: createdAt, userId: remoteUserServerBaned || remoteUser.banned ? (await deletedUser)?.id : remoteUser.id, remotePostId: postPetition.id, privacy: privacy, bskyUri: postPetition.blueskyUri, displayUrl: Array.isArray(postPetition.url) ? postPetition.url[0] : postPetition.url, bskyCid: postPetition.blueskyCid, ...(bskyCid && bskyUri ? { bskyCid, bskyUri, } : {}), }; if (postPetition.name) { postToCreate.title = postPetition.name; } const mentionedUsersIds: string[] = []; const quotes: any[] = []; try { if (!remoteUser.banned && !remoteUserServerBaned) { for await (const mention of fediMentions) { let mentionedUser; if ( mention.href?.indexOf(completeEnvironment.frontendUrl) !== -1 ) { const username = mention.href?.substring( `${completeEnvironment.frontendUrl}/fediverse/blog/`.length ) as string; mentionedUser = await User.findOne({ where: sequelize.where( sequelize.fn("lower", sequelize.col("url")), username.toLowerCase() ), }); } else { mentionedUser = await getRemoteActor(mention.href, user); } if ( mentionedUser?.id && mentionedUser.id != (await deletedUser)?.id && !mentionedUsersIds.includes(mentionedUser.id) ) { mentionedUsersIds.push(mentionedUser.id); } } } } catch (error) { logger.info({ message: "problem processing mentions", error }); } if ( postPetition.inReplyTo && postPetition.id !== postPetition.inReplyTo ) { const parent = await getPostThreadRecursive( user, postPetition.inReplyTo.id ? postPetition.inReplyTo.id : postPetition.inReplyTo ); postToCreate.parentId = parent?.id; } const existingPost = localPostToForceUpdate ? await Post.findByPk(localPostToForceUpdate) : undefined; if (existingPost) { existingPost.set(postToCreate); await existingPost.save(); await loadPoll(postPetition, existingPost, user); } const newPost = existingPost ? existingPost : await Post.create(postToCreate); try { if (!remoteUser.banned && !remoteUserServerBaned && fediEmojis) { processEmojis(newPost, fediEmojis); } } catch (error) { logger.debug("Problem processing emojis"); } newPost.setMedias(medias); try { if (postPetition.quote || postPetition.quoteUrl) { const urlQuote = postPetition.quoteUrl || postPetition.quote; const postToQuote = await getPostThreadRecursive(user, urlQuote); if (postToQuote && postToQuote.privacy != Privacy.DirectMessage) { quotes.push(postToQuote); } if (!postToQuote) { postToCreate.content = postToCreate.content + `

RE: ${urlQuote}

`; } const postsToQuotePromise: any[] = []; postPetition.tag ?.filter((elem: fediverseTag) => elem.type === "Link") .forEach((quote: fediverseTag) => { postsToQuotePromise.push( getPostThreadRecursive(user, quote.href as string) ); postToCreate.content = postToCreate.content.replace( quote.name, "" ); }); const quotesToAdd = await Promise.allSettled(postsToQuotePromise); const quotesThatWillGetAdded = quotesToAdd.filter( (elem) => elem.status === "fulfilled" && elem.value && elem.value.privacy !== 10 ); quotesThatWillGetAdded.forEach((quot) => { if ( quot.status === "fulfilled" && !quotes.map((q) => q.id).includes(quot.value.id) ) { quotes.push(quot.value); } }); } } catch (error) { logger.info("Error processing quotes"); logger.debug(error); } newPost.setQuoted(quotes); await newPost.save(); await bulkCreateNotifications( quotes.map((quote) => ({ notificationType: "QUOTE", notifiedUserId: quote.userId, userId: newPost.userId, postId: newPost.id, createdAt: new Date(newPost.createdAt), })), { postContent: newPost.content, userUrl: remoteUser.url, } ); try { if (!remoteUser.banned && !remoteUserServerBaned) { await addTagsToPost(newPost, fediTags); } } catch (error) { logger.info("problem processing tags"); } try { await addAsksToPost(newPost, fediTags); } catch (error) {} if (mentionedUsersIds.length != 0) { await processMentions(newPost, mentionedUsersIds); } await loadPoll(remotePostObject, newPost, user); const postCleanContent = dompurify .sanitize(newPost.content, { ALLOWED_TAGS: [] }) .trim(); const mentions = await newPost.getMentionPost(); if (postCleanContent.startsWith("!ask") && mentions.length === 1) { let askContent = postCleanContent.split( `!ask @${mentions[0].url}` )[1]; if (askContent.startsWith("@" + completeEnvironment.instanceUrl)) { askContent = askContent.split( "@" + completeEnvironment.instanceUrl )[1]; } await Ask.create({ question: askContent, userAsker: newPost.userId, userAsked: mentions[0].id, answered: false, apObject: JSON.stringify(postPetition), }); } if (existingBskyPost) { // very expensive updates! but only happens when bsky // post is already on db but the fedi post is not await EmojiReaction.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await Notification.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await PostReport.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); try { await PostAncestor.update( { postsId: newPost.id, }, { where: { postsId: existingBskyPost.id, }, } ); } catch {} await QuestionPoll.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await Quotes.update( { quoterPostId: newPost.id, }, { where: { quoterPostId: existingBskyPost.id, }, } ); if ( !(await Quotes.findOne({ where: { quotedPostId: newPost.id, }, })) ) { await Quotes.update( { quotedPostId: newPost.id, }, { where: { quotedPostId: existingBskyPost.id, }, } ); } await RemoteUserPostView.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await SilencedPost.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await SilencedPost.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await UserBitesPostRelation.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await UserBookmarkedPosts.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await UserLikesPostRelations.update( { postId: newPost.id, }, { where: { postId: existingBskyPost.id, }, } ); await Post.update( { parentId: newPost.id, }, { where: { parentId: existingBskyPost.id, }, } ); // now we delete the existing bsky post await existingBskyPost.destroy(); // THEN we merge it newPost.bskyCid = existingBskyPost.bskyCid; newPost.bskyUri = existingBskyPost.bskyUri; await newPost.save(); } return newPost; } } catch (error) { logger.trace({ message: "error getting remote post", url: remotePostId, user: user.url, problem: error, }); return null; } } } async function addAsksToPost(post: Post, tags: fediverseTag[]) { const asks = tags.filter((elem) => elem.type === "AskQuestion"); if (asks.length) { const ask = asks[0]; const userAsker = await getRemoteActor( ask.actor as string, await getAdminUser() ); const textToRemove = ask.representation as string; const askText = ask.name; if (textToRemove) { post.content = post.content.replace(textToRemove, ""); await Ask.create({ answered: true, postId: post.id, userAsker: userAsker ? userAsker.id : undefined, userAsked: post.userId, }); await post.save(); } } } async function addTagsToPost(post: any, originalTags: fediverseTag[]) { let tags = [...originalTags]; const res = await post.setPostTags([]); if (tags.some((elem) => elem.name == "WafrnHashtag")) { tags = tags.filter((elem) => elem.name == "WafrnHashtag"); } return await PostTag.bulkCreate( tags .filter((elem) => elem && post && elem.name && post.id) .map((elem) => { return { tagName: elem?.name?.replace("#", ""), postId: post.id, }; }) ); } async function processMentions(post: any, userIds: string[]) { await post.setMentionPost([]); await Notification.destroy({ where: { notificationType: "MENTION", postId: post.id, }, }); const blocks = await Blocks.findAll({ where: { blockerId: { [Op.in]: userIds, }, blockedId: post.userId, }, }); const remoteUser = await User.findByPk(post.userId, { attributes: ["url", "federatedHostId"], }); const userServerBlocks = await ServerBlock.findAll({ where: { userBlockerId: { [Op.in]: userIds, }, blockedServerId: remoteUser?.federatedHostId || "", }, }); const blockerIds: string[] = blocks .map((block: any) => block.blockerId) .concat(userServerBlocks.map((elem: any) => elem.userBlockerId)); await bulkCreateNotifications( userIds.map((mentionedUserId) => ({ notificationType: "MENTION", notifiedUserId: mentionedUserId, userId: post.userId, postId: post.id, createdAt: new Date(post.createdAt), })), { postContent: post.content, userUrl: remoteUser?.url, } ); return await PostMentionsUserRelation.bulkCreate( userIds .filter((elem) => !blockerIds.includes(elem)) .map((elem) => { return { postId: post.id, userId: elem, }; }), { ignoreDuplicates: true, } ); } async function processEmojis(post: any, fediEmojis: any[]) { let emojis: any[] = []; let res: any; const emojiIds: string[] = Array.from( new Set(fediEmojis.map((emoji: any) => emoji.id)) ); const foundEmojis = await Emoji.findAll({ where: { id: { [Op.in]: emojiIds, }, }, }); foundEmojis.forEach((emoji: any) => { const newData = fediEmojis.find( (foundEmoji: any) => foundEmoji.id === emoji.id ); if (newData && newData.icon?.url) { emoji.set({ url: newData.icon.url, }); emoji.save(); } else { logger.debug("issue with emoji"); logger.debug(emoji); logger.debug(newData); } }); emojis = emojis.concat(foundEmojis); const notFoundEmojis = fediEmojis.filter( (elem: any) => !foundEmojis.find((found: any) => found.id === elem.id) ); if (fediEmojis && notFoundEmojis && notFoundEmojis.length > 0) { try { const newEmojis = notFoundEmojis.map((newEmoji: any) => { return { id: newEmoji.id ? newEmoji.id : newEmoji.name + newEmoji.icon?.url, name: newEmoji.name, external: true, url: newEmoji.icon?.url, }; }); emojis = emojis.concat( await Emoji.bulkCreate(newEmojis, { ignoreDuplicates: true }) ); } catch (error) { logger.debug("Error with emojis"); logger.debug(error); } } return await post.setEmojis(emojis); } export { getPostThreadRecursive };