jbc.lol
unoffical wafrn mirror
wafrn.net
atproto
social-network
activitypub
1/* eslint-disable @typescript-eslint/no-explicit-any */
2// exports a user into an ActivityPub compatible backup file.
3
4import fs from 'fs'
5import { Post, User } from '../../models/index.js'
6import { userToJSONLD } from '../activitypub/userToJSONLD.js'
7import archiver from 'archiver'
8import { v4 as uuidv4 } from 'uuid'
9import { postToJSONLD } from '../activitypub/postToJSONLD.js'
10import axios from 'axios'
11import { completeEnvironment } from '../backendOptions.js'
12
13const archived: Record<string, boolean> = {}
14
15async function extractImages(archive: archiver.Archiver, data: any, remoteFetch: boolean) {
16 const promises: Promise<any>[] = []
17 if (Array.isArray(data)) {
18 for (const value of data) {
19 promises.push(extractImages(archive, value, remoteFetch))
20 }
21 } else if (typeof data === 'object') {
22 for (const key in data) {
23 const value = data[key]
24 if (key == 'url' && typeof value === 'string') {
25 // ugly hack to skip the main "url" objects in the post and blog events
26 if (value.indexOf('fediverse/post') !== -1) {
27 } else if (value.indexOf('fediverse/blog') !== -1) {
28 } else if (value.startsWith(completeEnvironment.mediaUrl)) {
29 const fileName = value.slice(completeEnvironment.mediaUrl.length + 1)
30 const newFileName = `media_attachments/files/${fileName}`
31 if (!archived[fileName]) {
32 console.log(`Media file ${value} - ${fileName}`)
33 archived[fileName] = true
34 archive.file(`uploads/${fileName}`, { name: newFileName })
35 }
36 data[key] = `/${newFileName}`
37 } else if (remoteFetch) {
38 const fileName = value.replaceAll(/[^.a-zA-Z0-9_-]/g, '_')
39 const downloadedFile = await axios
40 .get(completeEnvironment.externalCacheurl + value, { responseType: 'stream' })
41 .catch(() => null)
42 if (downloadedFile?.data) {
43 const newFileName = `media_attachments/files/${fileName}`
44 if (!archived[fileName]) {
45 console.log(`Remote media file ${value} - ${fileName}`)
46 archived[fileName] = true
47 archive.append(downloadedFile.data, { name: newFileName })
48 }
49 data[key] = `/${newFileName}`
50 } else {
51 console.log(`Could not download remote media file ${value} - ${fileName}`)
52 }
53 }
54 } else {
55 promises.push(extractImages(archive, value, remoteFetch))
56 }
57 }
58 }
59 return Promise.all(promises)
60}
61
62async function exportBackup(userUrl: string, exportType: string): Promise<string> {
63 return new Promise(async (resolve, reject) => {
64 const user = await User.findOne({ where: { url: userUrl } })
65 if (!user) return
66
67 const fileName = `backup-${userUrl}-${Date.now()}-${uuidv4()}.zip`
68 const output = fs.createWriteStream('uploads/' + fileName)
69
70 const archive = archiver('zip', { zlib: { level: 9 } })
71
72 output.on('close', () => {
73 resolve(fileName)
74 })
75
76 archive.on('error', (err: Error) => {
77 reject(err)
78 })
79
80 archive.pipe(output)
81
82 // Export Blog
83 const userData = await userToJSONLD(user)
84 await extractImages(archive, userData, exportType == '3')
85 archive.append(JSON.stringify(userData), { name: 'actor.json' })
86
87 // Export Posts
88 const outbox: any = {}
89 outbox['@context'] = [
90 'https://www.w3.org/ns/activitystreams',
91 'https://w3id.org/security/v1',
92 {
93 manuallyApprovesFollowers: 'as:manuallyApprovesFollowers',
94 sensitive: 'as:sensitive',
95 Hashtag: 'as:Hashtag',
96 movedTo: {
97 '@id': 'as:movedTo',
98 '@type': '@id'
99 },
100 alsoKnownAs: {
101 '@id': 'as:alsoKnownAs',
102 '@type': '@id'
103 },
104 toot: 'http://joinmastodon.org/ns#',
105 Emoji: 'toot:Emoji',
106 featured: {
107 '@id': 'toot:featured',
108 '@type': '@id'
109 },
110 featuredTags: {
111 '@id': 'toot:featuredTags',
112 '@type': '@id'
113 },
114 schema: 'http://schema.org#',
115 PropertyValue: 'schema:PropertyValue',
116 value: 'schema:value',
117 ostatus: 'http://ostatus.org#',
118 atomUri: 'ostatus:atomUri',
119 inReplyToAtomUri: 'ostatus:inReplyToAtomUri',
120 conversation: 'ostatus:conversation',
121 focalPoint: {
122 '@container': '@list',
123 '@id': 'toot:focalPoint'
124 },
125 blurhash: 'toot:blurhash',
126 discoverable: 'toot:discoverable',
127 indexable: 'toot:indexable',
128 memorial: 'toot:memorial',
129 votersCount: 'toot:votersCount',
130 suspended: 'toot:suspended',
131 attributionDomains: {
132 '@id': 'toot:attributionDomains',
133 '@type': '@id'
134 },
135 gts: 'https://gotosocial.org/ns#',
136 interactionPolicy: {
137 '@id': 'gts:interactionPolicy',
138 '@type': '@id'
139 },
140 canQuote: {
141 '@id': 'gts:canQuote',
142 '@type': '@id'
143 },
144 automaticApproval: {
145 '@id': 'gts:automaticApproval',
146 '@type': '@id'
147 },
148 manualApproval: {
149 '@id': 'gts:manualApproval',
150 '@type': '@id'
151 }
152 }
153 ]
154 outbox.id = 'outbox.json'
155 outbox.type = 'OrderedCollection'
156 outbox.orderedItems = []
157 for (const postItem of await user.getPosts({ order: [['createdAt', 'ASC']] })) {
158 const postsToExport: Post[] = [postItem]
159 if (exportType == '2' || exportType == '3') {
160 while (postsToExport[0].parentId && (await postsToExport[0].getParent())) {
161 const parentPost = await postsToExport[0].getParent()
162 postsToExport.unshift(parentPost)
163 }
164 }
165
166 for (const post of postsToExport) {
167 if (archived[post.id]) continue
168
169 archived[post.id] = true
170 console.log(`Processing ${post.id}`)
171 try {
172 const postData = await postToJSONLD(post.id)
173 if (postData) {
174 await extractImages(archive, postData, exportType == '3')
175 if (postData.type == 'Create') {
176 if (post.remotePostId) {
177 postData.object.id = post.remotePostId
178 }
179 const postUser = await post.getUser()
180
181 if (postUser.remoteId) {
182 postData.object.attributedTo = postUser.remoteId
183 }
184 if (postUser.isRemoteUser) {
185 // local posts also have bskyUri so this is to determine if this is a remote bluesky post&user
186 if (post.bskyUri) {
187 postData.object.id = post.bskyUri
188 }
189 if (postUser.bskyDid) {
190 postData.object.attributedTo = `at://${postUser.bskyDid}`
191 }
192 }
193
194 const postParent = post.parentId && (await post.getParent({ include: 'user' }))
195 if (postParent) {
196 if (postParent.isRemoteBlueskyPost) {
197 postData.object.inReplyTo = postParent.bskyUri
198 }
199 }
200 } else if (postData.type == 'Announce') {
201 const postParent = post.parentId && (await post.getParent({ include: 'user' }))
202 if (postParent) {
203 if (postParent.bskyUri) {
204 postData.object = (await post.getParent()).bskyUri
205 }
206 }
207 }
208 outbox.orderedItems.push(postData)
209 delete outbox.orderedItems[outbox.orderedItems.length - 1]['@context']
210 }
211 } catch (error) {
212 console.log('Error during JSONLD processing')
213 console.log(error)
214 }
215 }
216 }
217 outbox.totalItems = outbox.orderedItems.length
218 archive.append(JSON.stringify(outbox), { name: 'outbox.json' })
219
220 // Export Likes
221
222 const likes: any = {
223 '@context': 'https://www.w3.org/ns/activitystreams',
224 id: 'likes.json',
225 type: 'OrderedCollection',
226 orderedItems: []
227 }
228
229 for (const like of await user.getUserLikesPostRelations({ include: Post })) {
230 if (like.post) {
231 const postRemoteId = await like.post.fullUrlIncludingBsky()
232 likes.orderedItems.push(postRemoteId)
233 }
234 }
235
236 archive.append(JSON.stringify(likes), { name: 'likes.json' })
237
238 // Export Bookmarks
239
240 const bookmarks: any = {
241 '@context': 'https://www.w3.org/ns/activitystreams',
242 id: 'bookmarks.json',
243 type: 'OrderedCollection',
244 orderedItems: []
245 }
246
247 for (const bookmark of await user.getUserBookmarkedPosts({ include: Post })) {
248 if (bookmark.post) {
249 const postRemoteId = await bookmark.post.fullUrlIncludingBsky()
250 bookmarks.orderedItems.push(postRemoteId)
251 }
252 }
253
254 archive.append(JSON.stringify(bookmarks), { name: 'bookmarks.json' })
255
256 archive.finalize()
257 })
258}
259
260if (!process.argv[2]) {
261 console.log('Usage: tsx exportActivityPubBackup.ts <localUserName> <exportType>')
262 console.log('exportType:')
263 console.log(" 1: export blog's posts and it's media only (default)")
264 console.log(" 2: export blog's posts, the entire thread, and all local media")
265 console.log(" 3: export blog's posts, the entire thread, and both local and remote media files")
266 process.exit(0)
267}
268
269const fileName = await exportBackup(process.argv[2], process.argv[3])
270
271console.log(`Exported to: ${completeEnvironment.mediaUrl}/${fileName}`)
272process.exit(0)