jbc.lol
unoffical wafrn mirror
wafrn.net
atproto
social-network
activitypub
1import { Application, Request, Response } from "express";
2import crypto from "crypto";
3import fs from "fs";
4import axios, { AxiosResponse } from "axios";
5import { logger } from "../utils/logger.js";
6import optimizeMedia from "../utils/optimizeMedia.js";
7import { Resolver } from "did-resolver";
8import { getResolver } from "plc-did-resolver";
9import { redisCache } from "../utils/redis.js";
10import { getLinkPreview } from "link-preview-js";
11import { linkPreviewRateLimiter } from "../utils/rateLimiters.js";
12import { getMimeType } from "stream-mime-type";
13import { completeEnvironment } from "../utils/backendOptions.js";
14import { Media } from "../models/media.js";
15import { Op } from "sequelize";
16import { spawn } from "child_process";
17import sequelize from "sequelize/lib/sequelize";
18import { return404 } from "../utils/return404.js";
19import { User } from "../models/user.js";
20import { Emoji } from "../models/emoji.js";
21
22function sendWithCache(res: Response, localFileName: string) {
23 // Does the .mime file exist?
24 if (fs.existsSync(localFileName + ".mime")) {
25 let mime = fs.readFileSync(localFileName + ".mime").toString();
26 res.contentType(mime);
27 }
28 // 1 hour of cache
29 res.set("Cache-control", "public, max-age=3600");
30 res.set(
31 "Content-Disposition",
32 `inline; filename="${localFileName.split("/").pop()}"`
33 );
34 res.sendFile(localFileName, { root: "." });
35}
36
37// converting the stream parsing to a promise to be able to use async/await and catch the errors with the try/catch blocks
38function writeStream(
39 stream: NodeJS.ReadableStream,
40 localFileName: string,
41 mime: string,
42 altText: string
43) {
44 const writeStream = fs.createWriteStream(localFileName);
45 fs.writeFileSync(localFileName + ".mime", mime);
46 return new Promise((resolve, reject) => {
47 writeStream.on("finish", async () => {
48 writeStream.close();
49 if (altText != "") {
50 try {
51 const updateAltText = spawn("exiv2", [
52 "-M",
53 `set Exif.Photo.UserComment charset=Ascii ${altText
54 .replaceAll('"', "")
55 .replaceAll("'", "")
56 .replaceAll("\\", "")
57 .replaceAll("$", "")
58 .replaceAll("@", "")}`,
59 localFileName,
60 ]);
61 updateAltText.on("close", () => {
62 return resolve(localFileName);
63 });
64 } catch (error) {
65 return resolve(localFileName);
66 }
67 } else {
68 return resolve(localFileName);
69 }
70 });
71 writeStream.on("error", (error) => {
72 return reject(error);
73 });
74 stream.pipe(writeStream);
75 });
76}
77function cacheRoutes(app: Application) {
78 app.get("/api/cache", async (req: Request, res: Response) => {
79 let mediaUrl = String(req.query?.media);
80 if (!mediaUrl) {
81 res.sendStatus(404);
82 return;
83 }
84 logger.trace({
85 message: `Old cache use: ${mediaUrl}`,
86 });
87 await getMediaFromUrl(mediaUrl, res);
88 });
89
90 app.get("/api/v2/cache/media/:id", async (req: Request, res: Response) => {
91 let mediaId = req.params.id;
92 const media = mediaId ? await Media.findByPk(mediaId) : undefined;
93 if (media) {
94 await getMediaFromUrl(
95 media.external ? media.url : completeEnvironment.mediaUrl + media.url,
96 res
97 );
98 } else {
99 res.sendStatus(404);
100 }
101 });
102
103 app.get("/api/v2/cache/avatar/:id", async (req: Request, res: Response) => {
104 try {
105 let userId = req.params.id;
106 const user = userId
107 ? await User.scope("full").findOne({
108 attributes: ["email", "avatar"],
109 where: {
110 banned: false,
111 [Op.or]: [
112 {
113 id: userId,
114 },
115 {
116 url: userId,
117 },
118 ],
119 },
120 })
121 : undefined;
122 if (user) {
123 const url = user.email
124 ? `${completeEnvironment.mediaUrl}${user.avatar}`
125 : user.avatar;
126 await getMediaFromUrl(url, res);
127 } else {
128 res.sendStatus(404);
129 }
130 } catch (error) {
131 logger.debug({
132 message: `Error caching user avatar`,
133 error: error,
134 });
135 res.sendStatus(500);
136 }
137 });
138
139 app.get("/api/v2/cache/emoji/:id", async (req: Request, res: Response) => {
140 let emojiUUID = req.params.id;
141 const emoji = emojiUUID
142 ? await Emoji.findOne({
143 where: {
144 uuid: emojiUUID,
145 },
146 })
147 : undefined;
148 if (emoji) {
149 await getMediaFromUrl(
150 emoji.external ? emoji.url : completeEnvironment.mediaUrl + emoji.url,
151 res
152 );
153 } else {
154 res.sendStatus(404);
155 }
156 });
157
158 app.get("/api/v2/cache/youtube/:id", async (req: Request, res: Response) => {
159 const youtubeId = decodeURIComponent(req.params.id);
160 const ytRegex =
161 /((?:https?:\/\/)?(www.|m.)?(youtube(\-nocookie)?\.com|youtu\.be)\/(v\/|watch\?v=|embed\/)?([\S]{11}))([^\S]|\?[\S]*|\&[\S]*|\b)/g;
162 const match = youtubeId.matchAll(ytRegex).toArray();
163 if (match && match.length >= 7) {
164 await getMediaFromUrl(
165 `https://img.youtube.com/vi/${match[6]}/hqdefault.jpg`,
166 res
167 );
168 } else {
169 res.sendStatus(404);
170 }
171 });
172
173 app.get("/api/v2/cache/favicon/:id", async (req: Request, res: Response) => {
174 try {
175 const link = new URL(decodeURIComponent(req.params.id));
176 await getMediaFromUrl("https://" + link.hostname + "/favicon.ico", res);
177 } catch (error) {
178 res.sendStatus(500);
179 }
180 });
181
182 app.get("/api/v2/cache/imageurl/:id", async (req: Request, res: Response) => {
183 try {
184 const link = decodeURIComponent(req.params.id);
185
186 const shasum = crypto.createHash("sha1");
187 shasum.update(link.toLowerCase());
188 const urlHash = shasum.digest("hex");
189 // Here is the thing: for this to be asked, the link component has to load first so we can assume cache has been set
190 const cacheResult = JSON.parse(
191 (await redisCache.get("linkPreviewCache:" + urlHash)) || "{}"
192 );
193 if (cacheResult && cacheResult.images && cacheResult.images[0]) {
194 await getMediaFromUrl(cacheResult.images[0], res);
195 } else {
196 res.sendStatus(404);
197 }
198 } catch (error) {
199 res.sendStatus(500);
200 }
201 });
202
203 app.get(
204 "/api/linkPreview",
205 linkPreviewRateLimiter,
206 async (req: Request, res: Response) => {
207 const url = String(req.query?.url);
208 const shasum = crypto.createHash("sha1");
209 shasum.update(url.toLowerCase());
210 const urlHash = shasum.digest("hex");
211 const cacheResult = await redisCache.get("linkPreviewCache:" + urlHash);
212 if (cacheResult) {
213 res.send(cacheResult);
214 } else {
215 let result = {};
216 try {
217 result = await getLinkPreview(url, {
218 followRedirects: "follow",
219 headers: { "User-Agent": completeEnvironment.instanceUrl },
220 });
221 } catch (error) {}
222 // we cache the url 24 hours if success, 5 minutes if not
223 await redisCache.set(
224 "linkPreviewCache:" + urlHash,
225 JSON.stringify(result),
226 "EX",
227 result ? 3600 * 24 : 300
228 );
229 res.send(result);
230 }
231 }
232 );
233}
234
235async function getMediaFromUrl(mediaUrl: string, res?: Response) {
236 try {
237 const mediaLinkHash = crypto
238 .createHash("sha256")
239 .update(mediaUrl)
240 .digest("hex");
241 let localFileName = `cache/${mediaLinkHash}`;
242 // if file exists
243 if (fs.existsSync(localFileName) && res) {
244 return await sendWithCache(res, localFileName);
245 } else {
246 try {
247 if (mediaUrl.startsWith("?cid=")) {
248 try {
249 const did = decodeURIComponent(mediaUrl.split("&did=")[1]);
250 const cid = decodeURIComponent(
251 mediaUrl.split("&did=")[0].split("?cid=")[1]
252 );
253 if ((!did || !cid) && res) {
254 return res.sendStatus(400);
255 }
256 const plcResolver = getResolver();
257 const didResolver = new Resolver(plcResolver);
258 const didData = await didResolver.resolve(did);
259 if (didData?.didDocument?.service) {
260 const url =
261 didData.didDocument.service[0].serviceEndpoint +
262 "/xrpc/com.atproto.sync.getBlob?did=" +
263 encodeURIComponent(did) +
264 "&cid=" +
265 encodeURIComponent(cid);
266 mediaUrl = url;
267 } else if (did.startsWith("did:web")) {
268 // get did doc first
269 const docRes = await fetch(
270 `https://${did.split("did:web:")[1]}/.well-known/did.json`
271 );
272 const didDoc = await docRes.json();
273 const atProtoServer = didDoc.service.find(
274 (x: any) =>
275 x.id === "#atproto_pds" ||
276 x.type === "AtprotoPersonalDataServer"
277 );
278 if (!atProtoServer && res) {
279 return res.sendStatus(500);
280 }
281 const url =
282 atProtoServer.serviceEndpoint +
283 "/xrpc/com.atproto.sync.getBlob?did=" +
284 encodeURIComponent(did) +
285 "&cid=" +
286 encodeURIComponent(cid);
287 mediaUrl = url;
288 }
289 } catch (error) {
290 if (res) {
291 return res.sendStatus(500);
292 }
293 }
294 }
295 const response = await axios.get(mediaUrl, {
296 responseType: "stream",
297 headers: { "User-Agent": "wafrnCacher" },
298 });
299 let altText = "";
300 /*
301 let dbMediaUrl = String(req.query?.media).startsWith(
302 completeEnvironment.mediaUrl
303 )
304 ? String(req.query?.media).split(completeEnvironment.mediaUrl)[1]
305 : String(req.query?.media);
306 // we are disabling this feature temporarily
307 let media = true
308 ? undefined
309 : await Media.findOne({
310 where: sequelize.where(
311 sequelize.fn("md5", sequelize.col("url")),
312 crypto.createHash("md5").update(dbMediaUrl).digest("hex")
313 ),
314 });
315 if (media) {
316 altText = media.description;
317 }
318 */
319
320 const { stream, mime } = await getMimeType(response.data);
321 if (res) {
322 res.contentType(mime);
323 }
324 await writeStream(stream, localFileName, mime, altText);
325 if (res) {
326 return await sendWithCache(res, localFileName);
327 }
328 } catch (error) {
329 if (res) {
330 return res.sendStatus(500);
331 }
332 }
333 }
334 } catch (error) {
335 logger.debug({
336 message: "Error with cacher",
337 url: mediaUrl,
338 error: error,
339 });
340 if (res) {
341 res.sendStatus(500);
342 }
343 }
344}
345
346export { cacheRoutes, getMediaFromUrl };