[mirror of https://git.0x0.st/mia/0x0] No-bullshit file hosting and URL shortening service https://0x0.st

Avoid holding in-memory copies of file content

Werkzeug uses tempfile.SpooledTemporaryFile, so we can make use of
file-like object properties. This may result in more disk writes,
but that’s probably better than eating up RAM.

I hope this fixes #84.

Changed files
+10 -8
+10 -8
fhost.py
··· 27 27 import sqlalchemy.types as types 28 28 from jinja2.exceptions import * 29 29 from jinja2 import ChoiceLoader, FileSystemLoader 30 - from hashlib import sha256 30 + from hashlib import file_digest 31 31 from magic import Magic 32 32 from mimetypes import guess_extension 33 33 import click ··· 248 248 @staticmethod 249 249 def store(file_, requested_expiration: typing.Optional[int], addr, ua, 250 250 secret: bool): 251 - data = file_.read() 252 - digest = sha256(data).hexdigest() 251 + fstream = file_.stream 252 + digest = file_digest(fstream, "sha256").hexdigest() 253 + fstream.seek(0, os.SEEK_END) 254 + flen = fstream.tell() 255 + fstream.seek(0) 253 256 254 257 def get_mime(): 255 - guess = mimedetect.from_buffer(data) 258 + guess = mimedetect.from_descriptor(fstream.fileno()) 256 259 app.logger.debug(f"MIME - specified: '{file_.content_type}' - " 257 260 f"detected: '{guess}'") 258 261 ··· 295 298 296 299 return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" 297 300 298 - expiration = File.get_expiration(requested_expiration, len(data)) 301 + expiration = File.get_expiration(requested_expiration, flen) 299 302 isnew = True 300 303 301 304 f = File.query.filter_by(sha256=digest).first() ··· 334 337 p = storage / digest 335 338 336 339 if not p.is_file(): 337 - with open(p, "wb") as of: 338 - of.write(data) 340 + file_.save(p) 339 341 340 - f.size = len(data) 342 + f.size = flen 341 343 342 344 if not f.nsfw_score and app.config["NSFW_DETECT"]: 343 345 f.nsfw_score = nsfw.detect(str(p))