[mirror of https://git.0x0.st/mia/0x0] No-bullshit file hosting and URL shortening service https://0x0.st

Avoid holding in-memory copies of file content

Werkzeug uses tempfile.SpooledTemporaryFile, so we can make use of
file-like object properties. This may result in more disk writes,
but that’s probably better than eating up RAM.

I hope this fixes #84.

Changed files
+10 -8
+10 -8
fhost.py
··· 27 import sqlalchemy.types as types 28 from jinja2.exceptions import * 29 from jinja2 import ChoiceLoader, FileSystemLoader 30 - from hashlib import sha256 31 from magic import Magic 32 from mimetypes import guess_extension 33 import click ··· 248 @staticmethod 249 def store(file_, requested_expiration: typing.Optional[int], addr, ua, 250 secret: bool): 251 - data = file_.read() 252 - digest = sha256(data).hexdigest() 253 254 def get_mime(): 255 - guess = mimedetect.from_buffer(data) 256 app.logger.debug(f"MIME - specified: '{file_.content_type}' - " 257 f"detected: '{guess}'") 258 ··· 295 296 return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" 297 298 - expiration = File.get_expiration(requested_expiration, len(data)) 299 isnew = True 300 301 f = File.query.filter_by(sha256=digest).first() ··· 334 p = storage / digest 335 336 if not p.is_file(): 337 - with open(p, "wb") as of: 338 - of.write(data) 339 340 - f.size = len(data) 341 342 if not f.nsfw_score and app.config["NSFW_DETECT"]: 343 f.nsfw_score = nsfw.detect(str(p))
··· 27 import sqlalchemy.types as types 28 from jinja2.exceptions import * 29 from jinja2 import ChoiceLoader, FileSystemLoader 30 + from hashlib import file_digest 31 from magic import Magic 32 from mimetypes import guess_extension 33 import click ··· 248 @staticmethod 249 def store(file_, requested_expiration: typing.Optional[int], addr, ua, 250 secret: bool): 251 + fstream = file_.stream 252 + digest = file_digest(fstream, "sha256").hexdigest() 253 + fstream.seek(0, os.SEEK_END) 254 + flen = fstream.tell() 255 + fstream.seek(0) 256 257 def get_mime(): 258 + guess = mimedetect.from_descriptor(fstream.fileno()) 259 app.logger.debug(f"MIME - specified: '{file_.content_type}' - " 260 f"detected: '{guess}'") 261 ··· 298 299 return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" 300 301 + expiration = File.get_expiration(requested_expiration, flen) 302 isnew = True 303 304 f = File.query.filter_by(sha256=digest).first() ··· 337 p = storage / digest 338 339 if not p.is_file(): 340 + file_.save(p) 341 342 + f.size = flen 343 344 if not f.nsfw_score and app.config["NSFW_DETECT"]: 345 f.nsfw_score = nsfw.detect(str(p))