[mirror of https://git.0x0.st/mia/0x0] No-bullshit file hosting and URL shortening service https://0x0.st

PEP8 compliance

+1 -1
cleanup.py
··· 5 print("") 6 print(" $ FLASK_APP=fhost flask prune") 7 print("") 8 - exit(1);
··· 5 print("") 6 print(" $ FLASK_APP=fhost flask prune") 7 print("") 8 + exit(1)
+146 -104
fhost.py
··· 1 #!/usr/bin/env python3 2 - # -*- coding: utf-8 -*- 3 4 """ 5 - Copyright © 2020 Mia Herkt 6 Licensed under the EUPL, Version 1.2 or - as soon as approved 7 by the European Commission - subsequent versions of the EUPL 8 (the "License"); ··· 19 and limitations under the License. 20 """ 21 22 - from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template, Request 23 from flask_sqlalchemy import SQLAlchemy 24 from flask_migrate import Migrate 25 from sqlalchemy import and_, or_ ··· 46 47 app = Flask(__name__, instance_relative_config=True) 48 app.config.update( 49 - SQLALCHEMY_TRACK_MODIFICATIONS = False, 50 - PREFERRED_URL_SCHEME = "https", # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config 51 - MAX_CONTENT_LENGTH = 256 * 1024 * 1024, 52 - MAX_URL_LENGTH = 4096, 53 - USE_X_SENDFILE = False, 54 - FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default 55 - FHOST_STORAGE_PATH = "up", 56 - FHOST_MAX_EXT_LENGTH = 9, 57 - FHOST_SECRET_BYTES = 16, 58 - FHOST_EXT_OVERRIDE = { 59 - "audio/flac" : ".flac", 60 - "image/gif" : ".gif", 61 - "image/jpeg" : ".jpg", 62 - "image/png" : ".png", 63 - "image/svg+xml" : ".svg", 64 - "video/webm" : ".webm", 65 - "video/x-matroska" : ".mkv", 66 - "application/octet-stream" : ".bin", 67 - "text/plain" : ".log", 68 - "text/plain" : ".txt", 69 - "text/x-diff" : ".diff", 70 }, 71 - NSFW_DETECT = False, 72 - NSFW_THRESHOLD = 0.92, 73 - VSCAN_SOCKET = None, 74 - VSCAN_QUARANTINE_PATH = "quarantine", 75 - VSCAN_IGNORE = [ 76 "Eicar-Test-Signature", 77 "PUA.Win.Packer.XmMusicFile", 78 ], 79 - VSCAN_INTERVAL = datetime.timedelta(days=7), 80 - URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-", 81 ) 82 83 app.config.from_pyfile("config.py") ··· 95 96 try: 97 mimedetect = Magic(mime=True, mime_encoding=False) 98 - except: 99 print("""Error: You have installed the wrong version of the 'magic' module. 100 Please install python-magic.""") 101 sys.exit(1) ··· 103 db = SQLAlchemy(app) 104 migrate = Migrate(app, db) 105 106 class URL(db.Model): 107 __tablename__ = "URL" 108 - id = db.Column(db.Integer, primary_key = True) 109 - url = db.Column(db.UnicodeText, unique = True) 110 111 def __init__(self, url): 112 self.url = url ··· 117 def geturl(self): 118 return url_for("get", path=self.getname(), _external=True) + "\n" 119 120 def get(url): 121 u = URL.query.filter_by(url=url).first() 122 ··· 126 db.session.commit() 127 128 return u 129 130 class IPAddress(types.TypeDecorator): 131 impl = types.LargeBinary ··· 150 151 152 class File(db.Model): 153 - id = db.Column(db.Integer, primary_key = True) 154 - sha256 = db.Column(db.String, unique = True) 155 ext = db.Column(db.UnicodeText) 156 mime = db.Column(db.UnicodeText) 157 addr = db.Column(IPAddress(16)) ··· 175 176 @property 177 def is_nsfw(self) -> bool: 178 - return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"] 179 180 def getname(self): 181 return u"{0}{1}".format(su.enbase(self.id), self.ext) 182 183 def geturl(self): 184 n = self.getname() 185 186 - if self.is_nsfw: 187 - return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n" 188 - else: 189 - return url_for("get", path=n, secret=self.secret, _external=True) + "\n" 190 191 def getpath(self) -> Path: 192 return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256 ··· 197 self.removed = permanent 198 self.getpath().unlink(missing_ok=True) 199 200 - # Returns the epoch millisecond that a file should expire 201 - # 202 - # Uses the expiration time provided by the user (requested_expiration) 203 - # upper-bounded by an algorithm that computes the size based on the size of the 204 - # file. 205 - # 206 - # That is, all files are assigned a computed expiration, which can voluntarily 207 - # shortened by the user either by providing a timestamp in epoch millis or a 208 - # duration in hours. 209 def get_expiration(requested_expiration, size) -> int: 210 - current_epoch_millis = time.time() * 1000; 211 212 # Maximum lifetime of the file in milliseconds 213 - this_files_max_lifespan = get_max_lifespan(size); 214 215 # The latest allowed expiration date for this file, in epoch millis 216 - this_files_max_expiration = this_files_max_lifespan + 1000 * time.time(); 217 218 if requested_expiration is None: 219 - return this_files_max_expiration 220 elif requested_expiration < 1650460320000: 221 # Treat the requested expiration time as a duration in hours 222 requested_expiration_ms = requested_expiration * 60 * 60 * 1000 223 - return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms) 224 else: 225 - # Treat the requested expiration time as a timestamp in epoch millis 226 - return min(this_files_max_expiration, requested_expiration) 227 228 """ 229 requested_expiration can be: ··· 231 - a duration (in hours) that the file should live for 232 - a timestamp in epoch millis that the file should expire at 233 234 - Any value greater that the longest allowed file lifespan will be rounded down to that 235 - value. 236 """ 237 - def store(file_, requested_expiration: typing.Optional[int], addr, ua, secret: bool): 238 data = file_.read() 239 digest = sha256(data).hexdigest() 240 241 def get_mime(): 242 guess = mimedetect.from_buffer(data) 243 - app.logger.debug(f"MIME - specified: '{file_.content_type}' - detected: '{guess}'") 244 245 - if not file_.content_type or not "/" in file_.content_type or file_.content_type == "application/octet-stream": 246 mime = guess 247 else: 248 mime = file_.content_type ··· 254 if flt.check(guess): 255 abort(403, flt.reason) 256 257 - if mime.startswith("text/") and not "charset" in mime: 258 mime += "; charset=utf-8" 259 260 return mime ··· 266 gmime = mime.split(";")[0] 267 guess = guess_extension(gmime) 268 269 - app.logger.debug(f"extension - specified: '{ext}' - detected: '{guess}'") 270 271 if not ext: 272 if gmime in app.config["FHOST_EXT_OVERRIDE"]: ··· 309 if isnew: 310 f.secret = None 311 if secret: 312 - f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"]) 313 314 storage = Path(app.config["FHOST_STORAGE_PATH"]) 315 storage.mkdir(parents=True, exist_ok=True) ··· 451 452 453 class UrlEncoder(object): 454 - def __init__(self,alphabet, min_length): 455 self.alphabet = alphabet 456 self.min_length = min_length 457 ··· 471 result += self.alphabet.index(c) * (n ** i) 472 return result 473 474 su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1) 475 476 def fhost_url(scheme=None): 477 if not scheme: 478 return url_for(".fhost", _external=True).rstrip("/") 479 else: 480 return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/") 481 482 def is_fhost_url(url): 483 return url.startswith(fhost_url()) or url.startswith(fhost_url("https")) 484 485 def shorten(url): 486 if len(url) > app.config["MAX_URL_LENGTH"]: 487 abort(414) ··· 493 494 return u.geturl() 495 496 """ 497 requested_expiration can be: 498 - None, to use the longest allowed file lifespan 499 - a duration (in hours) that the file should live for 500 - a timestamp in epoch millis that the file should expire at 501 502 - Any value greater that the longest allowed file lifespan will be rounded down to that 503 - value. 504 """ 505 - def store_file(f, requested_expiration: typing.Optional[int], addr, ua, secret: bool): 506 sf, isnew = File.store(f, requested_expiration, addr, ua, secret) 507 508 response = make_response(sf.geturl()) ··· 513 514 return response 515 516 def store_url(url, addr, ua, secret: bool): 517 if is_fhost_url(url): 518 abort(400) 519 520 - h = { "Accept-Encoding" : "identity" } 521 r = requests.get(url, stream=True, verify=False, headers=h) 522 523 try: ··· 526 return str(e) + "\n" 527 528 if "content-length" in r.headers: 529 - l = int(r.headers["content-length"]) 530 531 - if l <= app.config["MAX_CONTENT_LENGTH"]: 532 def urlfile(**kwargs): 533 - return type('',(),kwargs)() 534 535 - f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="") 536 537 return store_file(f, None, addr, ua, secret) 538 else: ··· 540 else: 541 abort(411) 542 543 def manage_file(f): 544 - try: 545 - assert(request.form["token"] == f.mgmt_token) 546 - except: 547 abort(401) 548 549 if "delete" in request.form: ··· 561 return "", 202 562 563 abort(400) 564 565 @app.route("/<path:path>", methods=["GET", "POST"]) 566 @app.route("/s/<secret>/<path:path>", methods=["GET", "POST"]) ··· 598 response.headers["Content-Length"] = f.size 599 response.headers["X-Accel-Redirect"] = "/" + str(fpath) 600 else: 601 - response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime) 602 603 response.headers["X-Expires"] = f.expiration 604 return response ··· 615 return redirect(u.url) 616 617 abort(404) 618 619 @app.route("/", methods=["GET", "POST"]) 620 def fhost(): ··· 665 else: 666 return render_template("index.html") 667 668 @app.route("/robots.txt") 669 def robots(): 670 return """User-agent: * 671 Disallow: / 672 """ 673 674 @app.errorhandler(400) 675 @app.errorhandler(401) ··· 682 @app.errorhandler(451) 683 def ehandler(e): 684 try: 685 - return render_template(f"{e.code}.html", id=id, request=request, description=e.description), e.code 686 except TemplateNotFound: 687 return "Segmentation fault\n", e.code 688 689 @app.cli.command("prune") 690 def prune(): 691 """ 692 Clean up expired files 693 694 - Deletes any files from the filesystem which have hit their expiration time. This 695 - doesn't remove them from the database, only from the filesystem. It's recommended 696 - that server owners run this command regularly, or set it up on a timer. 697 """ 698 - current_time = time.time() * 1000; 699 700 # The path to where uploaded files are stored 701 storage = Path(app.config["FHOST_STORAGE_PATH"]) ··· 709 ) 710 ) 711 712 - files_removed = 0; 713 714 # For every expired file... 715 for file in expired_files: ··· 722 # Remove it from the file system 723 try: 724 os.remove(file_path) 725 - files_removed += 1; 726 except FileNotFoundError: 727 - pass # If the file was already gone, we're good 728 except OSError as e: 729 print(e) 730 print( 731 "\n------------------------------------" 732 - "Encountered an error while trying to remove file {file_path}. Double" 733 - "check to make sure the server is configured correctly, permissions are" 734 - "okay, and everything is ship shape, then try again.") 735 - return; 736 737 # Finally, mark that the file was removed 738 - file.expiration = None; 739 db.session.commit() 740 741 print(f"\nDone! {files_removed} file(s) removed") 742 743 - """ For a file of a given size, determine the largest allowed lifespan of that file 744 745 - Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well 746 - as FHOST_{MIN,MAX}_EXPIRATION. 747 748 - This lifespan may be shortened by a user's request, but no files should be allowed to 749 - expire at a point after this number. 750 751 Value returned is a duration in milliseconds. 752 """ ··· 756 max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) 757 return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) 758 759 def do_vscan(f): 760 if f["path"].is_file(): 761 with open(f["path"], "rb") as scanf: 762 try: 763 - f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0] 764 except: 765 f["result"] = ("SCAN FAILED", None) 766 else: ··· 768 769 return f 770 771 @app.cli.command("vscan") 772 def vscan(): 773 if not app.config["VSCAN_SOCKET"]: 774 - print("""Error: Virus scanning enabled but no connection method specified. 775 - Please set VSCAN_SOCKET.""") 776 sys.exit(1) 777 778 qp = Path(app.config["VSCAN_QUARANTINE_PATH"]) ··· 786 File.last_vscan == None), 787 File.removed == False) 788 else: 789 - res = File.query.filter(File.last_vscan == None, File.removed == False) 790 791 - work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res] 792 793 results = [] 794 for i, r in enumerate(p.imap_unordered(do_vscan, work)): ··· 802 found = True 803 804 results.append({ 805 - "id" : r["id"], 806 - "last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(), 807 - "removed" : found}) 808 809 db.session.bulk_update_mappings(File, results) 810 db.session.commit()
··· 1 #!/usr/bin/env python3 2 3 """ 4 + Copyright © 2024 Mia Herkt 5 Licensed under the EUPL, Version 1.2 or - as soon as approved 6 by the European Commission - subsequent versions of the EUPL 7 (the "License"); ··· 18 and limitations under the License. 19 """ 20 21 + from flask import Flask, abort, make_response, redirect, render_template, \ 22 + Request, request, Response, send_from_directory, url_for 23 from flask_sqlalchemy import SQLAlchemy 24 from flask_migrate import Migrate 25 from sqlalchemy import and_, or_ ··· 46 47 app = Flask(__name__, instance_relative_config=True) 48 app.config.update( 49 + SQLALCHEMY_TRACK_MODIFICATIONS=False, 50 + PREFERRED_URL_SCHEME="https", # nginx users: make sure to have 51 + # 'uwsgi_param UWSGI_SCHEME $scheme;' in 52 + # your config 53 + MAX_CONTENT_LENGTH=256 * 1024 * 1024, 54 + MAX_URL_LENGTH=4096, 55 + USE_X_SENDFILE=False, 56 + FHOST_USE_X_ACCEL_REDIRECT=True, # expect nginx by default 57 + FHOST_STORAGE_PATH="up", 58 + FHOST_MAX_EXT_LENGTH=9, 59 + FHOST_SECRET_BYTES=16, 60 + FHOST_EXT_OVERRIDE={ 61 + "audio/flac": ".flac", 62 + "image/gif": ".gif", 63 + "image/jpeg": ".jpg", 64 + "image/png": ".png", 65 + "image/svg+xml": ".svg", 66 + "video/webm": ".webm", 67 + "video/x-matroska": ".mkv", 68 + "application/octet-stream": ".bin", 69 + "text/plain": ".log", 70 + "text/plain": ".txt", 71 + "text/x-diff": ".diff", 72 }, 73 + NSFW_DETECT=False, 74 + NSFW_THRESHOLD=0.92, 75 + VSCAN_SOCKET=None, 76 + VSCAN_QUARANTINE_PATH="quarantine", 77 + VSCAN_IGNORE=[ 78 "Eicar-Test-Signature", 79 "PUA.Win.Packer.XmMusicFile", 80 ], 81 + VSCAN_INTERVAL=datetime.timedelta(days=7), 82 + URL_ALPHABET="DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMX" 83 + "y6Vx-", 84 ) 85 86 app.config.from_pyfile("config.py") ··· 98 99 try: 100 mimedetect = Magic(mime=True, mime_encoding=False) 101 + except TypeError: 102 print("""Error: You have installed the wrong version of the 'magic' module. 103 Please install python-magic.""") 104 sys.exit(1) ··· 106 db = SQLAlchemy(app) 107 migrate = Migrate(app, db) 108 109 + 110 class URL(db.Model): 111 __tablename__ = "URL" 112 + id = db.Column(db.Integer, primary_key=True) 113 + url = db.Column(db.UnicodeText, unique=True) 114 115 def __init__(self, url): 116 self.url = url ··· 121 def geturl(self): 122 return url_for("get", path=self.getname(), _external=True) + "\n" 123 124 + @staticmethod 125 def get(url): 126 u = URL.query.filter_by(url=url).first() 127 ··· 131 db.session.commit() 132 133 return u 134 + 135 136 class IPAddress(types.TypeDecorator): 137 impl = types.LargeBinary ··· 156 157 158 class File(db.Model): 159 + id = db.Column(db.Integer, primary_key=True) 160 + sha256 = db.Column(db.String, unique=True) 161 ext = db.Column(db.UnicodeText) 162 mime = db.Column(db.UnicodeText) 163 addr = db.Column(IPAddress(16)) ··· 181 182 @property 183 def is_nsfw(self) -> bool: 184 + if self.nsfw_score: 185 + return self.nsfw_score > app.config["NSFW_THRESHOLD"] 186 + return False 187 188 def getname(self): 189 return u"{0}{1}".format(su.enbase(self.id), self.ext) 190 191 def geturl(self): 192 n = self.getname() 193 + a = "nsfw" if self.is_nsfw else None 194 195 + return url_for("get", path=n, secret=self.secret, 196 + _external=True, _anchor=a) + "\n" 197 198 def getpath(self) -> Path: 199 return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256 ··· 204 self.removed = permanent 205 self.getpath().unlink(missing_ok=True) 206 207 + """ 208 + Returns the epoch millisecond that a file should expire 209 + 210 + Uses the expiration time provided by the user (requested_expiration) 211 + upper-bounded by an algorithm that computes the size based on the size of 212 + the file. 213 + 214 + That is, all files are assigned a computed expiration, which can be 215 + voluntarily shortened by the user either by providing a timestamp in 216 + milliseconds since epoch or a duration in hours. 217 + """ 218 + @staticmethod 219 def get_expiration(requested_expiration, size) -> int: 220 + current_epoch_millis = time.time() * 1000 221 222 # Maximum lifetime of the file in milliseconds 223 + max_lifespan = get_max_lifespan(size) 224 225 # The latest allowed expiration date for this file, in epoch millis 226 + max_expiration = max_lifespan + 1000 * time.time() 227 228 if requested_expiration is None: 229 + return max_expiration 230 elif requested_expiration < 1650460320000: 231 # Treat the requested expiration time as a duration in hours 232 requested_expiration_ms = requested_expiration * 60 * 60 * 1000 233 + return min(max_expiration, 234 + current_epoch_millis + requested_expiration_ms) 235 else: 236 + # Treat expiration time as a timestamp in epoch millis 237 + return min(max_expiration, requested_expiration) 238 239 """ 240 requested_expiration can be: ··· 242 - a duration (in hours) that the file should live for 243 - a timestamp in epoch millis that the file should expire at 244 245 + Any value greater that the longest allowed file lifespan will be rounded 246 + down to that value. 247 """ 248 + @staticmethod 249 + def store(file_, requested_expiration: typing.Optional[int], addr, ua, 250 + secret: bool): 251 data = file_.read() 252 digest = sha256(data).hexdigest() 253 254 def get_mime(): 255 guess = mimedetect.from_buffer(data) 256 + app.logger.debug(f"MIME - specified: '{file_.content_type}' - " 257 + f"detected: '{guess}'") 258 259 + if (not file_.content_type 260 + or "/" not in file_.content_type 261 + or file_.content_type == "application/octet-stream"): 262 mime = guess 263 else: 264 mime = file_.content_type ··· 270 if flt.check(guess): 271 abort(403, flt.reason) 272 273 + if mime.startswith("text/") and "charset" not in mime: 274 mime += "; charset=utf-8" 275 276 return mime ··· 282 gmime = mime.split(";")[0] 283 guess = guess_extension(gmime) 284 285 + app.logger.debug(f"extension - specified: '{ext}' - detected: " 286 + f"'{guess}'") 287 288 if not ext: 289 if gmime in app.config["FHOST_EXT_OVERRIDE"]: ··· 326 if isnew: 327 f.secret = None 328 if secret: 329 + f.secret = \ 330 + secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"]) 331 332 storage = Path(app.config["FHOST_STORAGE_PATH"]) 333 storage.mkdir(parents=True, exist_ok=True) ··· 469 470 471 class UrlEncoder(object): 472 + def __init__(self, alphabet, min_length): 473 self.alphabet = alphabet 474 self.min_length = min_length 475 ··· 489 result += self.alphabet.index(c) * (n ** i) 490 return result 491 492 + 493 su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1) 494 + 495 496 def fhost_url(scheme=None): 497 if not scheme: 498 return url_for(".fhost", _external=True).rstrip("/") 499 else: 500 return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/") 501 + 502 503 def is_fhost_url(url): 504 return url.startswith(fhost_url()) or url.startswith(fhost_url("https")) 505 506 + 507 def shorten(url): 508 if len(url) > app.config["MAX_URL_LENGTH"]: 509 abort(414) ··· 515 516 return u.geturl() 517 518 + 519 """ 520 requested_expiration can be: 521 - None, to use the longest allowed file lifespan 522 - a duration (in hours) that the file should live for 523 - a timestamp in epoch millis that the file should expire at 524 525 + Any value greater that the longest allowed file lifespan will be rounded down 526 + to that value. 527 """ 528 + def store_file(f, requested_expiration: typing.Optional[int], addr, ua, 529 + secret: bool): 530 sf, isnew = File.store(f, requested_expiration, addr, ua, secret) 531 532 response = make_response(sf.geturl()) ··· 537 538 return response 539 540 + 541 def store_url(url, addr, ua, secret: bool): 542 if is_fhost_url(url): 543 abort(400) 544 545 + h = {"Accept-Encoding": "identity"} 546 r = requests.get(url, stream=True, verify=False, headers=h) 547 548 try: ··· 551 return str(e) + "\n" 552 553 if "content-length" in r.headers: 554 + length = int(r.headers["content-length"]) 555 556 + if length <= app.config["MAX_CONTENT_LENGTH"]: 557 def urlfile(**kwargs): 558 + return type('', (), kwargs)() 559 560 + f = urlfile(read=r.raw.read, 561 + content_type=r.headers["content-type"], filename="") 562 563 return store_file(f, None, addr, ua, secret) 564 else: ··· 566 else: 567 abort(411) 568 569 + 570 def manage_file(f): 571 + if request.form["token"] != f.mgmt_token: 572 abort(401) 573 574 if "delete" in request.form: ··· 586 return "", 202 587 588 abort(400) 589 + 590 591 @app.route("/<path:path>", methods=["GET", "POST"]) 592 @app.route("/s/<secret>/<path:path>", methods=["GET", "POST"]) ··· 624 response.headers["Content-Length"] = f.size 625 response.headers["X-Accel-Redirect"] = "/" + str(fpath) 626 else: 627 + response = send_from_directory( 628 + app.config["FHOST_STORAGE_PATH"], f.sha256, 629 + mimetype=f.mime) 630 631 response.headers["X-Expires"] = f.expiration 632 return response ··· 643 return redirect(u.url) 644 645 abort(404) 646 + 647 648 @app.route("/", methods=["GET", "POST"]) 649 def fhost(): ··· 694 else: 695 return render_template("index.html") 696 697 + 698 @app.route("/robots.txt") 699 def robots(): 700 return """User-agent: * 701 Disallow: / 702 """ 703 + 704 705 @app.errorhandler(400) 706 @app.errorhandler(401) ··· 713 @app.errorhandler(451) 714 def ehandler(e): 715 try: 716 + return render_template(f"{e.code}.html", id=id, request=request, 717 + description=e.description), e.code 718 except TemplateNotFound: 719 return "Segmentation fault\n", e.code 720 + 721 722 @app.cli.command("prune") 723 def prune(): 724 """ 725 Clean up expired files 726 727 + Deletes any files from the filesystem which have hit their expiration time. 728 + This doesn't remove them from the database, only from the filesystem. 729 + It is recommended that server owners run this command regularly, or set it 730 + up on a timer. 731 """ 732 + current_time = time.time() * 1000 733 734 # The path to where uploaded files are stored 735 storage = Path(app.config["FHOST_STORAGE_PATH"]) ··· 743 ) 744 ) 745 746 + files_removed = 0 747 748 # For every expired file... 749 for file in expired_files: ··· 756 # Remove it from the file system 757 try: 758 os.remove(file_path) 759 + files_removed += 1 760 except FileNotFoundError: 761 + pass # If the file was already gone, we're good 762 except OSError as e: 763 print(e) 764 print( 765 "\n------------------------------------" 766 + "Encountered an error while trying to remove file {file_path}." 767 + "Make sure the server is configured correctly, permissions " 768 + "are okay, and everything is ship shape, then try again.") 769 + return 770 771 # Finally, mark that the file was removed 772 + file.expiration = None 773 db.session.commit() 774 775 print(f"\nDone! {files_removed} file(s) removed") 776 777 778 + """ 779 + For a file of a given size, determine the largest allowed lifespan of that file 780 781 + Based on the current app's configuration: 782 + Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION. 783 + 784 + This lifespan may be shortened by a user's request, but no files should be 785 + allowed to expire at a point after this number. 786 787 Value returned is a duration in milliseconds. 788 """ ··· 792 max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) 793 return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) 794 795 + 796 def do_vscan(f): 797 if f["path"].is_file(): 798 with open(f["path"], "rb") as scanf: 799 try: 800 + res = list(app.config["VSCAN_SOCKET"].instream(scanf).values()) 801 + f["result"] = res[0] 802 except: 803 f["result"] = ("SCAN FAILED", None) 804 else: ··· 806 807 return f 808 809 + 810 @app.cli.command("vscan") 811 def vscan(): 812 if not app.config["VSCAN_SOCKET"]: 813 + print("Error: Virus scanning enabled but no connection method " 814 + "specified.\nPlease set VSCAN_SOCKET.") 815 sys.exit(1) 816 817 qp = Path(app.config["VSCAN_QUARANTINE_PATH"]) ··· 825 File.last_vscan == None), 826 File.removed == False) 827 else: 828 + res = File.query.filter(File.last_vscan == None, 829 + File.removed == False) 830 831 + work = [{"path": f.getpath(), "name": f.getname(), "id": f.id} 832 + for f in res] 833 834 results = [] 835 for i, r in enumerate(p.imap_unordered(do_vscan, work)): ··· 843 found = True 844 845 results.append({ 846 + "id": r["id"], 847 + "last_vscan": None if r["result"][0] == "SCAN FAILED" 848 + else datetime.datetime.now(), 849 + "removed": found}) 850 851 db.session.bulk_update_mappings(File, results) 852 db.session.commit()
+1
migrations/env.py
··· 81 finally: 82 connection.close() 83 84 if context.is_offline_mode(): 85 run_migrations_offline() 86 else:
··· 81 finally: 82 connection.close() 83 84 + 85 if context.is_offline_mode(): 86 run_migrations_offline() 87 else:
-4
migrations/versions/0659d7b9eea8_.py
··· 15 16 17 def upgrade(): 18 - # ### commands auto generated by Alembic - please adjust! ### 19 op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True)) 20 - # ### end Alembic commands ### 21 22 23 def downgrade(): 24 - # ### commands auto generated by Alembic - please adjust! ### 25 op.drop_column('file', 'mgmt_token') 26 - # ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True)) 19 20 21 def downgrade(): 22 op.drop_column('file', 'mgmt_token')
+12 -18
migrations/versions/0cd36ecdd937_.py
··· 15 16 17 def upgrade(): 18 - ### commands auto generated by Alembic - please adjust! ### 19 op.create_table('URL', 20 - sa.Column('id', sa.Integer(), nullable=False), 21 - sa.Column('url', sa.UnicodeText(), nullable=True), 22 - sa.PrimaryKeyConstraint('id'), 23 - sa.UniqueConstraint('url') 24 - ) 25 op.create_table('file', 26 - sa.Column('id', sa.Integer(), nullable=False), 27 - sa.Column('sha256', sa.String(), nullable=True), 28 - sa.Column('ext', sa.UnicodeText(), nullable=True), 29 - sa.Column('mime', sa.UnicodeText(), nullable=True), 30 - sa.Column('addr', sa.UnicodeText(), nullable=True), 31 - sa.Column('removed', sa.Boolean(), nullable=True), 32 - sa.PrimaryKeyConstraint('id'), 33 - sa.UniqueConstraint('sha256') 34 - ) 35 - ### end Alembic commands ### 36 37 38 def downgrade(): 39 - ### commands auto generated by Alembic - please adjust! ### 40 op.drop_table('file') 41 op.drop_table('URL') 42 - ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 op.create_table('URL', 19 + sa.Column('id', sa.Integer(), nullable=False), 20 + sa.Column('url', sa.UnicodeText(), nullable=True), 21 + sa.PrimaryKeyConstraint('id'), 22 + sa.UniqueConstraint('url')) 23 op.create_table('file', 24 + sa.Column('id', sa.Integer(), nullable=False), 25 + sa.Column('sha256', sa.String(), nullable=True), 26 + sa.Column('ext', sa.UnicodeText(), nullable=True), 27 + sa.Column('mime', sa.UnicodeText(), nullable=True), 28 + sa.Column('addr', sa.UnicodeText(), nullable=True), 29 + sa.Column('removed', sa.Boolean(), nullable=True), 30 + sa.PrimaryKeyConstraint('id'), 31 + sa.UniqueConstraint('sha256')) 32 33 34 def downgrade(): 35 op.drop_table('file') 36 op.drop_table('URL')
+3 -2
migrations/versions/30bfe33aa328_add_file_size_field.py
··· 19 20 Base = automap_base() 21 22 def upgrade(): 23 op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True)) 24 bind = op.get_bind() ··· 34 p = storage / f.sha256 35 if p.is_file(): 36 updates.append({ 37 - "id" : f.id, 38 - "size" : p.stat().st_size 39 }) 40 41 session.bulk_update_mappings(File, updates)
··· 19 20 Base = automap_base() 21 22 + 23 def upgrade(): 24 op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True)) 25 bind = op.get_bind() ··· 35 p = storage / f.sha256 36 if p.is_file(): 37 updates.append({ 38 + "id": f.id, 39 + "size": p.stat().st_size 40 }) 41 42 session.bulk_update_mappings(File, updates)
+31 -32
migrations/versions/5cda1743b92d_add_request_filters.py
··· 19 20 Base = automap_base() 21 22 def upgrade(): 23 - # ### commands auto generated by Alembic - please adjust! ### 24 op.create_table('request_filter', 25 - sa.Column('id', sa.Integer(), nullable=False), 26 - sa.Column('type', sa.String(length=20), nullable=False), 27 - sa.Column('comment', sa.UnicodeText(), nullable=True), 28 - sa.Column('addr', sa.LargeBinary(length=16), nullable=True), 29 - sa.Column('net', sa.Text(), nullable=True), 30 - sa.Column('regex', sa.UnicodeText(), nullable=True), 31 - sa.PrimaryKeyConstraint('id'), 32 - sa.UniqueConstraint('addr') 33 - ) 34 - with op.batch_alter_table('request_filter', schema=None) as batch_op: 35 - batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], unique=False) 36 37 - # ### end Alembic commands ### 38 39 bind = op.get_bind() 40 Base.prepare(autoload_with=bind) 41 RequestFilter = Base.classes.request_filter 42 session = Session(bind=bind) 43 44 - if "FHOST_UPLOAD_BLACKLIST" in current_app.config: 45 - if current_app.config["FHOST_UPLOAD_BLACKLIST"]: 46 - with current_app.open_instance_resource(current_app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl: 47 - for l in bl.readlines(): 48 - if not l.startswith("#"): 49 - l = l.strip() 50 - if l.endswith(":"): 51 - # old implementation uses str.startswith, 52 - # which does not translate to networks 53 - current_app.logger.warning(f"Ignored address: {l}") 54 - continue 55 56 - flt = RequestFilter(type="addr", addr=ipaddress.ip_address(l).packed) 57 - session.add(flt) 58 59 - if "FHOST_MIME_BLACKLIST" in current_app.config: 60 - for mime in current_app.config["FHOST_MIME_BLACKLIST"]: 61 - flt = RequestFilter(type="mime", regex=mime) 62 - session.add(flt) 63 64 session.commit() 65 ··· 72 73 74 def downgrade(): 75 - # ### commands auto generated by Alembic - please adjust! ### 76 with op.batch_alter_table('request_filter', schema=None) as batch_op: 77 batch_op.drop_index(batch_op.f('ix_request_filter_type')) 78 79 op.drop_table('request_filter') 80 - # ### end Alembic commands ###
··· 19 20 Base = automap_base() 21 22 + 23 def upgrade(): 24 op.create_table('request_filter', 25 + sa.Column('id', sa.Integer(), nullable=False), 26 + sa.Column('type', sa.String(length=20), nullable=False), 27 + sa.Column('comment', sa.UnicodeText(), nullable=True), 28 + sa.Column('addr', sa.LargeBinary(length=16), 29 + nullable=True), 30 + sa.Column('net', sa.Text(), nullable=True), 31 + sa.Column('regex', sa.UnicodeText(), nullable=True), 32 + sa.PrimaryKeyConstraint('id'), 33 + sa.UniqueConstraint('addr')) 34 35 + with op.batch_alter_table('request_filter', schema=None) as batch_op: 36 + batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], 37 + unique=False) 38 39 bind = op.get_bind() 40 Base.prepare(autoload_with=bind) 41 RequestFilter = Base.classes.request_filter 42 session = Session(bind=bind) 43 44 + blp = current_app.config.get("FHOST_UPLOAD_BLACKLIST") 45 + if blp: 46 + with current_app.open_instance_resource(blp, "r") as bl: 47 + for line in bl.readlines(): 48 + if not line.startswith("#"): 49 + line = line.strip() 50 + if line.endswith(":"): 51 + # old implementation uses str.startswith, 52 + # which does not translate to networks 53 + current_app.logger.warning( 54 + f"Ignored address: {line}") 55 + continue 56 57 + addr = ipaddress.ip_address(line).packed 58 + flt = RequestFilter(type="addr", addr=addr) 59 + session.add(flt) 60 61 + for mime in current_app.config.get("FHOST_MIME_BLACKLIST", []): 62 + flt = RequestFilter(type="mime", regex=mime) 63 + session.add(flt) 64 65 session.commit() 66 ··· 73 74 75 def downgrade(): 76 with op.batch_alter_table('request_filter', schema=None) as batch_op: 77 batch_op.drop_index(batch_op.f('ix_request_filter_type')) 78 79 op.drop_table('request_filter')
+2 -5
migrations/versions/5cee97aab219_.py
··· 15 16 17 def upgrade(): 18 - # ### commands auto generated by Alembic - please adjust! ### 19 - op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True)) 20 - # ### end Alembic commands ### 21 22 23 def downgrade(): 24 - # ### commands auto generated by Alembic - please adjust! ### 25 op.drop_column('file', 'last_vscan') 26 - # ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 + op.add_column('file', sa.Column('last_vscan', sa.DateTime(), 19 + nullable=True)) 20 21 22 def downgrade(): 23 op.drop_column('file', 'last_vscan')
-4
migrations/versions/7e246705da6a_.py
··· 15 16 17 def upgrade(): 18 - # ### commands auto generated by Alembic - please adjust! ### 19 op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True)) 20 - # ### end Alembic commands ### 21 22 23 def downgrade(): 24 - # ### commands auto generated by Alembic - please adjust! ### 25 op.drop_column('file', 'nsfw_score') 26 - # ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True)) 19 20 21 def downgrade(): 22 op.drop_column('file', 'nsfw_score')
+24 -14
migrations/versions/939a08e1d6e5_.py
··· 21 import os 22 import time 23 24 - """ For a file of a given size, determine the largest allowed lifespan of that file 25 26 - Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well 27 - as FHOST_{MIN,MAX}_EXPIRATION. 28 29 - This lifespan may be shortened by a user's request, but no files should be allowed to 30 - expire at a point after this number. 31 32 Value returned is a duration in milliseconds. 33 """ 34 def get_max_lifespan(filesize: int) -> int: 35 - min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) 36 - max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) 37 - max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) 38 return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) 39 40 Base = automap_base() 41 42 def upgrade(): 43 op.add_column('file', sa.Column('expiration', sa.BigInteger())) ··· 48 session = Session(bind=bind) 49 50 storage = Path(current_app.config["FHOST_STORAGE_PATH"]) 51 - current_time = time.time() * 1000; 52 53 # List of file hashes which have not expired yet 54 # This could get really big for some servers 55 try: 56 unexpired_files = os.listdir(storage) 57 except FileNotFoundError: 58 - return # There are no currently unexpired files 59 60 # Calculate an expiration date for all existing files 61 ··· 65 sa.not_(File.removed) 66 ) 67 ) 68 - updates = [] # We coalesce updates to the database here 69 70 # SQLite has a hard limit on the number of variables so we 71 # need to do this the slow way ··· 74 for file in files: 75 file_path = storage / file.sha256 76 stat = os.stat(file_path) 77 - max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms 78 - file_birth = stat.st_mtime * 1000 # When the file was created, in ms 79 - updates.append({'id': file.id, 'expiration': int(file_birth + max_age)}) 80 81 # Apply coalesced updates 82 session.bulk_update_mappings(File, updates) 83 session.commit() 84 85 def downgrade(): 86 op.drop_column('file', 'expiration')
··· 21 import os 22 import time 23 24 25 + """ 26 + For a file of a given size, determine the largest allowed lifespan of that file 27 28 + Based on the current app's configuration: 29 + Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION. 30 + 31 + This lifespan may be shortened by a user's request, but no files should be 32 + allowed to expire at a point after this number. 33 34 Value returned is a duration in milliseconds. 35 """ 36 def get_max_lifespan(filesize: int) -> int: 37 + cfg = current_app.config 38 + min_exp = cfg.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) 39 + max_exp = cfg.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) 40 + max_size = cfg.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) 41 return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) 42 43 + 44 Base = automap_base() 45 + 46 47 def upgrade(): 48 op.add_column('file', sa.Column('expiration', sa.BigInteger())) ··· 53 session = Session(bind=bind) 54 55 storage = Path(current_app.config["FHOST_STORAGE_PATH"]) 56 + current_time = time.time() * 1000 57 58 # List of file hashes which have not expired yet 59 # This could get really big for some servers 60 try: 61 unexpired_files = os.listdir(storage) 62 except FileNotFoundError: 63 + return # There are no currently unexpired files 64 65 # Calculate an expiration date for all existing files 66 ··· 70 sa.not_(File.removed) 71 ) 72 ) 73 + updates = [] # We coalesce updates to the database here 74 75 # SQLite has a hard limit on the number of variables so we 76 # need to do this the slow way ··· 79 for file in files: 80 file_path = storage / file.sha256 81 stat = os.stat(file_path) 82 + # How long the file is allowed to live, in ms 83 + max_age = get_max_lifespan(stat.st_size) 84 + # When the file was created, in ms 85 + file_birth = stat.st_mtime * 1000 86 + updates.append({ 87 + 'id': file.id, 88 + 'expiration': int(file_birth + max_age)}) 89 90 # Apply coalesced updates 91 session.bulk_update_mappings(File, updates) 92 session.commit() 93 + 94 95 def downgrade(): 96 op.drop_column('file', 'expiration')
-6
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
··· 15 16 17 def upgrade(): 18 - # ### commands auto generated by Alembic - please adjust! ### 19 with op.batch_alter_table('file', schema=None) as batch_op: 20 batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True)) 21 - 22 - # ### end Alembic commands ### 23 24 25 def downgrade(): 26 - # ### commands auto generated by Alembic - please adjust! ### 27 with op.batch_alter_table('file', schema=None) as batch_op: 28 batch_op.drop_column('ua') 29 - 30 - # ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 with op.batch_alter_table('file', schema=None) as batch_op: 19 batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True)) 20 21 22 def downgrade(): 23 with op.batch_alter_table('file', schema=None) as batch_op: 24 batch_op.drop_column('ua')
-4
migrations/versions/e2e816056589_.py
··· 15 16 17 def upgrade(): 18 - # ### commands auto generated by Alembic - please adjust! ### 19 op.add_column('file', sa.Column('secret', sa.String(), nullable=True)) 20 - # ### end Alembic commands ### 21 22 23 def downgrade(): 24 - # ### commands auto generated by Alembic - please adjust! ### 25 op.drop_column('file', 'secret') 26 - # ### end Alembic commands ###
··· 15 16 17 def upgrade(): 18 op.add_column('file', sa.Column('secret', sa.String(), nullable=True)) 19 20 21 def downgrade(): 22 op.drop_column('file', 'secret')
+64 -37
mod.py
··· 14 import ipaddress 15 16 from fhost import db, File, AddrFilter, su, app as fhost_app 17 - from modui import * 18 19 fhost_app.app_context().push() 20 21 class NullptrMod(Screen): 22 BINDINGS = [ ··· 67 self.finput.display = False 68 ftable = self.query_one("#ftable") 69 ftable.focus() 70 71 if len(message.value): 72 match self.filter_col: 73 case 1: 74 - try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value)) 75 - except ValueError: pass 76 case 2: 77 try: 78 addr = ipaddress.ip_address(message.value) 79 if type(addr) is ipaddress.IPv6Address: 80 addr = addr.ipv4_mapped or addr 81 - q = ftable.base_query.filter(File.addr == addr) 82 - ftable.query = q 83 - except ValueError: pass 84 - case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value)) 85 - case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value)) 86 - case 5: ftable.query = ftable.base_query.filter(File.ua.like(message.value)) 87 - else: 88 - ftable.query = ftable.base_query 89 90 def action_remove_file(self, permanent: bool) -> None: 91 if self.current_file: 92 self.current_file.delete(permanent) 93 db.session.commit() 94 - self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}")) 95 self.action_refresh() 96 97 def action_ban_ip(self, nuke: bool) -> None: 98 if self.current_file: 99 - if AddrFilter.query.filter(AddrFilter.addr == 100 - self.current_file.addr).scalar(): 101 - txt = f"{self.current_file.addr.compressed} is already banned" 102 else: 103 - db.session.add(AddrFilter(self.current_file.addr)) 104 db.session.commit() 105 - txt = f"Banned {self.current_file.addr.compressed}" 106 107 if nuke: 108 tsize = 0 109 trm = 0 110 - for f in File.query.filter(File.addr == self.current_file.addr): 111 if f.getpath().is_file(): 112 tsize += f.size or f.getpath().stat().st_size 113 trm += 1 114 f.delete(True) 115 db.session.commit() 116 - txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}" 117 self.mount(Notification(txt)) 118 self._refresh_layout() 119 ftable = self.query_one("#ftable") ··· 131 DataTable(id="finfo", show_header=False, cursor_type="none"), 132 MpvWidget(id="mpv"), 133 RichLog(id="ftextlog", auto_scroll=False), 134 - id="infopane")) 135 yield Input(id="filter_input") 136 yield Footer() 137 ··· 150 self.finput = self.query_one("#filter_input") 151 152 self.mimehandler = mime.MIMEHandler() 153 - self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive) 154 self.mimehandler.register(mime.MIMECategory.Text, self.handle_text) 155 self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv) 156 - self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf) 157 - self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive) 158 self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv) 159 self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw) 160 ··· 166 167 def handle_text(self, cat): 168 with open(self.current_file.getpath(), "r") as sf: 169 - data = sf.read(1000000).replace("\033","") 170 self.ftlog.write(data) 171 return True 172 ··· 181 self.mpvw.styles.height = "40%" 182 self.mpvw.start_mpv("hex://" + imgdata, 0) 183 184 - self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}")) 185 self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]")) 186 for k, v in doc.metadata.items(): 187 self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}")) ··· 206 for k, v in c.metadata.items(): 207 self.ftlog.write(f" {k}: {v}") 208 for s in c.streams: 209 - self.ftlog.write(Text(f"Stream {s.index}:", style="bold")) 210 self.ftlog.write(f" Type: {s.type}") 211 if s.base_rate: 212 self.ftlog.write(f" Frame rate: {s.base_rate}") ··· 225 else: 226 c = chr(s) 227 s = c 228 - if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]" 229 - if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]" 230 if not c.isprintable(): 231 g = "grey50" if c == "\0" else "cadet_blue" 232 return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]" 233 return s 234 - return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}" 235 - f"{' ' * (16 - len(c))}" 236 - f" {''.join(map(fmt, c))}" 237 - for c in map(lambda x: bytes([n for n in x if n != None]), 238 - zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16)))) 239 240 with open(self.current_file.getpath(), "rb") as binf: 241 self.ftlog.write(hexdump(binf, self.current_file.size)) 242 if self.current_file.size > 16*10*2: 243 binf.seek(self.current_file.size-16*10) 244 self.ftlog.write(" [...] ".center(64, '─')) 245 - self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell())) 246 247 return True 248 ··· 253 self.finfo.add_rows([ 254 ("ID:", str(f.id)), 255 ("File name:", f.getname()), 256 - ("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"), 257 ("File size:", do_filesizeformat(f.size, True)), 258 ("MIME type:", f.mime), 259 ("SHA256 checksum:", f.sha256), ··· 261 ("User agent:", Text(f.ua or "")), 262 ("Management token:", f.mgmt_token), 263 ("Secret:", f.secret), 264 - ("Is NSFW:", ("Yes" if f.is_nsfw else "No") + (f" (Score: {f.nsfw_score:0.4f})" if f.nsfw_score else " (Not scanned)")), 265 ("Is banned:", "Yes" if f.removed else "No"), 266 - ("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000))) 267 ]) 268 269 self.mpvw.stop_mpv(True) ··· 272 if f.getpath().is_file(): 273 self.mimehandler.handle(f.mime, f.ext) 274 self.ftlog.scroll_to(x=0, y=0, animate=False) 275 276 class NullptrModApp(App): 277 CSS_PATH = "mod.css" ··· 281 self.main_screen = NullptrMod() 282 self.install_screen(self.main_screen, name="main") 283 self.push_screen("main") 284 285 if __name__ == "__main__": 286 app = NullptrModApp()
··· 14 import ipaddress 15 16 from fhost import db, File, AddrFilter, su, app as fhost_app 17 + from modui import FileTable, mime, MpvWidget, Notification 18 19 fhost_app.app_context().push() 20 + 21 22 class NullptrMod(Screen): 23 BINDINGS = [ ··· 68 self.finput.display = False 69 ftable = self.query_one("#ftable") 70 ftable.focus() 71 + q = ftable.base_query 72 73 if len(message.value): 74 match self.filter_col: 75 case 1: 76 + try: 77 + q = q.filter(File.id == su.debase(message.value)) 78 + except ValueError: 79 + return 80 case 2: 81 try: 82 addr = ipaddress.ip_address(message.value) 83 if type(addr) is ipaddress.IPv6Address: 84 addr = addr.ipv4_mapped or addr 85 + q = q.filter(File.addr == addr) 86 + except ValueError: 87 + return 88 + case 3: q = q.filter(File.mime.like(message.value)) 89 + case 4: q = q.filter(File.ext.like(message.value)) 90 + case 5: q = q.filter(File.ua.like(message.value)) 91 + 92 + ftable.query = q 93 94 def action_remove_file(self, permanent: bool) -> None: 95 if self.current_file: 96 self.current_file.delete(permanent) 97 db.session.commit() 98 + self.mount(Notification(f"{'Banned' if permanent else 'Removed'}" 99 + f"file {self.current_file.getname()}")) 100 self.action_refresh() 101 102 def action_ban_ip(self, nuke: bool) -> None: 103 if self.current_file: 104 + addr = self.current_file.addr 105 + if AddrFilter.query.filter(AddrFilter.addr == addr).scalar(): 106 + txt = f"{addr.compressed} is already banned" 107 else: 108 + db.session.add(AddrFilter(addr)) 109 db.session.commit() 110 + txt = f"Banned {addr.compressed}" 111 112 if nuke: 113 tsize = 0 114 trm = 0 115 + for f in File.query.filter(File.addr == addr): 116 if f.getpath().is_file(): 117 tsize += f.size or f.getpath().stat().st_size 118 trm += 1 119 f.delete(True) 120 db.session.commit() 121 + txt += f", removed {trm} {'files' if trm != 1 else 'file'} " \ 122 + f"totaling {do_filesizeformat(tsize, True)}" 123 self.mount(Notification(txt)) 124 self._refresh_layout() 125 ftable = self.query_one("#ftable") ··· 137 DataTable(id="finfo", show_header=False, cursor_type="none"), 138 MpvWidget(id="mpv"), 139 RichLog(id="ftextlog", auto_scroll=False), 140 + id="infopane")) 141 yield Input(id="filter_input") 142 yield Footer() 143 ··· 156 self.finput = self.query_one("#filter_input") 157 158 self.mimehandler = mime.MIMEHandler() 159 + self.mimehandler.register(mime.MIMECategory.Archive, 160 + self.handle_libarchive) 161 self.mimehandler.register(mime.MIMECategory.Text, self.handle_text) 162 self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv) 163 + self.mimehandler.register(mime.MIMECategory.Document, 164 + self.handle_mupdf) 165 + self.mimehandler.register(mime.MIMECategory.Fallback, 166 + self.handle_libarchive) 167 self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv) 168 self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw) 169 ··· 175 176 def handle_text(self, cat): 177 with open(self.current_file.getpath(), "r") as sf: 178 + data = sf.read(1000000).replace("\033", "") 179 self.ftlog.write(data) 180 return True 181 ··· 190 self.mpvw.styles.height = "40%" 191 self.mpvw.start_mpv("hex://" + imgdata, 0) 192 193 + self.ftlog.write( 194 + Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}")) 195 self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]")) 196 for k, v in doc.metadata.items(): 197 self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}")) ··· 216 for k, v in c.metadata.items(): 217 self.ftlog.write(f" {k}: {v}") 218 for s in c.streams: 219 + self.ftlog.write( 220 + Text(f"Stream {s.index}:", style="bold")) 221 self.ftlog.write(f" Type: {s.type}") 222 if s.base_rate: 223 self.ftlog.write(f" Frame rate: {s.base_rate}") ··· 236 else: 237 c = chr(s) 238 s = c 239 + if c.isalpha(): 240 + return f"\0[chartreuse1]{s}\0[/chartreuse1]" 241 + if c.isdigit(): 242 + return f"\0[gold1]{s}\0[/gold1]" 243 if not c.isprintable(): 244 g = "grey50" if c == "\0" else "cadet_blue" 245 return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]" 246 return s 247 + 248 + return Text.from_markup( 249 + "\n".join(' '.join( 250 + map(fmt, map(''.join, zip(*[iter(c.hex())] * 2)))) + 251 + f"{' ' * (16 - len(c))} {''.join(map(fmt, c))}" 252 + for c in 253 + map(lambda x: bytes([n for n in x if n is not None]), 254 + zip_longest( 255 + *[iter(binf.read(min(length, 16 * 10)))] * 16)))) 256 257 with open(self.current_file.getpath(), "rb") as binf: 258 self.ftlog.write(hexdump(binf, self.current_file.size)) 259 if self.current_file.size > 16*10*2: 260 binf.seek(self.current_file.size-16*10) 261 self.ftlog.write(" [...] ".center(64, '─')) 262 + self.ftlog.write(hexdump(binf, 263 + self.current_file.size - binf.tell())) 264 265 return True 266 ··· 271 self.finfo.add_rows([ 272 ("ID:", str(f.id)), 273 ("File name:", f.getname()), 274 + ("URL:", f.geturl() 275 + if fhost_app.config["SERVER_NAME"] 276 + else "⚠ Set SERVER_NAME in config.py to display"), 277 ("File size:", do_filesizeformat(f.size, True)), 278 ("MIME type:", f.mime), 279 ("SHA256 checksum:", f.sha256), ··· 281 ("User agent:", Text(f.ua or "")), 282 ("Management token:", f.mgmt_token), 283 ("Secret:", f.secret), 284 + ("Is NSFW:", ("Yes" if f.is_nsfw else "No") + 285 + (f" (Score: {f.nsfw_score:0.4f})" 286 + if f.nsfw_score else " (Not scanned)")), 287 ("Is banned:", "Yes" if f.removed else "No"), 288 + ("Expires:", 289 + time.strftime("%Y-%m-%d %H:%M:%S", 290 + time.gmtime(File.get_expiration(f.expiration, 291 + f.size)/1000))) 292 ]) 293 294 self.mpvw.stop_mpv(True) ··· 297 if f.getpath().is_file(): 298 self.mimehandler.handle(f.mime, f.ext) 299 self.ftlog.scroll_to(x=0, y=0, animate=False) 300 + 301 302 class NullptrModApp(App): 303 CSS_PATH = "mod.css" ··· 307 self.main_screen = NullptrMod() 308 self.install_screen(self.main_screen, name="main") 309 self.push_screen("main") 310 + 311 312 if __name__ == "__main__": 313 app = NullptrModApp()
+9 -4
modui/filetable.py
··· 7 from fhost import File 8 from modui import mime 9 10 class FileTable(DataTable): 11 query = Reactive(None) 12 order_col = Reactive(0) 13 order_desc = Reactive(True) 14 limit = 10000 15 - colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime] 16 17 def __init__(self, **kwargs): 18 super().__init__(**kwargs) ··· 33 34 def watch_query(self, old, value) -> None: 35 def fmt_file(f: File) -> tuple: 36 return ( 37 str(f.id), 38 "🔴" if f.removed else " ", ··· 40 "👻" if not f.getpath().is_file() else " ", 41 f.getname(), 42 do_filesizeformat(f.size, True), 43 - f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or ' '} " + f.mime, 44 ) 45 46 if (self.query): 47 - 48 order = FileTable.colmap[self.order_col] 49 q = self.query 50 - if order: q = q.order_by(order.desc() if self.order_desc else order, File.id) 51 qres = list(map(fmt_file, q.limit(self.limit))) 52 53 ri = 0
··· 7 from fhost import File 8 from modui import mime 9 10 + 11 class FileTable(DataTable): 12 query = Reactive(None) 13 order_col = Reactive(0) 14 order_desc = Reactive(True) 15 limit = 10000 16 + colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, 17 + File.size, File.mime] 18 19 def __init__(self, **kwargs): 20 super().__init__(**kwargs) ··· 35 36 def watch_query(self, old, value) -> None: 37 def fmt_file(f: File) -> tuple: 38 + mimemoji = mime.mimemoji.get(f.mime.split('/')[0], 39 + mime.mimemoji.get(f.mime)) or ' ' 40 return ( 41 str(f.id), 42 "🔴" if f.removed else " ", ··· 44 "👻" if not f.getpath().is_file() else " ", 45 f.getname(), 46 do_filesizeformat(f.size, True), 47 + f"{mimemoji} {f.mime}", 48 ) 49 50 if (self.query): 51 order = FileTable.colmap[self.order_col] 52 q = self.query 53 + if order: 54 + q = q.order_by(order.desc() if self.order_desc 55 + else order, File.id) 56 qres = list(map(fmt_file, q.limit(self.limit))) 57 58 ri = 0
+48 -46
modui/mime.py
··· 2 from textual import log 3 4 mimemoji = { 5 - "audio" : "🔈", 6 - "video" : "🎞", 7 - "text" : "📄", 8 - "image" : "🖼", 9 - "application/zip" : "🗜️", 10 - "application/x-zip-compressed" : "🗜️", 11 - "application/x-tar" : "🗄", 12 - "application/x-cpio" : "🗄", 13 - "application/x-xz" : "🗜️", 14 - "application/x-7z-compressed" : "🗜️", 15 - "application/gzip" : "🗜️", 16 - "application/zstd" : "🗜️", 17 - "application/x-rar" : "🗜️", 18 - "application/x-rar-compressed" : "🗜️", 19 - "application/vnd.ms-cab-compressed" : "🗜️", 20 - "application/x-bzip2" : "🗜️", 21 - "application/x-lzip" : "🗜️", 22 - "application/x-iso9660-image" : "💿", 23 - "application/pdf" : "📕", 24 - "application/epub+zip" : "📕", 25 - "application/mxf" : "🎞", 26 - "application/vnd.android.package-archive" : "📦", 27 - "application/vnd.debian.binary-package" : "📦", 28 - "application/x-rpm" : "📦", 29 - "application/x-dosexec" : "⚙", 30 - "application/x-execuftable" : "⚙", 31 - "application/x-sharedlib" : "⚙", 32 - "application/java-archive" : "☕", 33 - "application/x-qemu-disk" : "🖴", 34 - "application/pgp-encrypted" : "🔏", 35 } 36 37 - MIMECategory = Enum("MIMECategory", 38 - ["Archive", "Text", "AV", "Document", "Fallback"] 39 - ) 40 41 class MIMEHandler: 42 def __init__(self): 43 self.handlers = { 44 - MIMECategory.Archive : [[ 45 "application/zip", 46 "application/x-zip-compressed", 47 "application/x-tar", ··· 62 "application/java-archive", 63 "application/vnd.openxmlformats" 64 ], []], 65 - MIMECategory.Text : [[ 66 "text", 67 "application/json", 68 "application/xml", 69 ], []], 70 - MIMECategory.AV : [[ 71 "audio", "video", "image", 72 "application/mxf" 73 ], []], 74 - MIMECategory.Document : [[ 75 "application/pdf", 76 "application/epub", 77 "application/x-mobipocket-ebook", 78 ], []], 79 - MIMECategory.Fallback : [[], []] 80 } 81 82 self.exceptions = { 83 - MIMECategory.Archive : { 84 - ".cbz" : MIMECategory.Document, 85 - ".xps" : MIMECategory.Document, 86 - ".epub" : MIMECategory.Document, 87 }, 88 - MIMECategory.Text : { 89 - ".fb2" : MIMECategory.Document, 90 } 91 } 92 ··· 115 cat = getcat(mime) 116 for handler in self.handlers[cat][1]: 117 try: 118 - if handler(cat): return 119 except: pass 120 121 for handler in self.handlers[MIMECategory.Fallback][1]: 122 try: 123 - if handler(None): return 124 except: pass 125 126 raise RuntimeError(f"Unhandled MIME type category: {cat}")
··· 2 from textual import log 3 4 mimemoji = { 5 + "audio": "🔈", 6 + "video": "🎞", 7 + "text": "📄", 8 + "image": "🖼", 9 + "application/zip": "🗜️", 10 + "application/x-zip-compressed": "🗜️", 11 + "application/x-tar": "🗄", 12 + "application/x-cpio": "🗄", 13 + "application/x-xz": "🗜️", 14 + "application/x-7z-compressed": "🗜️", 15 + "application/gzip": "🗜️", 16 + "application/zstd": "🗜️", 17 + "application/x-rar": "🗜️", 18 + "application/x-rar-compressed": "🗜️", 19 + "application/vnd.ms-cab-compressed": "🗜️", 20 + "application/x-bzip2": "🗜️", 21 + "application/x-lzip": "🗜️", 22 + "application/x-iso9660-image": "💿", 23 + "application/pdf": "📕", 24 + "application/epub+zip": "📕", 25 + "application/mxf": "🎞", 26 + "application/vnd.android.package-archive": "📦", 27 + "application/vnd.debian.binary-package": "📦", 28 + "application/x-rpm": "📦", 29 + "application/x-dosexec": "⚙", 30 + "application/x-execuftable": "⚙", 31 + "application/x-sharedlib": "⚙", 32 + "application/java-archive": "☕", 33 + "application/x-qemu-disk": "🖴", 34 + "application/pgp-encrypted": "🔏", 35 } 36 37 + MIMECategory = Enum("MIMECategory", ["Archive", "Text", "AV", "Document", 38 + "Fallback"]) 39 + 40 41 class MIMEHandler: 42 def __init__(self): 43 self.handlers = { 44 + MIMECategory.Archive: [[ 45 "application/zip", 46 "application/x-zip-compressed", 47 "application/x-tar", ··· 62 "application/java-archive", 63 "application/vnd.openxmlformats" 64 ], []], 65 + MIMECategory.Text: [[ 66 "text", 67 "application/json", 68 "application/xml", 69 ], []], 70 + MIMECategory.AV: [[ 71 "audio", "video", "image", 72 "application/mxf" 73 ], []], 74 + MIMECategory.Document: [[ 75 "application/pdf", 76 "application/epub", 77 "application/x-mobipocket-ebook", 78 ], []], 79 + MIMECategory.Fallback: [[], []] 80 } 81 82 self.exceptions = { 83 + MIMECategory.Archive: { 84 + ".cbz": MIMECategory.Document, 85 + ".xps": MIMECategory.Document, 86 + ".epub": MIMECategory.Document, 87 }, 88 + MIMECategory.Text: { 89 + ".fb2": MIMECategory.Document, 90 } 91 } 92 ··· 115 cat = getcat(mime) 116 for handler in self.handlers[cat][1]: 117 try: 118 + if handler(cat): 119 + return 120 except: pass 121 122 for handler in self.handlers[MIMECategory.Fallback][1]: 123 try: 124 + if handler(None): 125 + return 126 except: pass 127 128 raise RuntimeError(f"Unhandled MIME type category: {cat}")
+24 -10
modui/mpvwidget.py
··· 1 import time 2 - import fcntl, struct, termios 3 from sys import stdout 4 5 from textual import events, log 6 from textual.widgets import Static 7 8 from fhost import app as fhost_app 9 10 class MpvWidget(Static): 11 def __init__(self, **kwargs): ··· 14 self.mpv = None 15 self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO") 16 17 - if not self.vo in ["sixel", "kitty"]: 18 - self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.") 19 else: 20 try: 21 import mpv ··· 27 self.mpv[f"vo-sixel-buffered"] = True 28 self.mpv["audio"] = False 29 self.mpv["loop-file"] = "inf" 30 - self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf" 31 except Exception as e: 32 self.mpv = None 33 - self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}") 34 35 - def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None: 36 self.display = True 37 self.screen._refresh_layout() 38 39 if self.mpv: 40 if self.content_region.x: 41 - r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678')) 42 width = int((w / c) * self.content_region.width) 43 - height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0))) 44 self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1 45 self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1 46 - self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0) 47 self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width 48 self.mpv[f"vo-{self.vo}-width"] = width 49 self.mpv[f"vo-{self.vo}-height"] = height 50 51 - if pos != None: 52 self.mpv["start"] = pos 53 54 if f:
··· 1 import time 2 + 3 + import fcntl 4 + import struct 5 + import termios 6 + 7 from sys import stdout 8 9 from textual import events, log 10 from textual.widgets import Static 11 12 from fhost import app as fhost_app 13 + 14 15 class MpvWidget(Static): 16 def __init__(self, **kwargs): ··· 19 self.mpv = None 20 self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO") 21 22 + if self.vo not in ["sixel", "kitty"]: 23 + self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO " 24 + "to 'sixel' or 'kitty' in config.py,\nwhichever is " 25 + "supported by your terminal.") 26 else: 27 try: 28 import mpv ··· 34 self.mpv[f"vo-sixel-buffered"] = True 35 self.mpv["audio"] = False 36 self.mpv["loop-file"] = "inf" 37 + self.mpv["image-display-duration"] = 0.5 \ 38 + if self.vo == "sixel" else "inf" 39 except Exception as e: 40 self.mpv = None 41 + self.update("⚠ Previews require python-mpv with libmpv " 42 + "0.36.0 or later \n\nError was:\n" 43 + f"{type(e).__name__}: {e}") 44 45 + def start_mpv(self, f: str | None = None, 46 + pos: float | str | None = None) -> None: 47 self.display = True 48 self.screen._refresh_layout() 49 50 if self.mpv: 51 if self.content_region.x: 52 + winsz = fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678') 53 + r, c, w, h = struct.unpack('hhhh', winsz) 54 width = int((w / c) * self.content_region.width) 55 + height = int((h / r) * (self.content_region.height + 56 + (1 if self.vo == "sixel" else 0))) 57 self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1 58 self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1 59 + self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + \ 60 + (1 if self.vo == "sixel" else 0) 61 self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width 62 self.mpv[f"vo-{self.vo}-width"] = width 63 self.mpv[f"vo-{self.vo}-height"] = height 64 65 + if pos is not None: 66 self.mpv["start"] = pos 67 68 if f:
+1
modui/notification.py
··· 1 from textual.widgets import Static 2 3 class Notification(Static): 4 def on_mount(self) -> None: 5 self.set_timer(3, self.remove)
··· 1 from textual.widgets import Static 2 3 + 4 class Notification(Static): 5 def on_mount(self) -> None: 6 self.set_timer(3, self.remove)
+8 -6
nsfw_detect.py
··· 18 and limitations under the License. 19 """ 20 21 - import os 22 import sys 23 - from pathlib import Path 24 - 25 import av 26 from transformers import pipeline 27 28 class NSFWDetector: 29 def __init__(self): 30 - self.classifier = pipeline("image-classification", model="giacomoarienti/nsfw-classifier") 31 32 def detect(self, fpath): 33 try: 34 with av.open(fpath) as container: 35 - try: container.seek(int(container.duration / 2)) 36 except: container.seek(0) 37 38 frame = next(container.decode(video=0)) 39 img = frame.to_image() 40 res = self.classifier(img) 41 42 - return max([x["score"] for x in res if x["label"] not in ["neutral", "drawings"]]) 43 except: pass 44 45 return -1.0 46 47 if __name__ == "__main__": 48 n = NSFWDetector()
··· 18 and limitations under the License. 19 """ 20 21 import sys 22 import av 23 from transformers import pipeline 24 + 25 26 class NSFWDetector: 27 def __init__(self): 28 + self.classifier = pipeline("image-classification", 29 + model="giacomoarienti/nsfw-classifier") 30 31 def detect(self, fpath): 32 try: 33 with av.open(fpath) as container: 34 + try: 35 + container.seek(int(container.duration / 2)) 36 except: container.seek(0) 37 38 frame = next(container.decode(video=0)) 39 img = frame.to_image() 40 res = self.classifier(img) 41 42 + return max([x["score"] for x in res 43 + if x["label"] not in ["neutral", "drawings"]]) 44 except: pass 45 46 return -1.0 47 + 48 49 if __name__ == "__main__": 50 n = NSFWDetector()