+1
-1
cleanup.py
+1
-1
cleanup.py
+146
-104
fhost.py
+146
-104
fhost.py
···
1
#!/usr/bin/env python3
2
-
# -*- coding: utf-8 -*-
3
4
"""
5
-
Copyright © 2020 Mia Herkt
6
Licensed under the EUPL, Version 1.2 or - as soon as approved
7
by the European Commission - subsequent versions of the EUPL
8
(the "License");
···
19
and limitations under the License.
20
"""
21
22
-
from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template, Request
23
from flask_sqlalchemy import SQLAlchemy
24
from flask_migrate import Migrate
25
from sqlalchemy import and_, or_
···
46
47
app = Flask(__name__, instance_relative_config=True)
48
app.config.update(
49
-
SQLALCHEMY_TRACK_MODIFICATIONS = False,
50
-
PREFERRED_URL_SCHEME = "https", # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config
51
-
MAX_CONTENT_LENGTH = 256 * 1024 * 1024,
52
-
MAX_URL_LENGTH = 4096,
53
-
USE_X_SENDFILE = False,
54
-
FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default
55
-
FHOST_STORAGE_PATH = "up",
56
-
FHOST_MAX_EXT_LENGTH = 9,
57
-
FHOST_SECRET_BYTES = 16,
58
-
FHOST_EXT_OVERRIDE = {
59
-
"audio/flac" : ".flac",
60
-
"image/gif" : ".gif",
61
-
"image/jpeg" : ".jpg",
62
-
"image/png" : ".png",
63
-
"image/svg+xml" : ".svg",
64
-
"video/webm" : ".webm",
65
-
"video/x-matroska" : ".mkv",
66
-
"application/octet-stream" : ".bin",
67
-
"text/plain" : ".log",
68
-
"text/plain" : ".txt",
69
-
"text/x-diff" : ".diff",
70
},
71
-
NSFW_DETECT = False,
72
-
NSFW_THRESHOLD = 0.92,
73
-
VSCAN_SOCKET = None,
74
-
VSCAN_QUARANTINE_PATH = "quarantine",
75
-
VSCAN_IGNORE = [
76
"Eicar-Test-Signature",
77
"PUA.Win.Packer.XmMusicFile",
78
],
79
-
VSCAN_INTERVAL = datetime.timedelta(days=7),
80
-
URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-",
81
)
82
83
app.config.from_pyfile("config.py")
···
95
96
try:
97
mimedetect = Magic(mime=True, mime_encoding=False)
98
-
except:
99
print("""Error: You have installed the wrong version of the 'magic' module.
100
Please install python-magic.""")
101
sys.exit(1)
···
103
db = SQLAlchemy(app)
104
migrate = Migrate(app, db)
105
106
class URL(db.Model):
107
__tablename__ = "URL"
108
-
id = db.Column(db.Integer, primary_key = True)
109
-
url = db.Column(db.UnicodeText, unique = True)
110
111
def __init__(self, url):
112
self.url = url
···
117
def geturl(self):
118
return url_for("get", path=self.getname(), _external=True) + "\n"
119
120
def get(url):
121
u = URL.query.filter_by(url=url).first()
122
···
126
db.session.commit()
127
128
return u
129
130
class IPAddress(types.TypeDecorator):
131
impl = types.LargeBinary
···
150
151
152
class File(db.Model):
153
-
id = db.Column(db.Integer, primary_key = True)
154
-
sha256 = db.Column(db.String, unique = True)
155
ext = db.Column(db.UnicodeText)
156
mime = db.Column(db.UnicodeText)
157
addr = db.Column(IPAddress(16))
···
175
176
@property
177
def is_nsfw(self) -> bool:
178
-
return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]
179
180
def getname(self):
181
return u"{0}{1}".format(su.enbase(self.id), self.ext)
182
183
def geturl(self):
184
n = self.getname()
185
186
-
if self.is_nsfw:
187
-
return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n"
188
-
else:
189
-
return url_for("get", path=n, secret=self.secret, _external=True) + "\n"
190
191
def getpath(self) -> Path:
192
return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256
···
197
self.removed = permanent
198
self.getpath().unlink(missing_ok=True)
199
200
-
# Returns the epoch millisecond that a file should expire
201
-
#
202
-
# Uses the expiration time provided by the user (requested_expiration)
203
-
# upper-bounded by an algorithm that computes the size based on the size of the
204
-
# file.
205
-
#
206
-
# That is, all files are assigned a computed expiration, which can voluntarily
207
-
# shortened by the user either by providing a timestamp in epoch millis or a
208
-
# duration in hours.
209
def get_expiration(requested_expiration, size) -> int:
210
-
current_epoch_millis = time.time() * 1000;
211
212
# Maximum lifetime of the file in milliseconds
213
-
this_files_max_lifespan = get_max_lifespan(size);
214
215
# The latest allowed expiration date for this file, in epoch millis
216
-
this_files_max_expiration = this_files_max_lifespan + 1000 * time.time();
217
218
if requested_expiration is None:
219
-
return this_files_max_expiration
220
elif requested_expiration < 1650460320000:
221
# Treat the requested expiration time as a duration in hours
222
requested_expiration_ms = requested_expiration * 60 * 60 * 1000
223
-
return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms)
224
else:
225
-
# Treat the requested expiration time as a timestamp in epoch millis
226
-
return min(this_files_max_expiration, requested_expiration)
227
228
"""
229
requested_expiration can be:
···
231
- a duration (in hours) that the file should live for
232
- a timestamp in epoch millis that the file should expire at
233
234
-
Any value greater that the longest allowed file lifespan will be rounded down to that
235
-
value.
236
"""
237
-
def store(file_, requested_expiration: typing.Optional[int], addr, ua, secret: bool):
238
data = file_.read()
239
digest = sha256(data).hexdigest()
240
241
def get_mime():
242
guess = mimedetect.from_buffer(data)
243
-
app.logger.debug(f"MIME - specified: '{file_.content_type}' - detected: '{guess}'")
244
245
-
if not file_.content_type or not "/" in file_.content_type or file_.content_type == "application/octet-stream":
246
mime = guess
247
else:
248
mime = file_.content_type
···
254
if flt.check(guess):
255
abort(403, flt.reason)
256
257
-
if mime.startswith("text/") and not "charset" in mime:
258
mime += "; charset=utf-8"
259
260
return mime
···
266
gmime = mime.split(";")[0]
267
guess = guess_extension(gmime)
268
269
-
app.logger.debug(f"extension - specified: '{ext}' - detected: '{guess}'")
270
271
if not ext:
272
if gmime in app.config["FHOST_EXT_OVERRIDE"]:
···
309
if isnew:
310
f.secret = None
311
if secret:
312
-
f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
313
314
storage = Path(app.config["FHOST_STORAGE_PATH"])
315
storage.mkdir(parents=True, exist_ok=True)
···
451
452
453
class UrlEncoder(object):
454
-
def __init__(self,alphabet, min_length):
455
self.alphabet = alphabet
456
self.min_length = min_length
457
···
471
result += self.alphabet.index(c) * (n ** i)
472
return result
473
474
su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1)
475
476
def fhost_url(scheme=None):
477
if not scheme:
478
return url_for(".fhost", _external=True).rstrip("/")
479
else:
480
return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
481
482
def is_fhost_url(url):
483
return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
484
485
def shorten(url):
486
if len(url) > app.config["MAX_URL_LENGTH"]:
487
abort(414)
···
493
494
return u.geturl()
495
496
"""
497
requested_expiration can be:
498
- None, to use the longest allowed file lifespan
499
- a duration (in hours) that the file should live for
500
- a timestamp in epoch millis that the file should expire at
501
502
-
Any value greater that the longest allowed file lifespan will be rounded down to that
503
-
value.
504
"""
505
-
def store_file(f, requested_expiration: typing.Optional[int], addr, ua, secret: bool):
506
sf, isnew = File.store(f, requested_expiration, addr, ua, secret)
507
508
response = make_response(sf.geturl())
···
513
514
return response
515
516
def store_url(url, addr, ua, secret: bool):
517
if is_fhost_url(url):
518
abort(400)
519
520
-
h = { "Accept-Encoding" : "identity" }
521
r = requests.get(url, stream=True, verify=False, headers=h)
522
523
try:
···
526
return str(e) + "\n"
527
528
if "content-length" in r.headers:
529
-
l = int(r.headers["content-length"])
530
531
-
if l <= app.config["MAX_CONTENT_LENGTH"]:
532
def urlfile(**kwargs):
533
-
return type('',(),kwargs)()
534
535
-
f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="")
536
537
return store_file(f, None, addr, ua, secret)
538
else:
···
540
else:
541
abort(411)
542
543
def manage_file(f):
544
-
try:
545
-
assert(request.form["token"] == f.mgmt_token)
546
-
except:
547
abort(401)
548
549
if "delete" in request.form:
···
561
return "", 202
562
563
abort(400)
564
565
@app.route("/<path:path>", methods=["GET", "POST"])
566
@app.route("/s/<secret>/<path:path>", methods=["GET", "POST"])
···
598
response.headers["Content-Length"] = f.size
599
response.headers["X-Accel-Redirect"] = "/" + str(fpath)
600
else:
601
-
response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
602
603
response.headers["X-Expires"] = f.expiration
604
return response
···
615
return redirect(u.url)
616
617
abort(404)
618
619
@app.route("/", methods=["GET", "POST"])
620
def fhost():
···
665
else:
666
return render_template("index.html")
667
668
@app.route("/robots.txt")
669
def robots():
670
return """User-agent: *
671
Disallow: /
672
"""
673
674
@app.errorhandler(400)
675
@app.errorhandler(401)
···
682
@app.errorhandler(451)
683
def ehandler(e):
684
try:
685
-
return render_template(f"{e.code}.html", id=id, request=request, description=e.description), e.code
686
except TemplateNotFound:
687
return "Segmentation fault\n", e.code
688
689
@app.cli.command("prune")
690
def prune():
691
"""
692
Clean up expired files
693
694
-
Deletes any files from the filesystem which have hit their expiration time. This
695
-
doesn't remove them from the database, only from the filesystem. It's recommended
696
-
that server owners run this command regularly, or set it up on a timer.
697
"""
698
-
current_time = time.time() * 1000;
699
700
# The path to where uploaded files are stored
701
storage = Path(app.config["FHOST_STORAGE_PATH"])
···
709
)
710
)
711
712
-
files_removed = 0;
713
714
# For every expired file...
715
for file in expired_files:
···
722
# Remove it from the file system
723
try:
724
os.remove(file_path)
725
-
files_removed += 1;
726
except FileNotFoundError:
727
-
pass # If the file was already gone, we're good
728
except OSError as e:
729
print(e)
730
print(
731
"\n------------------------------------"
732
-
"Encountered an error while trying to remove file {file_path}. Double"
733
-
"check to make sure the server is configured correctly, permissions are"
734
-
"okay, and everything is ship shape, then try again.")
735
-
return;
736
737
# Finally, mark that the file was removed
738
-
file.expiration = None;
739
db.session.commit()
740
741
print(f"\nDone! {files_removed} file(s) removed")
742
743
-
""" For a file of a given size, determine the largest allowed lifespan of that file
744
745
-
Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
746
-
as FHOST_{MIN,MAX}_EXPIRATION.
747
748
-
This lifespan may be shortened by a user's request, but no files should be allowed to
749
-
expire at a point after this number.
750
751
Value returned is a duration in milliseconds.
752
"""
···
756
max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
757
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
758
759
def do_vscan(f):
760
if f["path"].is_file():
761
with open(f["path"], "rb") as scanf:
762
try:
763
-
f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0]
764
except:
765
f["result"] = ("SCAN FAILED", None)
766
else:
···
768
769
return f
770
771
@app.cli.command("vscan")
772
def vscan():
773
if not app.config["VSCAN_SOCKET"]:
774
-
print("""Error: Virus scanning enabled but no connection method specified.
775
-
Please set VSCAN_SOCKET.""")
776
sys.exit(1)
777
778
qp = Path(app.config["VSCAN_QUARANTINE_PATH"])
···
786
File.last_vscan == None),
787
File.removed == False)
788
else:
789
-
res = File.query.filter(File.last_vscan == None, File.removed == False)
790
791
-
work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res]
792
793
results = []
794
for i, r in enumerate(p.imap_unordered(do_vscan, work)):
···
802
found = True
803
804
results.append({
805
-
"id" : r["id"],
806
-
"last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(),
807
-
"removed" : found})
808
809
db.session.bulk_update_mappings(File, results)
810
db.session.commit()
···
1
#!/usr/bin/env python3
2
3
"""
4
+
Copyright © 2024 Mia Herkt
5
Licensed under the EUPL, Version 1.2 or - as soon as approved
6
by the European Commission - subsequent versions of the EUPL
7
(the "License");
···
18
and limitations under the License.
19
"""
20
21
+
from flask import Flask, abort, make_response, redirect, render_template, \
22
+
Request, request, Response, send_from_directory, url_for
23
from flask_sqlalchemy import SQLAlchemy
24
from flask_migrate import Migrate
25
from sqlalchemy import and_, or_
···
46
47
app = Flask(__name__, instance_relative_config=True)
48
app.config.update(
49
+
SQLALCHEMY_TRACK_MODIFICATIONS=False,
50
+
PREFERRED_URL_SCHEME="https", # nginx users: make sure to have
51
+
# 'uwsgi_param UWSGI_SCHEME $scheme;' in
52
+
# your config
53
+
MAX_CONTENT_LENGTH=256 * 1024 * 1024,
54
+
MAX_URL_LENGTH=4096,
55
+
USE_X_SENDFILE=False,
56
+
FHOST_USE_X_ACCEL_REDIRECT=True, # expect nginx by default
57
+
FHOST_STORAGE_PATH="up",
58
+
FHOST_MAX_EXT_LENGTH=9,
59
+
FHOST_SECRET_BYTES=16,
60
+
FHOST_EXT_OVERRIDE={
61
+
"audio/flac": ".flac",
62
+
"image/gif": ".gif",
63
+
"image/jpeg": ".jpg",
64
+
"image/png": ".png",
65
+
"image/svg+xml": ".svg",
66
+
"video/webm": ".webm",
67
+
"video/x-matroska": ".mkv",
68
+
"application/octet-stream": ".bin",
69
+
"text/plain": ".log",
70
+
"text/plain": ".txt",
71
+
"text/x-diff": ".diff",
72
},
73
+
NSFW_DETECT=False,
74
+
NSFW_THRESHOLD=0.92,
75
+
VSCAN_SOCKET=None,
76
+
VSCAN_QUARANTINE_PATH="quarantine",
77
+
VSCAN_IGNORE=[
78
"Eicar-Test-Signature",
79
"PUA.Win.Packer.XmMusicFile",
80
],
81
+
VSCAN_INTERVAL=datetime.timedelta(days=7),
82
+
URL_ALPHABET="DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMX"
83
+
"y6Vx-",
84
)
85
86
app.config.from_pyfile("config.py")
···
98
99
try:
100
mimedetect = Magic(mime=True, mime_encoding=False)
101
+
except TypeError:
102
print("""Error: You have installed the wrong version of the 'magic' module.
103
Please install python-magic.""")
104
sys.exit(1)
···
106
db = SQLAlchemy(app)
107
migrate = Migrate(app, db)
108
109
+
110
class URL(db.Model):
111
__tablename__ = "URL"
112
+
id = db.Column(db.Integer, primary_key=True)
113
+
url = db.Column(db.UnicodeText, unique=True)
114
115
def __init__(self, url):
116
self.url = url
···
121
def geturl(self):
122
return url_for("get", path=self.getname(), _external=True) + "\n"
123
124
+
@staticmethod
125
def get(url):
126
u = URL.query.filter_by(url=url).first()
127
···
131
db.session.commit()
132
133
return u
134
+
135
136
class IPAddress(types.TypeDecorator):
137
impl = types.LargeBinary
···
156
157
158
class File(db.Model):
159
+
id = db.Column(db.Integer, primary_key=True)
160
+
sha256 = db.Column(db.String, unique=True)
161
ext = db.Column(db.UnicodeText)
162
mime = db.Column(db.UnicodeText)
163
addr = db.Column(IPAddress(16))
···
181
182
@property
183
def is_nsfw(self) -> bool:
184
+
if self.nsfw_score:
185
+
return self.nsfw_score > app.config["NSFW_THRESHOLD"]
186
+
return False
187
188
def getname(self):
189
return u"{0}{1}".format(su.enbase(self.id), self.ext)
190
191
def geturl(self):
192
n = self.getname()
193
+
a = "nsfw" if self.is_nsfw else None
194
195
+
return url_for("get", path=n, secret=self.secret,
196
+
_external=True, _anchor=a) + "\n"
197
198
def getpath(self) -> Path:
199
return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256
···
204
self.removed = permanent
205
self.getpath().unlink(missing_ok=True)
206
207
+
"""
208
+
Returns the epoch millisecond that a file should expire
209
+
210
+
Uses the expiration time provided by the user (requested_expiration)
211
+
upper-bounded by an algorithm that computes the size based on the size of
212
+
the file.
213
+
214
+
That is, all files are assigned a computed expiration, which can be
215
+
voluntarily shortened by the user either by providing a timestamp in
216
+
milliseconds since epoch or a duration in hours.
217
+
"""
218
+
@staticmethod
219
def get_expiration(requested_expiration, size) -> int:
220
+
current_epoch_millis = time.time() * 1000
221
222
# Maximum lifetime of the file in milliseconds
223
+
max_lifespan = get_max_lifespan(size)
224
225
# The latest allowed expiration date for this file, in epoch millis
226
+
max_expiration = max_lifespan + 1000 * time.time()
227
228
if requested_expiration is None:
229
+
return max_expiration
230
elif requested_expiration < 1650460320000:
231
# Treat the requested expiration time as a duration in hours
232
requested_expiration_ms = requested_expiration * 60 * 60 * 1000
233
+
return min(max_expiration,
234
+
current_epoch_millis + requested_expiration_ms)
235
else:
236
+
# Treat expiration time as a timestamp in epoch millis
237
+
return min(max_expiration, requested_expiration)
238
239
"""
240
requested_expiration can be:
···
242
- a duration (in hours) that the file should live for
243
- a timestamp in epoch millis that the file should expire at
244
245
+
Any value greater that the longest allowed file lifespan will be rounded
246
+
down to that value.
247
"""
248
+
@staticmethod
249
+
def store(file_, requested_expiration: typing.Optional[int], addr, ua,
250
+
secret: bool):
251
data = file_.read()
252
digest = sha256(data).hexdigest()
253
254
def get_mime():
255
guess = mimedetect.from_buffer(data)
256
+
app.logger.debug(f"MIME - specified: '{file_.content_type}' - "
257
+
f"detected: '{guess}'")
258
259
+
if (not file_.content_type
260
+
or "/" not in file_.content_type
261
+
or file_.content_type == "application/octet-stream"):
262
mime = guess
263
else:
264
mime = file_.content_type
···
270
if flt.check(guess):
271
abort(403, flt.reason)
272
273
+
if mime.startswith("text/") and "charset" not in mime:
274
mime += "; charset=utf-8"
275
276
return mime
···
282
gmime = mime.split(";")[0]
283
guess = guess_extension(gmime)
284
285
+
app.logger.debug(f"extension - specified: '{ext}' - detected: "
286
+
f"'{guess}'")
287
288
if not ext:
289
if gmime in app.config["FHOST_EXT_OVERRIDE"]:
···
326
if isnew:
327
f.secret = None
328
if secret:
329
+
f.secret = \
330
+
secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
331
332
storage = Path(app.config["FHOST_STORAGE_PATH"])
333
storage.mkdir(parents=True, exist_ok=True)
···
469
470
471
class UrlEncoder(object):
472
+
def __init__(self, alphabet, min_length):
473
self.alphabet = alphabet
474
self.min_length = min_length
475
···
489
result += self.alphabet.index(c) * (n ** i)
490
return result
491
492
+
493
su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1)
494
+
495
496
def fhost_url(scheme=None):
497
if not scheme:
498
return url_for(".fhost", _external=True).rstrip("/")
499
else:
500
return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
501
+
502
503
def is_fhost_url(url):
504
return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
505
506
+
507
def shorten(url):
508
if len(url) > app.config["MAX_URL_LENGTH"]:
509
abort(414)
···
515
516
return u.geturl()
517
518
+
519
"""
520
requested_expiration can be:
521
- None, to use the longest allowed file lifespan
522
- a duration (in hours) that the file should live for
523
- a timestamp in epoch millis that the file should expire at
524
525
+
Any value greater that the longest allowed file lifespan will be rounded down
526
+
to that value.
527
"""
528
+
def store_file(f, requested_expiration: typing.Optional[int], addr, ua,
529
+
secret: bool):
530
sf, isnew = File.store(f, requested_expiration, addr, ua, secret)
531
532
response = make_response(sf.geturl())
···
537
538
return response
539
540
+
541
def store_url(url, addr, ua, secret: bool):
542
if is_fhost_url(url):
543
abort(400)
544
545
+
h = {"Accept-Encoding": "identity"}
546
r = requests.get(url, stream=True, verify=False, headers=h)
547
548
try:
···
551
return str(e) + "\n"
552
553
if "content-length" in r.headers:
554
+
length = int(r.headers["content-length"])
555
556
+
if length <= app.config["MAX_CONTENT_LENGTH"]:
557
def urlfile(**kwargs):
558
+
return type('', (), kwargs)()
559
560
+
f = urlfile(read=r.raw.read,
561
+
content_type=r.headers["content-type"], filename="")
562
563
return store_file(f, None, addr, ua, secret)
564
else:
···
566
else:
567
abort(411)
568
569
+
570
def manage_file(f):
571
+
if request.form["token"] != f.mgmt_token:
572
abort(401)
573
574
if "delete" in request.form:
···
586
return "", 202
587
588
abort(400)
589
+
590
591
@app.route("/<path:path>", methods=["GET", "POST"])
592
@app.route("/s/<secret>/<path:path>", methods=["GET", "POST"])
···
624
response.headers["Content-Length"] = f.size
625
response.headers["X-Accel-Redirect"] = "/" + str(fpath)
626
else:
627
+
response = send_from_directory(
628
+
app.config["FHOST_STORAGE_PATH"], f.sha256,
629
+
mimetype=f.mime)
630
631
response.headers["X-Expires"] = f.expiration
632
return response
···
643
return redirect(u.url)
644
645
abort(404)
646
+
647
648
@app.route("/", methods=["GET", "POST"])
649
def fhost():
···
694
else:
695
return render_template("index.html")
696
697
+
698
@app.route("/robots.txt")
699
def robots():
700
return """User-agent: *
701
Disallow: /
702
"""
703
+
704
705
@app.errorhandler(400)
706
@app.errorhandler(401)
···
713
@app.errorhandler(451)
714
def ehandler(e):
715
try:
716
+
return render_template(f"{e.code}.html", id=id, request=request,
717
+
description=e.description), e.code
718
except TemplateNotFound:
719
return "Segmentation fault\n", e.code
720
+
721
722
@app.cli.command("prune")
723
def prune():
724
"""
725
Clean up expired files
726
727
+
Deletes any files from the filesystem which have hit their expiration time.
728
+
This doesn't remove them from the database, only from the filesystem.
729
+
It is recommended that server owners run this command regularly, or set it
730
+
up on a timer.
731
"""
732
+
current_time = time.time() * 1000
733
734
# The path to where uploaded files are stored
735
storage = Path(app.config["FHOST_STORAGE_PATH"])
···
743
)
744
)
745
746
+
files_removed = 0
747
748
# For every expired file...
749
for file in expired_files:
···
756
# Remove it from the file system
757
try:
758
os.remove(file_path)
759
+
files_removed += 1
760
except FileNotFoundError:
761
+
pass # If the file was already gone, we're good
762
except OSError as e:
763
print(e)
764
print(
765
"\n------------------------------------"
766
+
"Encountered an error while trying to remove file {file_path}."
767
+
"Make sure the server is configured correctly, permissions "
768
+
"are okay, and everything is ship shape, then try again.")
769
+
return
770
771
# Finally, mark that the file was removed
772
+
file.expiration = None
773
db.session.commit()
774
775
print(f"\nDone! {files_removed} file(s) removed")
776
777
778
+
"""
779
+
For a file of a given size, determine the largest allowed lifespan of that file
780
781
+
Based on the current app's configuration:
782
+
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION.
783
+
784
+
This lifespan may be shortened by a user's request, but no files should be
785
+
allowed to expire at a point after this number.
786
787
Value returned is a duration in milliseconds.
788
"""
···
792
max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
793
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
794
795
+
796
def do_vscan(f):
797
if f["path"].is_file():
798
with open(f["path"], "rb") as scanf:
799
try:
800
+
res = list(app.config["VSCAN_SOCKET"].instream(scanf).values())
801
+
f["result"] = res[0]
802
except:
803
f["result"] = ("SCAN FAILED", None)
804
else:
···
806
807
return f
808
809
+
810
@app.cli.command("vscan")
811
def vscan():
812
if not app.config["VSCAN_SOCKET"]:
813
+
print("Error: Virus scanning enabled but no connection method "
814
+
"specified.\nPlease set VSCAN_SOCKET.")
815
sys.exit(1)
816
817
qp = Path(app.config["VSCAN_QUARANTINE_PATH"])
···
825
File.last_vscan == None),
826
File.removed == False)
827
else:
828
+
res = File.query.filter(File.last_vscan == None,
829
+
File.removed == False)
830
831
+
work = [{"path": f.getpath(), "name": f.getname(), "id": f.id}
832
+
for f in res]
833
834
results = []
835
for i, r in enumerate(p.imap_unordered(do_vscan, work)):
···
843
found = True
844
845
results.append({
846
+
"id": r["id"],
847
+
"last_vscan": None if r["result"][0] == "SCAN FAILED"
848
+
else datetime.datetime.now(),
849
+
"removed": found})
850
851
db.session.bulk_update_mappings(File, results)
852
db.session.commit()
+1
migrations/env.py
+1
migrations/env.py
-4
migrations/versions/0659d7b9eea8_.py
-4
migrations/versions/0659d7b9eea8_.py
···
15
16
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True))
20
-
# ### end Alembic commands ###
21
22
23
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
op.drop_column('file', 'mgmt_token')
26
-
# ### end Alembic commands ###
+12
-18
migrations/versions/0cd36ecdd937_.py
+12
-18
migrations/versions/0cd36ecdd937_.py
···
15
16
17
def upgrade():
18
-
### commands auto generated by Alembic - please adjust! ###
19
op.create_table('URL',
20
-
sa.Column('id', sa.Integer(), nullable=False),
21
-
sa.Column('url', sa.UnicodeText(), nullable=True),
22
-
sa.PrimaryKeyConstraint('id'),
23
-
sa.UniqueConstraint('url')
24
-
)
25
op.create_table('file',
26
-
sa.Column('id', sa.Integer(), nullable=False),
27
-
sa.Column('sha256', sa.String(), nullable=True),
28
-
sa.Column('ext', sa.UnicodeText(), nullable=True),
29
-
sa.Column('mime', sa.UnicodeText(), nullable=True),
30
-
sa.Column('addr', sa.UnicodeText(), nullable=True),
31
-
sa.Column('removed', sa.Boolean(), nullable=True),
32
-
sa.PrimaryKeyConstraint('id'),
33
-
sa.UniqueConstraint('sha256')
34
-
)
35
-
### end Alembic commands ###
36
37
38
def downgrade():
39
-
### commands auto generated by Alembic - please adjust! ###
40
op.drop_table('file')
41
op.drop_table('URL')
42
-
### end Alembic commands ###
···
15
16
17
def upgrade():
18
op.create_table('URL',
19
+
sa.Column('id', sa.Integer(), nullable=False),
20
+
sa.Column('url', sa.UnicodeText(), nullable=True),
21
+
sa.PrimaryKeyConstraint('id'),
22
+
sa.UniqueConstraint('url'))
23
op.create_table('file',
24
+
sa.Column('id', sa.Integer(), nullable=False),
25
+
sa.Column('sha256', sa.String(), nullable=True),
26
+
sa.Column('ext', sa.UnicodeText(), nullable=True),
27
+
sa.Column('mime', sa.UnicodeText(), nullable=True),
28
+
sa.Column('addr', sa.UnicodeText(), nullable=True),
29
+
sa.Column('removed', sa.Boolean(), nullable=True),
30
+
sa.PrimaryKeyConstraint('id'),
31
+
sa.UniqueConstraint('sha256'))
32
33
34
def downgrade():
35
op.drop_table('file')
36
op.drop_table('URL')
+3
-2
migrations/versions/30bfe33aa328_add_file_size_field.py
+3
-2
migrations/versions/30bfe33aa328_add_file_size_field.py
···
19
20
Base = automap_base()
21
22
def upgrade():
23
op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True))
24
bind = op.get_bind()
···
34
p = storage / f.sha256
35
if p.is_file():
36
updates.append({
37
-
"id" : f.id,
38
-
"size" : p.stat().st_size
39
})
40
41
session.bulk_update_mappings(File, updates)
···
19
20
Base = automap_base()
21
22
+
23
def upgrade():
24
op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True))
25
bind = op.get_bind()
···
35
p = storage / f.sha256
36
if p.is_file():
37
updates.append({
38
+
"id": f.id,
39
+
"size": p.stat().st_size
40
})
41
42
session.bulk_update_mappings(File, updates)
+31
-32
migrations/versions/5cda1743b92d_add_request_filters.py
+31
-32
migrations/versions/5cda1743b92d_add_request_filters.py
···
19
20
Base = automap_base()
21
22
def upgrade():
23
-
# ### commands auto generated by Alembic - please adjust! ###
24
op.create_table('request_filter',
25
-
sa.Column('id', sa.Integer(), nullable=False),
26
-
sa.Column('type', sa.String(length=20), nullable=False),
27
-
sa.Column('comment', sa.UnicodeText(), nullable=True),
28
-
sa.Column('addr', sa.LargeBinary(length=16), nullable=True),
29
-
sa.Column('net', sa.Text(), nullable=True),
30
-
sa.Column('regex', sa.UnicodeText(), nullable=True),
31
-
sa.PrimaryKeyConstraint('id'),
32
-
sa.UniqueConstraint('addr')
33
-
)
34
-
with op.batch_alter_table('request_filter', schema=None) as batch_op:
35
-
batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], unique=False)
36
37
-
# ### end Alembic commands ###
38
39
bind = op.get_bind()
40
Base.prepare(autoload_with=bind)
41
RequestFilter = Base.classes.request_filter
42
session = Session(bind=bind)
43
44
-
if "FHOST_UPLOAD_BLACKLIST" in current_app.config:
45
-
if current_app.config["FHOST_UPLOAD_BLACKLIST"]:
46
-
with current_app.open_instance_resource(current_app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
47
-
for l in bl.readlines():
48
-
if not l.startswith("#"):
49
-
l = l.strip()
50
-
if l.endswith(":"):
51
-
# old implementation uses str.startswith,
52
-
# which does not translate to networks
53
-
current_app.logger.warning(f"Ignored address: {l}")
54
-
continue
55
56
-
flt = RequestFilter(type="addr", addr=ipaddress.ip_address(l).packed)
57
-
session.add(flt)
58
59
-
if "FHOST_MIME_BLACKLIST" in current_app.config:
60
-
for mime in current_app.config["FHOST_MIME_BLACKLIST"]:
61
-
flt = RequestFilter(type="mime", regex=mime)
62
-
session.add(flt)
63
64
session.commit()
65
···
72
73
74
def downgrade():
75
-
# ### commands auto generated by Alembic - please adjust! ###
76
with op.batch_alter_table('request_filter', schema=None) as batch_op:
77
batch_op.drop_index(batch_op.f('ix_request_filter_type'))
78
79
op.drop_table('request_filter')
80
-
# ### end Alembic commands ###
···
19
20
Base = automap_base()
21
22
+
23
def upgrade():
24
op.create_table('request_filter',
25
+
sa.Column('id', sa.Integer(), nullable=False),
26
+
sa.Column('type', sa.String(length=20), nullable=False),
27
+
sa.Column('comment', sa.UnicodeText(), nullable=True),
28
+
sa.Column('addr', sa.LargeBinary(length=16),
29
+
nullable=True),
30
+
sa.Column('net', sa.Text(), nullable=True),
31
+
sa.Column('regex', sa.UnicodeText(), nullable=True),
32
+
sa.PrimaryKeyConstraint('id'),
33
+
sa.UniqueConstraint('addr'))
34
35
+
with op.batch_alter_table('request_filter', schema=None) as batch_op:
36
+
batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'],
37
+
unique=False)
38
39
bind = op.get_bind()
40
Base.prepare(autoload_with=bind)
41
RequestFilter = Base.classes.request_filter
42
session = Session(bind=bind)
43
44
+
blp = current_app.config.get("FHOST_UPLOAD_BLACKLIST")
45
+
if blp:
46
+
with current_app.open_instance_resource(blp, "r") as bl:
47
+
for line in bl.readlines():
48
+
if not line.startswith("#"):
49
+
line = line.strip()
50
+
if line.endswith(":"):
51
+
# old implementation uses str.startswith,
52
+
# which does not translate to networks
53
+
current_app.logger.warning(
54
+
f"Ignored address: {line}")
55
+
continue
56
57
+
addr = ipaddress.ip_address(line).packed
58
+
flt = RequestFilter(type="addr", addr=addr)
59
+
session.add(flt)
60
61
+
for mime in current_app.config.get("FHOST_MIME_BLACKLIST", []):
62
+
flt = RequestFilter(type="mime", regex=mime)
63
+
session.add(flt)
64
65
session.commit()
66
···
73
74
75
def downgrade():
76
with op.batch_alter_table('request_filter', schema=None) as batch_op:
77
batch_op.drop_index(batch_op.f('ix_request_filter_type'))
78
79
op.drop_table('request_filter')
+2
-5
migrations/versions/5cee97aab219_.py
+2
-5
migrations/versions/5cee97aab219_.py
···
15
16
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
-
op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True))
20
-
# ### end Alembic commands ###
21
22
23
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
op.drop_column('file', 'last_vscan')
26
-
# ### end Alembic commands ###
-4
migrations/versions/7e246705da6a_.py
-4
migrations/versions/7e246705da6a_.py
···
15
16
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True))
20
-
# ### end Alembic commands ###
21
22
23
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
op.drop_column('file', 'nsfw_score')
26
-
# ### end Alembic commands ###
+24
-14
migrations/versions/939a08e1d6e5_.py
+24
-14
migrations/versions/939a08e1d6e5_.py
···
21
import os
22
import time
23
24
-
""" For a file of a given size, determine the largest allowed lifespan of that file
25
26
-
Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
27
-
as FHOST_{MIN,MAX}_EXPIRATION.
28
29
-
This lifespan may be shortened by a user's request, but no files should be allowed to
30
-
expire at a point after this number.
31
32
Value returned is a duration in milliseconds.
33
"""
34
def get_max_lifespan(filesize: int) -> int:
35
-
min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
36
-
max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
37
-
max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
38
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
39
40
Base = automap_base()
41
42
def upgrade():
43
op.add_column('file', sa.Column('expiration', sa.BigInteger()))
···
48
session = Session(bind=bind)
49
50
storage = Path(current_app.config["FHOST_STORAGE_PATH"])
51
-
current_time = time.time() * 1000;
52
53
# List of file hashes which have not expired yet
54
# This could get really big for some servers
55
try:
56
unexpired_files = os.listdir(storage)
57
except FileNotFoundError:
58
-
return # There are no currently unexpired files
59
60
# Calculate an expiration date for all existing files
61
···
65
sa.not_(File.removed)
66
)
67
)
68
-
updates = [] # We coalesce updates to the database here
69
70
# SQLite has a hard limit on the number of variables so we
71
# need to do this the slow way
···
74
for file in files:
75
file_path = storage / file.sha256
76
stat = os.stat(file_path)
77
-
max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms
78
-
file_birth = stat.st_mtime * 1000 # When the file was created, in ms
79
-
updates.append({'id': file.id, 'expiration': int(file_birth + max_age)})
80
81
# Apply coalesced updates
82
session.bulk_update_mappings(File, updates)
83
session.commit()
84
85
def downgrade():
86
op.drop_column('file', 'expiration')
···
21
import os
22
import time
23
24
25
+
"""
26
+
For a file of a given size, determine the largest allowed lifespan of that file
27
28
+
Based on the current app's configuration:
29
+
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION.
30
+
31
+
This lifespan may be shortened by a user's request, but no files should be
32
+
allowed to expire at a point after this number.
33
34
Value returned is a duration in milliseconds.
35
"""
36
def get_max_lifespan(filesize: int) -> int:
37
+
cfg = current_app.config
38
+
min_exp = cfg.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
39
+
max_exp = cfg.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
40
+
max_size = cfg.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
41
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
42
43
+
44
Base = automap_base()
45
+
46
47
def upgrade():
48
op.add_column('file', sa.Column('expiration', sa.BigInteger()))
···
53
session = Session(bind=bind)
54
55
storage = Path(current_app.config["FHOST_STORAGE_PATH"])
56
+
current_time = time.time() * 1000
57
58
# List of file hashes which have not expired yet
59
# This could get really big for some servers
60
try:
61
unexpired_files = os.listdir(storage)
62
except FileNotFoundError:
63
+
return # There are no currently unexpired files
64
65
# Calculate an expiration date for all existing files
66
···
70
sa.not_(File.removed)
71
)
72
)
73
+
updates = [] # We coalesce updates to the database here
74
75
# SQLite has a hard limit on the number of variables so we
76
# need to do this the slow way
···
79
for file in files:
80
file_path = storage / file.sha256
81
stat = os.stat(file_path)
82
+
# How long the file is allowed to live, in ms
83
+
max_age = get_max_lifespan(stat.st_size)
84
+
# When the file was created, in ms
85
+
file_birth = stat.st_mtime * 1000
86
+
updates.append({
87
+
'id': file.id,
88
+
'expiration': int(file_birth + max_age)})
89
90
# Apply coalesced updates
91
session.bulk_update_mappings(File, updates)
92
session.commit()
93
+
94
95
def downgrade():
96
op.drop_column('file', 'expiration')
-6
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
-6
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
···
15
16
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
with op.batch_alter_table('file', schema=None) as batch_op:
20
batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True))
21
-
22
-
# ### end Alembic commands ###
23
24
25
def downgrade():
26
-
# ### commands auto generated by Alembic - please adjust! ###
27
with op.batch_alter_table('file', schema=None) as batch_op:
28
batch_op.drop_column('ua')
29
-
30
-
# ### end Alembic commands ###
-4
migrations/versions/e2e816056589_.py
-4
migrations/versions/e2e816056589_.py
···
15
16
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
op.add_column('file', sa.Column('secret', sa.String(), nullable=True))
20
-
# ### end Alembic commands ###
21
22
23
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
op.drop_column('file', 'secret')
26
-
# ### end Alembic commands ###
+64
-37
mod.py
+64
-37
mod.py
···
14
import ipaddress
15
16
from fhost import db, File, AddrFilter, su, app as fhost_app
17
-
from modui import *
18
19
fhost_app.app_context().push()
20
21
class NullptrMod(Screen):
22
BINDINGS = [
···
67
self.finput.display = False
68
ftable = self.query_one("#ftable")
69
ftable.focus()
70
71
if len(message.value):
72
match self.filter_col:
73
case 1:
74
-
try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value))
75
-
except ValueError: pass
76
case 2:
77
try:
78
addr = ipaddress.ip_address(message.value)
79
if type(addr) is ipaddress.IPv6Address:
80
addr = addr.ipv4_mapped or addr
81
-
q = ftable.base_query.filter(File.addr == addr)
82
-
ftable.query = q
83
-
except ValueError: pass
84
-
case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value))
85
-
case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value))
86
-
case 5: ftable.query = ftable.base_query.filter(File.ua.like(message.value))
87
-
else:
88
-
ftable.query = ftable.base_query
89
90
def action_remove_file(self, permanent: bool) -> None:
91
if self.current_file:
92
self.current_file.delete(permanent)
93
db.session.commit()
94
-
self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}"))
95
self.action_refresh()
96
97
def action_ban_ip(self, nuke: bool) -> None:
98
if self.current_file:
99
-
if AddrFilter.query.filter(AddrFilter.addr ==
100
-
self.current_file.addr).scalar():
101
-
txt = f"{self.current_file.addr.compressed} is already banned"
102
else:
103
-
db.session.add(AddrFilter(self.current_file.addr))
104
db.session.commit()
105
-
txt = f"Banned {self.current_file.addr.compressed}"
106
107
if nuke:
108
tsize = 0
109
trm = 0
110
-
for f in File.query.filter(File.addr == self.current_file.addr):
111
if f.getpath().is_file():
112
tsize += f.size or f.getpath().stat().st_size
113
trm += 1
114
f.delete(True)
115
db.session.commit()
116
-
txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}"
117
self.mount(Notification(txt))
118
self._refresh_layout()
119
ftable = self.query_one("#ftable")
···
131
DataTable(id="finfo", show_header=False, cursor_type="none"),
132
MpvWidget(id="mpv"),
133
RichLog(id="ftextlog", auto_scroll=False),
134
-
id="infopane"))
135
yield Input(id="filter_input")
136
yield Footer()
137
···
150
self.finput = self.query_one("#filter_input")
151
152
self.mimehandler = mime.MIMEHandler()
153
-
self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive)
154
self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
155
self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
156
-
self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf)
157
-
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive)
158
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
159
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
160
···
166
167
def handle_text(self, cat):
168
with open(self.current_file.getpath(), "r") as sf:
169
-
data = sf.read(1000000).replace("\033","")
170
self.ftlog.write(data)
171
return True
172
···
181
self.mpvw.styles.height = "40%"
182
self.mpvw.start_mpv("hex://" + imgdata, 0)
183
184
-
self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
185
self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
186
for k, v in doc.metadata.items():
187
self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}"))
···
206
for k, v in c.metadata.items():
207
self.ftlog.write(f" {k}: {v}")
208
for s in c.streams:
209
-
self.ftlog.write(Text(f"Stream {s.index}:", style="bold"))
210
self.ftlog.write(f" Type: {s.type}")
211
if s.base_rate:
212
self.ftlog.write(f" Frame rate: {s.base_rate}")
···
225
else:
226
c = chr(s)
227
s = c
228
-
if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]"
229
-
if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]"
230
if not c.isprintable():
231
g = "grey50" if c == "\0" else "cadet_blue"
232
return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
233
return s
234
-
return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}"
235
-
f"{' ' * (16 - len(c))}"
236
-
f" {''.join(map(fmt, c))}"
237
-
for c in map(lambda x: bytes([n for n in x if n != None]),
238
-
zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16))))
239
240
with open(self.current_file.getpath(), "rb") as binf:
241
self.ftlog.write(hexdump(binf, self.current_file.size))
242
if self.current_file.size > 16*10*2:
243
binf.seek(self.current_file.size-16*10)
244
self.ftlog.write(" [...] ".center(64, '─'))
245
-
self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell()))
246
247
return True
248
···
253
self.finfo.add_rows([
254
("ID:", str(f.id)),
255
("File name:", f.getname()),
256
-
("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"),
257
("File size:", do_filesizeformat(f.size, True)),
258
("MIME type:", f.mime),
259
("SHA256 checksum:", f.sha256),
···
261
("User agent:", Text(f.ua or "")),
262
("Management token:", f.mgmt_token),
263
("Secret:", f.secret),
264
-
("Is NSFW:", ("Yes" if f.is_nsfw else "No") + (f" (Score: {f.nsfw_score:0.4f})" if f.nsfw_score else " (Not scanned)")),
265
("Is banned:", "Yes" if f.removed else "No"),
266
-
("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000)))
267
])
268
269
self.mpvw.stop_mpv(True)
···
272
if f.getpath().is_file():
273
self.mimehandler.handle(f.mime, f.ext)
274
self.ftlog.scroll_to(x=0, y=0, animate=False)
275
276
class NullptrModApp(App):
277
CSS_PATH = "mod.css"
···
281
self.main_screen = NullptrMod()
282
self.install_screen(self.main_screen, name="main")
283
self.push_screen("main")
284
285
if __name__ == "__main__":
286
app = NullptrModApp()
···
14
import ipaddress
15
16
from fhost import db, File, AddrFilter, su, app as fhost_app
17
+
from modui import FileTable, mime, MpvWidget, Notification
18
19
fhost_app.app_context().push()
20
+
21
22
class NullptrMod(Screen):
23
BINDINGS = [
···
68
self.finput.display = False
69
ftable = self.query_one("#ftable")
70
ftable.focus()
71
+
q = ftable.base_query
72
73
if len(message.value):
74
match self.filter_col:
75
case 1:
76
+
try:
77
+
q = q.filter(File.id == su.debase(message.value))
78
+
except ValueError:
79
+
return
80
case 2:
81
try:
82
addr = ipaddress.ip_address(message.value)
83
if type(addr) is ipaddress.IPv6Address:
84
addr = addr.ipv4_mapped or addr
85
+
q = q.filter(File.addr == addr)
86
+
except ValueError:
87
+
return
88
+
case 3: q = q.filter(File.mime.like(message.value))
89
+
case 4: q = q.filter(File.ext.like(message.value))
90
+
case 5: q = q.filter(File.ua.like(message.value))
91
+
92
+
ftable.query = q
93
94
def action_remove_file(self, permanent: bool) -> None:
95
if self.current_file:
96
self.current_file.delete(permanent)
97
db.session.commit()
98
+
self.mount(Notification(f"{'Banned' if permanent else 'Removed'}"
99
+
f"file {self.current_file.getname()}"))
100
self.action_refresh()
101
102
def action_ban_ip(self, nuke: bool) -> None:
103
if self.current_file:
104
+
addr = self.current_file.addr
105
+
if AddrFilter.query.filter(AddrFilter.addr == addr).scalar():
106
+
txt = f"{addr.compressed} is already banned"
107
else:
108
+
db.session.add(AddrFilter(addr))
109
db.session.commit()
110
+
txt = f"Banned {addr.compressed}"
111
112
if nuke:
113
tsize = 0
114
trm = 0
115
+
for f in File.query.filter(File.addr == addr):
116
if f.getpath().is_file():
117
tsize += f.size or f.getpath().stat().st_size
118
trm += 1
119
f.delete(True)
120
db.session.commit()
121
+
txt += f", removed {trm} {'files' if trm != 1 else 'file'} " \
122
+
f"totaling {do_filesizeformat(tsize, True)}"
123
self.mount(Notification(txt))
124
self._refresh_layout()
125
ftable = self.query_one("#ftable")
···
137
DataTable(id="finfo", show_header=False, cursor_type="none"),
138
MpvWidget(id="mpv"),
139
RichLog(id="ftextlog", auto_scroll=False),
140
+
id="infopane"))
141
yield Input(id="filter_input")
142
yield Footer()
143
···
156
self.finput = self.query_one("#filter_input")
157
158
self.mimehandler = mime.MIMEHandler()
159
+
self.mimehandler.register(mime.MIMECategory.Archive,
160
+
self.handle_libarchive)
161
self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
162
self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
163
+
self.mimehandler.register(mime.MIMECategory.Document,
164
+
self.handle_mupdf)
165
+
self.mimehandler.register(mime.MIMECategory.Fallback,
166
+
self.handle_libarchive)
167
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
168
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
169
···
175
176
def handle_text(self, cat):
177
with open(self.current_file.getpath(), "r") as sf:
178
+
data = sf.read(1000000).replace("\033", "")
179
self.ftlog.write(data)
180
return True
181
···
190
self.mpvw.styles.height = "40%"
191
self.mpvw.start_mpv("hex://" + imgdata, 0)
192
193
+
self.ftlog.write(
194
+
Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
195
self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
196
for k, v in doc.metadata.items():
197
self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}"))
···
216
for k, v in c.metadata.items():
217
self.ftlog.write(f" {k}: {v}")
218
for s in c.streams:
219
+
self.ftlog.write(
220
+
Text(f"Stream {s.index}:", style="bold"))
221
self.ftlog.write(f" Type: {s.type}")
222
if s.base_rate:
223
self.ftlog.write(f" Frame rate: {s.base_rate}")
···
236
else:
237
c = chr(s)
238
s = c
239
+
if c.isalpha():
240
+
return f"\0[chartreuse1]{s}\0[/chartreuse1]"
241
+
if c.isdigit():
242
+
return f"\0[gold1]{s}\0[/gold1]"
243
if not c.isprintable():
244
g = "grey50" if c == "\0" else "cadet_blue"
245
return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
246
return s
247
+
248
+
return Text.from_markup(
249
+
"\n".join(' '.join(
250
+
map(fmt, map(''.join, zip(*[iter(c.hex())] * 2)))) +
251
+
f"{' ' * (16 - len(c))} {''.join(map(fmt, c))}"
252
+
for c in
253
+
map(lambda x: bytes([n for n in x if n is not None]),
254
+
zip_longest(
255
+
*[iter(binf.read(min(length, 16 * 10)))] * 16))))
256
257
with open(self.current_file.getpath(), "rb") as binf:
258
self.ftlog.write(hexdump(binf, self.current_file.size))
259
if self.current_file.size > 16*10*2:
260
binf.seek(self.current_file.size-16*10)
261
self.ftlog.write(" [...] ".center(64, '─'))
262
+
self.ftlog.write(hexdump(binf,
263
+
self.current_file.size - binf.tell()))
264
265
return True
266
···
271
self.finfo.add_rows([
272
("ID:", str(f.id)),
273
("File name:", f.getname()),
274
+
("URL:", f.geturl()
275
+
if fhost_app.config["SERVER_NAME"]
276
+
else "⚠ Set SERVER_NAME in config.py to display"),
277
("File size:", do_filesizeformat(f.size, True)),
278
("MIME type:", f.mime),
279
("SHA256 checksum:", f.sha256),
···
281
("User agent:", Text(f.ua or "")),
282
("Management token:", f.mgmt_token),
283
("Secret:", f.secret),
284
+
("Is NSFW:", ("Yes" if f.is_nsfw else "No") +
285
+
(f" (Score: {f.nsfw_score:0.4f})"
286
+
if f.nsfw_score else " (Not scanned)")),
287
("Is banned:", "Yes" if f.removed else "No"),
288
+
("Expires:",
289
+
time.strftime("%Y-%m-%d %H:%M:%S",
290
+
time.gmtime(File.get_expiration(f.expiration,
291
+
f.size)/1000)))
292
])
293
294
self.mpvw.stop_mpv(True)
···
297
if f.getpath().is_file():
298
self.mimehandler.handle(f.mime, f.ext)
299
self.ftlog.scroll_to(x=0, y=0, animate=False)
300
+
301
302
class NullptrModApp(App):
303
CSS_PATH = "mod.css"
···
307
self.main_screen = NullptrMod()
308
self.install_screen(self.main_screen, name="main")
309
self.push_screen("main")
310
+
311
312
if __name__ == "__main__":
313
app = NullptrModApp()
+9
-4
modui/filetable.py
+9
-4
modui/filetable.py
···
7
from fhost import File
8
from modui import mime
9
10
class FileTable(DataTable):
11
query = Reactive(None)
12
order_col = Reactive(0)
13
order_desc = Reactive(True)
14
limit = 10000
15
-
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime]
16
17
def __init__(self, **kwargs):
18
super().__init__(**kwargs)
···
33
34
def watch_query(self, old, value) -> None:
35
def fmt_file(f: File) -> tuple:
36
return (
37
str(f.id),
38
"🔴" if f.removed else " ",
···
40
"👻" if not f.getpath().is_file() else " ",
41
f.getname(),
42
do_filesizeformat(f.size, True),
43
-
f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or ' '} " + f.mime,
44
)
45
46
if (self.query):
47
-
48
order = FileTable.colmap[self.order_col]
49
q = self.query
50
-
if order: q = q.order_by(order.desc() if self.order_desc else order, File.id)
51
qres = list(map(fmt_file, q.limit(self.limit)))
52
53
ri = 0
···
7
from fhost import File
8
from modui import mime
9
10
+
11
class FileTable(DataTable):
12
query = Reactive(None)
13
order_col = Reactive(0)
14
order_desc = Reactive(True)
15
limit = 10000
16
+
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext,
17
+
File.size, File.mime]
18
19
def __init__(self, **kwargs):
20
super().__init__(**kwargs)
···
35
36
def watch_query(self, old, value) -> None:
37
def fmt_file(f: File) -> tuple:
38
+
mimemoji = mime.mimemoji.get(f.mime.split('/')[0],
39
+
mime.mimemoji.get(f.mime)) or ' '
40
return (
41
str(f.id),
42
"🔴" if f.removed else " ",
···
44
"👻" if not f.getpath().is_file() else " ",
45
f.getname(),
46
do_filesizeformat(f.size, True),
47
+
f"{mimemoji} {f.mime}",
48
)
49
50
if (self.query):
51
order = FileTable.colmap[self.order_col]
52
q = self.query
53
+
if order:
54
+
q = q.order_by(order.desc() if self.order_desc
55
+
else order, File.id)
56
qres = list(map(fmt_file, q.limit(self.limit)))
57
58
ri = 0
+48
-46
modui/mime.py
+48
-46
modui/mime.py
···
2
from textual import log
3
4
mimemoji = {
5
-
"audio" : "🔈",
6
-
"video" : "🎞",
7
-
"text" : "📄",
8
-
"image" : "🖼",
9
-
"application/zip" : "🗜️",
10
-
"application/x-zip-compressed" : "🗜️",
11
-
"application/x-tar" : "🗄",
12
-
"application/x-cpio" : "🗄",
13
-
"application/x-xz" : "🗜️",
14
-
"application/x-7z-compressed" : "🗜️",
15
-
"application/gzip" : "🗜️",
16
-
"application/zstd" : "🗜️",
17
-
"application/x-rar" : "🗜️",
18
-
"application/x-rar-compressed" : "🗜️",
19
-
"application/vnd.ms-cab-compressed" : "🗜️",
20
-
"application/x-bzip2" : "🗜️",
21
-
"application/x-lzip" : "🗜️",
22
-
"application/x-iso9660-image" : "💿",
23
-
"application/pdf" : "📕",
24
-
"application/epub+zip" : "📕",
25
-
"application/mxf" : "🎞",
26
-
"application/vnd.android.package-archive" : "📦",
27
-
"application/vnd.debian.binary-package" : "📦",
28
-
"application/x-rpm" : "📦",
29
-
"application/x-dosexec" : "⚙",
30
-
"application/x-execuftable" : "⚙",
31
-
"application/x-sharedlib" : "⚙",
32
-
"application/java-archive" : "☕",
33
-
"application/x-qemu-disk" : "🖴",
34
-
"application/pgp-encrypted" : "🔏",
35
}
36
37
-
MIMECategory = Enum("MIMECategory",
38
-
["Archive", "Text", "AV", "Document", "Fallback"]
39
-
)
40
41
class MIMEHandler:
42
def __init__(self):
43
self.handlers = {
44
-
MIMECategory.Archive : [[
45
"application/zip",
46
"application/x-zip-compressed",
47
"application/x-tar",
···
62
"application/java-archive",
63
"application/vnd.openxmlformats"
64
], []],
65
-
MIMECategory.Text : [[
66
"text",
67
"application/json",
68
"application/xml",
69
], []],
70
-
MIMECategory.AV : [[
71
"audio", "video", "image",
72
"application/mxf"
73
], []],
74
-
MIMECategory.Document : [[
75
"application/pdf",
76
"application/epub",
77
"application/x-mobipocket-ebook",
78
], []],
79
-
MIMECategory.Fallback : [[], []]
80
}
81
82
self.exceptions = {
83
-
MIMECategory.Archive : {
84
-
".cbz" : MIMECategory.Document,
85
-
".xps" : MIMECategory.Document,
86
-
".epub" : MIMECategory.Document,
87
},
88
-
MIMECategory.Text : {
89
-
".fb2" : MIMECategory.Document,
90
}
91
}
92
···
115
cat = getcat(mime)
116
for handler in self.handlers[cat][1]:
117
try:
118
-
if handler(cat): return
119
except: pass
120
121
for handler in self.handlers[MIMECategory.Fallback][1]:
122
try:
123
-
if handler(None): return
124
except: pass
125
126
raise RuntimeError(f"Unhandled MIME type category: {cat}")
···
2
from textual import log
3
4
mimemoji = {
5
+
"audio": "🔈",
6
+
"video": "🎞",
7
+
"text": "📄",
8
+
"image": "🖼",
9
+
"application/zip": "🗜️",
10
+
"application/x-zip-compressed": "🗜️",
11
+
"application/x-tar": "🗄",
12
+
"application/x-cpio": "🗄",
13
+
"application/x-xz": "🗜️",
14
+
"application/x-7z-compressed": "🗜️",
15
+
"application/gzip": "🗜️",
16
+
"application/zstd": "🗜️",
17
+
"application/x-rar": "🗜️",
18
+
"application/x-rar-compressed": "🗜️",
19
+
"application/vnd.ms-cab-compressed": "🗜️",
20
+
"application/x-bzip2": "🗜️",
21
+
"application/x-lzip": "🗜️",
22
+
"application/x-iso9660-image": "💿",
23
+
"application/pdf": "📕",
24
+
"application/epub+zip": "📕",
25
+
"application/mxf": "🎞",
26
+
"application/vnd.android.package-archive": "📦",
27
+
"application/vnd.debian.binary-package": "📦",
28
+
"application/x-rpm": "📦",
29
+
"application/x-dosexec": "⚙",
30
+
"application/x-execuftable": "⚙",
31
+
"application/x-sharedlib": "⚙",
32
+
"application/java-archive": "☕",
33
+
"application/x-qemu-disk": "🖴",
34
+
"application/pgp-encrypted": "🔏",
35
}
36
37
+
MIMECategory = Enum("MIMECategory", ["Archive", "Text", "AV", "Document",
38
+
"Fallback"])
39
+
40
41
class MIMEHandler:
42
def __init__(self):
43
self.handlers = {
44
+
MIMECategory.Archive: [[
45
"application/zip",
46
"application/x-zip-compressed",
47
"application/x-tar",
···
62
"application/java-archive",
63
"application/vnd.openxmlformats"
64
], []],
65
+
MIMECategory.Text: [[
66
"text",
67
"application/json",
68
"application/xml",
69
], []],
70
+
MIMECategory.AV: [[
71
"audio", "video", "image",
72
"application/mxf"
73
], []],
74
+
MIMECategory.Document: [[
75
"application/pdf",
76
"application/epub",
77
"application/x-mobipocket-ebook",
78
], []],
79
+
MIMECategory.Fallback: [[], []]
80
}
81
82
self.exceptions = {
83
+
MIMECategory.Archive: {
84
+
".cbz": MIMECategory.Document,
85
+
".xps": MIMECategory.Document,
86
+
".epub": MIMECategory.Document,
87
},
88
+
MIMECategory.Text: {
89
+
".fb2": MIMECategory.Document,
90
}
91
}
92
···
115
cat = getcat(mime)
116
for handler in self.handlers[cat][1]:
117
try:
118
+
if handler(cat):
119
+
return
120
except: pass
121
122
for handler in self.handlers[MIMECategory.Fallback][1]:
123
try:
124
+
if handler(None):
125
+
return
126
except: pass
127
128
raise RuntimeError(f"Unhandled MIME type category: {cat}")
+24
-10
modui/mpvwidget.py
+24
-10
modui/mpvwidget.py
···
1
import time
2
-
import fcntl, struct, termios
3
from sys import stdout
4
5
from textual import events, log
6
from textual.widgets import Static
7
8
from fhost import app as fhost_app
9
10
class MpvWidget(Static):
11
def __init__(self, **kwargs):
···
14
self.mpv = None
15
self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
16
17
-
if not self.vo in ["sixel", "kitty"]:
18
-
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.")
19
else:
20
try:
21
import mpv
···
27
self.mpv[f"vo-sixel-buffered"] = True
28
self.mpv["audio"] = False
29
self.mpv["loop-file"] = "inf"
30
-
self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf"
31
except Exception as e:
32
self.mpv = None
33
-
self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}")
34
35
-
def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None:
36
self.display = True
37
self.screen._refresh_layout()
38
39
if self.mpv:
40
if self.content_region.x:
41
-
r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678'))
42
width = int((w / c) * self.content_region.width)
43
-
height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0)))
44
self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
45
self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
46
-
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0)
47
self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
48
self.mpv[f"vo-{self.vo}-width"] = width
49
self.mpv[f"vo-{self.vo}-height"] = height
50
51
-
if pos != None:
52
self.mpv["start"] = pos
53
54
if f:
···
1
import time
2
+
3
+
import fcntl
4
+
import struct
5
+
import termios
6
+
7
from sys import stdout
8
9
from textual import events, log
10
from textual.widgets import Static
11
12
from fhost import app as fhost_app
13
+
14
15
class MpvWidget(Static):
16
def __init__(self, **kwargs):
···
19
self.mpv = None
20
self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
21
22
+
if self.vo not in ["sixel", "kitty"]:
23
+
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO "
24
+
"to 'sixel' or 'kitty' in config.py,\nwhichever is "
25
+
"supported by your terminal.")
26
else:
27
try:
28
import mpv
···
34
self.mpv[f"vo-sixel-buffered"] = True
35
self.mpv["audio"] = False
36
self.mpv["loop-file"] = "inf"
37
+
self.mpv["image-display-duration"] = 0.5 \
38
+
if self.vo == "sixel" else "inf"
39
except Exception as e:
40
self.mpv = None
41
+
self.update("⚠ Previews require python-mpv with libmpv "
42
+
"0.36.0 or later \n\nError was:\n"
43
+
f"{type(e).__name__}: {e}")
44
45
+
def start_mpv(self, f: str | None = None,
46
+
pos: float | str | None = None) -> None:
47
self.display = True
48
self.screen._refresh_layout()
49
50
if self.mpv:
51
if self.content_region.x:
52
+
winsz = fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678')
53
+
r, c, w, h = struct.unpack('hhhh', winsz)
54
width = int((w / c) * self.content_region.width)
55
+
height = int((h / r) * (self.content_region.height +
56
+
(1 if self.vo == "sixel" else 0)))
57
self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
58
self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
59
+
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + \
60
+
(1 if self.vo == "sixel" else 0)
61
self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
62
self.mpv[f"vo-{self.vo}-width"] = width
63
self.mpv[f"vo-{self.vo}-height"] = height
64
65
+
if pos is not None:
66
self.mpv["start"] = pos
67
68
if f:
+1
modui/notification.py
+1
modui/notification.py
+8
-6
nsfw_detect.py
+8
-6
nsfw_detect.py
···
18
and limitations under the License.
19
"""
20
21
-
import os
22
import sys
23
-
from pathlib import Path
24
-
25
import av
26
from transformers import pipeline
27
28
class NSFWDetector:
29
def __init__(self):
30
-
self.classifier = pipeline("image-classification", model="giacomoarienti/nsfw-classifier")
31
32
def detect(self, fpath):
33
try:
34
with av.open(fpath) as container:
35
-
try: container.seek(int(container.duration / 2))
36
except: container.seek(0)
37
38
frame = next(container.decode(video=0))
39
img = frame.to_image()
40
res = self.classifier(img)
41
42
-
return max([x["score"] for x in res if x["label"] not in ["neutral", "drawings"]])
43
except: pass
44
45
return -1.0
46
47
if __name__ == "__main__":
48
n = NSFWDetector()
···
18
and limitations under the License.
19
"""
20
21
import sys
22
import av
23
from transformers import pipeline
24
+
25
26
class NSFWDetector:
27
def __init__(self):
28
+
self.classifier = pipeline("image-classification",
29
+
model="giacomoarienti/nsfw-classifier")
30
31
def detect(self, fpath):
32
try:
33
with av.open(fpath) as container:
34
+
try:
35
+
container.seek(int(container.duration / 2))
36
except: container.seek(0)
37
38
frame = next(container.decode(video=0))
39
img = frame.to_image()
40
res = self.classifier(img)
41
42
+
return max([x["score"] for x in res
43
+
if x["label"] not in ["neutral", "drawings"]])
44
except: pass
45
46
return -1.0
47
+
48
49
if __name__ == "__main__":
50
n = NSFWDetector()