+1
-1
cleanup.py
+1
-1
cleanup.py
+146
-104
fhost.py
+146
-104
fhost.py
···
1
1
#!/usr/bin/env python3
2
-
# -*- coding: utf-8 -*-
3
2
4
3
"""
5
-
Copyright © 2020 Mia Herkt
4
+
Copyright © 2024 Mia Herkt
6
5
Licensed under the EUPL, Version 1.2 or - as soon as approved
7
6
by the European Commission - subsequent versions of the EUPL
8
7
(the "License");
···
19
18
and limitations under the License.
20
19
"""
21
20
22
-
from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template, Request
21
+
from flask import Flask, abort, make_response, redirect, render_template, \
22
+
Request, request, Response, send_from_directory, url_for
23
23
from flask_sqlalchemy import SQLAlchemy
24
24
from flask_migrate import Migrate
25
25
from sqlalchemy import and_, or_
···
46
46
47
47
app = Flask(__name__, instance_relative_config=True)
48
48
app.config.update(
49
-
SQLALCHEMY_TRACK_MODIFICATIONS = False,
50
-
PREFERRED_URL_SCHEME = "https", # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config
51
-
MAX_CONTENT_LENGTH = 256 * 1024 * 1024,
52
-
MAX_URL_LENGTH = 4096,
53
-
USE_X_SENDFILE = False,
54
-
FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default
55
-
FHOST_STORAGE_PATH = "up",
56
-
FHOST_MAX_EXT_LENGTH = 9,
57
-
FHOST_SECRET_BYTES = 16,
58
-
FHOST_EXT_OVERRIDE = {
59
-
"audio/flac" : ".flac",
60
-
"image/gif" : ".gif",
61
-
"image/jpeg" : ".jpg",
62
-
"image/png" : ".png",
63
-
"image/svg+xml" : ".svg",
64
-
"video/webm" : ".webm",
65
-
"video/x-matroska" : ".mkv",
66
-
"application/octet-stream" : ".bin",
67
-
"text/plain" : ".log",
68
-
"text/plain" : ".txt",
69
-
"text/x-diff" : ".diff",
49
+
SQLALCHEMY_TRACK_MODIFICATIONS=False,
50
+
PREFERRED_URL_SCHEME="https", # nginx users: make sure to have
51
+
# 'uwsgi_param UWSGI_SCHEME $scheme;' in
52
+
# your config
53
+
MAX_CONTENT_LENGTH=256 * 1024 * 1024,
54
+
MAX_URL_LENGTH=4096,
55
+
USE_X_SENDFILE=False,
56
+
FHOST_USE_X_ACCEL_REDIRECT=True, # expect nginx by default
57
+
FHOST_STORAGE_PATH="up",
58
+
FHOST_MAX_EXT_LENGTH=9,
59
+
FHOST_SECRET_BYTES=16,
60
+
FHOST_EXT_OVERRIDE={
61
+
"audio/flac": ".flac",
62
+
"image/gif": ".gif",
63
+
"image/jpeg": ".jpg",
64
+
"image/png": ".png",
65
+
"image/svg+xml": ".svg",
66
+
"video/webm": ".webm",
67
+
"video/x-matroska": ".mkv",
68
+
"application/octet-stream": ".bin",
69
+
"text/plain": ".log",
70
+
"text/plain": ".txt",
71
+
"text/x-diff": ".diff",
70
72
},
71
-
NSFW_DETECT = False,
72
-
NSFW_THRESHOLD = 0.92,
73
-
VSCAN_SOCKET = None,
74
-
VSCAN_QUARANTINE_PATH = "quarantine",
75
-
VSCAN_IGNORE = [
73
+
NSFW_DETECT=False,
74
+
NSFW_THRESHOLD=0.92,
75
+
VSCAN_SOCKET=None,
76
+
VSCAN_QUARANTINE_PATH="quarantine",
77
+
VSCAN_IGNORE=[
76
78
"Eicar-Test-Signature",
77
79
"PUA.Win.Packer.XmMusicFile",
78
80
],
79
-
VSCAN_INTERVAL = datetime.timedelta(days=7),
80
-
URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-",
81
+
VSCAN_INTERVAL=datetime.timedelta(days=7),
82
+
URL_ALPHABET="DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMX"
83
+
"y6Vx-",
81
84
)
82
85
83
86
app.config.from_pyfile("config.py")
···
95
98
96
99
try:
97
100
mimedetect = Magic(mime=True, mime_encoding=False)
98
-
except:
101
+
except TypeError:
99
102
print("""Error: You have installed the wrong version of the 'magic' module.
100
103
Please install python-magic.""")
101
104
sys.exit(1)
···
103
106
db = SQLAlchemy(app)
104
107
migrate = Migrate(app, db)
105
108
109
+
106
110
class URL(db.Model):
107
111
__tablename__ = "URL"
108
-
id = db.Column(db.Integer, primary_key = True)
109
-
url = db.Column(db.UnicodeText, unique = True)
112
+
id = db.Column(db.Integer, primary_key=True)
113
+
url = db.Column(db.UnicodeText, unique=True)
110
114
111
115
def __init__(self, url):
112
116
self.url = url
···
117
121
def geturl(self):
118
122
return url_for("get", path=self.getname(), _external=True) + "\n"
119
123
124
+
@staticmethod
120
125
def get(url):
121
126
u = URL.query.filter_by(url=url).first()
122
127
···
126
131
db.session.commit()
127
132
128
133
return u
134
+
129
135
130
136
class IPAddress(types.TypeDecorator):
131
137
impl = types.LargeBinary
···
150
156
151
157
152
158
class File(db.Model):
153
-
id = db.Column(db.Integer, primary_key = True)
154
-
sha256 = db.Column(db.String, unique = True)
159
+
id = db.Column(db.Integer, primary_key=True)
160
+
sha256 = db.Column(db.String, unique=True)
155
161
ext = db.Column(db.UnicodeText)
156
162
mime = db.Column(db.UnicodeText)
157
163
addr = db.Column(IPAddress(16))
···
175
181
176
182
@property
177
183
def is_nsfw(self) -> bool:
178
-
return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]
184
+
if self.nsfw_score:
185
+
return self.nsfw_score > app.config["NSFW_THRESHOLD"]
186
+
return False
179
187
180
188
def getname(self):
181
189
return u"{0}{1}".format(su.enbase(self.id), self.ext)
182
190
183
191
def geturl(self):
184
192
n = self.getname()
193
+
a = "nsfw" if self.is_nsfw else None
185
194
186
-
if self.is_nsfw:
187
-
return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n"
188
-
else:
189
-
return url_for("get", path=n, secret=self.secret, _external=True) + "\n"
195
+
return url_for("get", path=n, secret=self.secret,
196
+
_external=True, _anchor=a) + "\n"
190
197
191
198
def getpath(self) -> Path:
192
199
return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256
···
197
204
self.removed = permanent
198
205
self.getpath().unlink(missing_ok=True)
199
206
200
-
# Returns the epoch millisecond that a file should expire
201
-
#
202
-
# Uses the expiration time provided by the user (requested_expiration)
203
-
# upper-bounded by an algorithm that computes the size based on the size of the
204
-
# file.
205
-
#
206
-
# That is, all files are assigned a computed expiration, which can voluntarily
207
-
# shortened by the user either by providing a timestamp in epoch millis or a
208
-
# duration in hours.
207
+
"""
208
+
Returns the epoch millisecond that a file should expire
209
+
210
+
Uses the expiration time provided by the user (requested_expiration)
211
+
upper-bounded by an algorithm that computes the size based on the size of
212
+
the file.
213
+
214
+
That is, all files are assigned a computed expiration, which can be
215
+
voluntarily shortened by the user either by providing a timestamp in
216
+
milliseconds since epoch or a duration in hours.
217
+
"""
218
+
@staticmethod
209
219
def get_expiration(requested_expiration, size) -> int:
210
-
current_epoch_millis = time.time() * 1000;
220
+
current_epoch_millis = time.time() * 1000
211
221
212
222
# Maximum lifetime of the file in milliseconds
213
-
this_files_max_lifespan = get_max_lifespan(size);
223
+
max_lifespan = get_max_lifespan(size)
214
224
215
225
# The latest allowed expiration date for this file, in epoch millis
216
-
this_files_max_expiration = this_files_max_lifespan + 1000 * time.time();
226
+
max_expiration = max_lifespan + 1000 * time.time()
217
227
218
228
if requested_expiration is None:
219
-
return this_files_max_expiration
229
+
return max_expiration
220
230
elif requested_expiration < 1650460320000:
221
231
# Treat the requested expiration time as a duration in hours
222
232
requested_expiration_ms = requested_expiration * 60 * 60 * 1000
223
-
return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms)
233
+
return min(max_expiration,
234
+
current_epoch_millis + requested_expiration_ms)
224
235
else:
225
-
# Treat the requested expiration time as a timestamp in epoch millis
226
-
return min(this_files_max_expiration, requested_expiration)
236
+
# Treat expiration time as a timestamp in epoch millis
237
+
return min(max_expiration, requested_expiration)
227
238
228
239
"""
229
240
requested_expiration can be:
···
231
242
- a duration (in hours) that the file should live for
232
243
- a timestamp in epoch millis that the file should expire at
233
244
234
-
Any value greater that the longest allowed file lifespan will be rounded down to that
235
-
value.
245
+
Any value greater that the longest allowed file lifespan will be rounded
246
+
down to that value.
236
247
"""
237
-
def store(file_, requested_expiration: typing.Optional[int], addr, ua, secret: bool):
248
+
@staticmethod
249
+
def store(file_, requested_expiration: typing.Optional[int], addr, ua,
250
+
secret: bool):
238
251
data = file_.read()
239
252
digest = sha256(data).hexdigest()
240
253
241
254
def get_mime():
242
255
guess = mimedetect.from_buffer(data)
243
-
app.logger.debug(f"MIME - specified: '{file_.content_type}' - detected: '{guess}'")
256
+
app.logger.debug(f"MIME - specified: '{file_.content_type}' - "
257
+
f"detected: '{guess}'")
244
258
245
-
if not file_.content_type or not "/" in file_.content_type or file_.content_type == "application/octet-stream":
259
+
if (not file_.content_type
260
+
or "/" not in file_.content_type
261
+
or file_.content_type == "application/octet-stream"):
246
262
mime = guess
247
263
else:
248
264
mime = file_.content_type
···
254
270
if flt.check(guess):
255
271
abort(403, flt.reason)
256
272
257
-
if mime.startswith("text/") and not "charset" in mime:
273
+
if mime.startswith("text/") and "charset" not in mime:
258
274
mime += "; charset=utf-8"
259
275
260
276
return mime
···
266
282
gmime = mime.split(";")[0]
267
283
guess = guess_extension(gmime)
268
284
269
-
app.logger.debug(f"extension - specified: '{ext}' - detected: '{guess}'")
285
+
app.logger.debug(f"extension - specified: '{ext}' - detected: "
286
+
f"'{guess}'")
270
287
271
288
if not ext:
272
289
if gmime in app.config["FHOST_EXT_OVERRIDE"]:
···
309
326
if isnew:
310
327
f.secret = None
311
328
if secret:
312
-
f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
329
+
f.secret = \
330
+
secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
313
331
314
332
storage = Path(app.config["FHOST_STORAGE_PATH"])
315
333
storage.mkdir(parents=True, exist_ok=True)
···
451
469
452
470
453
471
class UrlEncoder(object):
454
-
def __init__(self,alphabet, min_length):
472
+
def __init__(self, alphabet, min_length):
455
473
self.alphabet = alphabet
456
474
self.min_length = min_length
457
475
···
471
489
result += self.alphabet.index(c) * (n ** i)
472
490
return result
473
491
492
+
474
493
su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1)
494
+
475
495
476
496
def fhost_url(scheme=None):
477
497
if not scheme:
478
498
return url_for(".fhost", _external=True).rstrip("/")
479
499
else:
480
500
return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
501
+
481
502
482
503
def is_fhost_url(url):
483
504
return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
484
505
506
+
485
507
def shorten(url):
486
508
if len(url) > app.config["MAX_URL_LENGTH"]:
487
509
abort(414)
···
493
515
494
516
return u.geturl()
495
517
518
+
496
519
"""
497
520
requested_expiration can be:
498
521
- None, to use the longest allowed file lifespan
499
522
- a duration (in hours) that the file should live for
500
523
- a timestamp in epoch millis that the file should expire at
501
524
502
-
Any value greater that the longest allowed file lifespan will be rounded down to that
503
-
value.
525
+
Any value greater that the longest allowed file lifespan will be rounded down
526
+
to that value.
504
527
"""
505
-
def store_file(f, requested_expiration: typing.Optional[int], addr, ua, secret: bool):
528
+
def store_file(f, requested_expiration: typing.Optional[int], addr, ua,
529
+
secret: bool):
506
530
sf, isnew = File.store(f, requested_expiration, addr, ua, secret)
507
531
508
532
response = make_response(sf.geturl())
···
513
537
514
538
return response
515
539
540
+
516
541
def store_url(url, addr, ua, secret: bool):
517
542
if is_fhost_url(url):
518
543
abort(400)
519
544
520
-
h = { "Accept-Encoding" : "identity" }
545
+
h = {"Accept-Encoding": "identity"}
521
546
r = requests.get(url, stream=True, verify=False, headers=h)
522
547
523
548
try:
···
526
551
return str(e) + "\n"
527
552
528
553
if "content-length" in r.headers:
529
-
l = int(r.headers["content-length"])
554
+
length = int(r.headers["content-length"])
530
555
531
-
if l <= app.config["MAX_CONTENT_LENGTH"]:
556
+
if length <= app.config["MAX_CONTENT_LENGTH"]:
532
557
def urlfile(**kwargs):
533
-
return type('',(),kwargs)()
558
+
return type('', (), kwargs)()
534
559
535
-
f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="")
560
+
f = urlfile(read=r.raw.read,
561
+
content_type=r.headers["content-type"], filename="")
536
562
537
563
return store_file(f, None, addr, ua, secret)
538
564
else:
···
540
566
else:
541
567
abort(411)
542
568
569
+
543
570
def manage_file(f):
544
-
try:
545
-
assert(request.form["token"] == f.mgmt_token)
546
-
except:
571
+
if request.form["token"] != f.mgmt_token:
547
572
abort(401)
548
573
549
574
if "delete" in request.form:
···
561
586
return "", 202
562
587
563
588
abort(400)
589
+
564
590
565
591
@app.route("/<path:path>", methods=["GET", "POST"])
566
592
@app.route("/s/<secret>/<path:path>", methods=["GET", "POST"])
···
598
624
response.headers["Content-Length"] = f.size
599
625
response.headers["X-Accel-Redirect"] = "/" + str(fpath)
600
626
else:
601
-
response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
627
+
response = send_from_directory(
628
+
app.config["FHOST_STORAGE_PATH"], f.sha256,
629
+
mimetype=f.mime)
602
630
603
631
response.headers["X-Expires"] = f.expiration
604
632
return response
···
615
643
return redirect(u.url)
616
644
617
645
abort(404)
646
+
618
647
619
648
@app.route("/", methods=["GET", "POST"])
620
649
def fhost():
···
665
694
else:
666
695
return render_template("index.html")
667
696
697
+
668
698
@app.route("/robots.txt")
669
699
def robots():
670
700
return """User-agent: *
671
701
Disallow: /
672
702
"""
703
+
673
704
674
705
@app.errorhandler(400)
675
706
@app.errorhandler(401)
···
682
713
@app.errorhandler(451)
683
714
def ehandler(e):
684
715
try:
685
-
return render_template(f"{e.code}.html", id=id, request=request, description=e.description), e.code
716
+
return render_template(f"{e.code}.html", id=id, request=request,
717
+
description=e.description), e.code
686
718
except TemplateNotFound:
687
719
return "Segmentation fault\n", e.code
720
+
688
721
689
722
@app.cli.command("prune")
690
723
def prune():
691
724
"""
692
725
Clean up expired files
693
726
694
-
Deletes any files from the filesystem which have hit their expiration time. This
695
-
doesn't remove them from the database, only from the filesystem. It's recommended
696
-
that server owners run this command regularly, or set it up on a timer.
727
+
Deletes any files from the filesystem which have hit their expiration time.
728
+
This doesn't remove them from the database, only from the filesystem.
729
+
It is recommended that server owners run this command regularly, or set it
730
+
up on a timer.
697
731
"""
698
-
current_time = time.time() * 1000;
732
+
current_time = time.time() * 1000
699
733
700
734
# The path to where uploaded files are stored
701
735
storage = Path(app.config["FHOST_STORAGE_PATH"])
···
709
743
)
710
744
)
711
745
712
-
files_removed = 0;
746
+
files_removed = 0
713
747
714
748
# For every expired file...
715
749
for file in expired_files:
···
722
756
# Remove it from the file system
723
757
try:
724
758
os.remove(file_path)
725
-
files_removed += 1;
759
+
files_removed += 1
726
760
except FileNotFoundError:
727
-
pass # If the file was already gone, we're good
761
+
pass # If the file was already gone, we're good
728
762
except OSError as e:
729
763
print(e)
730
764
print(
731
765
"\n------------------------------------"
732
-
"Encountered an error while trying to remove file {file_path}. Double"
733
-
"check to make sure the server is configured correctly, permissions are"
734
-
"okay, and everything is ship shape, then try again.")
735
-
return;
766
+
"Encountered an error while trying to remove file {file_path}."
767
+
"Make sure the server is configured correctly, permissions "
768
+
"are okay, and everything is ship shape, then try again.")
769
+
return
736
770
737
771
# Finally, mark that the file was removed
738
-
file.expiration = None;
772
+
file.expiration = None
739
773
db.session.commit()
740
774
741
775
print(f"\nDone! {files_removed} file(s) removed")
742
776
743
-
""" For a file of a given size, determine the largest allowed lifespan of that file
744
777
745
-
Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
746
-
as FHOST_{MIN,MAX}_EXPIRATION.
778
+
"""
779
+
For a file of a given size, determine the largest allowed lifespan of that file
747
780
748
-
This lifespan may be shortened by a user's request, but no files should be allowed to
749
-
expire at a point after this number.
781
+
Based on the current app's configuration:
782
+
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION.
783
+
784
+
This lifespan may be shortened by a user's request, but no files should be
785
+
allowed to expire at a point after this number.
750
786
751
787
Value returned is a duration in milliseconds.
752
788
"""
···
756
792
max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
757
793
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
758
794
795
+
759
796
def do_vscan(f):
760
797
if f["path"].is_file():
761
798
with open(f["path"], "rb") as scanf:
762
799
try:
763
-
f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0]
800
+
res = list(app.config["VSCAN_SOCKET"].instream(scanf).values())
801
+
f["result"] = res[0]
764
802
except:
765
803
f["result"] = ("SCAN FAILED", None)
766
804
else:
···
768
806
769
807
return f
770
808
809
+
771
810
@app.cli.command("vscan")
772
811
def vscan():
773
812
if not app.config["VSCAN_SOCKET"]:
774
-
print("""Error: Virus scanning enabled but no connection method specified.
775
-
Please set VSCAN_SOCKET.""")
813
+
print("Error: Virus scanning enabled but no connection method "
814
+
"specified.\nPlease set VSCAN_SOCKET.")
776
815
sys.exit(1)
777
816
778
817
qp = Path(app.config["VSCAN_QUARANTINE_PATH"])
···
786
825
File.last_vscan == None),
787
826
File.removed == False)
788
827
else:
789
-
res = File.query.filter(File.last_vscan == None, File.removed == False)
828
+
res = File.query.filter(File.last_vscan == None,
829
+
File.removed == False)
790
830
791
-
work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res]
831
+
work = [{"path": f.getpath(), "name": f.getname(), "id": f.id}
832
+
for f in res]
792
833
793
834
results = []
794
835
for i, r in enumerate(p.imap_unordered(do_vscan, work)):
···
802
843
found = True
803
844
804
845
results.append({
805
-
"id" : r["id"],
806
-
"last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(),
807
-
"removed" : found})
846
+
"id": r["id"],
847
+
"last_vscan": None if r["result"][0] == "SCAN FAILED"
848
+
else datetime.datetime.now(),
849
+
"removed": found})
808
850
809
851
db.session.bulk_update_mappings(File, results)
810
852
db.session.commit()
+1
migrations/env.py
+1
migrations/env.py
-4
migrations/versions/0659d7b9eea8_.py
-4
migrations/versions/0659d7b9eea8_.py
···
15
15
16
16
17
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
18
op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True))
20
-
# ### end Alembic commands ###
21
19
22
20
23
21
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
22
op.drop_column('file', 'mgmt_token')
26
-
# ### end Alembic commands ###
+12
-18
migrations/versions/0cd36ecdd937_.py
+12
-18
migrations/versions/0cd36ecdd937_.py
···
15
15
16
16
17
17
def upgrade():
18
-
### commands auto generated by Alembic - please adjust! ###
19
18
op.create_table('URL',
20
-
sa.Column('id', sa.Integer(), nullable=False),
21
-
sa.Column('url', sa.UnicodeText(), nullable=True),
22
-
sa.PrimaryKeyConstraint('id'),
23
-
sa.UniqueConstraint('url')
24
-
)
19
+
sa.Column('id', sa.Integer(), nullable=False),
20
+
sa.Column('url', sa.UnicodeText(), nullable=True),
21
+
sa.PrimaryKeyConstraint('id'),
22
+
sa.UniqueConstraint('url'))
25
23
op.create_table('file',
26
-
sa.Column('id', sa.Integer(), nullable=False),
27
-
sa.Column('sha256', sa.String(), nullable=True),
28
-
sa.Column('ext', sa.UnicodeText(), nullable=True),
29
-
sa.Column('mime', sa.UnicodeText(), nullable=True),
30
-
sa.Column('addr', sa.UnicodeText(), nullable=True),
31
-
sa.Column('removed', sa.Boolean(), nullable=True),
32
-
sa.PrimaryKeyConstraint('id'),
33
-
sa.UniqueConstraint('sha256')
34
-
)
35
-
### end Alembic commands ###
24
+
sa.Column('id', sa.Integer(), nullable=False),
25
+
sa.Column('sha256', sa.String(), nullable=True),
26
+
sa.Column('ext', sa.UnicodeText(), nullable=True),
27
+
sa.Column('mime', sa.UnicodeText(), nullable=True),
28
+
sa.Column('addr', sa.UnicodeText(), nullable=True),
29
+
sa.Column('removed', sa.Boolean(), nullable=True),
30
+
sa.PrimaryKeyConstraint('id'),
31
+
sa.UniqueConstraint('sha256'))
36
32
37
33
38
34
def downgrade():
39
-
### commands auto generated by Alembic - please adjust! ###
40
35
op.drop_table('file')
41
36
op.drop_table('URL')
42
-
### end Alembic commands ###
+3
-2
migrations/versions/30bfe33aa328_add_file_size_field.py
+3
-2
migrations/versions/30bfe33aa328_add_file_size_field.py
···
19
19
20
20
Base = automap_base()
21
21
22
+
22
23
def upgrade():
23
24
op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True))
24
25
bind = op.get_bind()
···
34
35
p = storage / f.sha256
35
36
if p.is_file():
36
37
updates.append({
37
-
"id" : f.id,
38
-
"size" : p.stat().st_size
38
+
"id": f.id,
39
+
"size": p.stat().st_size
39
40
})
40
41
41
42
session.bulk_update_mappings(File, updates)
+31
-32
migrations/versions/5cda1743b92d_add_request_filters.py
+31
-32
migrations/versions/5cda1743b92d_add_request_filters.py
···
19
19
20
20
Base = automap_base()
21
21
22
+
22
23
def upgrade():
23
-
# ### commands auto generated by Alembic - please adjust! ###
24
24
op.create_table('request_filter',
25
-
sa.Column('id', sa.Integer(), nullable=False),
26
-
sa.Column('type', sa.String(length=20), nullable=False),
27
-
sa.Column('comment', sa.UnicodeText(), nullable=True),
28
-
sa.Column('addr', sa.LargeBinary(length=16), nullable=True),
29
-
sa.Column('net', sa.Text(), nullable=True),
30
-
sa.Column('regex', sa.UnicodeText(), nullable=True),
31
-
sa.PrimaryKeyConstraint('id'),
32
-
sa.UniqueConstraint('addr')
33
-
)
34
-
with op.batch_alter_table('request_filter', schema=None) as batch_op:
35
-
batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'], unique=False)
25
+
sa.Column('id', sa.Integer(), nullable=False),
26
+
sa.Column('type', sa.String(length=20), nullable=False),
27
+
sa.Column('comment', sa.UnicodeText(), nullable=True),
28
+
sa.Column('addr', sa.LargeBinary(length=16),
29
+
nullable=True),
30
+
sa.Column('net', sa.Text(), nullable=True),
31
+
sa.Column('regex', sa.UnicodeText(), nullable=True),
32
+
sa.PrimaryKeyConstraint('id'),
33
+
sa.UniqueConstraint('addr'))
36
34
37
-
# ### end Alembic commands ###
35
+
with op.batch_alter_table('request_filter', schema=None) as batch_op:
36
+
batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'],
37
+
unique=False)
38
38
39
39
bind = op.get_bind()
40
40
Base.prepare(autoload_with=bind)
41
41
RequestFilter = Base.classes.request_filter
42
42
session = Session(bind=bind)
43
43
44
-
if "FHOST_UPLOAD_BLACKLIST" in current_app.config:
45
-
if current_app.config["FHOST_UPLOAD_BLACKLIST"]:
46
-
with current_app.open_instance_resource(current_app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
47
-
for l in bl.readlines():
48
-
if not l.startswith("#"):
49
-
l = l.strip()
50
-
if l.endswith(":"):
51
-
# old implementation uses str.startswith,
52
-
# which does not translate to networks
53
-
current_app.logger.warning(f"Ignored address: {l}")
54
-
continue
44
+
blp = current_app.config.get("FHOST_UPLOAD_BLACKLIST")
45
+
if blp:
46
+
with current_app.open_instance_resource(blp, "r") as bl:
47
+
for line in bl.readlines():
48
+
if not line.startswith("#"):
49
+
line = line.strip()
50
+
if line.endswith(":"):
51
+
# old implementation uses str.startswith,
52
+
# which does not translate to networks
53
+
current_app.logger.warning(
54
+
f"Ignored address: {line}")
55
+
continue
55
56
56
-
flt = RequestFilter(type="addr", addr=ipaddress.ip_address(l).packed)
57
-
session.add(flt)
57
+
addr = ipaddress.ip_address(line).packed
58
+
flt = RequestFilter(type="addr", addr=addr)
59
+
session.add(flt)
58
60
59
-
if "FHOST_MIME_BLACKLIST" in current_app.config:
60
-
for mime in current_app.config["FHOST_MIME_BLACKLIST"]:
61
-
flt = RequestFilter(type="mime", regex=mime)
62
-
session.add(flt)
61
+
for mime in current_app.config.get("FHOST_MIME_BLACKLIST", []):
62
+
flt = RequestFilter(type="mime", regex=mime)
63
+
session.add(flt)
63
64
64
65
session.commit()
65
66
···
72
73
73
74
74
75
def downgrade():
75
-
# ### commands auto generated by Alembic - please adjust! ###
76
76
with op.batch_alter_table('request_filter', schema=None) as batch_op:
77
77
batch_op.drop_index(batch_op.f('ix_request_filter_type'))
78
78
79
79
op.drop_table('request_filter')
80
-
# ### end Alembic commands ###
+2
-5
migrations/versions/5cee97aab219_.py
+2
-5
migrations/versions/5cee97aab219_.py
···
15
15
16
16
17
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
-
op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True))
20
-
# ### end Alembic commands ###
18
+
op.add_column('file', sa.Column('last_vscan', sa.DateTime(),
19
+
nullable=True))
21
20
22
21
23
22
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
23
op.drop_column('file', 'last_vscan')
26
-
# ### end Alembic commands ###
-4
migrations/versions/7e246705da6a_.py
-4
migrations/versions/7e246705da6a_.py
···
15
15
16
16
17
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
18
op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True))
20
-
# ### end Alembic commands ###
21
19
22
20
23
21
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
22
op.drop_column('file', 'nsfw_score')
26
-
# ### end Alembic commands ###
+24
-14
migrations/versions/939a08e1d6e5_.py
+24
-14
migrations/versions/939a08e1d6e5_.py
···
21
21
import os
22
22
import time
23
23
24
-
""" For a file of a given size, determine the largest allowed lifespan of that file
25
24
26
-
Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
27
-
as FHOST_{MIN,MAX}_EXPIRATION.
25
+
"""
26
+
For a file of a given size, determine the largest allowed lifespan of that file
28
27
29
-
This lifespan may be shortened by a user's request, but no files should be allowed to
30
-
expire at a point after this number.
28
+
Based on the current app's configuration:
29
+
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION.
30
+
31
+
This lifespan may be shortened by a user's request, but no files should be
32
+
allowed to expire at a point after this number.
31
33
32
34
Value returned is a duration in milliseconds.
33
35
"""
34
36
def get_max_lifespan(filesize: int) -> int:
35
-
min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
36
-
max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
37
-
max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
37
+
cfg = current_app.config
38
+
min_exp = cfg.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
39
+
max_exp = cfg.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
40
+
max_size = cfg.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
38
41
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
39
42
43
+
40
44
Base = automap_base()
45
+
41
46
42
47
def upgrade():
43
48
op.add_column('file', sa.Column('expiration', sa.BigInteger()))
···
48
53
session = Session(bind=bind)
49
54
50
55
storage = Path(current_app.config["FHOST_STORAGE_PATH"])
51
-
current_time = time.time() * 1000;
56
+
current_time = time.time() * 1000
52
57
53
58
# List of file hashes which have not expired yet
54
59
# This could get really big for some servers
55
60
try:
56
61
unexpired_files = os.listdir(storage)
57
62
except FileNotFoundError:
58
-
return # There are no currently unexpired files
63
+
return # There are no currently unexpired files
59
64
60
65
# Calculate an expiration date for all existing files
61
66
···
65
70
sa.not_(File.removed)
66
71
)
67
72
)
68
-
updates = [] # We coalesce updates to the database here
73
+
updates = [] # We coalesce updates to the database here
69
74
70
75
# SQLite has a hard limit on the number of variables so we
71
76
# need to do this the slow way
···
74
79
for file in files:
75
80
file_path = storage / file.sha256
76
81
stat = os.stat(file_path)
77
-
max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms
78
-
file_birth = stat.st_mtime * 1000 # When the file was created, in ms
79
-
updates.append({'id': file.id, 'expiration': int(file_birth + max_age)})
82
+
# How long the file is allowed to live, in ms
83
+
max_age = get_max_lifespan(stat.st_size)
84
+
# When the file was created, in ms
85
+
file_birth = stat.st_mtime * 1000
86
+
updates.append({
87
+
'id': file.id,
88
+
'expiration': int(file_birth + max_age)})
80
89
81
90
# Apply coalesced updates
82
91
session.bulk_update_mappings(File, updates)
83
92
session.commit()
93
+
84
94
85
95
def downgrade():
86
96
op.drop_column('file', 'expiration')
-6
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
-6
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
···
15
15
16
16
17
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
18
with op.batch_alter_table('file', schema=None) as batch_op:
20
19
batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True))
21
-
22
-
# ### end Alembic commands ###
23
20
24
21
25
22
def downgrade():
26
-
# ### commands auto generated by Alembic - please adjust! ###
27
23
with op.batch_alter_table('file', schema=None) as batch_op:
28
24
batch_op.drop_column('ua')
29
-
30
-
# ### end Alembic commands ###
-4
migrations/versions/e2e816056589_.py
-4
migrations/versions/e2e816056589_.py
···
15
15
16
16
17
17
def upgrade():
18
-
# ### commands auto generated by Alembic - please adjust! ###
19
18
op.add_column('file', sa.Column('secret', sa.String(), nullable=True))
20
-
# ### end Alembic commands ###
21
19
22
20
23
21
def downgrade():
24
-
# ### commands auto generated by Alembic - please adjust! ###
25
22
op.drop_column('file', 'secret')
26
-
# ### end Alembic commands ###
+64
-37
mod.py
+64
-37
mod.py
···
14
14
import ipaddress
15
15
16
16
from fhost import db, File, AddrFilter, su, app as fhost_app
17
-
from modui import *
17
+
from modui import FileTable, mime, MpvWidget, Notification
18
18
19
19
fhost_app.app_context().push()
20
+
20
21
21
22
class NullptrMod(Screen):
22
23
BINDINGS = [
···
67
68
self.finput.display = False
68
69
ftable = self.query_one("#ftable")
69
70
ftable.focus()
71
+
q = ftable.base_query
70
72
71
73
if len(message.value):
72
74
match self.filter_col:
73
75
case 1:
74
-
try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value))
75
-
except ValueError: pass
76
+
try:
77
+
q = q.filter(File.id == su.debase(message.value))
78
+
except ValueError:
79
+
return
76
80
case 2:
77
81
try:
78
82
addr = ipaddress.ip_address(message.value)
79
83
if type(addr) is ipaddress.IPv6Address:
80
84
addr = addr.ipv4_mapped or addr
81
-
q = ftable.base_query.filter(File.addr == addr)
82
-
ftable.query = q
83
-
except ValueError: pass
84
-
case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value))
85
-
case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value))
86
-
case 5: ftable.query = ftable.base_query.filter(File.ua.like(message.value))
87
-
else:
88
-
ftable.query = ftable.base_query
85
+
q = q.filter(File.addr == addr)
86
+
except ValueError:
87
+
return
88
+
case 3: q = q.filter(File.mime.like(message.value))
89
+
case 4: q = q.filter(File.ext.like(message.value))
90
+
case 5: q = q.filter(File.ua.like(message.value))
91
+
92
+
ftable.query = q
89
93
90
94
def action_remove_file(self, permanent: bool) -> None:
91
95
if self.current_file:
92
96
self.current_file.delete(permanent)
93
97
db.session.commit()
94
-
self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}"))
98
+
self.mount(Notification(f"{'Banned' if permanent else 'Removed'}"
99
+
f"file {self.current_file.getname()}"))
95
100
self.action_refresh()
96
101
97
102
def action_ban_ip(self, nuke: bool) -> None:
98
103
if self.current_file:
99
-
if AddrFilter.query.filter(AddrFilter.addr ==
100
-
self.current_file.addr).scalar():
101
-
txt = f"{self.current_file.addr.compressed} is already banned"
104
+
addr = self.current_file.addr
105
+
if AddrFilter.query.filter(AddrFilter.addr == addr).scalar():
106
+
txt = f"{addr.compressed} is already banned"
102
107
else:
103
-
db.session.add(AddrFilter(self.current_file.addr))
108
+
db.session.add(AddrFilter(addr))
104
109
db.session.commit()
105
-
txt = f"Banned {self.current_file.addr.compressed}"
110
+
txt = f"Banned {addr.compressed}"
106
111
107
112
if nuke:
108
113
tsize = 0
109
114
trm = 0
110
-
for f in File.query.filter(File.addr == self.current_file.addr):
115
+
for f in File.query.filter(File.addr == addr):
111
116
if f.getpath().is_file():
112
117
tsize += f.size or f.getpath().stat().st_size
113
118
trm += 1
114
119
f.delete(True)
115
120
db.session.commit()
116
-
txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}"
121
+
txt += f", removed {trm} {'files' if trm != 1 else 'file'} " \
122
+
f"totaling {do_filesizeformat(tsize, True)}"
117
123
self.mount(Notification(txt))
118
124
self._refresh_layout()
119
125
ftable = self.query_one("#ftable")
···
131
137
DataTable(id="finfo", show_header=False, cursor_type="none"),
132
138
MpvWidget(id="mpv"),
133
139
RichLog(id="ftextlog", auto_scroll=False),
134
-
id="infopane"))
140
+
id="infopane"))
135
141
yield Input(id="filter_input")
136
142
yield Footer()
137
143
···
150
156
self.finput = self.query_one("#filter_input")
151
157
152
158
self.mimehandler = mime.MIMEHandler()
153
-
self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive)
159
+
self.mimehandler.register(mime.MIMECategory.Archive,
160
+
self.handle_libarchive)
154
161
self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
155
162
self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
156
-
self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf)
157
-
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive)
163
+
self.mimehandler.register(mime.MIMECategory.Document,
164
+
self.handle_mupdf)
165
+
self.mimehandler.register(mime.MIMECategory.Fallback,
166
+
self.handle_libarchive)
158
167
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
159
168
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
160
169
···
166
175
167
176
def handle_text(self, cat):
168
177
with open(self.current_file.getpath(), "r") as sf:
169
-
data = sf.read(1000000).replace("\033","")
178
+
data = sf.read(1000000).replace("\033", "")
170
179
self.ftlog.write(data)
171
180
return True
172
181
···
181
190
self.mpvw.styles.height = "40%"
182
191
self.mpvw.start_mpv("hex://" + imgdata, 0)
183
192
184
-
self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
193
+
self.ftlog.write(
194
+
Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
185
195
self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
186
196
for k, v in doc.metadata.items():
187
197
self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}"))
···
206
216
for k, v in c.metadata.items():
207
217
self.ftlog.write(f" {k}: {v}")
208
218
for s in c.streams:
209
-
self.ftlog.write(Text(f"Stream {s.index}:", style="bold"))
219
+
self.ftlog.write(
220
+
Text(f"Stream {s.index}:", style="bold"))
210
221
self.ftlog.write(f" Type: {s.type}")
211
222
if s.base_rate:
212
223
self.ftlog.write(f" Frame rate: {s.base_rate}")
···
225
236
else:
226
237
c = chr(s)
227
238
s = c
228
-
if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]"
229
-
if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]"
239
+
if c.isalpha():
240
+
return f"\0[chartreuse1]{s}\0[/chartreuse1]"
241
+
if c.isdigit():
242
+
return f"\0[gold1]{s}\0[/gold1]"
230
243
if not c.isprintable():
231
244
g = "grey50" if c == "\0" else "cadet_blue"
232
245
return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
233
246
return s
234
-
return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}"
235
-
f"{' ' * (16 - len(c))}"
236
-
f" {''.join(map(fmt, c))}"
237
-
for c in map(lambda x: bytes([n for n in x if n != None]),
238
-
zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16))))
247
+
248
+
return Text.from_markup(
249
+
"\n".join(' '.join(
250
+
map(fmt, map(''.join, zip(*[iter(c.hex())] * 2)))) +
251
+
f"{' ' * (16 - len(c))} {''.join(map(fmt, c))}"
252
+
for c in
253
+
map(lambda x: bytes([n for n in x if n is not None]),
254
+
zip_longest(
255
+
*[iter(binf.read(min(length, 16 * 10)))] * 16))))
239
256
240
257
with open(self.current_file.getpath(), "rb") as binf:
241
258
self.ftlog.write(hexdump(binf, self.current_file.size))
242
259
if self.current_file.size > 16*10*2:
243
260
binf.seek(self.current_file.size-16*10)
244
261
self.ftlog.write(" [...] ".center(64, '─'))
245
-
self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell()))
262
+
self.ftlog.write(hexdump(binf,
263
+
self.current_file.size - binf.tell()))
246
264
247
265
return True
248
266
···
253
271
self.finfo.add_rows([
254
272
("ID:", str(f.id)),
255
273
("File name:", f.getname()),
256
-
("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"),
274
+
("URL:", f.geturl()
275
+
if fhost_app.config["SERVER_NAME"]
276
+
else "⚠ Set SERVER_NAME in config.py to display"),
257
277
("File size:", do_filesizeformat(f.size, True)),
258
278
("MIME type:", f.mime),
259
279
("SHA256 checksum:", f.sha256),
···
261
281
("User agent:", Text(f.ua or "")),
262
282
("Management token:", f.mgmt_token),
263
283
("Secret:", f.secret),
264
-
("Is NSFW:", ("Yes" if f.is_nsfw else "No") + (f" (Score: {f.nsfw_score:0.4f})" if f.nsfw_score else " (Not scanned)")),
284
+
("Is NSFW:", ("Yes" if f.is_nsfw else "No") +
285
+
(f" (Score: {f.nsfw_score:0.4f})"
286
+
if f.nsfw_score else " (Not scanned)")),
265
287
("Is banned:", "Yes" if f.removed else "No"),
266
-
("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000)))
288
+
("Expires:",
289
+
time.strftime("%Y-%m-%d %H:%M:%S",
290
+
time.gmtime(File.get_expiration(f.expiration,
291
+
f.size)/1000)))
267
292
])
268
293
269
294
self.mpvw.stop_mpv(True)
···
272
297
if f.getpath().is_file():
273
298
self.mimehandler.handle(f.mime, f.ext)
274
299
self.ftlog.scroll_to(x=0, y=0, animate=False)
300
+
275
301
276
302
class NullptrModApp(App):
277
303
CSS_PATH = "mod.css"
···
281
307
self.main_screen = NullptrMod()
282
308
self.install_screen(self.main_screen, name="main")
283
309
self.push_screen("main")
310
+
284
311
285
312
if __name__ == "__main__":
286
313
app = NullptrModApp()
+9
-4
modui/filetable.py
+9
-4
modui/filetable.py
···
7
7
from fhost import File
8
8
from modui import mime
9
9
10
+
10
11
class FileTable(DataTable):
11
12
query = Reactive(None)
12
13
order_col = Reactive(0)
13
14
order_desc = Reactive(True)
14
15
limit = 10000
15
-
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime]
16
+
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext,
17
+
File.size, File.mime]
16
18
17
19
def __init__(self, **kwargs):
18
20
super().__init__(**kwargs)
···
33
35
34
36
def watch_query(self, old, value) -> None:
35
37
def fmt_file(f: File) -> tuple:
38
+
mimemoji = mime.mimemoji.get(f.mime.split('/')[0],
39
+
mime.mimemoji.get(f.mime)) or ' '
36
40
return (
37
41
str(f.id),
38
42
"🔴" if f.removed else " ",
···
40
44
"👻" if not f.getpath().is_file() else " ",
41
45
f.getname(),
42
46
do_filesizeformat(f.size, True),
43
-
f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or ' '} " + f.mime,
47
+
f"{mimemoji} {f.mime}",
44
48
)
45
49
46
50
if (self.query):
47
-
48
51
order = FileTable.colmap[self.order_col]
49
52
q = self.query
50
-
if order: q = q.order_by(order.desc() if self.order_desc else order, File.id)
53
+
if order:
54
+
q = q.order_by(order.desc() if self.order_desc
55
+
else order, File.id)
51
56
qres = list(map(fmt_file, q.limit(self.limit)))
52
57
53
58
ri = 0
+48
-46
modui/mime.py
+48
-46
modui/mime.py
···
2
2
from textual import log
3
3
4
4
mimemoji = {
5
-
"audio" : "🔈",
6
-
"video" : "🎞",
7
-
"text" : "📄",
8
-
"image" : "🖼",
9
-
"application/zip" : "🗜️",
10
-
"application/x-zip-compressed" : "🗜️",
11
-
"application/x-tar" : "🗄",
12
-
"application/x-cpio" : "🗄",
13
-
"application/x-xz" : "🗜️",
14
-
"application/x-7z-compressed" : "🗜️",
15
-
"application/gzip" : "🗜️",
16
-
"application/zstd" : "🗜️",
17
-
"application/x-rar" : "🗜️",
18
-
"application/x-rar-compressed" : "🗜️",
19
-
"application/vnd.ms-cab-compressed" : "🗜️",
20
-
"application/x-bzip2" : "🗜️",
21
-
"application/x-lzip" : "🗜️",
22
-
"application/x-iso9660-image" : "💿",
23
-
"application/pdf" : "📕",
24
-
"application/epub+zip" : "📕",
25
-
"application/mxf" : "🎞",
26
-
"application/vnd.android.package-archive" : "📦",
27
-
"application/vnd.debian.binary-package" : "📦",
28
-
"application/x-rpm" : "📦",
29
-
"application/x-dosexec" : "⚙",
30
-
"application/x-execuftable" : "⚙",
31
-
"application/x-sharedlib" : "⚙",
32
-
"application/java-archive" : "☕",
33
-
"application/x-qemu-disk" : "🖴",
34
-
"application/pgp-encrypted" : "🔏",
5
+
"audio": "🔈",
6
+
"video": "🎞",
7
+
"text": "📄",
8
+
"image": "🖼",
9
+
"application/zip": "🗜️",
10
+
"application/x-zip-compressed": "🗜️",
11
+
"application/x-tar": "🗄",
12
+
"application/x-cpio": "🗄",
13
+
"application/x-xz": "🗜️",
14
+
"application/x-7z-compressed": "🗜️",
15
+
"application/gzip": "🗜️",
16
+
"application/zstd": "🗜️",
17
+
"application/x-rar": "🗜️",
18
+
"application/x-rar-compressed": "🗜️",
19
+
"application/vnd.ms-cab-compressed": "🗜️",
20
+
"application/x-bzip2": "🗜️",
21
+
"application/x-lzip": "🗜️",
22
+
"application/x-iso9660-image": "💿",
23
+
"application/pdf": "📕",
24
+
"application/epub+zip": "📕",
25
+
"application/mxf": "🎞",
26
+
"application/vnd.android.package-archive": "📦",
27
+
"application/vnd.debian.binary-package": "📦",
28
+
"application/x-rpm": "📦",
29
+
"application/x-dosexec": "⚙",
30
+
"application/x-execuftable": "⚙",
31
+
"application/x-sharedlib": "⚙",
32
+
"application/java-archive": "☕",
33
+
"application/x-qemu-disk": "🖴",
34
+
"application/pgp-encrypted": "🔏",
35
35
}
36
36
37
-
MIMECategory = Enum("MIMECategory",
38
-
["Archive", "Text", "AV", "Document", "Fallback"]
39
-
)
37
+
MIMECategory = Enum("MIMECategory", ["Archive", "Text", "AV", "Document",
38
+
"Fallback"])
39
+
40
40
41
41
class MIMEHandler:
42
42
def __init__(self):
43
43
self.handlers = {
44
-
MIMECategory.Archive : [[
44
+
MIMECategory.Archive: [[
45
45
"application/zip",
46
46
"application/x-zip-compressed",
47
47
"application/x-tar",
···
62
62
"application/java-archive",
63
63
"application/vnd.openxmlformats"
64
64
], []],
65
-
MIMECategory.Text : [[
65
+
MIMECategory.Text: [[
66
66
"text",
67
67
"application/json",
68
68
"application/xml",
69
69
], []],
70
-
MIMECategory.AV : [[
70
+
MIMECategory.AV: [[
71
71
"audio", "video", "image",
72
72
"application/mxf"
73
73
], []],
74
-
MIMECategory.Document : [[
74
+
MIMECategory.Document: [[
75
75
"application/pdf",
76
76
"application/epub",
77
77
"application/x-mobipocket-ebook",
78
78
], []],
79
-
MIMECategory.Fallback : [[], []]
79
+
MIMECategory.Fallback: [[], []]
80
80
}
81
81
82
82
self.exceptions = {
83
-
MIMECategory.Archive : {
84
-
".cbz" : MIMECategory.Document,
85
-
".xps" : MIMECategory.Document,
86
-
".epub" : MIMECategory.Document,
83
+
MIMECategory.Archive: {
84
+
".cbz": MIMECategory.Document,
85
+
".xps": MIMECategory.Document,
86
+
".epub": MIMECategory.Document,
87
87
},
88
-
MIMECategory.Text : {
89
-
".fb2" : MIMECategory.Document,
88
+
MIMECategory.Text: {
89
+
".fb2": MIMECategory.Document,
90
90
}
91
91
}
92
92
···
115
115
cat = getcat(mime)
116
116
for handler in self.handlers[cat][1]:
117
117
try:
118
-
if handler(cat): return
118
+
if handler(cat):
119
+
return
119
120
except: pass
120
121
121
122
for handler in self.handlers[MIMECategory.Fallback][1]:
122
123
try:
123
-
if handler(None): return
124
+
if handler(None):
125
+
return
124
126
except: pass
125
127
126
128
raise RuntimeError(f"Unhandled MIME type category: {cat}")
+24
-10
modui/mpvwidget.py
+24
-10
modui/mpvwidget.py
···
1
1
import time
2
-
import fcntl, struct, termios
2
+
3
+
import fcntl
4
+
import struct
5
+
import termios
6
+
3
7
from sys import stdout
4
8
5
9
from textual import events, log
6
10
from textual.widgets import Static
7
11
8
12
from fhost import app as fhost_app
13
+
9
14
10
15
class MpvWidget(Static):
11
16
def __init__(self, **kwargs):
···
14
19
self.mpv = None
15
20
self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
16
21
17
-
if not self.vo in ["sixel", "kitty"]:
18
-
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.")
22
+
if self.vo not in ["sixel", "kitty"]:
23
+
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO "
24
+
"to 'sixel' or 'kitty' in config.py,\nwhichever is "
25
+
"supported by your terminal.")
19
26
else:
20
27
try:
21
28
import mpv
···
27
34
self.mpv[f"vo-sixel-buffered"] = True
28
35
self.mpv["audio"] = False
29
36
self.mpv["loop-file"] = "inf"
30
-
self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf"
37
+
self.mpv["image-display-duration"] = 0.5 \
38
+
if self.vo == "sixel" else "inf"
31
39
except Exception as e:
32
40
self.mpv = None
33
-
self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}")
41
+
self.update("⚠ Previews require python-mpv with libmpv "
42
+
"0.36.0 or later \n\nError was:\n"
43
+
f"{type(e).__name__}: {e}")
34
44
35
-
def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None:
45
+
def start_mpv(self, f: str | None = None,
46
+
pos: float | str | None = None) -> None:
36
47
self.display = True
37
48
self.screen._refresh_layout()
38
49
39
50
if self.mpv:
40
51
if self.content_region.x:
41
-
r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678'))
52
+
winsz = fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678')
53
+
r, c, w, h = struct.unpack('hhhh', winsz)
42
54
width = int((w / c) * self.content_region.width)
43
-
height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0)))
55
+
height = int((h / r) * (self.content_region.height +
56
+
(1 if self.vo == "sixel" else 0)))
44
57
self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
45
58
self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
46
-
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0)
59
+
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + \
60
+
(1 if self.vo == "sixel" else 0)
47
61
self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
48
62
self.mpv[f"vo-{self.vo}-width"] = width
49
63
self.mpv[f"vo-{self.vo}-height"] = height
50
64
51
-
if pos != None:
65
+
if pos is not None:
52
66
self.mpv["start"] = pos
53
67
54
68
if f:
+1
modui/notification.py
+1
modui/notification.py
+8
-6
nsfw_detect.py
+8
-6
nsfw_detect.py
···
18
18
and limitations under the License.
19
19
"""
20
20
21
-
import os
22
21
import sys
23
-
from pathlib import Path
24
-
25
22
import av
26
23
from transformers import pipeline
24
+
27
25
28
26
class NSFWDetector:
29
27
def __init__(self):
30
-
self.classifier = pipeline("image-classification", model="giacomoarienti/nsfw-classifier")
28
+
self.classifier = pipeline("image-classification",
29
+
model="giacomoarienti/nsfw-classifier")
31
30
32
31
def detect(self, fpath):
33
32
try:
34
33
with av.open(fpath) as container:
35
-
try: container.seek(int(container.duration / 2))
34
+
try:
35
+
container.seek(int(container.duration / 2))
36
36
except: container.seek(0)
37
37
38
38
frame = next(container.decode(video=0))
39
39
img = frame.to_image()
40
40
res = self.classifier(img)
41
41
42
-
return max([x["score"] for x in res if x["label"] not in ["neutral", "drawings"]])
42
+
return max([x["score"] for x in res
43
+
if x["label"] not in ["neutral", "drawings"]])
43
44
except: pass
44
45
45
46
return -1.0
47
+
46
48
47
49
if __name__ == "__main__":
48
50
n = NSFWDetector()