mia/0x0
3
8
Fork 3
Code Issues 11 Pull requests 2 Projects 1 Wiki Activity

Compare commits

base: mia:master
Branches Tags
mia:master
jonas-w:master
..
compare: jonas-w:master
Branches Tags
jonas-w:master
mia:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

30 changed files with 4035 additions and 808 deletions
Show stats Expand all files Collapse all files

13
README.rst
View file

@ -95,15 +95,12 @@ Optional:
NSFW Detection NSFW Detection
-------------- --------------
0x0 supports classification of NSFW content via 0x0 supports classification of NSFW content via Yahoo’s open_nsfw Caffe
`a machine learning model <https://huggingface.co/giacomoarienti/nsfw-classifier>`_. neural network model. This works for images and video files and requires
This works for images and video files and requires the following the following:
Python modules:
* torch * Caffe Python module (built for Python 3)
* transformers * `PyAV <https://github.com/PyAV-Org/PyAV>`_
* pillow
* `av <https://github.com/PyAV-Org/PyAV>`_
Virus Scanning Virus Scanning

2
cleanup.py
View file

@ -5,4 +5,4 @@ print("Instead, please run")
print("") print("")
print(" $ FLASK_APP=fhost flask prune") print(" $ FLASK_APP=fhost flask prune")
print("") print("")
exit(1) exit(1);

489
fhost.py
View file

@ -1,7 +1,8 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*-
""" """
Copyright © 2024 Mia Herkt Copyright © 2020 Mia Herkt
Licensed under the EUPL, Version 1.2 or - as soon as approved Licensed under the EUPL, Version 1.2 or - as soon as approved
by the European Commission - subsequent versions of the EUPL by the European Commission - subsequent versions of the EUPL
(the "License"); (the "License");
@ -18,79 +19,77 @@
and limitations under the License. and limitations under the License.
""" """
from flask import Flask, abort, make_response, redirect, render_template, \ from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template
Request, request, Response, send_from_directory, url_for
from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate from flask_migrate import Migrate
from sqlalchemy import and_, or_ from sqlalchemy import and_, or_
from sqlalchemy.orm import declared_attr
import sqlalchemy.types as types
from jinja2.exceptions import * from jinja2.exceptions import *
from jinja2 import ChoiceLoader, FileSystemLoader from jinja2 import ChoiceLoader, FileSystemLoader
from hashlib import file_digest from hashlib import sha256
from magic import Magic from magic import Magic
from mimetypes import guess_extension from mimetypes import guess_extension
import click import click
import enum
import os import os
import sys import sys
import time import time
import datetime import datetime
import ipaddress
import typing import typing
import requests import requests
import secrets import secrets
import re
from validators import url as url_valid from validators import url as url_valid
from pathlib import Path from pathlib import Path
app = Flask(__name__, instance_relative_config=True) app = Flask(__name__, instance_relative_config=True)
app.config.update( app.config.update(
SQLALCHEMY_TRACK_MODIFICATIONS=False, SQLALCHEMY_TRACK_MODIFICATIONS = False,
PREFERRED_URL_SCHEME="https", # nginx users: make sure to have PREFERRED_URL_SCHEME = "https", # nginx users: make sure to have 'uwsgi_param UWSGI_SCHEME $scheme;' in your config
# 'uwsgi_param UWSGI_SCHEME $scheme;' in MAX_CONTENT_LENGTH = 256 * 1024 * 1024,
# your config MAX_URL_LENGTH = 4096,
MAX_CONTENT_LENGTH=256 * 1024 * 1024, USE_X_SENDFILE = False,
MAX_URL_LENGTH=4096, FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default
USE_X_SENDFILE=False, FHOST_STORAGE_PATH = "up",
FHOST_USE_X_ACCEL_REDIRECT=True, # expect nginx by default FHOST_MAX_EXT_LENGTH = 9,
FHOST_STORAGE_PATH="up", FHOST_SECRET_BYTES = 16,
FHOST_MAX_EXT_LENGTH=9, FHOST_EXT_OVERRIDE = {
FHOST_SECRET_BYTES=16, "audio/flac" : ".flac",
FHOST_EXT_OVERRIDE={ "image/gif" : ".gif",
"audio/flac": ".flac", "image/jpeg" : ".jpg",
"image/gif": ".gif", "image/png" : ".png",
"image/jpeg": ".jpg", "image/svg+xml" : ".svg",
"image/png": ".png", "video/webm" : ".webm",
"image/svg+xml": ".svg", "video/x-matroska" : ".mkv",
"video/webm": ".webm", "application/octet-stream" : ".bin",
"video/x-matroska": ".mkv", "text/plain" : ".log",
"application/octet-stream": ".bin", "text/plain" : ".txt",
"text/plain": ".log", "text/x-diff" : ".diff",
"text/plain": ".txt",
"text/x-diff": ".diff",
}, },
NSFW_DETECT=False, FHOST_MIME_BLACKLIST = [
NSFW_THRESHOLD=0.92, "application/x-dosexec",
VSCAN_SOCKET=None, "application/java-archive",
VSCAN_QUARANTINE_PATH="quarantine", "application/java-vm"
VSCAN_IGNORE=[ ],
FHOST_UPLOAD_BLACKLIST = None,
NSFW_DETECT = False,
NSFW_THRESHOLD = 0.608,
VSCAN_SOCKET = None,
VSCAN_QUARANTINE_PATH = "quarantine",
VSCAN_IGNORE = [
"Eicar-Test-Signature", "Eicar-Test-Signature",
"PUA.Win.Packer.XmMusicFile", "PUA.Win.Packer.XmMusicFile",
], ],
VSCAN_INTERVAL=datetime.timedelta(days=7), VSCAN_INTERVAL = datetime.timedelta(days=7),
URL_ALPHABET="DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMX" URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-",
"y6Vx-",
) )
app.config.from_pyfile("config.py") if not app.config["TESTING"]:
app.jinja_loader = ChoiceLoader([ app.config.from_pyfile("config.py")
FileSystemLoader(str(Path(app.instance_path) / "templates")), app.jinja_loader = ChoiceLoader([
app.jinja_loader FileSystemLoader(str(Path(app.instance_path) / "templates")),
]) app.jinja_loader
])
if app.config["DEBUG"]: if app.config["DEBUG"]:
app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False app.config["FHOST_USE_X_ACCEL_REDIRECT"] = False
if app.config["NSFW_DETECT"]: if app.config["NSFW_DETECT"]:
from nsfw_detect import NSFWDetector from nsfw_detect import NSFWDetector
@ -98,7 +97,7 @@ if app.config["NSFW_DETECT"]:
try: try:
mimedetect = Magic(mime=True, mime_encoding=False) mimedetect = Magic(mime=True, mime_encoding=False)
except TypeError: except:
print("""Error: You have installed the wrong version of the 'magic' module. print("""Error: You have installed the wrong version of the 'magic' module.
Please install python-magic.""") Please install python-magic.""")
sys.exit(1) sys.exit(1)
@ -106,11 +105,9 @@ Please install python-magic.""")
db = SQLAlchemy(app) db = SQLAlchemy(app)
migrate = Migrate(app, db) migrate = Migrate(app, db)
class URL(db.Model): class URL(db.Model):
__tablename__ = "URL" id = db.Column(db.Integer, primary_key = True)
id = db.Column(db.Integer, primary_key=True) url = db.Column(db.UnicodeText, unique = True)
url = db.Column(db.UnicodeText, unique=True)
def __init__(self, url): def __init__(self, url):
self.url = url self.url = url
@ -121,7 +118,6 @@ class URL(db.Model):
def geturl(self): def geturl(self):
return url_for("get", path=self.getname(), _external=True) + "\n" return url_for("get", path=self.getname(), _external=True) + "\n"
@staticmethod
def get(url): def get(url):
u = URL.query.filter_by(url=url).first() u = URL.query.filter_by(url=url).first()
@ -132,36 +128,12 @@ class URL(db.Model):
return u return u
class IPAddress(types.TypeDecorator):
impl = types.LargeBinary
cache_ok = True
def process_bind_param(self, value, dialect):
match value:
case ipaddress.IPv6Address():
value = (value.ipv4_mapped or value).packed
case ipaddress.IPv4Address():
value = value.packed
return value
def process_result_value(self, value, dialect):
if value is not None:
value = ipaddress.ip_address(value)
if type(value) is ipaddress.IPv6Address:
value = value.ipv4_mapped or value
return value
class File(db.Model): class File(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key = True)
sha256 = db.Column(db.String, unique=True) sha256 = db.Column(db.String, unique = True)
ext = db.Column(db.UnicodeText) ext = db.Column(db.UnicodeText)
mime = db.Column(db.UnicodeText) mime = db.Column(db.UnicodeText)
addr = db.Column(IPAddress(16)) addr = db.Column(db.UnicodeText)
ua = db.Column(db.UnicodeText)
removed = db.Column(db.Boolean, default=False) removed = db.Column(db.Boolean, default=False)
nsfw_score = db.Column(db.Float) nsfw_score = db.Column(db.Float)
expiration = db.Column(db.BigInteger) expiration = db.Column(db.BigInteger)
@ -170,30 +142,28 @@ class File(db.Model):
last_vscan = db.Column(db.DateTime) last_vscan = db.Column(db.DateTime)
size = db.Column(db.BigInteger) size = db.Column(db.BigInteger)
def __init__(self, sha256, ext, mime, addr, ua, expiration, mgmt_token): def __init__(self, sha256, ext, mime, addr, expiration, mgmt_token):
self.sha256 = sha256 self.sha256 = sha256
self.ext = ext self.ext = ext
self.mime = mime self.mime = mime
self.addr = addr self.addr = addr
self.ua = ua
self.expiration = expiration self.expiration = expiration
self.mgmt_token = mgmt_token self.mgmt_token = mgmt_token
@property @property
def is_nsfw(self) -> bool: def is_nsfw(self) -> bool:
if self.nsfw_score: return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]
return self.nsfw_score > app.config["NSFW_THRESHOLD"]
return False
def getname(self): def getname(self):
return u"{0}{1}".format(su.enbase(self.id), self.ext) return u"{0}{1}".format(su.enbase(self.id), self.ext)
def geturl(self): def geturl(self):
n = self.getname() n = self.getname()
a = "nsfw" if self.is_nsfw else None
return url_for("get", path=n, secret=self.secret, if self.is_nsfw:
_external=True, _anchor=a) + "\n" return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n"
else:
return url_for("get", path=n, secret=self.secret, _external=True) + "\n"
def getpath(self) -> Path: def getpath(self) -> Path:
return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256 return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256
@ -204,37 +174,33 @@ class File(db.Model):
self.removed = permanent self.removed = permanent
self.getpath().unlink(missing_ok=True) self.getpath().unlink(missing_ok=True)
""" # Returns the epoch millisecond that a file should expire
Returns the epoch millisecond that a file should expire #
# Uses the expiration time provided by the user (requested_expiration)
Uses the expiration time provided by the user (requested_expiration) # upper-bounded by an algorithm that computes the size based on the size of the
upper-bounded by an algorithm that computes the size based on the size of # file.
the file. #
# That is, all files are assigned a computed expiration, which can voluntarily
That is, all files are assigned a computed expiration, which can be # shortened by the user either by providing a timestamp in epoch millis or a
voluntarily shortened by the user either by providing a timestamp in # duration in hours.
milliseconds since epoch or a duration in hours.
"""
@staticmethod
def get_expiration(requested_expiration, size) -> int: def get_expiration(requested_expiration, size) -> int:
current_epoch_millis = time.time() * 1000 current_epoch_millis = time.time() * 1000;
# Maximum lifetime of the file in milliseconds # Maximum lifetime of the file in milliseconds
max_lifespan = get_max_lifespan(size) this_files_max_lifespan = get_max_lifespan(size);
# The latest allowed expiration date for this file, in epoch millis # The latest allowed expiration date for this file, in epoch millis
max_expiration = max_lifespan + 1000 * time.time() this_files_max_expiration = this_files_max_lifespan + 1000 * time.time();
if requested_expiration is None: if requested_expiration is None:
return max_expiration return this_files_max_expiration
elif requested_expiration < 1650460320000: elif requested_expiration < 1650460320000:
# Treat the requested expiration time as a duration in hours # Treat the requested expiration time as a duration in hours
requested_expiration_ms = requested_expiration * 60 * 60 * 1000 requested_expiration_ms = requested_expiration * 60 * 60 * 1000
return min(max_expiration, return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms)
current_epoch_millis + requested_expiration_ms)
else: else:
# Treat expiration time as a timestamp in epoch millis # Treat the requested expiration time as a timestamp in epoch millis
return min(max_expiration, requested_expiration) return min(this_files_max_expiration, requested_expiration)
""" """
requested_expiration can be: requested_expiration can be:
@ -242,38 +208,29 @@ class File(db.Model):
- a duration (in hours) that the file should live for - a duration (in hours) that the file should live for
- a timestamp in epoch millis that the file should expire at - a timestamp in epoch millis that the file should expire at
Any value greater that the longest allowed file lifespan will be rounded Any value greater that the longest allowed file lifespan will be rounded down to that
down to that value. value.
""" """
@staticmethod def store(file_, requested_expiration: typing.Optional[int], addr, secret: bool):
def store(file_, requested_expiration: typing.Optional[int], addr, ua, data = file_.read()
secret: bool): digest = sha256(data).hexdigest()
fstream = file_.stream
digest = file_digest(fstream, "sha256").hexdigest()
fstream.seek(0, os.SEEK_END)
flen = fstream.tell()
fstream.seek(0)
def get_mime(): def get_mime():
guess = mimedetect.from_descriptor(fstream.fileno()) guess = mimedetect.from_buffer(data)
app.logger.debug(f"MIME - specified: '{file_.content_type}' - " app.logger.debug(f"MIME - specified: '{file_.content_type}' - detected: '{guess}'")
f"detected: '{guess}'")
if (not file_.content_type if not file_.content_type or not "/" in file_.content_type or file_.content_type == "application/octet-stream":
or "/" not in file_.content_type
or file_.content_type == "application/octet-stream"):
mime = guess mime = guess
else: else:
mime = file_.content_type mime = file_.content_type
if mime in app.config["FHOST_MIME_BLACKLIST"] or guess in app.config["FHOST_MIME_BLACKLIST"]:
abort(415)
if len(mime) > 128: if len(mime) > 128:
abort(400) abort(400)
for flt in MIMEFilter.query.all(): if mime.startswith("text/") and not "charset" in mime:
if flt.check(guess):
abort(403, flt.reason)
if mime.startswith("text/") and "charset" not in mime:
mime += "; charset=utf-8" mime += "; charset=utf-8"
return mime return mime
@ -285,8 +242,7 @@ class File(db.Model):
gmime = mime.split(";")[0] gmime = mime.split(";")[0]
guess = guess_extension(gmime) guess = guess_extension(gmime)
app.logger.debug(f"extension - specified: '{ext}' - detected: " app.logger.debug(f"extension - specified: '{ext}' - detected: '{guess}'")
f"'{guess}'")
if not ext: if not ext:
if gmime in app.config["FHOST_EXT_OVERRIDE"]: if gmime in app.config["FHOST_EXT_OVERRIDE"]:
@ -298,7 +254,7 @@ class File(db.Model):
return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin" return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin"
expiration = File.get_expiration(requested_expiration, flen) expiration = File.get_expiration(requested_expiration, len(data))
isnew = True isnew = True
f = File.query.filter_by(sha256=digest).first() f = File.query.filter_by(sha256=digest).first()
@ -321,25 +277,24 @@ class File(db.Model):
mime = get_mime() mime = get_mime()
ext = get_ext(mime) ext = get_ext(mime)
mgmt_token = secrets.token_urlsafe() mgmt_token = secrets.token_urlsafe()
f = File(digest, ext, mime, addr, ua, expiration, mgmt_token) f = File(digest, ext, mime, addr, expiration, mgmt_token)
f.addr = addr f.addr = addr
f.ua = ua
if isnew: if isnew:
f.secret = None f.secret = None
if secret: if secret:
f.secret = \ f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
storage = Path(app.config["FHOST_STORAGE_PATH"]) storage = Path(app.config["FHOST_STORAGE_PATH"])
storage.mkdir(parents=True, exist_ok=True) storage.mkdir(parents=True, exist_ok=True)
p = storage / digest p = storage / digest
if not p.is_file(): if not p.is_file():
file_.save(p) with open(p, "wb") as of:
of.write(data)
f.size = flen f.size = len(data)
if not f.nsfw_score and app.config["NSFW_DETECT"]: if not f.nsfw_score and app.config["NSFW_DETECT"]:
f.nsfw_score = nsfw.detect(str(p)) f.nsfw_score = nsfw.detect(str(p))
@ -349,129 +304,8 @@ class File(db.Model):
return f, isnew return f, isnew
class RequestFilter(db.Model):
__tablename__ = "request_filter"
id = db.Column(db.Integer, primary_key=True)
type = db.Column(db.String(20), index=True, nullable=False)
comment = db.Column(db.UnicodeText)
__mapper_args__ = {
"polymorphic_on": type,
"with_polymorphic": "*",
"polymorphic_identity": "empty"
}
def __init__(self, comment: str = None):
self.comment = comment
class AddrFilter(RequestFilter):
addr = db.Column(IPAddress(16), unique=True)
__mapper_args__ = {"polymorphic_identity": "addr"}
def __init__(self, addr: ipaddress._BaseAddress, comment: str = None):
self.addr = addr
super().__init__(comment=comment)
def check(self, addr: ipaddress._BaseAddress) -> bool:
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
return addr == self.addr
def check_request(self, r: Request) -> bool:
return self.check(ipaddress.ip_address(r.remote_addr))
@property
def reason(self) -> str:
return f"Your IP Address ({self.addr.compressed}) is blocked from " \
"uploading files."
class IPNetwork(types.TypeDecorator):
impl = types.Text
cache_ok = True
def process_bind_param(self, value, dialect):
if value is not None:
value = value.compressed
return value
def process_result_value(self, value, dialect):
if value is not None:
value = ipaddress.ip_network(value)
return value
class NetFilter(RequestFilter):
net = db.Column(IPNetwork)
__mapper_args__ = {"polymorphic_identity": "net"}
def __init__(self, net: ipaddress._BaseNetwork, comment: str = None):
self.net = net
super().__init__(comment=comment)
def check(self, addr: ipaddress._BaseAddress) -> bool:
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
return addr in self.net
def check_request(self, r: Request) -> bool:
return self.check(ipaddress.ip_address(r.remote_addr))
@property
def reason(self) -> str:
return f"Your network ({self.net.compressed}) is blocked from " \
"uploading files."
class HasRegex:
@declared_attr
def regex(cls):
return cls.__table__.c.get("regex", db.Column(db.UnicodeText))
def check(self, s: str) -> bool:
return re.match(self.regex, s) is not None
class MIMEFilter(HasRegex, RequestFilter):
__mapper_args__ = {"polymorphic_identity": "mime"}
def __init__(self, mime_regex: str, comment: str = None):
self.regex = mime_regex
super().__init__(comment=comment)
def check_request(self, r: Request) -> bool:
if "file" in r.files:
return self.check(r.files["file"].mimetype)
return False
@property
def reason(self) -> str:
return "File MIME type not allowed."
class UAFilter(HasRegex, RequestFilter):
__mapper_args__ = {"polymorphic_identity": "ua"}
def __init__(self, ua_regex: str, comment: str = None):
self.regex = ua_regex
super().__init__(comment=comment)
def check_request(self, r: Request) -> bool:
return self.check(r.user_agent.string)
@property
def reason(self) -> str:
return "User agent not allowed."
class UrlEncoder(object): class UrlEncoder(object):
def __init__(self, alphabet, min_length): def __init__(self,alphabet, min_length):
self.alphabet = alphabet self.alphabet = alphabet
self.min_length = min_length self.min_length = min_length
@ -491,21 +325,17 @@ class UrlEncoder(object):
result += self.alphabet.index(c) * (n ** i) result += self.alphabet.index(c) * (n ** i)
return result return result
su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1) su = UrlEncoder(alphabet=app.config["URL_ALPHABET"], min_length=1)
def fhost_url(scheme=None): def fhost_url(scheme=None):
if not scheme: if not scheme:
return url_for(".fhost", _external=True).rstrip("/") return url_for(".fhost", _external=True).rstrip("/")
else: else:
return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/") return url_for(".fhost", _external=True, _scheme=scheme).rstrip("/")
def is_fhost_url(url): def is_fhost_url(url):
return url.startswith(fhost_url()) or url.startswith(fhost_url("https")) return url.startswith(fhost_url()) or url.startswith(fhost_url("https"))
def shorten(url): def shorten(url):
if len(url) > app.config["MAX_URL_LENGTH"]: if len(url) > app.config["MAX_URL_LENGTH"]:
abort(414) abort(414)
@ -517,6 +347,16 @@ def shorten(url):
return u.geturl() return u.geturl()
def in_upload_bl(addr):
if app.config["FHOST_UPLOAD_BLACKLIST"]:
with app.open_instance_resource(app.config["FHOST_UPLOAD_BLACKLIST"], "r") as bl:
check = addr.lstrip("::ffff:")
for l in bl.readlines():
if not l.startswith("#"):
if check == l.rstrip():
return True
return False
""" """
requested_expiration can be: requested_expiration can be:
@ -524,12 +364,14 @@ requested_expiration can be:
- a duration (in hours) that the file should live for - a duration (in hours) that the file should live for
- a timestamp in epoch millis that the file should expire at - a timestamp in epoch millis that the file should expire at
Any value greater that the longest allowed file lifespan will be rounded down Any value greater that the longest allowed file lifespan will be rounded down to that
to that value. value.
""" """
def store_file(f, requested_expiration: typing.Optional[int], addr, ua, def store_file(f, requested_expiration: typing.Optional[int], addr, secret: bool):
secret: bool): if in_upload_bl(addr):
sf, isnew = File.store(f, requested_expiration, addr, ua, secret) return "Your host is blocked from uploading files.\n", 451
sf, isnew = File.store(f, requested_expiration, addr, secret)
response = make_response(sf.geturl()) response = make_response(sf.geturl())
response.headers["X-Expires"] = sf.expiration response.headers["X-Expires"] = sf.expiration
@ -539,12 +381,11 @@ def store_file(f, requested_expiration: typing.Optional[int], addr, ua,
return response return response
def store_url(url, addr, secret: bool):
def store_url(url, addr, ua, secret: bool):
if is_fhost_url(url): if is_fhost_url(url):
abort(400) abort(400)
h = {"Accept-Encoding": "identity"} h = { "Accept-Encoding" : "identity" }
r = requests.get(url, stream=True, verify=False, headers=h) r = requests.get(url, stream=True, verify=False, headers=h)
try: try:
@ -553,24 +394,24 @@ def store_url(url, addr, ua, secret: bool):
return str(e) + "\n" return str(e) + "\n"
if "content-length" in r.headers: if "content-length" in r.headers:
length = int(r.headers["content-length"]) l = int(r.headers["content-length"])
if length <= app.config["MAX_CONTENT_LENGTH"]: if l < app.config["MAX_CONTENT_LENGTH"]:
def urlfile(**kwargs): def urlfile(**kwargs):
return type('', (), kwargs)() return type('',(),kwargs)()
f = urlfile(read=r.raw.read, f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="")
content_type=r.headers["content-type"], filename="")
return store_file(f, None, addr, ua, secret) return store_file(f, None, addr, secret)
else: else:
abort(413) abort(413)
else: else:
abort(411) abort(411)
def manage_file(f): def manage_file(f):
if request.form["token"] != f.mgmt_token: try:
assert(request.form["token"] == f.mgmt_token)
except:
abort(401) abort(401)
if "delete" in request.form: if "delete" in request.form:
@ -589,7 +430,6 @@ def manage_file(f):
abort(400) abort(400)
@app.route("/<path:path>", methods=["GET", "POST"]) @app.route("/<path:path>", methods=["GET", "POST"])
@app.route("/s/<secret>/<path:path>", methods=["GET", "POST"]) @app.route("/s/<secret>/<path:path>", methods=["GET", "POST"])
def get(path, secret=None): def get(path, secret=None):
@ -626,9 +466,7 @@ def get(path, secret=None):
response.headers["Content-Length"] = f.size response.headers["Content-Length"] = f.size
response.headers["X-Accel-Redirect"] = "/" + str(fpath) response.headers["X-Accel-Redirect"] = "/" + str(fpath)
else: else:
response = send_from_directory( response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
app.config["FHOST_STORAGE_PATH"], f.sha256,
mimetype=f.mime)
response.headers["X-Expires"] = f.expiration response.headers["X-Expires"] = f.expiration
return response return response
@ -646,19 +484,11 @@ def get(path, secret=None):
abort(404) abort(404)
@app.route("/", methods=["GET", "POST"]) @app.route("/", methods=["GET", "POST"])
def fhost(): def fhost():
if request.method == "POST": if request.method == "POST":
for flt in RequestFilter.query.all():
if flt.check_request(request):
abort(403, flt.reason)
sf = None sf = None
secret = "secret" in request.form secret = "secret" in request.form
addr = ipaddress.ip_address(request.remote_addr)
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
if "file" in request.files: if "file" in request.files:
try: try:
@ -666,8 +496,7 @@ def fhost():
return store_file( return store_file(
request.files["file"], request.files["file"],
int(request.form["expires"]), int(request.form["expires"]),
addr, request.remote_addr,
request.user_agent.string,
secret secret
) )
except ValueError: except ValueError:
@ -678,15 +507,13 @@ def fhost():
return store_file( return store_file(
request.files["file"], request.files["file"],
None, None,
addr, request.remote_addr,
request.user_agent.string,
secret secret
) )
elif "url" in request.form: elif "url" in request.form:
return store_url( return store_url(
request.form["url"], request.form["url"],
addr, request.remote_addr,
request.user_agent.string,
secret secret
) )
elif "shorten" in request.form: elif "shorten" in request.form:
@ -696,17 +523,14 @@ def fhost():
else: else:
return render_template("index.html") return render_template("index.html")
@app.route("/robots.txt") @app.route("/robots.txt")
def robots(): def robots():
return """User-agent: * return """User-agent: *
Disallow: / Disallow: /
""" """
@app.errorhandler(400) @app.errorhandler(400)
@app.errorhandler(401) @app.errorhandler(401)
@app.errorhandler(403)
@app.errorhandler(404) @app.errorhandler(404)
@app.errorhandler(411) @app.errorhandler(411)
@app.errorhandler(413) @app.errorhandler(413)
@ -715,23 +539,20 @@ Disallow: /
@app.errorhandler(451) @app.errorhandler(451)
def ehandler(e): def ehandler(e):
try: try:
return render_template(f"{e.code}.html", id=id, request=request, return render_template(f"{e.code}.html", id=id, request=request), e.code
description=e.description), e.code
except TemplateNotFound: except TemplateNotFound:
return "Segmentation fault\n", e.code return "Segmentation fault\n", e.code
@app.cli.command("prune") @app.cli.command("prune")
def prune(): def prune():
""" """
Clean up expired files Clean up expired files
Deletes any files from the filesystem which have hit their expiration time. Deletes any files from the filesystem which have hit their expiration time. This
This doesn't remove them from the database, only from the filesystem. doesn't remove them from the database, only from the filesystem. It's recommended
It is recommended that server owners run this command regularly, or set it that server owners run this command regularly, or set it up on a timer.
up on a timer.
""" """
current_time = time.time() * 1000 current_time = time.time() * 1000;
# The path to where uploaded files are stored # The path to where uploaded files are stored
storage = Path(app.config["FHOST_STORAGE_PATH"]) storage = Path(app.config["FHOST_STORAGE_PATH"])
@ -745,7 +566,7 @@ def prune():
) )
) )
files_removed = 0 files_removed = 0;
# For every expired file... # For every expired file...
for file in expired_files: for file in expired_files:
@ -758,33 +579,31 @@ def prune():
# Remove it from the file system # Remove it from the file system
try: try:
os.remove(file_path) os.remove(file_path)
files_removed += 1 files_removed += 1;
except FileNotFoundError: except FileNotFoundError:
pass # If the file was already gone, we're good pass # If the file was already gone, we're good
except OSError as e: except OSError as e:
print(e) print(e)
print( print(
"\n------------------------------------" "\n------------------------------------"
"Encountered an error while trying to remove file {file_path}." "Encountered an error while trying to remove file {file_path}. Double"
"Make sure the server is configured correctly, permissions " "check to make sure the server is configured correctly, permissions are"
"are okay, and everything is ship shape, then try again.") "okay, and everything is ship shape, then try again.")
return return;
# Finally, mark that the file was removed # Finally, mark that the file was removed
file.expiration = None file.expiration = None;
db.session.commit() db.session.commit()
print(f"\nDone! {files_removed} file(s) removed") print(f"\nDone! {files_removed} file(s) removed")
""" For a file of a given size, determine the largest allowed lifespan of that file
""" Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
For a file of a given size, determine the largest allowed lifespan of that file as FHOST_{MIN,MAX}_EXPIRATION.
Based on the current app's configuration: This lifespan may be shortened by a user's request, but no files should be allowed to
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION. expire at a point after this number.
This lifespan may be shortened by a user's request, but no files should be
allowed to expire at a point after this number.
Value returned is a duration in milliseconds. Value returned is a duration in milliseconds.
""" """
@ -794,13 +613,11 @@ def get_max_lifespan(filesize: int) -> int:
max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024) max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
def do_vscan(f): def do_vscan(f):
if f["path"].is_file(): if f["path"].is_file():
with open(f["path"], "rb") as scanf: with open(f["path"], "rb") as scanf:
try: try:
res = list(app.config["VSCAN_SOCKET"].instream(scanf).values()) f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0]
f["result"] = res[0]
except: except:
f["result"] = ("SCAN FAILED", None) f["result"] = ("SCAN FAILED", None)
else: else:
@ -808,12 +625,11 @@ def do_vscan(f):
return f return f
@app.cli.command("vscan") @app.cli.command("vscan")
def vscan(): def vscan():
if not app.config["VSCAN_SOCKET"]: if not app.config["VSCAN_SOCKET"]:
print("Error: Virus scanning enabled but no connection method " print("""Error: Virus scanning enabled but no connection method specified.
"specified.\nPlease set VSCAN_SOCKET.") Please set VSCAN_SOCKET.""")
sys.exit(1) sys.exit(1)
qp = Path(app.config["VSCAN_QUARANTINE_PATH"]) qp = Path(app.config["VSCAN_QUARANTINE_PATH"])
@ -827,11 +643,9 @@ def vscan():
File.last_vscan == None), File.last_vscan == None),
File.removed == False) File.removed == False)
else: else:
res = File.query.filter(File.last_vscan == None, res = File.query.filter(File.last_vscan == None, File.removed == False)
File.removed == False)
work = [{"path": f.getpath(), "name": f.getname(), "id": f.id} work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res]
for f in res]
results = [] results = []
for i, r in enumerate(p.imap_unordered(do_vscan, work)): for i, r in enumerate(p.imap_unordered(do_vscan, work)):
@ -845,10 +659,9 @@ def vscan():
found = True found = True
results.append({ results.append({
"id": r["id"], "id" : r["id"],
"last_vscan": None if r["result"][0] == "SCAN FAILED" "last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(),
else datetime.datetime.now(), "removed" : found})
"removed": found})
db.session.bulk_update_mappings(File, results) db.session.bulk_update_mappings(File, results)
db.session.commit() db.session.commit()

26
instance/config.example.py
View file

@ -139,6 +139,30 @@ FHOST_EXT_OVERRIDE = {
"text/x-diff" : ".diff", "text/x-diff" : ".diff",
} }
# Control which files aren't allowed to be uploaded
#
# Certain kinds of files are never accepted. If the file claims to be one of
# these types of files, or if we look at the contents of the file and it looks
# like one of these filetypes, then we reject the file outright with a 415
# UNSUPPORTED MEDIA EXCEPTION
FHOST_MIME_BLACKLIST = [
"application/x-dosexec",
"application/java-archive",
"application/java-vm"
]
# A list of IP addresses which are blacklisted from uploading files
#
# Can be set to the path of a file with an IP address on each line. The file
# can also include comment lines using a pound sign (#). Paths are resolved
# relative to the instance/ directory.
#
# If this is set to None, then no IP blacklist will be consulted.
FHOST_UPLOAD_BLACKLIST = None
# Enables support for detecting NSFW images # Enables support for detecting NSFW images
# #
# Consult README.md for additional dependencies before setting to True # Consult README.md for additional dependencies before setting to True
@ -152,7 +176,7 @@ NSFW_DETECT = False
# are marked as NSFW. # are marked as NSFW.
# #
# If NSFW_DETECT is set to False, then this has no effect. # If NSFW_DETECT is set to False, then this has no effect.
NSFW_THRESHOLD = 0.92 NSFW_THRESHOLD = 0.608
# If you want to scan files for viruses using ClamAV, specify the socket used # If you want to scan files for viruses using ClamAV, specify the socket used

1
migrations/env.py
View file

@ -81,7 +81,6 @@ def run_migrations_online():
finally: finally:
connection.close() connection.close()
if context.is_offline_mode(): if context.is_offline_mode():
run_migrations_offline() run_migrations_offline()
else: else:

4
migrations/versions/0659d7b9eea8_.py
View file

@ -15,8 +15,12 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True)) op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True))
# ### end Alembic commands ###
def downgrade(): def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('file', 'mgmt_token') op.drop_column('file', 'mgmt_token')
# ### end Alembic commands ###

30
migrations/versions/0cd36ecdd937_.py
View file

@ -15,22 +15,28 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('URL', op.create_table('URL',
sa.Column('id', sa.Integer(), nullable=False), sa.Column('id', sa.Integer(), nullable=False),
sa.Column('url', sa.UnicodeText(), nullable=True), sa.Column('url', sa.UnicodeText(), nullable=True),
sa.PrimaryKeyConstraint('id'), sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('url')) sa.UniqueConstraint('url')
)
op.create_table('file', op.create_table('file',
sa.Column('id', sa.Integer(), nullable=False), sa.Column('id', sa.Integer(), nullable=False),
sa.Column('sha256', sa.String(), nullable=True), sa.Column('sha256', sa.String(), nullable=True),
sa.Column('ext', sa.UnicodeText(), nullable=True), sa.Column('ext', sa.UnicodeText(), nullable=True),
sa.Column('mime', sa.UnicodeText(), nullable=True), sa.Column('mime', sa.UnicodeText(), nullable=True),
sa.Column('addr', sa.UnicodeText(), nullable=True), sa.Column('addr', sa.UnicodeText(), nullable=True),
sa.Column('removed', sa.Boolean(), nullable=True), sa.Column('removed', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id'), sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('sha256')) sa.UniqueConstraint('sha256')
)
### end Alembic commands ###
def downgrade(): def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('file') op.drop_table('file')
op.drop_table('URL') op.drop_table('URL')
### end Alembic commands ###

5
migrations/versions/30bfe33aa328_add_file_size_field.py
View file

@ -19,7 +19,6 @@ from pathlib import Path
Base = automap_base() Base = automap_base()
def upgrade(): def upgrade():
op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True)) op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True))
bind = op.get_bind() bind = op.get_bind()
@ -35,8 +34,8 @@ def upgrade():
p = storage / f.sha256 p = storage / f.sha256
if p.is_file(): if p.is_file():
updates.append({ updates.append({
"id": f.id, "id" : f.id,
"size": p.stat().st_size "size" : p.stat().st_size
}) })
session.bulk_update_mappings(File, updates) session.bulk_update_mappings(File, updates)

79
migrations/versions/5cda1743b92d_add_request_filters.py
View file

@ -1,79 +0,0 @@
"""Add request filters
Revision ID: 5cda1743b92d
Revises: dd0766afb7d2
Create Date: 2024-09-27 12:13:16.845981
"""
# revision identifiers, used by Alembic.
revision = '5cda1743b92d'
down_revision = 'dd0766afb7d2'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from flask import current_app
import ipaddress
Base = automap_base()
def upgrade():
op.create_table('request_filter',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('type', sa.String(length=20), nullable=False),
sa.Column('comment', sa.UnicodeText(), nullable=True),
sa.Column('addr', sa.LargeBinary(length=16),
nullable=True),
sa.Column('net', sa.Text(), nullable=True),
sa.Column('regex', sa.UnicodeText(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('addr'))
with op.batch_alter_table('request_filter', schema=None) as batch_op:
batch_op.create_index(batch_op.f('ix_request_filter_type'), ['type'],
unique=False)
bind = op.get_bind()
Base.prepare(autoload_with=bind)
RequestFilter = Base.classes.request_filter
session = Session(bind=bind)
blp = current_app.config.get("FHOST_UPLOAD_BLACKLIST")
if blp:
with current_app.open_instance_resource(blp, "r") as bl:
for line in bl.readlines():
if not line.startswith("#"):
line = line.strip()
if line.endswith(":"):
# old implementation uses str.startswith,
# which does not translate to networks
current_app.logger.warning(
f"Ignored address: {line}")
continue
addr = ipaddress.ip_address(line).packed
flt = RequestFilter(type="addr", addr=addr)
session.add(flt)
for mime in current_app.config.get("FHOST_MIME_BLACKLIST", []):
flt = RequestFilter(type="mime", regex=mime)
session.add(flt)
session.commit()
w = "Entries in your host and MIME blacklists have been migrated to " \
"request filters and stored in the databaes, where possible. " \
"The corresponding files and config options may now be deleted. " \
"Note that you may have to manually restore them if you wish to " \
"revert this with a db downgrade operation."
current_app.logger.warning(w)
def downgrade():
with op.batch_alter_table('request_filter', schema=None) as batch_op:
batch_op.drop_index(batch_op.f('ix_request_filter_type'))
op.drop_table('request_filter')

7
migrations/versions/5cee97aab219_.py
View file

@ -15,9 +15,12 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
op.add_column('file', sa.Column('last_vscan', sa.DateTime(), # ### commands auto generated by Alembic - please adjust! ###
nullable=True)) op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True))
# ### end Alembic commands ###
def downgrade(): def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('file', 'last_vscan') op.drop_column('file', 'last_vscan')
# ### end Alembic commands ###

4
migrations/versions/7e246705da6a_.py
View file

@ -15,8 +15,12 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True)) op.add_column('file', sa.Column('nsfw_score', sa.Float(), nullable=True))
# ### end Alembic commands ###
def downgrade(): def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('file', 'nsfw_score') op.drop_column('file', 'nsfw_score')
# ### end Alembic commands ###

38
migrations/versions/939a08e1d6e5_.py
View file

@ -21,29 +21,24 @@ from sqlalchemy.orm import Session
import os import os
import time import time
""" For a file of a given size, determine the largest allowed lifespan of that file
""" Based on the current app's configuration: Specifically, the MAX_CONTENT_LENGTH, as well
For a file of a given size, determine the largest allowed lifespan of that file as FHOST_{MIN,MAX}_EXPIRATION.
Based on the current app's configuration: This lifespan may be shortened by a user's request, but no files should be allowed to
Specifically, the MAX_CONTENT_LENGTH, as well as FHOST_{MIN,MAX}_EXPIRATION. expire at a point after this number.
This lifespan may be shortened by a user's request, but no files should be
allowed to expire at a point after this number.
Value returned is a duration in milliseconds. Value returned is a duration in milliseconds.
""" """
def get_max_lifespan(filesize: int) -> int: def get_max_lifespan(filesize: int) -> int:
cfg = current_app.config min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
min_exp = cfg.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000) max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
max_exp = cfg.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000) max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
max_size = cfg.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3) return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
Base = automap_base() Base = automap_base()
def upgrade(): def upgrade():
op.add_column('file', sa.Column('expiration', sa.BigInteger())) op.add_column('file', sa.Column('expiration', sa.BigInteger()))
@ -53,14 +48,14 @@ def upgrade():
session = Session(bind=bind) session = Session(bind=bind)
storage = Path(current_app.config["FHOST_STORAGE_PATH"]) storage = Path(current_app.config["FHOST_STORAGE_PATH"])
current_time = time.time() * 1000 current_time = time.time() * 1000;
# List of file hashes which have not expired yet # List of file hashes which have not expired yet
# This could get really big for some servers # This could get really big for some servers
try: try:
unexpired_files = os.listdir(storage) unexpired_files = os.listdir(storage)
except FileNotFoundError: except FileNotFoundError:
return # There are no currently unexpired files return # There are no currently unexpired files
# Calculate an expiration date for all existing files # Calculate an expiration date for all existing files
@ -70,7 +65,7 @@ def upgrade():
sa.not_(File.removed) sa.not_(File.removed)
) )
) )
updates = [] # We coalesce updates to the database here updates = [] # We coalesce updates to the database here
# SQLite has a hard limit on the number of variables so we # SQLite has a hard limit on the number of variables so we
# need to do this the slow way # need to do this the slow way
@ -79,18 +74,13 @@ def upgrade():
for file in files: for file in files:
file_path = storage / file.sha256 file_path = storage / file.sha256
stat = os.stat(file_path) stat = os.stat(file_path)
# How long the file is allowed to live, in ms max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms
max_age = get_max_lifespan(stat.st_size) file_birth = stat.st_mtime * 1000 # When the file was created, in ms
# When the file was created, in ms updates.append({'id': file.id, 'expiration': int(file_birth + max_age)})
file_birth = stat.st_mtime * 1000
updates.append({
'id': file.id,
'expiration': int(file_birth + max_age)})
# Apply coalesced updates # Apply coalesced updates
session.bulk_update_mappings(File, updates) session.bulk_update_mappings(File, updates)
session.commit() session.commit()
def downgrade(): def downgrade():
op.drop_column('file', 'expiration') op.drop_column('file', 'expiration')

78
migrations/versions/d9a53a28ba54_change_file_addr_to_ipaddress_type.py
View file

@ -1,78 +0,0 @@
"""Change File.addr to IPAddress type
Revision ID: d9a53a28ba54
Revises: 5cda1743b92d
Create Date: 2024-09-27 14:03:06.764764
"""
# revision identifiers, used by Alembic.
revision = 'd9a53a28ba54'
down_revision = '5cda1743b92d'
from alembic import op
import sqlalchemy as sa
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from flask import current_app
import ipaddress
Base = automap_base()
def upgrade():
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.add_column(sa.Column('addr_tmp', sa.LargeBinary(16),
nullable=True))
bind = op.get_bind()
Base.prepare(autoload_with=bind)
File = Base.classes.file
session = Session(bind=bind)
updates = []
stmt = sa.select(File).where(sa.not_(File.addr == None))
for f in session.scalars(stmt.execution_options(yield_per=1000)):
addr = ipaddress.ip_address(f.addr)
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
updates.append({
"id": f.id,
"addr_tmp": addr.packed
})
session.execute(sa.update(File), updates)
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.drop_column('addr')
batch_op.alter_column('addr_tmp', new_column_name='addr')
def downgrade():
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.add_column(sa.Column('addr_tmp', sa.UnicodeText,
nullable=True))
bind = op.get_bind()
Base.prepare(autoload_with=bind)
File = Base.classes.file
session = Session(bind=bind)
updates = []
stmt = sa.select(File).where(sa.not_(File.addr == None))
for f in session.scalars(stmt.execution_options(yield_per=1000)):
addr = ipaddress.ip_address(f.addr)
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
updates.append({
"id": f.id,
"addr_tmp": addr.compressed
})
session.execute(sa.update(File), updates)
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.drop_column('addr')
batch_op.alter_column('addr_tmp', new_column_name='addr')

24
migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py
View file

@ -1,24 +0,0 @@
"""Store user agent string with files
Revision ID: dd0766afb7d2
Revises: 30bfe33aa328
Create Date: 2023-03-29 07:18:49.113200
"""
# revision identifiers, used by Alembic.
revision = 'dd0766afb7d2'
down_revision = '30bfe33aa328'
from alembic import op
import sqlalchemy as sa
def upgrade():
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True))
def downgrade():
with op.batch_alter_table('file', schema=None) as batch_op:
batch_op.drop_column('ua')

4
migrations/versions/e2e816056589_.py
View file

@ -15,8 +15,12 @@ import sqlalchemy as sa
def upgrade(): def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('file', sa.Column('secret', sa.String(), nullable=True)) op.add_column('file', sa.Column('secret', sa.String(), nullable=True))
# ### end Alembic commands ###
def downgrade(): def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('file', 'secret') op.drop_column('file', 'secret')
# ### end Alembic commands ###

16
mod.css
View file

@ -1,6 +1,5 @@
#ftable { #ftable {
width: 1fr; width: 1fr;
height: 100%;
} }
#infopane { #infopane {
@ -11,7 +10,7 @@
#finfo { #finfo {
background: $boost; background: $boost;
height: 14; height: 12;
width: 1fr; width: 1fr;
box-sizing: content-box; box-sizing: content-box;
} }
@ -28,9 +27,20 @@
width: 1fr; width: 1fr;
} }
#filter_container {
height: auto;
display: none;
}
#filter_label {
content-align: right middle;
height: 1fr;
width: 20%;
margin: 0 1 0 2;
}
#filter_input { #filter_input {
width: 1fr; width: 1fr;
display: none;
} }
Notification { Notification {

177
mod.py
View file

@ -5,30 +5,27 @@ from sys import stdout
import time import time
from textual.app import App, ComposeResult from textual.app import App, ComposeResult
from textual.widgets import DataTable, Header, Footer, RichLog, Static, Input from textual.widgets import DataTable, Header, Footer, TextLog, Static, Input
from textual.containers import Horizontal, Vertical from textual.containers import Horizontal, Vertical
from textual.screen import Screen from textual.screen import Screen
from textual import log from textual import log
from rich.text import Text from rich.text import Text
from jinja2.filters import do_filesizeformat from jinja2.filters import do_filesizeformat
import ipaddress
from fhost import db, File, AddrFilter, su, app as fhost_app from fhost import db, File, su, app as fhost_app, in_upload_bl
from modui import FileTable, mime, MpvWidget, Notification from modui import *
fhost_app.app_context().push() fhost_app.app_context().push()
class NullptrMod(Screen): class NullptrMod(Screen):
BINDINGS = [ BINDINGS = [
("q", "quit_app", "Quit"), ("q", "quit_app", "Quit"),
("f1", "filter(1, 'Name')", "Lookup name"), ("f1", "filter(1, 'Lookup name:')", "Lookup name"),
("f2", "filter(2, 'IP address')", "Filter IP"), ("f2", "filter(2, 'Filter IP address:')", "Filter IP"),
("f3", "filter(3, 'MIME Type')", "Filter MIME"), ("f3", "filter(3, 'Filter MIME Type:')", "Filter MIME"),
("f4", "filter(4, 'Extension')", "Filter Ext."), ("f4", "filter(4, 'Filter extension:')", "Filter Ext."),
("f5", "refresh", "Refresh"), ("f5", "refresh", "Refresh"),
("f6", "filter_clear", "Clear filter"), ("f6", "filter_clear", "Clear filter"),
("f7", "filter(5, 'User agent')", "Filter UA"),
("r", "remove_file(False)", "Remove file"), ("r", "remove_file(False)", "Remove file"),
("ctrl+r", "remove_file(True)", "Ban file"), ("ctrl+r", "remove_file(True)", "Ban file"),
("p", "ban_ip(False)", "Ban IP"), ("p", "ban_ip(False)", "Ban IP"),
@ -44,82 +41,72 @@ class NullptrMod(Screen):
ftable.watch_query(None, None) ftable.watch_query(None, None)
def action_filter_clear(self): def action_filter_clear(self):
self.finput.display = False self.query_one("#filter_container").display = False
ftable = self.query_one("#ftable") ftable = self.query_one("#ftable")
ftable.focus() ftable.focus()
ftable.query = ftable.base_query ftable.query = ftable.base_query
def action_filter(self, fcol: int, label: str): def action_filter(self, fcol: int, label: str):
self.finput.placeholder = label self.query_one("#filter_label").update(label)
self.finput.display = True finput = self.query_one("#filter_input")
self.finput.focus()
self.filter_col = fcol self.filter_col = fcol
self.query_one("#filter_container").display = True
finput.focus()
self._refresh_layout() self._refresh_layout()
if self.current_file: if self.current_file:
match fcol: match fcol:
case 1: self.finput.value = "" case 1: finput.value = ""
case 2: self.finput.value = self.current_file.addr.compressed case 2: finput.value = self.current_file.addr
case 3: self.finput.value = self.current_file.mime case 3: finput.value = self.current_file.mime
case 4: self.finput.value = self.current_file.ext case 4: finput.value = self.current_file.ext
case 5: self.finput.value = self.current_file.ua or ""
def on_input_submitted(self, message: Input.Submitted) -> None: def on_input_submitted(self, message: Input.Submitted) -> None:
self.finput.display = False self.query_one("#filter_container").display = False
ftable = self.query_one("#ftable") ftable = self.query_one("#ftable")
ftable.focus() ftable.focus()
q = ftable.base_query
if len(message.value): if len(message.value):
match self.filter_col: match self.filter_col:
case 1: case 1:
try: try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value))
q = q.filter(File.id == su.debase(message.value)) except ValueError: pass
except ValueError: case 2: ftable.query = ftable.base_query.filter(File.addr == message.value)
return case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value))
case 2: case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value))
try: else:
addr = ipaddress.ip_address(message.value) ftable.query = ftable.base_query
if type(addr) is ipaddress.IPv6Address:
addr = addr.ipv4_mapped or addr
q = q.filter(File.addr == addr)
except ValueError:
return
case 3: q = q.filter(File.mime.like(message.value))
case 4: q = q.filter(File.ext.like(message.value))
case 5: q = q.filter(File.ua.like(message.value))
ftable.query = q
def action_remove_file(self, permanent: bool) -> None: def action_remove_file(self, permanent: bool) -> None:
if self.current_file: if self.current_file:
self.current_file.delete(permanent) self.current_file.delete(permanent)
db.session.commit() db.session.commit()
self.mount(Notification(f"{'Banned' if permanent else 'Removed'}" self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}"))
f"file {self.current_file.getname()}"))
self.action_refresh() self.action_refresh()
def action_ban_ip(self, nuke: bool) -> None: def action_ban_ip(self, nuke: bool) -> None:
if self.current_file: if self.current_file:
addr = self.current_file.addr if not fhost_app.config["FHOST_UPLOAD_BLACKLIST"]:
if AddrFilter.query.filter(AddrFilter.addr == addr).scalar(): self.mount(Notification("Failed: FHOST_UPLOAD_BLACKLIST not set!"))
txt = f"{addr.compressed} is already banned" return
else: else:
db.session.add(AddrFilter(addr)) if in_upload_bl(self.current_file.addr):
db.session.commit() txt = f"{self.current_file.addr} is already banned"
txt = f"Banned {addr.compressed}" else:
with fhost_app.open_instance_resource(fhost_app.config["FHOST_UPLOAD_BLACKLIST"], "a") as bl:
print(self.current_file.addr.lstrip("::ffff:"), file=bl)
txt = f"Banned {self.current_file.addr}"
if nuke: if nuke:
tsize = 0 tsize = 0
trm = 0 trm = 0
for f in File.query.filter(File.addr == addr): for f in File.query.filter(File.addr == self.current_file.addr):
if f.getpath().is_file(): if f.getpath().is_file():
tsize += f.size or f.getpath().stat().st_size tsize += f.size or f.getpath().stat().st_size
trm += 1 trm += 1
f.delete(True) f.delete(True)
db.session.commit() db.session.commit()
txt += f", removed {trm} {'files' if trm != 1 else 'file'} " \ txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}"
f"totaling {do_filesizeformat(tsize, True)}"
self.mount(Notification(txt)) self.mount(Notification(txt))
self._refresh_layout() self._refresh_layout()
ftable = self.query_one("#ftable") ftable = self.query_one("#ftable")
@ -132,13 +119,13 @@ class NullptrMod(Screen):
def compose(self) -> ComposeResult: def compose(self) -> ComposeResult:
yield Header() yield Header()
yield Horizontal( yield Horizontal(
FileTable(id="ftable", zebra_stripes=True, cursor_type="row"), FileTable(id="ftable", zebra_stripes=True),
Vertical( Vertical(
DataTable(id="finfo", show_header=False, cursor_type="none"), DataTable(id="finfo", show_header=False),
MpvWidget(id="mpv"), MpvWidget(id="mpv"),
RichLog(id="ftextlog", auto_scroll=False), TextLog(id="ftextlog"),
id="infopane")) id="infopane"))
yield Input(id="filter_input") yield Horizontal(Static("Filter:", id="filter_label"), Input(id="filter_input"), id="filter_container")
yield Footer() yield Footer()
def on_mount(self) -> None: def on_mount(self) -> None:
@ -153,17 +140,12 @@ class NullptrMod(Screen):
self.mpvw = self.query_one("#mpv") self.mpvw = self.query_one("#mpv")
self.ftlog = self.query_one("#ftextlog") self.ftlog = self.query_one("#ftextlog")
self.finput = self.query_one("#filter_input")
self.mimehandler = mime.MIMEHandler() self.mimehandler = mime.MIMEHandler()
self.mimehandler.register(mime.MIMECategory.Archive, self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive)
self.handle_libarchive)
self.mimehandler.register(mime.MIMECategory.Text, self.handle_text) self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv) self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
self.mimehandler.register(mime.MIMECategory.Document, self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf)
self.handle_mupdf) self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive)
self.mimehandler.register(mime.MIMECategory.Fallback,
self.handle_libarchive)
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv) self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw) self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
@ -175,7 +157,7 @@ class NullptrMod(Screen):
def handle_text(self, cat): def handle_text(self, cat):
with open(self.current_file.getpath(), "r") as sf: with open(self.current_file.getpath(), "r") as sf:
data = sf.read(1000000).replace("\033", "") data = sf.read(1000000).replace("\033","")
self.ftlog.write(data) self.ftlog.write(data)
return True return True
@ -190,8 +172,7 @@ class NullptrMod(Screen):
self.mpvw.styles.height = "40%" self.mpvw.styles.height = "40%"
self.mpvw.start_mpv("hex://" + imgdata, 0) self.mpvw.start_mpv("hex://" + imgdata, 0)
self.ftlog.write( self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]")) self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
for k, v in doc.metadata.items(): for k, v in doc.metadata.items():
self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}")) self.ftlog.write(Text.from_markup(f" [bold]{k}:[/bold] {v}"))
@ -216,8 +197,7 @@ class NullptrMod(Screen):
for k, v in c.metadata.items(): for k, v in c.metadata.items():
self.ftlog.write(f" {k}: {v}") self.ftlog.write(f" {k}: {v}")
for s in c.streams: for s in c.streams:
self.ftlog.write( self.ftlog.write(Text(f"Stream {s.index}:", style="bold"))
Text(f"Stream {s.index}:", style="bold"))
self.ftlog.write(f" Type: {s.type}") self.ftlog.write(f" Type: {s.type}")
if s.base_rate: if s.base_rate:
self.ftlog.write(f" Frame rate: {s.base_rate}") self.ftlog.write(f" Frame rate: {s.base_rate}")
@ -236,31 +216,24 @@ class NullptrMod(Screen):
else: else:
c = chr(s) c = chr(s)
s = c s = c
if c.isalpha(): if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]"
return f"\0[chartreuse1]{s}\0[/chartreuse1]" if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]"
if c.isdigit():
return f"\0[gold1]{s}\0[/gold1]"
if not c.isprintable(): if not c.isprintable():
g = "grey50" if c == "\0" else "cadet_blue" g = "grey50" if c == "\0" else "cadet_blue"
return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]" return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
return s return s
return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}"
return Text.from_markup( f"{' ' * (16 - len(c))}"
"\n".join(' '.join( f" {''.join(map(fmt, c))}"
map(fmt, map(''.join, zip(*[iter(c.hex())] * 2)))) + for c in map(lambda x: bytes([n for n in x if n != None]),
f"{' ' * (16 - len(c))} {''.join(map(fmt, c))}" zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16))))
for c in
map(lambda x: bytes([n for n in x if n is not None]),
zip_longest(
*[iter(binf.read(min(length, 16 * 10)))] * 16))))
with open(self.current_file.getpath(), "rb") as binf: with open(self.current_file.getpath(), "rb") as binf:
self.ftlog.write(hexdump(binf, self.current_file.size)) self.ftlog.write(hexdump(binf, self.current_file.size))
if self.current_file.size > 16*10*2: if self.current_file.size > 16*10*2:
binf.seek(self.current_file.size-16*10) binf.seek(self.current_file.size-16*10)
self.ftlog.write(" [...] ".center(64, '─')) self.ftlog.write(" [...] ".center(64, '─'))
self.ftlog.write(hexdump(binf, self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell()))
self.current_file.size - binf.tell()))
return True return True
@ -271,33 +244,26 @@ class NullptrMod(Screen):
self.finfo.add_rows([ self.finfo.add_rows([
("ID:", str(f.id)), ("ID:", str(f.id)),
("File name:", f.getname()), ("File name:", f.getname()),
("URL:", f.geturl() ("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"),
if fhost_app.config["SERVER_NAME"]
else "⚠ Set SERVER_NAME in config.py to display"),
("File size:", do_filesizeformat(f.size, True)), ("File size:", do_filesizeformat(f.size, True)),
("MIME type:", f.mime), ("MIME type:", f.mime),
("SHA256 checksum:", f.sha256), ("SHA256 checksum:", f.sha256),
("Uploaded by:", Text(f.addr.compressed)), ("Uploaded by:", Text(f.addr)),
("User agent:", Text(f.ua or "")),
("Management token:", f.mgmt_token), ("Management token:", f.mgmt_token),
("Secret:", f.secret), ("Secret:", f.secret),
("Is NSFW:", ("Yes" if f.is_nsfw else "No") + ("Is NSFW:", ("Yes" if f.is_nsfw else "No") + (f" (Score: {f.nsfw_score:0.4f})" if f.nsfw_score else " (Not scanned)")),
(f" (Score: {f.nsfw_score:0.4f})"
if f.nsfw_score else " (Not scanned)")),
("Is banned:", "Yes" if f.removed else "No"), ("Is banned:", "Yes" if f.removed else "No"),
("Expires:", ("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000)))
time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(File.get_expiration(f.expiration,
f.size)/1000)))
]) ])
self.mpvw.stop_mpv(True) self.mpvw.stop_mpv(True)
self.ftlog.clear() self.ftlog.remove()
self.query_one("#infopane").mount(TextLog(id="ftextlog"))
self.ftlog = self.query_one("#ftextlog")
if f.getpath().is_file(): if f.getpath().is_file():
self.mimehandler.handle(f.mime, f.ext) self.mimehandler.handle(f.mime, f.ext)
self.ftlog.scroll_to(x=0, y=0, animate=False) self.ftlog.scroll_home(animate=False)
class NullptrModApp(App): class NullptrModApp(App):
CSS_PATH = "mod.css" CSS_PATH = "mod.css"
@ -308,7 +274,6 @@ class NullptrModApp(App):
self.install_screen(self.main_screen, name="main") self.install_screen(self.main_screen, name="main")
self.push_screen("main") self.push_screen("main")
if __name__ == "__main__": if __name__ == "__main__":
app = NullptrModApp() app = NullptrModApp()
app.run() app.run()

55
modui/filetable.py
View file

@ -1,20 +1,18 @@
from textual.widgets import DataTable, Static from textual.widgets import DataTable, Static
from textual.reactive import Reactive from textual.reactive import Reactive
from textual.message import Message from textual.message import Message, MessageTarget
from textual import events, log from textual import events, log
from jinja2.filters import do_filesizeformat from jinja2.filters import do_filesizeformat
from fhost import File from fhost import File
from modui import mime from modui import mime
class FileTable(DataTable): class FileTable(DataTable):
query = Reactive(None) query = Reactive(None)
order_col = Reactive(0) order_col = Reactive(0)
order_desc = Reactive(True) order_desc = Reactive(True)
limit = 10000 limit = 10000
colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime]
File.size, File.mime]
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
@ -23,9 +21,9 @@ class FileTable(DataTable):
self.query = self.base_query self.query = self.base_query
class Selected(Message): class Selected(Message):
def __init__(self, f: File) -> None: def __init__(self, sender: MessageTarget, f: File) -> None:
self.file = f self.file = f
super().__init__() super().__init__(sender)
def watch_order_col(self, old, value) -> None: def watch_order_col(self, old, value) -> None:
self.watch_query(None, None) self.watch_query(None, None)
@ -35,8 +33,6 @@ class FileTable(DataTable):
def watch_query(self, old, value) -> None: def watch_query(self, old, value) -> None:
def fmt_file(f: File) -> tuple: def fmt_file(f: File) -> tuple:
mimemoji = mime.mimemoji.get(f.mime.split('/')[0],
mime.mimemoji.get(f.mime)) or ' '
return ( return (
str(f.id), str(f.id),
"🔴" if f.removed else " ", "🔴" if f.removed else " ",
@ -44,44 +40,29 @@ class FileTable(DataTable):
"👻" if not f.getpath().is_file() else " ", "👻" if not f.getpath().is_file() else " ",
f.getname(), f.getname(),
do_filesizeformat(f.size, True), do_filesizeformat(f.size, True),
f"{mimemoji} {f.mime}", f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or ' '} " + f.mime,
) )
if (self.query): if (self.query):
self.clear()
order = FileTable.colmap[self.order_col] order = FileTable.colmap[self.order_col]
q = self.query q = self.query
if order: if order: q = q.order_by(order.desc() if self.order_desc else order, File.id)
q = q.order_by(order.desc() if self.order_desc self.add_rows(map(fmt_file, q.limit(self.limit)))
else order, File.id)
qres = list(map(fmt_file, q.limit(self.limit)))
ri = 0 def _scroll_cursor_in_to_view(self, animate: bool = False) -> None:
row = self.cursor_coordinate.row region = self._get_cell_region(self.cursor_row, 0)
if row < self.row_count and row >= 0: spacing = self._get_cell_border()
ri = int(self.get_row_at(row)[0]) self.scroll_to_region(region, animate=animate, spacing=spacing)
self.clear() async def watch_cursor_cell(self, old, value) -> None:
self.add_rows(qres) super().watch_cursor_cell(old, value)
if value[0] < len(self.data) and value[0] >= 0:
for i, v in enumerate(qres): f = File.query.get(int(self.data[value[0]][0]))
if int(v[0]) == ri: await self.emit(self.Selected(self, f))
self.move_cursor(row=i)
break
self.on_selected()
def on_selected(self) -> Selected:
row = self.cursor_coordinate.row
if row < self.row_count and row >= 0:
f = File.query.get(int(self.get_row_at(row)[0]))
self.post_message(self.Selected(f))
def watch_cursor_coordinate(self, old, value) -> None:
super().watch_cursor_coordinate(old, value)
if old != value:
self.on_selected()
def on_click(self, event: events.Click) -> None: def on_click(self, event: events.Click) -> None:
super().on_click(event)
meta = self.get_style_at(event.x, event.y).meta meta = self.get_style_at(event.x, event.y).meta
if meta: if meta:
if meta["row"] == -1: if meta["row"] == -1:

94
modui/mime.py
View file

@ -2,46 +2,46 @@ from enum import Enum
from textual import log from textual import log
mimemoji = { mimemoji = {
"audio": "🔈", "audio" : "🔈",
"video": "🎞", "video" : "🎞",
"text": "📄", "text" : "📄",
"image": "🖼", "image" : "🖼",
"application/zip": "🗜️", "application/zip" : "🗜️",
"application/x-zip-compressed": "🗜️", "application/x-zip-compressed" : "🗜️",
"application/x-tar": "🗄", "application/x-tar" : "🗄",
"application/x-cpio": "🗄", "application/x-cpio" : "🗄",
"application/x-xz": "🗜️", "application/x-xz" : "🗜️",
"application/x-7z-compressed": "🗜️", "application/x-7z-compressed" : "🗜️",
"application/gzip": "🗜️", "application/gzip" : "🗜️",
"application/zstd": "🗜️", "application/zstd" : "🗜️",
"application/x-rar": "🗜️", "application/x-rar" : "🗜️",
"application/x-rar-compressed": "🗜️", "application/x-rar-compressed" : "🗜️",
"application/vnd.ms-cab-compressed": "🗜️", "application/vnd.ms-cab-compressed" : "🗜️",
"application/x-bzip2": "🗜️", "application/x-bzip2" : "🗜️",
"application/x-lzip": "🗜️", "application/x-lzip" : "🗜️",
"application/x-iso9660-image": "💿", "application/x-iso9660-image" : "💿",
"application/pdf": "📕", "application/pdf" : "📕",
"application/epub+zip": "📕", "application/epub+zip" : "📕",
"application/mxf": "🎞", "application/mxf" : "🎞",
"application/vnd.android.package-archive": "📦", "application/vnd.android.package-archive" : "📦",
"application/vnd.debian.binary-package": "📦", "application/vnd.debian.binary-package" : "📦",
"application/x-rpm": "📦", "application/x-rpm" : "📦",
"application/x-dosexec": "⚙", "application/x-dosexec" : "⚙",
"application/x-execuftable": "⚙", "application/x-execuftable" : "⚙",
"application/x-sharedlib": "⚙", "application/x-sharedlib" : "⚙",
"application/java-archive": "☕", "application/java-archive" : "☕",
"application/x-qemu-disk": "🖴", "application/x-qemu-disk" : "🖴",
"application/pgp-encrypted": "🔏", "application/pgp-encrypted" : "🔏",
} }
MIMECategory = Enum("MIMECategory", ["Archive", "Text", "AV", "Document", MIMECategory = Enum("MIMECategory",
"Fallback"]) ["Archive", "Text", "AV", "Document", "Fallback"]
)
class MIMEHandler: class MIMEHandler:
def __init__(self): def __init__(self):
self.handlers = { self.handlers = {
MIMECategory.Archive: [[ MIMECategory.Archive : [[
"application/zip", "application/zip",
"application/x-zip-compressed", "application/x-zip-compressed",
"application/x-tar", "application/x-tar",
@ -62,31 +62,31 @@ class MIMEHandler:
"application/java-archive", "application/java-archive",
"application/vnd.openxmlformats" "application/vnd.openxmlformats"
], []], ], []],
MIMECategory.Text: [[ MIMECategory.Text : [[
"text", "text",
"application/json", "application/json",
"application/xml", "application/xml",
], []], ], []],
MIMECategory.AV: [[ MIMECategory.AV : [[
"audio", "video", "image", "audio", "video", "image",
"application/mxf" "application/mxf"
], []], ], []],
MIMECategory.Document: [[ MIMECategory.Document : [[
"application/pdf", "application/pdf",
"application/epub", "application/epub",
"application/x-mobipocket-ebook", "application/x-mobipocket-ebook",
], []], ], []],
MIMECategory.Fallback: [[], []] MIMECategory.Fallback : [[], []]
} }
self.exceptions = { self.exceptions = {
MIMECategory.Archive: { MIMECategory.Archive : {
".cbz": MIMECategory.Document, ".cbz" : MIMECategory.Document,
".xps": MIMECategory.Document, ".xps" : MIMECategory.Document,
".epub": MIMECategory.Document, ".epub" : MIMECategory.Document,
}, },
MIMECategory.Text: { MIMECategory.Text : {
".fb2": MIMECategory.Document, ".fb2" : MIMECategory.Document,
} }
} }
@ -115,14 +115,12 @@ class MIMEHandler:
cat = getcat(mime) cat = getcat(mime)
for handler in self.handlers[cat][1]: for handler in self.handlers[cat][1]:
try: try:
if handler(cat): if handler(cat): return
return
except: pass except: pass
for handler in self.handlers[MIMECategory.Fallback][1]: for handler in self.handlers[MIMECategory.Fallback][1]:
try: try:
if handler(None): if handler(None): return
return
except: pass except: pass
raise RuntimeError(f"Unhandled MIME type category: {cat}") raise RuntimeError(f"Unhandled MIME type category: {cat}")

34
modui/mpvwidget.py
View file

@ -1,9 +1,5 @@
import time import time
import fcntl, struct, termios
import fcntl
import struct
import termios
from sys import stdout from sys import stdout
from textual import events, log from textual import events, log
@ -11,7 +7,6 @@ from textual.widgets import Static
from fhost import app as fhost_app from fhost import app as fhost_app
class MpvWidget(Static): class MpvWidget(Static):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
@ -19,10 +14,8 @@ class MpvWidget(Static):
self.mpv = None self.mpv = None
self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO") self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
if self.vo not in ["sixel", "kitty"]: if not self.vo in ["sixel", "kitty"]:
self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO " self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.")
"to 'sixel' or 'kitty' in config.py,\nwhichever is "
"supported by your terminal.")
else: else:
try: try:
import mpv import mpv
@ -34,35 +27,28 @@ class MpvWidget(Static):
self.mpv[f"vo-sixel-buffered"] = True self.mpv[f"vo-sixel-buffered"] = True
self.mpv["audio"] = False self.mpv["audio"] = False
self.mpv["loop-file"] = "inf" self.mpv["loop-file"] = "inf"
self.mpv["image-display-duration"] = 0.5 \ self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf"
if self.vo == "sixel" else "inf"
except Exception as e: except Exception as e:
self.mpv = None self.mpv = None
self.update("⚠ Previews require python-mpv with libmpv " self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}")
"0.36.0 or later \n\nError was:\n"
f"{type(e).__name__}: {e}")
def start_mpv(self, f: str | None = None, def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None:
pos: float | str | None = None) -> None:
self.display = True self.display = True
self.screen._refresh_layout() self.screen._refresh_layout()
if self.mpv: if self.mpv:
if self.content_region.x: if self.content_region.x:
winsz = fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678') r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678'))
r, c, w, h = struct.unpack('hhhh', winsz)
width = int((w / c) * self.content_region.width) width = int((w / c) * self.content_region.width)
height = int((h / r) * (self.content_region.height + height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0)))
(1 if self.vo == "sixel" else 0)))
self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1 self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1 self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + \ self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0)
(1 if self.vo == "sixel" else 0)
self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
self.mpv[f"vo-{self.vo}-width"] = width self.mpv[f"vo-{self.vo}-width"] = width
self.mpv[f"vo-{self.vo}-height"] = height self.mpv[f"vo-{self.vo}-height"] = height
if pos is not None: if pos != None:
self.mpv["start"] = pos self.mpv["start"] = pos
if f: if f:

1
modui/notification.py
View file

@ -1,6 +1,5 @@
from textual.widgets import Static from textual.widgets import Static
class Notification(Static): class Notification(Static):
def on_mount(self) -> None: def on_mount(self) -> None:
self.set_timer(3, self.remove) self.set_timer(3, self.remove)

75
nsfw_detect.py
View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
""" """
Copyright © 2024 Mia Herkt Copyright © 2020 Mia Herkt
Licensed under the EUPL, Version 1.2 or - as soon as approved Licensed under the EUPL, Version 1.2 or - as soon as approved
by the European Commission - subsequent versions of the EUPL by the European Commission - subsequent versions of the EUPL
(the "License"); (the "License");
@ -18,32 +18,81 @@
and limitations under the License. and limitations under the License.
""" """
import numpy as np
import os
import sys import sys
import av from io import BytesIO
from transformers import pipeline from pathlib import Path
os.environ["GLOG_minloglevel"] = "2" # seriously :|
import caffe
import av
av.logging.set_level(av.logging.PANIC)
class NSFWDetector: class NSFWDetector:
def __init__(self): def __init__(self):
self.classifier = pipeline("image-classification", npath = Path(__file__).parent / "nsfw_model"
model="giacomoarienti/nsfw-classifier") self.nsfw_net = caffe.Net(
str(npath / "deploy.prototxt"),
caffe.TEST,
weights = str(npath / "resnet_50_1by2_nsfw.caffemodel")
)
self.caffe_transformer = caffe.io.Transformer({
'data': self.nsfw_net.blobs['data'].data.shape
})
# move image channels to outermost
self.caffe_transformer.set_transpose('data', (2, 0, 1))
# subtract the dataset-mean value in each channel
self.caffe_transformer.set_mean('data', np.array([104, 117, 123]))
# rescale from [0, 1] to [0, 255]
self.caffe_transformer.set_raw_scale('data', 255)
# swap channels from RGB to BGR
self.caffe_transformer.set_channel_swap('data', (2, 1, 0))
def _compute(self, img):
image = caffe.io.load_image(img)
H, W, _ = image.shape
_, _, h, w = self.nsfw_net.blobs["data"].data.shape
h_off = int(max((H - h) / 2, 0))
w_off = int(max((W - w) / 2, 0))
crop = image[h_off:h_off + h, w_off:w_off + w, :]
transformed_image = self.caffe_transformer.preprocess('data', crop)
transformed_image.shape = (1,) + transformed_image.shape
input_name = self.nsfw_net.inputs[0]
output_layers = ["prob"]
all_outputs = self.nsfw_net.forward_all(
blobs=output_layers, **{input_name: transformed_image})
outputs = all_outputs[output_layers[0]][0].astype(float)
return outputs
def detect(self, fpath): def detect(self, fpath):
try: try:
with av.open(fpath) as container: with av.open(fpath) as container:
try: try: container.seek(int(container.duration / 2))
container.seek(int(container.duration / 2))
except: container.seek(0) except: container.seek(0)
frame = next(container.decode(video=0)) frame = next(container.decode(video=0))
img = frame.to_image()
res = self.classifier(img)
return max([x["score"] for x in res if frame.width >= frame.height:
if x["label"] not in ["neutral", "drawings"]]) w = 256
except: pass h = int(frame.height * (256 / frame.width))
else:
w = int(frame.width * (256 / frame.height))
h = 256
frame = frame.reformat(width=w, height=h, format="rgb24")
img = BytesIO()
frame.to_image().save(img, format="ppm")
return -1.0 scores = self._compute(img)
except:
return -1.0
return scores[1]
if __name__ == "__main__": if __name__ == "__main__":

11
nsfw_model/LICENSE.md Normal file
View file

@ -0,0 +1,11 @@
Copyright 2016, Yahoo Inc.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3488
nsfw_model/deploy.prototxt Normal file
View file

File diff suppressed because it is too large Load diff

BIN
nsfw_model/resnet_50_1by2_nsfw.caffemodel Normal file
View file

Binary file not shown.

5
requirements.txt
View file

@ -7,15 +7,12 @@ Jinja2
Flask Flask
flask_sqlalchemy flask_sqlalchemy
python_magic python_magic
ipaddress
# vscan # vscan
clamd clamd
# nsfw detection # nsfw detection
torch numpy
transformers
pillow
# mod ui # mod ui
av av

1
templates/403.html
View file

@ -1 +0,0 @@
{{ description if description else "Your host is banned." }}

1
templates/index.html
View file

@ -37,6 +37,7 @@ To change the expiration date (see above):
{% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %} {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %}
Maximum file size: {{ max_size }} Maximum file size: {{ max_size }}
Not allowed: {{ config["FHOST_MIME_BLACKLIST"]|join(", ") }}
FILE RETENTION PERIOD FILE RETENTION PERIOD

0
tests/__init__.py Normal file
View file

81
tests/test_client.py Normal file
View file

@ -0,0 +1,81 @@
import pytest
import tempfile
import os
from flask_migrate import upgrade as db_upgrade
from io import BytesIO
from fhost import app, db, url_for, File, URL
@pytest.fixture
def client():
with tempfile.TemporaryDirectory() as tmpdir:
app.config["SQLALCHEMY_DATABASE_URI"] = f"sqlite:///{tmpdir}/db.sqlite"
app.config["FHOST_STORAGE_PATH"] = os.path.join(tmpdir, "up")
app.config["TESTING"] = True
with app.test_client() as client:
with app.app_context():
db_upgrade()
yield client
def test_client(client):
payloads = [
({ "file" : (BytesIO(b"hello"), "hello.txt") }, 200, b"https://localhost/E.txt\n"),
({ "file" : (BytesIO(b"hello"), "hello.ignorethis") }, 200, b"https://localhost/E.txt\n"),
({ "file" : (BytesIO(b"bye"), "bye.truncatethis") }, 200, b"https://localhost/Q.truncate\n"),
({ "file" : (BytesIO(b"hi"), "hi.tar.gz") }, 200, b"https://localhost/h.tar.gz\n"),
({ "file" : (BytesIO(b"lea!"), "lea!") }, 200, b"https://localhost/d.txt\n"),
({ "file" : (BytesIO(b"why?"), "balls", "application/x-dosexec") }, 415, None),
({ "shorten" : "https://0x0.st" }, 200, b"https://localhost/E\n"),
({ "shorten" : "https://localhost" }, 400, None),
({}, 400, None),
]
for p, s, r in payloads:
rv = client.post("/", buffered=True,
content_type="multipart/form-data",
data=p)
assert rv.status_code == s
if r:
assert rv.data == r
f = File.query.get(2)
f.removed = True
db.session.add(f)
db.session.commit()
rq = [
(200, [
"/",
"robots.txt",
"E.txt",
"E.txt/test",
"E.txt/test.py",
"d.txt",
"h.tar.gz",
]),
(302, [
"E",
]),
(404, [
"test.bin",
"test.bin/test",
"test.bin/test.py",
"test",
"test/test",
"test.bin/test.py",
"E.bin",
"E/test",
"E/test.bin",
]),
(451, [
"Q.truncate",
]),
]
for code, paths in rq:
for p in paths:
app.logger.info(f"GET {p}")
rv = client.get(p)
assert rv.status_code == code