ModUI: Update for Textual 0.54.0

Fixes #85

0x0-prune.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Prune 0x0 files
+After=remote-fs.target
+
+[Service]
+Type=oneshot
+User=nullptr
+WorkingDirectory=/path/to/0x0
+BindPaths=/path/to/0x0
+
+Environment=FLASK_APP=fhost
+ExecStart=/usr/bin/flask prune
+ProtectProc=noaccess
+ProtectSystem=strict
+ProtectHome=tmpfs
+PrivateTmp=true
+PrivateUsers=true
+ProtectKernelLogs=true
+LockPersonality=true
+
+[Install]
+WantedBy=multi-user.target

0x0-prune.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Prune 0x0 files
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

0x0-vscan.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Scan 0x0 files with ClamAV
+After=remote-fs.target clamd.service
+
+[Service]
+Type=oneshot
+User=nullptr
+WorkingDirectory=/path/to/0x0
+BindPaths=/path/to/0x0
+
+Environment=FLASK_APP=fhost
+ExecStart=/usr/bin/flask vscan
+ProtectProc=noaccess
+ProtectSystem=strict
+ProtectHome=tmpfs
+PrivateTmp=true
+PrivateUsers=true
+ProtectKernelLogs=true
+LockPersonality=true
+
+[Install]
+WantedBy=multi-user.target

0x0-vscan.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Scan 0x0 files with ClamAV
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target

README.rst

@@ -35,10 +35,61 @@ downsides, one of them being that range requests will not work. This is a
 problem for example when streaming media files: It won’t be possible to seek,
 and some ISOBMFF (MP4) files will not play at all.
 
-To make files expire, simply create a cronjob that runs ``cleanup.py`` every
-now and then.
+To make files expire, simply run ``FLASK_APP=fhost flask prune`` every
+now and then. You can use the provided systemd unit files for this::
 
-Before running the service for the first time, run ``FLASK_APP=fhost flask db upgrade``.
+    0x0-prune.service
+    0x0-prune.timer
+
+Make sure to edit them to match your system configuration. In particular,
+set the user and paths in ``0x0-prune.service``.
+
+Before running the service for the first time and every time you update it
+from this git repository, run ``FLASK_APP=fhost flask db upgrade``.
+
+
+Moderation UI
+-------------
+
+.. image:: modui.webp
+  :height: 300
+
+0x0 features a TUI program for file moderation. With it, you can view a list
+of uploaded files, as well as extended information on them. It allows you to
+take actions like removing files temporarily or permanently, as well as
+blocking IP addresses and associated files.
+
+If a sufficiently recent version of python-mpv with libmpv is present and
+your terminal supports it, you also get graphical file previews, including
+video playback. Upstream mpv currently supports sixels and the
+`kitty graphics protocol <https://sw.kovidgoyal.net/kitty/graphics-protocol/>`_.
+For this to work, set the ``MOD_PREVIEW_PROTO`` option in ``instance/config.py``.
+
+Requirements:
+
+* `Textual <https://textual.textualize.io/>`_
+
+Optional:
+
+* `python-mpv <https://github.com/jaseg/python-mpv>`_
+  (graphical previews)
+* `PyAV <https://github.com/PyAV-Org/PyAV>`_
+  (information on multimedia files)
+* `PyMuPDF <https://github.com/pymupdf/PyMuPDF>`_
+  (previews and file information for PDF, XPS, EPUB, MOBI and FB2)
+* `libarchive-c <https://github.com/Changaco/python-libarchive-c>`_
+  (archive content listing)
+
+.. note::
+    `Mosh <https://mosh.org/>`_ currently does not support sixels or kitty graphics.
+
+.. hint::
+    You may need to set the ``COLORTERM`` environment variable to
+    ``truecolor``.
+
+.. tip::
+    Using compression with SSH (``-C`` option) can significantly
+    reduce the bandwidth requirements for graphics.
 
 
 NSFW Detection
@@ -49,7 +100,24 @@ neural network model. This works for images and video files and requires
 the following:
 
 * Caffe Python module (built for Python 3)
-* ``ffmpegthumbnailer`` executable in ``$PATH``
+* `PyAV <https://github.com/PyAV-Org/PyAV>`_
+
+
+Virus Scanning
+--------------
+
+0x0 can scan its files with ClamAV’s daemon. As this can take a long time
+for larger files, this does not happen immediately but instead every time
+you run the ``vscan`` command. It is recommended to configure a systemd
+timer or cronjob to do this periodically. Examples are included::
+
+    0x0-vscan.service
+    0x0-vscan.timer
+
+Remember to adjust your size limits in clamd.conf, including
+``StreamMaxLength``!
+
+This feature requires the `clamd module <https://pypi.org/project/clamd/>`_.
 
 
 Network Security Considerations

cleanup.py

@@ -1,44 +1,8 @@
 #!/usr/bin/env python3
 
-"""
-    Copyright © 2020 Mia Herkt
-    Licensed under the EUPL, Version 1.2 or - as soon as approved
-    by the European Commission - subsequent versions of the EUPL
-    (the "License");
-    You may not use this work except in compliance with the License.
-    You may obtain a copy of the license at:
-
-        https://joinup.ec.europa.eu/software/page/eupl
-
-    Unless required by applicable law or agreed to in writing,
-    software distributed under the License is distributed on an
-    "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-    either express or implied.
-    See the License for the specific language governing permissions
-    and limitations under the License.
-"""
-
-import os
-import sys
-import time
-import datetime
-from fhost import app
-
-os.chdir(os.path.dirname(sys.argv[0]))
-os.chdir(app.config["FHOST_STORAGE_PATH"])
-
-files = [f for f in os.listdir(".")]
-
-maxs = app.config["MAX_CONTENT_LENGTH"]
-mind = 30
-maxd = 365
-
-for f in files:
-    stat = os.stat(f)
-    systime = time.time()
-    age = datetime.timedelta(seconds=(systime - stat.st_mtime)).days
-
-    maxage = mind + (-maxd + mind) * (stat.st_size / maxs - 1) ** 3
-
-    if age >= maxage:
-        os.remove(f)
+print("This script has been replaced!!")
+print("Instead, please run")
+print("")
+print("   $ FLASK_APP=fhost flask prune")
+print("")
+exit(1);

fhost.py

@@ -22,13 +22,20 @@
 from flask import Flask, abort, make_response, redirect, request, send_from_directory, url_for, Response, render_template
 from flask_sqlalchemy import SQLAlchemy
 from flask_migrate import Migrate
+from sqlalchemy import and_, or_
 from jinja2.exceptions import *
 from jinja2 import ChoiceLoader, FileSystemLoader
 from hashlib import sha256
 from magic import Magic
 from mimetypes import guess_extension
+import click
+import os
 import sys
+import time
+import datetime
+import typing
 import requests
+import secrets
 from validators import url as url_valid
 from pathlib import Path
 
@@ -42,6 +49,7 @@ app.config.update(
     FHOST_USE_X_ACCEL_REDIRECT = True, # expect nginx by default
     FHOST_STORAGE_PATH = "up",
     FHOST_MAX_EXT_LENGTH = 9,
+    FHOST_SECRET_BYTES = 16,
     FHOST_EXT_OVERRIDE = {
         "audio/flac" : ".flac",
         "image/gif" : ".gif",
@@ -63,6 +71,13 @@ app.config.update(
     FHOST_UPLOAD_BLACKLIST = None,
     NSFW_DETECT = False,
     NSFW_THRESHOLD = 0.608,
+    VSCAN_SOCKET = None,
+    VSCAN_QUARANTINE_PATH = "quarantine",
+    VSCAN_IGNORE = [
+        "Eicar-Test-Signature",
+        "PUA.Win.Packer.XmMusicFile",
+    ],
+    VSCAN_INTERVAL = datetime.timedelta(days=7),
     URL_ALPHABET = "DEQhd2uFteibPwq0SWBInTpA_jcZL5GKz3YCR14Ulk87Jors9vNHgfaOmMXy6Vx-",
 )
 
@@ -91,6 +106,7 @@ db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 
 class URL(db.Model):
+    __tablename__ = "URL"
     id = db.Column(db.Integer, primary_key = True)
     url = db.Column(db.UnicodeText, unique = True)
 
@@ -119,14 +135,27 @@ class File(db.Model):
     ext = db.Column(db.UnicodeText)
     mime = db.Column(db.UnicodeText)
     addr = db.Column(db.UnicodeText)
+    ua = db.Column(db.UnicodeText)
     removed = db.Column(db.Boolean, default=False)
     nsfw_score = db.Column(db.Float)
+    expiration = db.Column(db.BigInteger)
+    mgmt_token = db.Column(db.String)
+    secret = db.Column(db.String)
+    last_vscan = db.Column(db.DateTime)
+    size = db.Column(db.BigInteger)
 
-    def __init__(self, sha256, ext, mime, addr):
+    def __init__(self, sha256, ext, mime, addr, ua, expiration, mgmt_token):
         self.sha256 = sha256
         self.ext = ext
         self.mime = mime
         self.addr = addr
+        self.ua = ua
+        self.expiration = expiration
+        self.mgmt_token = mgmt_token
+
+    @property
+    def is_nsfw(self) -> bool:
+        return self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]
 
     def getname(self):
         return u"{0}{1}".format(su.enbase(self.id), self.ext)
@@ -134,12 +163,58 @@ class File(db.Model):
     def geturl(self):
         n = self.getname()
 
-        if self.nsfw_score and self.nsfw_score > app.config["NSFW_THRESHOLD"]:
-            return url_for("get", path=n, _external=True, _anchor="nsfw") + "\n"
+        if self.is_nsfw:
+            return url_for("get", path=n, secret=self.secret, _external=True, _anchor="nsfw") + "\n"
         else:
-            return url_for("get", path=n, _external=True) + "\n"
+            return url_for("get", path=n, secret=self.secret, _external=True) + "\n"
 
-    def store(file_, addr):
+    def getpath(self) -> Path:
+        return Path(app.config["FHOST_STORAGE_PATH"]) / self.sha256
+
+    def delete(self, permanent=False):
+        self.expiration = None
+        self.mgmt_token = None
+        self.removed = permanent
+        self.getpath().unlink(missing_ok=True)
+
+    # Returns the epoch millisecond that a file should expire
+    #
+    # Uses the expiration time provided by the user (requested_expiration)
+    # upper-bounded by an algorithm that computes the size based on the size of the
+    # file.
+    #
+    # That is, all files are assigned a computed expiration, which can voluntarily
+    # shortened by the user either by providing a timestamp in epoch millis or a
+    # duration in hours.
+    def get_expiration(requested_expiration, size) -> int:
+        current_epoch_millis = time.time() * 1000;
+
+        # Maximum lifetime of the file in milliseconds
+        this_files_max_lifespan = get_max_lifespan(size);
+
+        # The latest allowed expiration date for this file, in epoch millis
+        this_files_max_expiration = this_files_max_lifespan + 1000 * time.time();
+
+        if requested_expiration is None:
+            return this_files_max_expiration
+        elif requested_expiration < 1650460320000:
+            # Treat the requested expiration time as a duration in hours
+            requested_expiration_ms = requested_expiration * 60 * 60 * 1000
+            return min(this_files_max_expiration, current_epoch_millis + requested_expiration_ms)
+        else:
+            # Treat the requested expiration time as a timestamp in epoch millis
+            return min(this_files_max_expiration, requested_expiration)
+
+    """
+    requested_expiration can be:
+        - None, to use the longest allowed file lifespan
+        - a duration (in hours) that the file should live for
+        - a timestamp in epoch millis that the file should expire at
+
+    Any value greater that the longest allowed file lifespan will be rounded down to that
+    value.
+    """
+    def store(file_, requested_expiration: typing.Optional[int], addr, ua, secret: bool):
         data = file_.read()
         digest = sha256(data).hexdigest()
 
@@ -155,6 +230,9 @@ class File(db.Model):
             if mime in app.config["FHOST_MIME_BLACKLIST"] or guess in app.config["FHOST_MIME_BLACKLIST"]:
                 abort(415)
 
+            if len(mime) > 128:
+                abort(400)
+
             if mime.startswith("text/") and not "charset" in mime:
                 mime += "; charset=utf-8"
 
@@ -162,6 +240,8 @@ class File(db.Model):
 
         def get_ext(mime):
             ext = "".join(Path(file_.filename).suffixes[-2:])
+            if len(ext) > app.config["FHOST_MAX_EXT_LENGTH"]:
+                ext = Path(file_.filename).suffixes[-1]
             gmime = mime.split(";")[0]
             guess = guess_extension(gmime)
 
@@ -170,22 +250,45 @@ class File(db.Model):
             if not ext:
                 if gmime in app.config["FHOST_EXT_OVERRIDE"]:
                     ext = app.config["FHOST_EXT_OVERRIDE"][gmime]
+                elif guess:
+                    ext = guess
                 else:
-                    ext = guess_extension(gmime)
+                    ext = ""
 
             return ext[:app.config["FHOST_MAX_EXT_LENGTH"]] or ".bin"
 
-        f = File.query.filter_by(sha256=digest).first()
+        expiration = File.get_expiration(requested_expiration, len(data))
+        isnew = True
 
+        f = File.query.filter_by(sha256=digest).first()
         if f:
+            # If the file already exists
             if f.removed:
+                # The file was removed by moderation, so don't accept it back
                 abort(451)
+            if f.expiration is None:
+                # The file has expired, so give it a new expiration date
+                f.expiration = expiration
+
+                # Also generate a new management token
+                f.mgmt_token = secrets.token_urlsafe()
+            else:
+                # The file already exists, update the expiration if needed
+                f.expiration = max(f.expiration, expiration)
+                isnew = False
         else:
             mime = get_mime()
             ext = get_ext(mime)
-            f = File(digest, ext, mime, addr)
+            mgmt_token = secrets.token_urlsafe()
+            f = File(digest, ext, mime, addr, ua, expiration, mgmt_token)
 
         f.addr = addr
+        f.ua = ua
+
+        if isnew:
+            f.secret = None
+            if secret:
+                f.secret = secrets.token_urlsafe(app.config["FHOST_SECRET_BYTES"])
 
         storage = Path(app.config["FHOST_STORAGE_PATH"])
         storage.mkdir(parents=True, exist_ok=True)
@@ -194,16 +297,15 @@ class File(db.Model):
         if not p.is_file():
             with open(p, "wb") as of:
                 of.write(data)
-        else:
-            p.touch()
+
+        f.size = len(data)
 
         if not f.nsfw_score and app.config["NSFW_DETECT"]:
-            f.nsfw_score = nsfw.detect(p)
+            f.nsfw_score = nsfw.detect(str(p))
 
         db.session.add(f)
         db.session.commit()
-        return f
-
+        return f, isnew
 
 
 class UrlEncoder(object):
@@ -260,15 +362,30 @@ def in_upload_bl(addr):
 
     return False
 
-def store_file(f, addr):
+"""
+requested_expiration can be:
+    - None, to use the longest allowed file lifespan
+    - a duration (in hours) that the file should live for
+    - a timestamp in epoch millis that the file should expire at
+
+Any value greater that the longest allowed file lifespan will be rounded down to that
+value.
+"""
+def store_file(f, requested_expiration:  typing.Optional[int], addr, ua, secret: bool):
     if in_upload_bl(addr):
         return "Your host is blocked from uploading files.\n", 451
 
-    sf = File.store(f, addr)
+    sf, isnew = File.store(f, requested_expiration, addr, ua, secret)
 
-    return sf.geturl()
+    response = make_response(sf.geturl())
+    response.headers["X-Expires"] = sf.expiration
 
-def store_url(url, addr):
+    if isnew:
+        response.headers["X-Token"] = sf.mgmt_token
+
+    return response
+
+def store_url(url, addr, ua, secret: bool):
     if is_fhost_url(url):
         abort(400)
 
@@ -283,46 +400,87 @@ def store_url(url, addr):
     if "content-length" in r.headers:
         l = int(r.headers["content-length"])
 
-        if l < app.config["MAX_CONTENT_LENGTH"]:
+        if l <= app.config["MAX_CONTENT_LENGTH"]:
             def urlfile(**kwargs):
                 return type('',(),kwargs)()
 
             f = urlfile(read=r.raw.read, content_type=r.headers["content-type"], filename="")
 
-            return store_file(f, addr)
+            return store_file(f, None, addr, ua, secret)
         else:
             abort(413)
     else:
         abort(411)
 
-@app.route("/<path:path>")
-def get(path):
-    path = Path(path.split("/", 1)[0])
-    sufs = "".join(path.suffixes[-2:])
-    name = path.name[:-len(sufs) or None]
+def manage_file(f):
+    try:
+        assert(request.form["token"] == f.mgmt_token)
+    except:
+        abort(401)
+
+    if "delete" in request.form:
+        f.delete()
+        db.session.commit()
+        return ""
+    if "expires" in request.form:
+        try:
+            requested_expiration = int(request.form["expires"])
+        except ValueError:
+            abort(400)
+
+        f.expiration = File.get_expiration(requested_expiration, f.size)
+        db.session.commit()
+        return "", 202
+
+    abort(400)
+
+@app.route("/<path:path>", methods=["GET", "POST"])
+@app.route("/s/<secret>/<path:path>", methods=["GET", "POST"])
+def get(path, secret=None):
+    p = Path(path.split("/", 1)[0])
+    sufs = "".join(p.suffixes[-2:])
+    name = p.name[:-len(sufs) or None]
+
+    if "." in name:
+        abort(404)
+
     id = su.debase(name)
 
     if sufs:
         f = File.query.get(id)
 
         if f and f.ext == sufs:
+            if f.secret != secret:
+                abort(404)
+
             if f.removed:
                 abort(451)
 
-            fpath = Path(app.config["FHOST_STORAGE_PATH"]) / f.sha256
+            fpath = f.getpath()
 
             if not fpath.is_file():
                 abort(404)
 
+            if request.method == "POST":
+                return manage_file(f)
+
             if app.config["FHOST_USE_X_ACCEL_REDIRECT"]:
                 response = make_response()
                 response.headers["Content-Type"] = f.mime
-                response.headers["Content-Length"] = fpath.stat().st_size
+                response.headers["Content-Length"] = f.size
                 response.headers["X-Accel-Redirect"] = "/" + str(fpath)
-                return response
             else:
-                return send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
+                response = send_from_directory(app.config["FHOST_STORAGE_PATH"], f.sha256, mimetype = f.mime)
+
+            response.headers["X-Expires"] = f.expiration
+            return response
     else:
+        if request.method == "POST":
+            abort(405)
+
+        if "/" in path:
+            abort(404)
+
         u = URL.query.get(id)
 
         if u:
@@ -334,11 +492,37 @@ def get(path):
 def fhost():
     if request.method == "POST":
         sf = None
+        secret = "secret" in request.form
 
         if "file" in request.files:
-            return store_file(request.files["file"], request.remote_addr)
+            try:
+                # Store the file with the requested expiration date
+                return store_file(
+                    request.files["file"],
+                    int(request.form["expires"]),
+                    request.remote_addr,
+                    request.user_agent.string,
+                    secret
+                )
+            except ValueError:
+                # The requested expiration date wasn't properly formed
+                abort(400)
+            except KeyError:
+                # No expiration date was requested, store with the max lifespan
+                return store_file(
+                    request.files["file"],
+                    None,
+                    request.remote_addr,
+                    request.user_agent.string,
+                    secret
+                )
         elif "url" in request.form:
-            return store_url(request.form["url"], request.remote_addr)
+            return store_url(
+                request.form["url"],
+                request.remote_addr,
+                request.user_agent.string,
+                secret
+            )
         elif "shorten" in request.form:
             return shorten(request.form["shorten"])
 
@@ -353,6 +537,7 @@ Disallow: /
 """
 
 @app.errorhandler(400)
+@app.errorhandler(401)
 @app.errorhandler(404)
 @app.errorhandler(411)
 @app.errorhandler(413)
@@ -361,6 +546,129 @@ Disallow: /
 @app.errorhandler(451)
 def ehandler(e):
     try:
-        return render_template(f"{e.code}.html", id=id), e.code
+        return render_template(f"{e.code}.html", id=id, request=request), e.code
     except TemplateNotFound:
         return "Segmentation fault\n", e.code
+
+@app.cli.command("prune")
+def prune():
+    """
+    Clean up expired files
+
+    Deletes any files from the filesystem which have hit their expiration time.  This
+    doesn't remove them from the database, only from the filesystem.  It's recommended
+    that server owners run this command regularly, or set it up on a timer.
+    """
+    current_time = time.time() * 1000;
+
+    # The path to where uploaded files are stored
+    storage = Path(app.config["FHOST_STORAGE_PATH"])
+
+    # A list of all files who've passed their expiration times
+    expired_files = File.query\
+        .where(
+            and_(
+                File.expiration.is_not(None),
+                File.expiration < current_time
+            )
+        )
+
+    files_removed = 0;
+
+    # For every expired file...
+    for file in expired_files:
+        # Log the file we're about to remove
+        file_name = file.getname()
+        file_hash = file.sha256
+        file_path = storage / file_hash
+        print(f"Removing expired file {file_name} [{file_hash}]")
+
+        # Remove it from the file system
+        try:
+            os.remove(file_path)
+            files_removed += 1;
+        except FileNotFoundError:
+            pass # If the file was already gone, we're good
+        except OSError as e:
+            print(e)
+            print(
+                "\n------------------------------------"
+                "Encountered an error while trying to remove file {file_path}.  Double"
+                "check to make sure the server is configured correctly, permissions are"
+                "okay, and everything is ship shape, then try again.")
+            return;
+
+        # Finally, mark that the file was removed
+        file.expiration = None;
+    db.session.commit()
+
+    print(f"\nDone!  {files_removed} file(s) removed")
+
+""" For a file of a given size, determine the largest allowed lifespan of that file
+
+Based on the current app's configuration:  Specifically, the MAX_CONTENT_LENGTH, as well
+as FHOST_{MIN,MAX}_EXPIRATION.
+
+This lifespan may be shortened by a user's request, but no files should be allowed to
+expire at a point after this number.
+
+Value returned is a duration in milliseconds.
+"""
+def get_max_lifespan(filesize: int) -> int:
+    min_exp = app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
+    max_exp = app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
+    max_size = app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
+    return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
+
+def do_vscan(f):
+    if f["path"].is_file():
+        with open(f["path"], "rb") as scanf:
+            try:
+                f["result"] = list(app.config["VSCAN_SOCKET"].instream(scanf).values())[0]
+            except:
+                f["result"] = ("SCAN FAILED", None)
+    else:
+        f["result"] = ("FILE NOT FOUND", None)
+
+    return f
+
+@app.cli.command("vscan")
+def vscan():
+    if not app.config["VSCAN_SOCKET"]:
+        print("""Error: Virus scanning enabled but no connection method specified.
+Please set VSCAN_SOCKET.""")
+        sys.exit(1)
+
+    qp = Path(app.config["VSCAN_QUARANTINE_PATH"])
+    qp.mkdir(parents=True, exist_ok=True)
+
+    from multiprocessing import Pool
+    with Pool() as p:
+        if isinstance(app.config["VSCAN_INTERVAL"], datetime.timedelta):
+            scandate = datetime.datetime.now() - app.config["VSCAN_INTERVAL"]
+            res = File.query.filter(or_(File.last_vscan < scandate,
+                                        File.last_vscan == None),
+                                    File.removed == False)
+        else:
+            res = File.query.filter(File.last_vscan == None, File.removed == False)
+
+        work = [{"path" : f.getpath(), "name" : f.getname(), "id" : f.id} for f in res]
+
+        results = []
+        for i, r in enumerate(p.imap_unordered(do_vscan, work)):
+            if r["result"][0] != "OK":
+                print(f"{r['name']}: {r['result'][0]} {r['result'][1] or ''}")
+
+            found = False
+            if r["result"][0] == "FOUND":
+                if not r["result"][1] in app.config["VSCAN_IGNORE"]:
+                    r["path"].rename(qp / r["name"])
+                    found = True
+
+            results.append({
+                "id" : r["id"],
+                "last_vscan" : None if r["result"][0] == "SCAN FAILED" else datetime.datetime.now(),
+                "removed" : found})
+
+        db.session.bulk_update_mappings(File, results)
+        db.session.commit()

instance/config.example.py

@@ -45,6 +45,30 @@ MAX_CONTENT_LENGTH = 256 * 1024 * 1024 # Default: 256MiB
 MAX_URL_LENGTH = 4096
 
 
+# The minimum and maximum amount of time we'll retain a file for
+#
+# Small files (nearing zero bytes) are stored for the longest possible expiration date,
+# while larger files (nearing MAX_CONTENT_LENGTH bytes) are stored for the shortest amount
+# of time.  Values between these two extremes are interpolated with an exponential curve,
+# like the one shown on the index page.
+#
+# All times are in milliseconds.  If you want all files to be stored for the same amount
+# of time, set these to the same value.
+FHOST_MIN_EXPIRATION = 30  * 24 * 60 * 60 * 1000
+FHOST_MAX_EXPIRATION = 365 * 24 * 60 * 60 * 1000
+
+
+# This should be detected automatically when running behind a reverse proxy, but needs
+# to be set for URL resolution to work in e.g. the moderation UI.
+# SERVER_NAME = "example.com"
+
+
+# Specifies which graphics protocol to use for the media previews in the moderation UI.
+# Requires pympv with libmpv >= 0.36.0 and terminal support.
+# Available choices are "sixel" and "kitty".
+# MOD_PREVIEW_PROTO = "sixel"
+
+
 # Use the X-SENDFILE header to speed up serving files w/ compatible webservers
 #
 # Some webservers can be configured use the X-Sendfile header to handle sending
@@ -81,6 +105,13 @@ FHOST_STORAGE_PATH = "up"
 FHOST_MAX_EXT_LENGTH = 9
 
 
+# The number of bytes used for "secret" URLs
+#
+# When a user uploads a file with the "secret" option, 0x0 generates a string
+# from this many bytes of random data. It is base64-encoded, so on average
+# each byte results in approximately 1.3 characters.
+FHOST_SECRET_BYTES = 16
+
 # A list of filetypes to use when the uploader doesn't specify one
 #
 # When a user uploads a file with no file extension, we try to find an extension that
@@ -148,6 +179,37 @@ NSFW_DETECT = False
 NSFW_THRESHOLD = 0.608
 
 
+# If you want to scan files for viruses using ClamAV, specify the socket used
+# for connections here. You will need the clamd module.
+# Since this can take a very long time on larger files, it is not done
+# immediately but every time you run the vscan command. It is recommended to
+# configure a systemd timer or cronjob to do this periodically.
+# Remember to adjust your size limits in clamd.conf, including StreamMaxLength!
+#
+# Example:
+# from clamd import ClamdUnixSocket
+# VSCAN_SOCKET = ClamdUnixSocket("/run/clamav/clamd-socket")
+
+# This is the directory that files flagged as malicious are moved to.
+# Relative paths are resolved relative to the working directory
+# of the 0x0 process.
+VSCAN_QUARANTINE_PATH = "quarantine"
+
+# Since updated virus definitions might catch some files that were previously
+# reported as clean, you may want to rescan old files periodically.
+# Set this to a datetime.timedelta to specify the frequency, or None to
+# disable rescanning.
+from datetime import timedelta
+VSCAN_INTERVAL = timedelta(days=7)
+
+# Some files flagged by ClamAV are usually not malicious, especially if the
+# DetectPUA option is enabled in clamd.conf. This is a list of signatures
+# that will be ignored.
+VSCAN_IGNORE = [
+    "Eicar-Test-Signature",
+    "PUA.Win.Packer.XmMusicFile",
+]
+
 # A list of all characters which can appear in a URL
 #
 # If this list is too short, then URLs can very quickly become long.

migrations/versions/0659d7b9eea8_.py

@@ -0,0 +1,26 @@
+"""add file management token
+
+Revision ID: 0659d7b9eea8
+Revises: 939a08e1d6e5
+Create Date: 2022-11-30 01:06:53.362973
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0659d7b9eea8'
+down_revision = '939a08e1d6e5'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('file', sa.Column('mgmt_token', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('file', 'mgmt_token')
+    # ### end Alembic commands ###

migrations/versions/30bfe33aa328_add_file_size_field.py

@@ -0,0 +1,46 @@
+"""add file size field
+
+Revision ID: 30bfe33aa328
+Revises: 5cee97aab219
+Create Date: 2022-12-13 22:32:12.242394
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '30bfe33aa328'
+down_revision = '5cee97aab219'
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+from flask import current_app
+from pathlib import Path
+
+Base = automap_base()
+
+def upgrade():
+    op.add_column('file', sa.Column('size', sa.BigInteger(), nullable=True))
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    File = Base.classes.file
+    session = Session(bind=bind)
+
+    storage = Path(current_app.config["FHOST_STORAGE_PATH"])
+
+    updates = []
+    files = session.scalars(sa.select(File).where(sa.not_(File.removed)))
+    for f in files:
+        p = storage / f.sha256
+        if p.is_file():
+            updates.append({
+                "id" : f.id,
+                "size" : p.stat().st_size
+            })
+
+    session.bulk_update_mappings(File, updates)
+    session.commit()
+
+
+def downgrade():
+    op.drop_column('file', 'size')

migrations/versions/5cee97aab219_.py

@@ -0,0 +1,26 @@
+"""add date of last virus scan
+
+Revision ID: 5cee97aab219
+Revises: e2e816056589
+Create Date: 2022-12-10 16:39:56.388259
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5cee97aab219'
+down_revision = 'e2e816056589'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('file', sa.Column('last_vscan', sa.DateTime(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('file', 'last_vscan')
+    # ### end Alembic commands ###

migrations/versions/939a08e1d6e5_.py

@@ -0,0 +1,86 @@
+"""add file expirations
+
+Revision ID: 939a08e1d6e5
+Revises: 7e246705da6a
+Create Date: 2022-11-22 12:16:32.517184
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '939a08e1d6e5'
+down_revision = '7e246705da6a'
+
+from alembic import op
+from flask import current_app
+from flask_sqlalchemy import SQLAlchemy
+from pathlib import Path
+import sqlalchemy as sa
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.orm import Session
+
+import os
+import time
+
+""" For a file of a given size, determine the largest allowed lifespan of that file
+
+Based on the current app's configuration:  Specifically, the MAX_CONTENT_LENGTH, as well
+as FHOST_{MIN,MAX}_EXPIRATION.
+
+This lifespan may be shortened by a user's request, but no files should be allowed to
+expire at a point after this number.
+
+Value returned is a duration in milliseconds.
+"""
+def get_max_lifespan(filesize: int) -> int:
+    min_exp = current_app.config.get("FHOST_MIN_EXPIRATION", 30 * 24 * 60 * 60 * 1000)
+    max_exp = current_app.config.get("FHOST_MAX_EXPIRATION", 365 * 24 * 60 * 60 * 1000)
+    max_size = current_app.config.get("MAX_CONTENT_LENGTH", 256 * 1024 * 1024)
+    return min_exp + int((-max_exp + min_exp) * (filesize / max_size - 1) ** 3)
+
+Base = automap_base()
+
+def upgrade():
+    op.add_column('file', sa.Column('expiration', sa.BigInteger()))
+
+    bind = op.get_bind()
+    Base.prepare(autoload_with=bind)
+    File = Base.classes.file
+    session = Session(bind=bind)
+
+    storage = Path(current_app.config["FHOST_STORAGE_PATH"])
+    current_time = time.time() * 1000;
+
+    # List of file hashes which have not expired yet
+    # This could get really big for some servers
+    try:
+        unexpired_files = os.listdir(storage)
+    except FileNotFoundError:
+        return # There are no currently unexpired files
+
+    # Calculate an expiration date for all existing files
+
+    q = session.scalars(
+            sa.select(File)
+            .where(
+                sa.not_(File.removed)
+            )
+        )
+    updates = [] # We coalesce updates to the database here
+
+    # SQLite has a hard limit on the number of variables so we
+    # need to do this the slow way
+    files = [f for f in q if f.sha256 in unexpired_files]
+
+    for file in files:
+        file_path = storage / file.sha256
+        stat = os.stat(file_path)
+        max_age = get_max_lifespan(stat.st_size) # How long the file is allowed to live, in ms
+        file_birth = stat.st_mtime * 1000 # When the file was created, in ms
+        updates.append({'id': file.id, 'expiration': int(file_birth + max_age)})
+
+    # Apply coalesced updates
+    session.bulk_update_mappings(File, updates)
+    session.commit()
+
+def downgrade():
+    op.drop_column('file', 'expiration')

migrations/versions/dd0766afb7d2_store_user_agent_string_with_files.py

@@ -0,0 +1,30 @@
+"""Store user agent string with files
+
+Revision ID: dd0766afb7d2
+Revises: 30bfe33aa328
+Create Date: 2023-03-29 07:18:49.113200
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'dd0766afb7d2'
+down_revision = '30bfe33aa328'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('ua', sa.UnicodeText(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('file', schema=None) as batch_op:
+        batch_op.drop_column('ua')
+
+    # ### end Alembic commands ###

migrations/versions/e2e816056589_.py

@@ -0,0 +1,26 @@
+"""add URL secret
+
+Revision ID: e2e816056589
+Revises: 0659d7b9eea8
+Create Date: 2022-12-01 02:16:15.976864
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e2e816056589'
+down_revision = '0659d7b9eea8'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('file', sa.Column('secret', sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('file', 'secret')
+    # ### end Alembic commands ###

mod.css

@@ -0,0 +1,46 @@
+#ftable {
+    width: 1fr;
+    height: 100%;
+}
+
+#infopane {
+    width: 50%;
+    outline-top: hkey $primary;
+    background: $panel;
+}
+
+#finfo {
+    background: $boost;
+    height: 14;
+    width: 1fr;
+    box-sizing: content-box;
+}
+
+#mpv {
+    display: none;
+    height: 20%;
+    width: 1fr;
+    content-align: center middle;
+}
+
+#ftextlog {
+    height: 1fr;
+    width: 1fr;
+}
+
+#filter_input {
+    width: 1fr;
+    display: none;
+}
+
+Notification {
+    dock: bottom;
+    layer: notification;
+    width: auto;
+    margin: 2 4;
+    padding: 1 2;
+    background: $background;
+    color: $text;
+    height: auto;
+
+}

mod.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+
+from itertools import zip_longest
+from sys import stdout
+import time
+
+from textual.app import App, ComposeResult
+from textual.widgets import DataTable, Header, Footer, RichLog, Static, Input
+from textual.containers import Horizontal, Vertical
+from textual.screen import Screen
+from textual import log
+from rich.text import Text
+from jinja2.filters import do_filesizeformat
+
+from fhost import db, File, su, app as fhost_app, in_upload_bl
+from modui import *
+
+fhost_app.app_context().push()
+
+class NullptrMod(Screen):
+    BINDINGS = [
+        ("q", "quit_app", "Quit"),
+        ("f1", "filter(1, 'Name')", "Lookup name"),
+        ("f2", "filter(2, 'IP address')", "Filter IP"),
+        ("f3", "filter(3, 'MIME Type')", "Filter MIME"),
+        ("f4", "filter(4, 'Extension')", "Filter Ext."),
+        ("f5", "refresh", "Refresh"),
+        ("f6", "filter_clear", "Clear filter"),
+        ("f7", "filter(5, 'User agent')", "Filter UA"),
+        ("r", "remove_file(False)", "Remove file"),
+        ("ctrl+r", "remove_file(True)", "Ban file"),
+        ("p", "ban_ip(False)", "Ban IP"),
+        ("ctrl+p", "ban_ip(True)", "Nuke IP"),
+    ]
+
+    async def action_quit_app(self):
+        self.mpvw.shutdown()
+        await self.app.action_quit()
+
+    def action_refresh(self):
+        ftable = self.query_one("#ftable")
+        ftable.watch_query(None, None)
+
+    def action_filter_clear(self):
+        self.finput.display = False
+        ftable = self.query_one("#ftable")
+        ftable.focus()
+        ftable.query = ftable.base_query
+
+    def action_filter(self, fcol: int, label: str):
+        self.finput.placeholder = label
+        self.finput.display = True
+        self.finput.focus()
+        self.filter_col = fcol
+        self._refresh_layout()
+
+        if self.current_file:
+            match fcol:
+                case 1: self.finput.value = ""
+                case 2: self.finput.value = self.current_file.addr
+                case 3: self.finput.value = self.current_file.mime
+                case 4: self.finput.value = self.current_file.ext
+                case 5: self.finput.value = self.current_file.ua or ""
+
+    def on_input_submitted(self, message: Input.Submitted) -> None:
+        self.finput.display = False
+        ftable = self.query_one("#ftable")
+        ftable.focus()
+
+        if len(message.value):
+            match self.filter_col:
+                case 1:
+                    try: ftable.query = ftable.base_query.filter(File.id == su.debase(message.value))
+                    except ValueError: pass
+                case 2: ftable.query = ftable.base_query.filter(File.addr.like(message.value))
+                case 3: ftable.query = ftable.base_query.filter(File.mime.like(message.value))
+                case 4: ftable.query = ftable.base_query.filter(File.ext.like(message.value))
+                case 5: ftable.query = ftable.base_query.filter(File.ua.like(message.value))
+        else:
+            ftable.query = ftable.base_query
+
+    def action_remove_file(self, permanent: bool) -> None:
+        if self.current_file:
+            self.current_file.delete(permanent)
+            db.session.commit()
+            self.mount(Notification(f"{'Banned' if permanent else 'Removed'} file {self.current_file.getname()}"))
+            self.action_refresh()
+
+    def action_ban_ip(self, nuke: bool) -> None:
+        if self.current_file:
+            if not fhost_app.config["FHOST_UPLOAD_BLACKLIST"]:
+                self.mount(Notification("Failed: FHOST_UPLOAD_BLACKLIST not set!"))
+                return
+            else:
+                if in_upload_bl(self.current_file.addr):
+                    txt = f"{self.current_file.addr} is already banned"
+                else:
+                    with fhost_app.open_instance_resource(fhost_app.config["FHOST_UPLOAD_BLACKLIST"], "a") as bl:
+                        print(self.current_file.addr.lstrip("::ffff:"), file=bl)
+                    txt = f"Banned {self.current_file.addr}"
+
+                if nuke:
+                    tsize = 0
+                    trm = 0
+                    for f in File.query.filter(File.addr == self.current_file.addr):
+                        if f.getpath().is_file():
+                            tsize += f.size or f.getpath().stat().st_size
+                            trm += 1
+                        f.delete(True)
+                    db.session.commit()
+                    txt += f", removed {trm} {'files' if trm != 1 else 'file'} totaling {do_filesizeformat(tsize, True)}"
+            self.mount(Notification(txt))
+            self._refresh_layout()
+            ftable = self.query_one("#ftable")
+            ftable.watch_query(None, None)
+
+    def on_update(self) -> None:
+        stdout.write("\033[?25l")
+        stdout.flush()
+
+    def compose(self) -> ComposeResult:
+        yield Header()
+        yield Horizontal(
+            FileTable(id="ftable", zebra_stripes=True, cursor_type="row"),
+            Vertical(
+                DataTable(id="finfo", show_header=False, cursor_type="none"),
+                MpvWidget(id="mpv"),
+                RichLog(id="ftextlog", auto_scroll=False),
+            id="infopane"))
+        yield Input(id="filter_input")
+        yield Footer()
+
+    def on_mount(self) -> None:
+        self.current_file = None
+
+        self.ftable = self.query_one("#ftable")
+        self.ftable.focus()
+
+        self.finfo = self.query_one("#finfo")
+        self.finfo.add_columns("key", "value")
+
+        self.mpvw = self.query_one("#mpv")
+        self.ftlog = self.query_one("#ftextlog")
+
+        self.finput = self.query_one("#filter_input")
+
+        self.mimehandler = mime.MIMEHandler()
+        self.mimehandler.register(mime.MIMECategory.Archive, self.handle_libarchive)
+        self.mimehandler.register(mime.MIMECategory.Text, self.handle_text)
+        self.mimehandler.register(mime.MIMECategory.AV, self.handle_mpv)
+        self.mimehandler.register(mime.MIMECategory.Document, self.handle_mupdf)
+        self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_libarchive)
+        self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_mpv)
+        self.mimehandler.register(mime.MIMECategory.Fallback, self.handle_raw)
+
+    def handle_libarchive(self, cat):
+        import libarchive
+        with libarchive.file_reader(str(self.current_file.getpath())) as a:
+            self.ftlog.write("\n".join(e.path for e in a))
+        return True
+
+    def handle_text(self, cat):
+        with open(self.current_file.getpath(), "r") as sf:
+            data = sf.read(1000000).replace("\033","")
+            self.ftlog.write(data)
+        return True
+
+    def handle_mupdf(self, cat):
+        import fitz
+        with fitz.open(self.current_file.getpath(),
+                       filetype=self.current_file.ext.lstrip(".")) as doc:
+            p = doc.load_page(0)
+            pix = p.get_pixmap(dpi=72)
+            imgdata = pix.tobytes("ppm").hex()
+
+            self.mpvw.styles.height = "40%"
+            self.mpvw.start_mpv("hex://" + imgdata, 0)
+
+            self.ftlog.write(Text.from_markup(f"[bold]Pages:[/bold] {doc.page_count}"))
+            self.ftlog.write(Text.from_markup("[bold]Metadata:[/bold]"))
+            for k, v in doc.metadata.items():
+                self.ftlog.write(Text.from_markup(f"  [bold]{k}:[/bold] {v}"))
+            toc = doc.get_toc()
+            if len(toc):
+                self.ftlog.write(Text.from_markup("[bold]TOC:[/bold]"))
+                for lvl, title, page in toc:
+                    self.ftlog.write(f"{'  ' * lvl} {page}: {title}")
+        return True
+
+    def handle_mpv(self, cat):
+        if cat == mime.MIMECategory.AV or self.current_file.nsfw_score >= 0:
+            self.mpvw.styles.height = "20%"
+            self.mpvw.start_mpv(str(self.current_file.getpath()), 0)
+
+            import av
+            with av.open(str(self.current_file.getpath())) as c:
+                self.ftlog.write(Text("Format:", style="bold"))
+                self.ftlog.write(f"  {c.format.long_name}")
+                if len(c.metadata):
+                    self.ftlog.write(Text("Metadata:", style="bold"))
+                    for k, v in c.metadata.items():
+                        self.ftlog.write(f"  {k}: {v}")
+                    for s in c.streams:
+                        self.ftlog.write(Text(f"Stream {s.index}:", style="bold"))
+                        self.ftlog.write(f"  Type: {s.type}")
+                        if s.base_rate:
+                            self.ftlog.write(f"  Frame rate: {s.base_rate}")
+                        if len(s.metadata):
+                            self.ftlog.write(Text("  Metadata:", style="bold"))
+                            for k, v in s.metadata.items():
+                                self.ftlog.write(f"    {k}: {v}")
+            return True
+        return False
+
+    def handle_raw(self, cat):
+        def hexdump(binf, length):
+            def fmt(s):
+                if isinstance(s, str):
+                    c = chr(int(s, 16))
+                else:
+                    c = chr(s)
+                    s = c
+                if c.isalpha(): return f"\0[chartreuse1]{s}\0[/chartreuse1]"
+                if c.isdigit(): return f"\0[gold1]{s}\0[/gold1]"
+                if not c.isprintable():
+                    g = "grey50" if c == "\0" else "cadet_blue"
+                    return f"\0[{g}]{s if len(s) == 2 else '.'}\0[/{g}]"
+                return s
+            return Text.from_markup("\n".join(f"{' '.join(map(fmt, map(''.join, zip(*[iter(c.hex())] * 2))))}"
+                                    f"{'   ' * (16 - len(c))}"
+                                    f" {''.join(map(fmt, c))}"
+                                    for c in map(lambda x: bytes([n for n in x if n != None]),
+                                                zip_longest(*[iter(binf.read(min(length, 16 * 10)))] * 16))))
+
+        with open(self.current_file.getpath(), "rb") as binf:
+            self.ftlog.write(hexdump(binf, self.current_file.size))
+            if self.current_file.size > 16*10*2:
+                binf.seek(self.current_file.size-16*10)
+                self.ftlog.write("  [...]  ".center(64, '─'))
+            self.ftlog.write(hexdump(binf, self.current_file.size - binf.tell()))
+
+        return True
+
+    def on_file_table_selected(self, message: FileTable.Selected) -> None:
+        f = message.file
+        self.current_file = f
+        self.finfo.clear()
+        self.finfo.add_rows([
+            ("ID:", str(f.id)),
+            ("File name:", f.getname()),
+            ("URL:", f.geturl() if fhost_app.config["SERVER_NAME"] else "⚠ Set SERVER_NAME in config.py to display"),
+            ("File size:", do_filesizeformat(f.size, True)),
+            ("MIME type:", f.mime),
+            ("SHA256 checksum:", f.sha256),
+            ("Uploaded by:", Text(f.addr)),
+            ("User agent:", Text(f.ua or "")),
+            ("Management token:", f.mgmt_token),
+            ("Secret:", f.secret),
+            ("Is NSFW:", ("Yes" if f.is_nsfw else "No") + (f" (Score: {f.nsfw_score:0.4f})" if f.nsfw_score else " (Not scanned)")),
+            ("Is banned:", "Yes" if f.removed else "No"),
+            ("Expires:", time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(File.get_expiration(f.expiration, f.size)/1000)))
+        ])
+
+        self.mpvw.stop_mpv(True)
+        self.ftlog.clear()
+
+        if f.getpath().is_file():
+            self.mimehandler.handle(f.mime, f.ext)
+            self.ftlog.scroll_to(x=0, y=0, animate=False)
+
+class NullptrModApp(App):
+    CSS_PATH = "mod.css"
+
+    def on_mount(self) -> None:
+        self.title = "0x0 File Moderation Interface"
+        self.main_screen = NullptrMod()
+        self.install_screen(self.main_screen, name="main")
+        self.push_screen("main")
+
+if __name__ == "__main__":
+    app = NullptrModApp()
+    app.run()

modui/__init__.py

@@ -0,0 +1,3 @@
+from .filetable import FileTable
+from .notification import Notification
+from .mpvwidget import MpvWidget

modui/filetable.py

@@ -0,0 +1,86 @@
+from textual.widgets import DataTable, Static
+from textual.reactive import Reactive
+from textual.message import Message
+from textual import events, log
+from jinja2.filters import do_filesizeformat
+
+from fhost import File
+from modui import mime
+
+class FileTable(DataTable):
+    query = Reactive(None)
+    order_col = Reactive(0)
+    order_desc = Reactive(True)
+    limit = 10000
+    colmap = [File.id, File.removed, File.nsfw_score, None, File.ext, File.size, File.mime]
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_columns("#", "☣️", "🔞", "📂", "name", "size", "mime")
+        self.base_query = File.query.filter(File.size != None)
+        self.query = self.base_query
+
+    class Selected(Message):
+        def __init__(self, f: File) -> None:
+            self.file = f
+            super().__init__()
+
+    def watch_order_col(self, old, value) -> None:
+        self.watch_query(None, None)
+
+    def watch_order_desc(self, old, value) -> None:
+        self.watch_query(None, None)
+
+    def watch_query(self, old, value) -> None:
+        def fmt_file(f: File) -> tuple:
+            return (
+                str(f.id),
+                "🔴" if f.removed else "  ",
+                "🚩" if f.is_nsfw else "  ",
+                "👻" if not f.getpath().is_file() else "  ",
+                f.getname(),
+                do_filesizeformat(f.size, True),
+                f"{mime.mimemoji.get(f.mime.split('/')[0], mime.mimemoji.get(f.mime)) or '  '} " + f.mime,
+            )
+
+        if (self.query):
+
+            order = FileTable.colmap[self.order_col]
+            q = self.query
+            if order: q = q.order_by(order.desc() if self.order_desc else order, File.id)
+            qres = list(map(fmt_file, q.limit(self.limit)))
+
+            ri = 0
+            row = self.cursor_coordinate.row
+            if row < self.row_count and row >= 0:
+                ri = int(self.get_row_at(row)[0])
+
+            self.clear()
+            self.add_rows(qres)
+
+            for i, v in enumerate(qres):
+                if int(v[0]) == ri:
+                    self.move_cursor(row=i)
+                    break
+
+            self.on_selected()
+
+    def on_selected(self) -> Selected:
+        row = self.cursor_coordinate.row
+        if row < self.row_count and row >= 0:
+            f = File.query.get(int(self.get_row_at(row)[0]))
+            self.post_message(self.Selected(f))
+
+    def watch_cursor_coordinate(self, old, value) -> None:
+        super().watch_cursor_coordinate(old, value)
+        if old != value:
+            self.on_selected()
+
+    def on_click(self, event: events.Click) -> None:
+        meta = self.get_style_at(event.x, event.y).meta
+        if meta:
+            if meta["row"] == -1:
+                qi = FileTable.colmap[meta["column"]]
+                if meta["column"] == self.order_col:
+                    self.order_desc = not self.order_desc
+                self.order_col = meta["column"]

modui/mime.py

@@ -0,0 +1,126 @@
+from enum import Enum
+from textual import log
+
+mimemoji = {
+    "audio" : "🔈",
+    "video" : "🎞",
+    "text"  : "📄",
+    "image" : "🖼",
+    "application/zip" : "🗜️",
+    "application/x-zip-compressed" : "🗜️",
+    "application/x-tar" : "🗄",
+    "application/x-cpio" : "🗄",
+    "application/x-xz" : "🗜️",
+    "application/x-7z-compressed" : "🗜️",
+    "application/gzip" : "🗜️",
+    "application/zstd" : "🗜️",
+    "application/x-rar" : "🗜️",
+    "application/x-rar-compressed" : "🗜️",
+    "application/vnd.ms-cab-compressed" : "🗜️",
+    "application/x-bzip2" : "🗜️",
+    "application/x-lzip" : "🗜️",
+    "application/x-iso9660-image" : "💿",
+    "application/pdf" : "📕",
+    "application/epub+zip" : "📕",
+    "application/mxf" : "🎞",
+    "application/vnd.android.package-archive" : "📦",
+    "application/vnd.debian.binary-package" : "📦",
+    "application/x-rpm" : "📦",
+    "application/x-dosexec" : "⚙",
+    "application/x-execuftable" : "⚙",
+    "application/x-sharedlib" : "⚙",
+    "application/java-archive" : "☕",
+    "application/x-qemu-disk" : "🖴",
+    "application/pgp-encrypted" : "🔏",
+}
+
+MIMECategory = Enum("MIMECategory",
+    ["Archive", "Text", "AV", "Document", "Fallback"]
+)
+
+class MIMEHandler:
+    def __init__(self):
+        self.handlers = {
+            MIMECategory.Archive : [[
+                "application/zip",
+                "application/x-zip-compressed",
+                "application/x-tar",
+                "application/x-cpio",
+                "application/x-xz",
+                "application/x-7z-compressed",
+                "application/gzip",
+                "application/zstd",
+                "application/x-rar",
+                "application/x-rar-compressed",
+                "application/vnd.ms-cab-compressed",
+                "application/x-bzip2",
+                "application/x-lzip",
+                "application/x-iso9660-image",
+                "application/vnd.android.package-archive",
+                "application/vnd.debian.binary-package",
+                "application/x-rpm",
+                "application/java-archive",
+                "application/vnd.openxmlformats"
+            ], []],
+            MIMECategory.Text : [[
+                "text",
+                "application/json",
+                "application/xml",
+            ], []],
+            MIMECategory.AV : [[
+                "audio", "video", "image",
+                "application/mxf"
+            ], []],
+            MIMECategory.Document : [[
+                "application/pdf",
+                "application/epub",
+                "application/x-mobipocket-ebook",
+            ], []],
+            MIMECategory.Fallback : [[], []]
+        }
+
+        self.exceptions = {
+            MIMECategory.Archive : {
+                ".cbz" : MIMECategory.Document,
+                ".xps" : MIMECategory.Document,
+                ".epub" : MIMECategory.Document,
+            },
+            MIMECategory.Text : {
+                ".fb2" : MIMECategory.Document,
+            }
+        }
+
+    def register(self, category, handler):
+        self.handlers[category][1].append(handler)
+
+    def handle(self, mime, ext):
+        def getcat(s):
+            cat = MIMECategory.Fallback
+            for k, v in self.handlers.items():
+                s = s.split(";")[0]
+                if s in v[0] or s.split("/")[0] in v[0]:
+                    cat = k
+                    break
+
+                for x in v[0]:
+                    if s.startswith(x):
+                        cat = k
+                        break
+
+            if cat in self.exceptions:
+                cat = self.exceptions[cat].get(ext) or cat
+
+            return cat
+
+        cat = getcat(mime)
+        for handler in self.handlers[cat][1]:
+            try:
+                if handler(cat): return
+            except: pass
+
+        for handler in self.handlers[MIMECategory.Fallback][1]:
+            try:
+                if handler(None): return
+            except: pass
+
+        raise RuntimeError(f"Unhandled MIME type category: {cat}")

modui/mpvwidget.py

@@ -0,0 +1,88 @@
+import time
+import fcntl, struct, termios
+from sys import stdout
+
+from textual import events, log
+from textual.widgets import Static
+
+from fhost import app as fhost_app
+
+class MpvWidget(Static):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.mpv = None
+        self.vo = fhost_app.config.get("MOD_PREVIEW_PROTO")
+
+        if not self.vo in ["sixel", "kitty"]:
+            self.update("⚠ Previews not enabled. \n\nSet MOD_PREVIEW_PROTO to 'sixel' or 'kitty' in config.py,\nwhichever is supported by your terminal.")
+        else:
+            try:
+                import mpv
+                self.mpv = mpv.MPV()
+                self.mpv.profile = "sw-fast"
+                self.mpv["vo"] = self.vo
+                self.mpv[f"vo-{self.vo}-config-clear"] = False
+                self.mpv[f"vo-{self.vo}-alt-screen"] = False
+                self.mpv[f"vo-sixel-buffered"] = True
+                self.mpv["audio"] = False
+                self.mpv["loop-file"] = "inf"
+                self.mpv["image-display-duration"] = 0.5 if self.vo == "sixel" else "inf"
+            except Exception as e:
+                self.mpv = None
+                self.update(f"⚠ Previews require python-mpv with libmpv 0.36.0 or later \n\nError was:\n{type(e).__name__}: {e}")
+
+    def start_mpv(self, f: str|None = None, pos: float|str|None = None) -> None:
+        self.display = True
+        self.screen._refresh_layout()
+
+        if self.mpv:
+            if self.content_region.x:
+                r, c, w, h = struct.unpack('hhhh', fcntl.ioctl(0, termios.TIOCGWINSZ, '12345678'))
+                width = int((w / c) * self.content_region.width)
+                height = int((h / r) * (self.content_region.height + (1 if self.vo == "sixel" else 0)))
+                self.mpv[f"vo-{self.vo}-left"] = self.content_region.x + 1
+                self.mpv[f"vo-{self.vo}-top"] = self.content_region.y + 1
+                self.mpv[f"vo-{self.vo}-rows"] = self.content_region.height + (1 if self.vo == "sixel" else 0)
+                self.mpv[f"vo-{self.vo}-cols"] = self.content_region.width
+                self.mpv[f"vo-{self.vo}-width"] = width
+                self.mpv[f"vo-{self.vo}-height"] = height
+
+                if pos != None:
+                    self.mpv["start"] = pos
+
+                if f:
+                    self.mpv.loadfile(f)
+                else:
+                    self.mpv.playlist_play_index(0)
+
+    def stop_mpv(self, wait: bool = False) -> None:
+        if self.mpv:
+            if not self.mpv.idle_active:
+                self.mpv.stop(True)
+                if wait:
+                    time.sleep(0.1)
+        self.clear_mpv()
+        self.display = False
+
+    def on_resize(self, size) -> None:
+        if self.mpv:
+            if not self.mpv.idle_active:
+                t = self.mpv.time_pos
+                self.stop_mpv()
+                if t:
+                    self.mpv["start"] = t
+                self.start_mpv()
+
+    def clear_mpv(self) -> None:
+        if self.vo == "kitty":
+            stdout.write("\033_Ga=d;\033\\")
+            stdout.flush()
+
+    def shutdown(self) -> None:
+        if self.mpv:
+            self.mpv.stop()
+            del self.mpv
+            if self.vo == "kitty":
+                stdout.write("\033_Ga=d;\033\\\033[?25l")
+                stdout.flush()

modui/notification.py

@@ -0,0 +1,8 @@
+from textual.widgets import Static
+
+class Notification(Static):
+    def on_mount(self) -> None:
+        self.set_timer(3, self.remove)
+
+    def on_click(self) -> None:
+        self.remove()

nsfw_detect.py

@@ -22,21 +22,21 @@ import numpy as np
 import os
 import sys
 from io import BytesIO
-from subprocess import run, PIPE, DEVNULL
-
-import caffe
+from pathlib import Path
 
 os.environ["GLOG_minloglevel"] = "2"  # seriously :|
-
+import caffe
+import av
+av.logging.set_level(av.logging.PANIC)
 
 class NSFWDetector:
     def __init__(self):
-
-        npath = os.path.join(os.path.dirname(__file__), "nsfw_model")
+        npath = Path(__file__).parent / "nsfw_model"
         self.nsfw_net = caffe.Net(
-            os.path.join(npath, "deploy.prototxt"),
-            os.path.join(npath, "resnet_50_1by2_nsfw.caffemodel"),
-            caffe.TEST)
+            str(npath / "deploy.prototxt"),
+            caffe.TEST,
+            weights = str(npath / "resnet_50_1by2_nsfw.caffemodel")
+        )
         self.caffe_transformer = caffe.io.Transformer({
             'data': self.nsfw_net.blobs['data'].data.shape
         })
@@ -50,7 +50,7 @@ class NSFWDetector:
         self.caffe_transformer.set_channel_swap('data', (2, 1, 0))
 
     def _compute(self, img):
-        image = caffe.io.load_image(BytesIO(img))
+        image = caffe.io.load_image(img)
 
         H, W, _ = image.shape
         _, _, h, w = self.nsfw_net.blobs["data"].data.shape
@@ -72,16 +72,26 @@ class NSFWDetector:
 
     def detect(self, fpath):
         try:
-            ff = run([
-                "ffmpegthumbnailer", "-m", "-o-", "-s256", "-t50%", "-a",
-                "-cpng", "-i", fpath
-            ], stdout=PIPE, stderr=DEVNULL, check=True)
-            image_data = ff.stdout
+            with av.open(fpath) as container:
+                try: container.seek(int(container.duration / 2))
+                except: container.seek(0)
+
+                frame = next(container.decode(video=0))
+
+                if frame.width >= frame.height:
+                    w = 256
+                    h = int(frame.height * (256 / frame.width))
+                else:
+                    w = int(frame.width * (256 / frame.height))
+                    h = 256
+                frame = frame.reformat(width=w, height=h, format="rgb24")
+                img = BytesIO()
+                frame.to_image().save(img, format="ppm")
+
+            scores = self._compute(img)
         except:
             return -1.0
 
-        scores = self._compute(image_data)
-
         return scores[1]
 
 

requirements.txt

@@ -1,10 +1,25 @@
+click
+Flask_Migrate
+validators
 alembic
+requests
 Jinja2
 Flask
-numpy
-SQLAlchemy
-requests
-Flask_SQLAlchemy
-validators
-flask_migrate
+flask_sqlalchemy
 python_magic
+
+# vscan
+clamd
+
+# nsfw detection
+numpy
+
+# mod ui
+av
+PyMuPDF
+libarchive_c
+textual
+python-mpv
+
+# dev
+pytest

templates/401.html

@@ -0,0 +1,2 @@
+rm: cannot remove '{{ request.path.split("/")[1] }}': Permission denied
+

templates/index.html

@@ -6,11 +6,35 @@ HTTP POST files here:
     curl -F'file=@yourfile.png' {{ fhost_url }}
 You can also POST remote URLs:
     curl -F'url=http://example.com/image.jpg' {{ fhost_url }}
+If you don't want the resulting URL to be easy to guess:
+    curl -F'file=@yourfile.png' -Fsecret= {{ fhost_url }}
+    curl -F'url=http://example.com/image.jpg' -Fsecret= {{ fhost_url }}
 Or you can shorten URLs:
     curl -F'shorten=http://example.com/some/long/url' {{ fhost_url }}
 
+It is possible to append your own file name to the URL:
+    {{ fhost_url }}/aaa.jpg/image.jpeg
+
 File URLs are valid for at least 30 days and up to a year (see below).
 Shortened URLs do not expire.
+
+Files can be set to expire sooner by adding an "expires" parameter (in hours)
+    curl -F'file=@yourfile.png' -Fexpires=24 {{ fhost_url }}
+OR by setting "expires" to a timestamp in epoch milliseconds
+    curl -F'file=@yourfile.png' -Fexpires=1681996320000 {{ fhost_url }}
+
+Expired files won't be removed immediately, but will be removed as part of
+the next purge.
+
+Whenever a file that does not already exist or has expired is uploaded,
+the HTTP response header includes an X-Token field. You can use this
+to perform management operations on the file.
+
+To delete the file immediately:
+    curl -Ftoken=token_here -Fdelete= {{ fhost_url }}/abc.txt
+To change the expiration date (see above):
+    curl -Ftoken=token_here -Fexpires=3 {{ fhost_url }}/abc.txt
+
 {% set max_size = config["MAX_CONTENT_LENGTH"]|filesizeformat(True) %}
 Maximum file size: {{ max_size }}
 Not allowed: {{ config["FHOST_MIME_BLACKLIST"]|join(", ") }}
@@ -22,24 +46,24 @@ FILE RETENTION PERIOD
 retention = min_age + (-max_age + min_age) * pow((file_size / max_size - 1), 3)
 
    days
-    365 |  \\
-        |   \\
-        |    \\
-        |     \\
-        |      \\
-        |       \\
+ {{'{: 6}'.format(config.get("FHOST_MAX_EXPIRATION", 31536000000)//86400000)}} |  \
+        |   \
+        |    \
+        |     \
+        |      \
+        |       \
         |        ..
-        |          \\
-  197.5 | ----------..-------------------------------------------
+        |          \
+ {{'{: 6.1f}'.format((config.get("FHOST_MIN_EXPIRATION", 2592000000)/2 + config.get("FHOST_MAX_EXPIRATION", 31536000000)/2)/86400000)}} | ----------..-------------------------------------------
         |             ..
-        |               \\
+        |               \
         |                ..
         |                  ...
         |                     ..
         |                       ...
         |                          ....
         |                              ......
-     30 |                                    ....................
+ {{'{: 6}'.format(config.get("FHOST_MIN_EXPIRATION", 2592000000)//86400000)}} |                                    ....................
           0{{ ((config["MAX_CONTENT_LENGTH"]/2)|filesizeformat(True)).split(" ")[0].rjust(27) }}{{ max_size.split(" ")[0].rjust(27) }}
            {{ max_size.split(" ")[1].rjust(54) }}
 </pre>

tests/test_client.py

@@ -56,8 +56,6 @@ def test_client(client):
         ]),
         (302, [
             "E",
-            "E/test",
-            "E/test.bin",
         ]),
         (404, [
             "test.bin",
@@ -67,6 +65,8 @@ def test_client(client):
             "test/test",
             "test.bin/test.py",
             "E.bin",
+            "E/test",
+            "E/test.bin",
         ]),
         (451, [
             "Q.truncate",