kaledonya clips: a Discord-friendly clip site on top of Zipline

Co-Authored-By: Claude Opus 5.5 <[email protected]>

intake/app/errors.py

@@ -0,0 +1,15 @@
+"""JSON error responses shared by /upload and the site API."""
+import httpx
+from fastapi.responses import JSONResponse
+
+
+def error(status: int, message: str) -> JSONResponse:
+    return JSONResponse({"error": message}, status_code=status)
+
+
+def zipline_message(r: httpx.Response) -> str:
+    try:
+        body = r.json()
+        return str(body.get("error") or body.get("message") or body)
+    except ValueError:
+        return r.text[:200] or f"Zipline returned {r.status_code}"

intake/app/main.py

@@ -0,0 +1,3 @@
+from .server import create_app
+
+app = create_app()

intake/app/media.py

@@ -0,0 +1,38 @@
+"""Size and a still frame of a clip, read straight from Zipline with ffprobe/ffmpeg, for Discord's link preview."""
+import json
+import subprocess
+import threading
+from functools import lru_cache
+
+# /v/ and /_site/thumb need no login: cap how many ffmpeg/ffprobe processes they can start at once
+SLOTS = threading.BoundedSemaphore(2)
+
+
+@lru_cache(maxsize=256)
+def video_size(url: str, length: str) -> tuple[int, int] | None:
+    """(width, height) of the first video stream; `length` is only part of the cache key (a reused name)."""
+    cmd = ["ffprobe", "-v", "error", "-select_streams", "v:0",
+           "-show_entries", "stream=width,height", "-of", "json", url]
+    try:
+        with SLOTS:
+            result = subprocess.run(cmd, capture_output=True, timeout=10)
+        stream = json.loads(result.stdout)["streams"][0]
+        return int(stream["width"]), int(stream["height"])
+    except (OSError, subprocess.TimeoutExpired, ValueError, KeyError, IndexError):
+        return None
+
+
+@lru_cache(maxsize=64)
+def still_jpeg(url: str, length: str) -> bytes | None:
+    """One frame (1 s in, or the first one for very short clips) as a JPEG at most 720 px tall."""
+    for seek in ("1", "0"):
+        cmd = ["ffmpeg", "-v", "error", "-ss", seek, "-i", url, "-frames:v", "1",
+               "-vf", "scale=-2:'min(720,ih)'", "-f", "image2", "-c:v", "mjpeg", "pipe:1"]
+        try:
+            with SLOTS:
+                result = subprocess.run(cmd, capture_output=True, timeout=30)
+        except (OSError, subprocess.TimeoutExpired):
+            return None
+        if result.returncode == 0 and result.stdout:
+            return result.stdout
+    return None

intake/app/names.py

@@ -0,0 +1,25 @@
+"""Filename rules for uploads: spaces become dashes, duplicates get -2, -3, ..."""
+import os
+import re
+from collections.abc import Callable
+
+
+def clean_name(filename: str) -> str:
+    """Keep only the base name and turn every run of whitespace into one dash."""
+    base = filename.replace("\\", "/").rsplit("/", 1)[-1].strip()
+    base = re.sub(r"\s+", "-", base)
+    return base or "upload"
+
+
+def split_name(name: str) -> tuple[str, str]:
+    """('clip', '.mp4'); a name with no extension gives ('README', '')."""
+    return os.path.splitext(name)
+
+
+def next_free_name(stem: str, ext: str, exists: Callable[[str], bool], limit: int = 1000) -> str:
+    """First of stem+ext, stem-2+ext, stem-3+ext, ... that does not exist yet."""
+    for n in range(1, limit + 1):
+        name = f"{stem}{ext}" if n == 1 else f"{stem}-{n}{ext}"
+        if not exists(name):
+            return name
+    raise RuntimeError(f"no free name for {stem}{ext} after {limit} tries")

intake/app/previews.py

@@ -0,0 +1,216 @@
+"""Small Discord copies of big clips.
+
+Discord's crawler gives a linked video about 8 seconds to download and a home upload link of ~4 MB/s means a clip
+over ~30 MB never gets a player. For those, the /v/ page points Discord at a 720p copy that fits in ~22 MB
+(clips over 4 minutes: 480p in ~18 MB, so they download with room to spare);
+the original stays untouched. Copies are made one at a time in the background and live in WORK_DIR/previews.
+"""
+import hashlib
+import json
+import logging
+import queue
+import re
+import shutil
+import subprocess
+import threading
+import time
+from pathlib import Path
+from urllib.parse import quote
+
+log = logging.getLogger("uvicorn.error")  # uvicorn only prints its own loggers
+
+TARGET_BYTES = 22 * 1000 * 1000
+LONG_SECONDS = 240  # longer clips get a smaller, 480p copy
+LONG_TARGET_BYTES = 18 * 1000 * 1000
+AUDIO_KBPS = 128
+MAX_VIDEO_KBPS = 4000
+MIN_VIDEO_KBPS = 250  # below this a clip is too long to be worth watching in Discord; it gets no copy
+KEEP_DAYS = 95  # clips delete after 90 days
+KEY_RE = re.compile(r"^[0-9a-f]{24}$")
+
+
+def target_bytes(duration: float) -> int:
+    return LONG_TARGET_BYTES if duration > LONG_SECONDS else TARGET_BYTES
+
+
+def video_kbps(duration: float) -> int | None:
+    """Video bitrate that makes `duration` seconds land near target_bytes(), or None when it can't sensibly fit."""
+    if duration <= 0:
+        return None
+    kbps = min(MAX_VIDEO_KBPS, int(target_bytes(duration) * 8 / 1000 / duration) - AUDIO_KBPS)
+    return kbps if kbps >= MIN_VIDEO_KBPS else None
+
+
+def encode_cmd(src: str, dst: Path, kbps: int, duration: float) -> list[str]:
+    height = 480 if duration > LONG_SECONDS else 720
+    return [
+        "ffmpeg", "-v", "error", "-y", "-threads", "4", "-i", src,
+        "-map", "0:v:0", "-map", "0:a:0?",  # OBS clips carry a second audio track; Discord plays one
+        "-vf", f"scale=-2:'min({height},ih)'", "-fpsmax", "30",
+        "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
+        "-b:v", f"{kbps}k", "-maxrate", f"{kbps * 6 // 5}k", "-bufsize", f"{kbps * 2}k",
+        "-c:a", "aac", "-b:a", f"{AUDIO_KBPS}k", "-ac", "2",
+        "-movflags", "+faststart", "-progress", "pipe:1", "-nostats", str(dst),
+    ]
+
+
+def follow_progress(lines, duration: float, report) -> None:
+    """Turn ffmpeg's `-progress` lines into fractions done (0..1) for the site's upload dialog."""
+    for line in lines:
+        key, _, value = line.strip().partition("=")
+        if key == "out_time_us" and value.isdigit() and duration > 0:
+            report(min(1.0, round(int(value) / 1e6 / duration, 3)))
+
+
+def duration_of(src: str) -> float:
+    cmd = ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", src]
+    try:
+        out = subprocess.run(cmd, capture_output=True, timeout=30).stdout
+        return float(json.loads(out)["format"]["duration"])
+    except (OSError, subprocess.TimeoutExpired, ValueError, KeyError):
+        return 0.0
+
+
+def niced(cmd: list[str]) -> list[str]:
+    """Run at low CPU priority where `nice` exists, so uploads and everything else on the box come first."""
+    return ["nice", "-n", "10", *cmd] if shutil.which("nice") else cmd
+
+
+class Previews:
+    def __init__(self, directory: Path, zipline_url: str, over_bytes: int):
+        self.dir = directory
+        self.zipline_url = zipline_url.rstrip("/")
+        self.over_bytes = over_bytes
+        self.jobs: queue.Queue[tuple[str, int]] = queue.Queue()
+        self.pending: set[str] = set()
+        self.lock = threading.Lock()
+        self.worker: threading.Thread | None = None
+        self.current: str | None = None  # key being encoded right now, and how far along it is
+        self.progress = 0.0
+
+    @staticmethod
+    def key(name: str, length: int) -> str:
+        # the size is part of the key so a deleted clip's name reused for another video never gets the old copy
+        return hashlib.sha256(f"{name}\0{length}".encode()).hexdigest()[:24]
+
+    def wanted(self, length: int) -> bool:
+        return length > self.over_bytes
+
+    def file(self, key: str) -> Path | None:
+        path = self.dir / f"{key}.mp4"
+        return path if KEY_RE.match(key) and path.is_file() else None
+
+    def ready(self, name: str, length: int) -> str | None:
+        """The key of a finished copy, or None (asking for one in the background if it's missing)."""
+        key = self.key(name, length)
+        if self.file(key):
+            return key
+        self.request(name, length)
+        return None
+
+    def status(self, name: str, length: int) -> dict:
+        """For the site's upload dialog. state: "small" (plays as is), "ready" (copy made), "none" (too long, no
+        copy) or "making" (Discord would still get the big original), with how far the encode is and whether
+        it's still waiting behind another clip."""
+        done = lambda state: {"state": state, "progress": 1.0, "queued": False}
+        if not self.wanted(length):
+            return done("small")
+        key = self.key(name, length)
+        if self.file(key):
+            return done("ready")
+        if (self.dir / f"{key}.failed").exists():
+            return done("none")
+        self.request(name, length)
+        with self.lock:
+            now = self.current == key
+            return {"state": "making", "progress": self.progress if now else 0.0, "queued": not now}
+
+    def renamed(self, old: str, new: str, length: int) -> None:
+        """Carry a clip's copy over to its new name; without it a renamed clip's link sends Discord the big original."""
+        if not self.wanted(length):
+            return
+        old_key, new_key = self.key(old, length), self.key(new, length)
+        for ext in (".mp4", ".failed"):
+            try:
+                (self.dir / f"{old_key}{ext}").replace(self.dir / f"{new_key}{ext}")
+            except OSError:
+                pass
+        self.request(new, length)  # still being made under the old name: start the new one now, not on the first view
+
+    def request(self, name: str, length: int) -> None:
+        key = self.key(name, length)
+        if not self.wanted(length) or self.file(key) or (self.dir / f"{key}.failed").exists():
+            return
+        with self.lock:
+            if key in self.pending:
+                return
+            self.pending.add(key)
+            self.jobs.put((name, length))
+            if self.worker is None or not self.worker.is_alive():
+                self.worker = threading.Thread(target=self.run, name="previews", daemon=True)
+                self.worker.start()
+
+    def run(self) -> None:
+        while True:
+            try:
+                name, length = self.jobs.get(timeout=60)
+            except queue.Empty:
+                with self.lock:
+                    if self.jobs.empty():
+                        self.worker = None
+                        return
+                continue
+            key = self.key(name, length)
+            with self.lock:
+                self.current, self.progress = key, 0.0
+            try:
+                self.prune()
+                self.make(name, key)
+            except Exception:  # one bad clip must not kill the worker
+                log.exception("preview for %s failed", name)
+            finally:
+                with self.lock:
+                    self.pending.discard(key)
+                    self.current = None
+
+    def make(self, name: str, key: str) -> None:
+        self.dir.mkdir(parents=True, exist_ok=True)
+        src = f"{self.zipline_url}/{quote(name)}"
+        duration = duration_of(src)
+        kbps = video_kbps(duration)
+        tmp, dst = self.dir / f"{key}.part.mp4", self.dir / f"{key}.mp4"
+        ok = False
+        if kbps:
+            started = time.monotonic()
+            try:
+                proc = subprocess.Popen(niced(encode_cmd(src, tmp, kbps, duration)), stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL, text=True)
+                watchdog = threading.Timer(3 * 3600, proc.kill)
+                watchdog.start()
+                try:
+                    follow_progress(proc.stdout, duration, self.set_progress)
+                    code = proc.wait()
+                finally:
+                    watchdog.cancel()
+                ok = code == 0 and tmp.is_file() and tmp.stat().st_size < TARGET_BYTES * 13 // 10
+            except OSError:
+                ok = False
+            log.info("preview %s for %s: %s in %.0fs", key, name, "ok" if ok else "failed", time.monotonic() - started)
+        if ok:
+            tmp.replace(dst)
+        else:
+            tmp.unlink(missing_ok=True)
+            (self.dir / f"{key}.failed").touch()  # don't burn CPU on it again
+
+    def set_progress(self, fraction: float) -> None:
+        with self.lock:
+            self.progress = fraction
+
+    def prune(self) -> None:
+        cutoff = time.time() - KEEP_DAYS * 86400
+        for path in self.dir.glob("*"):
+            try:
+                if path.stat().st_mtime < cutoff:
+                    path.unlink()
+            except OSError:
+                pass

intake/app/remux.py

@@ -0,0 +1,19 @@
+"""Stream-copy an .mkv into an .mp4 so Discord can play it inline (no re-encode)."""
+import subprocess
+from pathlib import Path
+
+
+def remux_to_mp4(src: Path, dst: Path, timeout: int = 1800) -> bool:
+    cmd = [
+        "ffmpeg", "-v", "error", "-y", "-i", str(src),
+        "-map", "0:v", "-map", "0:a?", "-c", "copy",
+        "-movflags", "+faststart", str(dst),
+    ]
+    try:
+        result = subprocess.run(cmd, capture_output=True, timeout=timeout)
+        ok = result.returncode == 0 and dst.exists() and dst.stat().st_size > 0
+    except (OSError, subprocess.TimeoutExpired):
+        ok = False
+    if not ok:
+        dst.unlink(missing_ok=True)
+    return ok

intake/app/server.py

@@ -0,0 +1,178 @@
+"""intake: the upload door in front of Zipline (rename, remux .mkv, pick a free name, forward)."""
+import asyncio
+import http.cookiejar
+import mimetypes
+import os
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.datastructures import UploadFile
+
+from .errors import error, zipline_message
+from .names import clean_name, next_free_name, split_name
+from .previews import Previews
+from .remux import remux_to_mp4
+from .site_api import SITE_DIR, build_site_router
+from .site_auth import LoginLimiter, TicketStore
+
+FORM_SLACK = 1024 * 1024  # room for the multipart headers around the file
+UPLOAD_TRIES = 3
+
+
+@dataclass
+class Settings:
+    zipline_url: str = "http://zipline:3000"
+    max_bytes: int = 5 * 1024**3
+    deletes_at: str = "90d"
+    work_dir: Path = field(default_factory=lambda: Path(tempfile.gettempdir()))
+    site_origin: str = "http://localhost:3020"
+    upload_url: str = "http://localhost:3020"
+    site_name: str = "kaledonya clips"  # its first word is the big logo word on the login screen
+    server_name: str = "the server"  # shown as "Sending to <server_name>"
+    upload_hint: str = "Can't reach the upload server."  # shown when an upload can't connect at all
+    preview_over: int = 25 * 1000 * 1000  # bigger clips get a small copy for Discord (see previews.py)
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        return cls(
+            zipline_url=os.environ.get("ZIPLINE_URL", cls.zipline_url),
+            max_bytes=int(os.environ.get("MAX_BYTES", cls.max_bytes)),
+            deletes_at=os.environ.get("DELETES_AT", cls.deletes_at),
+            work_dir=Path(os.environ.get("WORK_DIR", tempfile.gettempdir())),
+            site_origin=os.environ.get("SITE_ORIGIN", cls.site_origin),
+            upload_url=os.environ.get("UPLOAD_URL", cls.upload_url),
+            site_name=os.environ.get("SITE_NAME", cls.site_name),
+            server_name=os.environ.get("SERVER_NAME", cls.server_name),
+            upload_hint=os.environ.get("UPLOAD_HINT", cls.upload_hint),
+            preview_over=int(os.environ.get("PREVIEW_OVER", cls.preview_over)),
+        )
+
+
+def create_app(settings: Settings | None = None, transport: httpx.BaseTransport | None = None) -> FastAPI:
+    settings = settings or Settings.from_env()
+    app = FastAPI(title="zipline-intake")
+    # the site (SITE_ORIGIN) uploads straight to this service at UPLOAD_URL, which may be another hostname
+    app.add_middleware(CORSMiddleware, allow_origins=[settings.site_origin],
+                       allow_methods=["POST"], allow_headers=["x-upload-ticket"])
+    client = httpx.Client(
+        base_url=settings.zipline_url,
+        transport=transport,
+        timeout=httpx.Timeout(60.0, read=900.0, write=None),
+    )
+    # never keep cookies: a Zipline login session must not ride along on anyone else's request
+    client.cookies.jar.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[]))
+    too_big = lambda: error(413, f"file is larger than the {settings.max_bytes / 1024**3:g} GB limit")
+
+    def exists(name: str) -> bool:
+        # 302: Zipline sends text and password-protected files to /view/<name>, but the name is taken
+        return client.head("/" + quote(name)).status_code in (200, 302)
+
+    tickets, limiter = TicketStore(), LoginLimiter()
+    previews = Previews(settings.work_dir / "previews", settings.zipline_url, settings.preview_over)
+    app.state.tickets, app.state.limiter, app.state.previews = tickets, limiter, previews
+    app.include_router(build_site_router(client, exists, tickets, limiter, settings, previews))
+    app.mount("/_site/static", StaticFiles(directory=SITE_DIR), name="site-static")
+
+    @app.middleware("http")
+    async def no_store_site_api(request: Request, call_next):
+        # Cloudflare caches anything ending in .mp4, even /_site/api/files/clip.mp4: never let it keep these
+        response = await call_next(request)
+        if request.url.path.startswith("/_site/api/"):
+            response.headers["cache-control"] = "no-store, private"
+        return response
+
+    def forward(body, stem: str, ext: str, token: str, content_type: str, remuxed: bool) -> JSONResponse:
+        taken: set[str] = set()
+        for _ in range(UPLOAD_TRIES):
+            try:
+                name = next_free_name(stem, ext, lambda n: n in taken or exists(n))
+            except httpx.HTTPError as e:
+                return error(502, f"could not reach Zipline: {e}")
+            except RuntimeError:
+                return error(409, "could not get a free filename, try again")
+            headers = {
+                "authorization": token,
+                "x-zipline-format": "name",
+                "x-zipline-filename": quote(name[: len(name) - len(ext)] if ext else name),
+                "x-zipline-original-name": "true",
+                "x-zipline-deletes-at": settings.deletes_at,
+            }
+            if ext:
+                headers["x-zipline-file-extension"] = ext
+            body.seek(0)
+            try:
+                r = client.post("/api/upload", headers=headers, files={"file": (name, body, content_type)})
+            except httpx.HTTPError as e:
+                return error(502, f"could not reach Zipline: {e}")
+            if r.status_code == 200:
+                # build the link ourselves: a '#' or '?' left raw in Zipline's url would cut it off
+                url = settings.site_origin + "/" + quote(name)
+                if content_type.startswith("video/"):
+                    previews.request(name, body.seek(0, os.SEEK_END))
+                return JSONResponse({"url": url, "name": name, "remuxed": remuxed})
+            if "already exists" in r.text:
+                taken.add(name)
+                continue
+            return error(r.status_code if r.status_code < 500 else 502, zipline_message(r))
+        return error(409, "could not get a free filename, try again")
+
+    def process(up: UploadFile, token: str) -> JSONResponse:
+        stem, ext = split_name(clean_name(up.filename or "upload"))
+        content_type = up.content_type or "application/octet-stream"
+        if content_type == "application/octet-stream":
+            # curl/ShareX don't say what the file is; Discord only plays video/mp4 etc. inline
+            content_type = mimetypes.guess_type(stem + ext)[0] or content_type
+        with tempfile.TemporaryDirectory(dir=settings.work_dir) as tmp:
+            body, remuxed = up.file, False
+            if ext.lower() == ".mkv":
+                src, dst = Path(tmp) / "in.mkv", Path(tmp) / "out.mp4"
+                up.file.seek(0)
+                with src.open("wb") as f:
+                    shutil.copyfileobj(up.file, f, 1024 * 1024)
+                if remux_to_mp4(src, dst):
+                    src.unlink()
+                    body, ext, content_type, remuxed = dst.open("rb"), ".mp4", "video/mp4", True
+            try:
+                return forward(body, stem, ext, token, content_type, remuxed)
+            finally:
+                if remuxed:
+                    body.close()
+
+    @app.get("/health")
+    def health():
+        return {"ok": True}
+
+    @app.post("/upload")
+    async def upload(request: Request):
+        token = request.headers.get("authorization")
+        if not token:
+            ticket = request.headers.get("x-upload-ticket")
+            if ticket:
+                token = tickets.redeem(ticket)
+                if not token:
+                    return error(401, "upload ticket expired, try again")
+        if not token:
+            return error(401, "missing Authorization header (your Zipline token)")
+        length = request.headers.get("content-length", "")
+        if length.isdigit() and int(length) > settings.max_bytes + FORM_SLACK:
+            return too_big()
+        form = await request.form(max_files=1, max_fields=10)
+        try:
+            up = form.get("file")
+            if not isinstance(up, UploadFile):
+                return error(400, "send the file in a form field named 'file'")
+            if up.size is not None and up.size > settings.max_bytes:
+                return too_big()
+            return await asyncio.to_thread(process, up, token)
+        finally:
+            await form.close()
+
+    return app

intake/app/site_api.py

@@ -0,0 +1,296 @@
+"""The clips site: its pages plus a small API that talks to Zipline with the user's token from a cookie."""
+import colorsys
+import hashlib
+import html
+import logging
+import random
+import re
+from collections.abc import Callable
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+from fastapi import APIRouter, Request
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Response
+from pydantic import BaseModel
+
+from .errors import error, zipline_message
+from .media import still_jpeg, video_size
+from .names import clean_name
+from .previews import Previews
+from .site_auth import LoginLimiter, TicketStore
+
+log = logging.getLogger("uvicorn.error")
+COOKIE = "ys"
+COOKIE_AGE = 30 * 24 * 3600
+PER_PAGE = 24
+SITE_DIR = Path(__file__).resolve().parent.parent / "site"
+SESSION_RE = re.compile(r"zipline_session=([^;]+)")
+
+
+class Login(BaseModel):
+    username: str
+    password: str
+
+
+class Rename(BaseModel):
+    name: str
+
+
+def client_ip(request: Request) -> str:
+    # Cloudflare sets CF-Connecting-IP; over Tailscale, tailscale serve sets X-Forwarded-For
+    forwarded = request.headers.get("x-forwarded-for", "").split(",")[0].strip()
+    return request.headers.get("cf-connecting-ip") or forwarded or (request.client.host if request.client else "?")
+
+
+def file_view(f: dict) -> dict:
+    thumb = (f.get("thumbnail") or {}).get("path")
+    return {
+        "id": f["id"], "name": f["name"], "size": f.get("size", 0), "type": f.get("type", ""),
+        "views": f.get("views", 0), "createdAt": f.get("createdAt"), "deletesAt": f.get("deletesAt"),
+        "url": "/" + quote(f["name"]), "thumbnail": "/raw/" + quote(thumb) if thumb else None,
+    }
+
+
+def preview_tags(name: str, origin: str, video: str, size: tuple[int, int] | None, site_name: str) -> str:
+    """Open Graph tags for /v/<name>: Discord's player, a still and a title. `video` is the clip itself,
+    or for a big clip its small copy (Discord gives up on files it can't download in ~8 s)."""
+    tags = [("og:type", "video.other"), ("og:site_name", site_name), ("og:title", name),
+            ("og:image", f"{origin}/_site/thumb/{quote(name)}"),
+            ("og:video", video), ("og:video:url", video),
+            ("og:video:secure_url", video), ("og:video:type", "video/mp4")]
+    if size:
+        tags += [("og:video:width", str(size[0])), ("og:video:height", str(size[1]))]
+    meta = "".join(f'<meta property="{k}" content="{html.escape(v, quote=True)}">\n' for k, v in tags)
+    return meta + '<meta name="twitter:card" content="player">\n'
+
+
+def random_color() -> str:
+    """A bright color for the stripe Discord draws beside the embed (it reads the page's theme-color)."""
+    r, g, b = colorsys.hls_to_rgb(random.random(), 0.55, 0.75)
+    return "#%02x%02x%02x" % (round(r * 255), round(g * 255), round(b * 255))
+
+
+def site_page(site_name: str, server_name: str) -> str:
+    """index.html with {{v}} replaced by a hash of the assets, so browsers never keep a stale app.js/style.css,
+    and {{name}}/{{server}} by the configured names (app.js reads them before /config loads)."""
+    digest = hashlib.sha256()
+    for name in ("app.js", "style.css"):
+        digest.update((SITE_DIR / name).read_bytes())
+    return ((SITE_DIR / "index.html").read_text(encoding="utf-8").replace("{{v}}", digest.hexdigest()[:10])
+            .replace("{{name}}", html.escape(site_name)).replace("{{server}}", html.escape(server_name, quote=True)))
+
+
+def build_site_router(client: httpx.Client, exists: Callable[[str], bool], tickets: TicketStore,
+                      limiter: LoginLimiter, settings, previews: Previews) -> APIRouter:
+    r = APIRouter()
+    page = site_page(settings.site_name, settings.server_name)
+
+    def logged_out() -> JSONResponse:
+        resp = error(401, "not logged in")
+        resp.delete_cookie(COOKIE, path="/", secure=True, httponly=True, samesite="lax")
+        return resp
+
+    def call(method: str, path: str, token: str, **kw) -> httpx.Response | None:
+        try:
+            return client.request(method, path, headers={"authorization": token}, **kw)
+        except httpx.HTTPError:
+            return None
+
+    def relay(resp: httpx.Response | None, ok: Callable[[dict], dict]):
+        if resp is None:
+            return error(502, "Zipline isn't reachable right now.")
+        if resp.status_code == 401:
+            return logged_out()
+        if resp.status_code == 404:
+            return error(404, "This clip doesn't exist (or isn't yours).")
+        if resp.status_code >= 400:
+            return error(resp.status_code if resp.status_code < 500 else 502, zipline_message(resp))
+        return ok(resp.json())
+
+    def file_path(ident: str) -> str:
+        return "/api/user/files/" + quote(ident, safe="")
+
+    def bad_ident(ident: str) -> bool:
+        # an empty or nested id would reach other Zipline routes (the list, /transaction, ...)
+        return not ident or "/" in ident
+
+    no_clip = lambda: error(404, "This clip doesn't exist (or isn't yours).")
+
+    @r.get("/", include_in_schema=False)
+    def index():
+        return HTMLResponse(page, headers={"cache-control": "no-store"})
+
+    def video_head(name: str) -> httpx.Response | None:
+        """Zipline's answer for the raw clip when it's a video, else None."""
+        try:
+            resp = client.head("/" + quote(name))
+        except httpx.HTTPError:
+            return None
+        ok = resp.status_code == 200 and resp.headers.get("content-type", "").startswith("video/")
+        return resp if ok else None
+
+    def raw_url(name: str) -> str:
+        return settings.zipline_url.rstrip("/") + "/" + quote(name)
+
+    @r.get("/v/{name:path}", include_in_schema=False)
+    def watch(name: str, request: Request):
+        head = video_head(name) if name else None
+        if head is None:
+            return HTMLResponse(page, headers={"cache-control": "no-store"})
+        length = head.headers.get("content-length", "")
+        key = previews.ready(name, int(length)) if length.isdigit() else None
+        if key:
+            video, size = f"{settings.site_origin}/_site/preview/{key}.mp4", video_size(str(previews.file(key)), key)
+        else:
+            video, size = f"{settings.site_origin}/{quote(name)}", video_size(raw_url(name), length)
+        log.info("/v/%s -> %s for %s", name, "copy" if key else "original", request.headers.get("user-agent", "?"))
+        tags = preview_tags(name, settings.site_origin, video, size, settings.site_name)
+        html_page = page.replace('<meta name="theme-color" content="#3e4637">',
+                                 f'<meta name="theme-color" content="{random_color()}">', 1)
+        return HTMLResponse(html_page.replace("</head>", tags + "</head>", 1), headers={"cache-control": "no-store"})
+
+    @r.get("/_site/api/discord/{name:path}")
+    def discord_state(name: str):
+        # the upload dialog holds a clip's link back while this says "making": Discord caches whatever it sees first
+        head = video_head(name) if name else None
+        length = head.headers.get("content-length", "") if head is not None else ""
+        return previews.status(name, int(length) if length.isdigit() else 0)
+
+    @r.get("/_site/thumb/{name:path}", include_in_schema=False)
+    def thumb(name: str):
+        head = video_head(name) if name else None
+        jpeg = head and still_jpeg(raw_url(name), head.headers.get("content-length", ""))
+        if not jpeg:
+            return error(404, "no preview for this clip")
+        return Response(jpeg, media_type="image/jpeg", headers={"cache-control": "public, max-age=3600"})
+
+    @r.get("/_site/preview/{key}.mp4", include_in_schema=False)
+    def preview(key: str):
+        path = previews.file(key)
+        if path is None:
+            return error(404, "no such preview")
+        return FileResponse(path, media_type="video/mp4")
+
+    @r.get("/_site/api/config")
+    def config():
+        return {"uploadUrl": settings.upload_url, "publicUrl": settings.site_origin, "uploadHint": settings.upload_hint}
+
+    @r.get("/_site/api/public/{name:path}")
+    def public_file(name: str):
+        """What a visitor from a shared link may see: the raw file is public already, this only names it."""
+        try:
+            head = client.head("/" + quote(name)) if name else None
+        except httpx.HTTPError:
+            return error(502, "Zipline isn't reachable right now.")
+        if head is None or head.status_code != 200:
+            return error(404, "This clip doesn't exist (anymore).")
+        kind = head.headers.get("content-type", "").split(";")[0]
+        length = head.headers.get("content-length", "0")
+        return {"name": name, "type": kind, "size": int(length) if length.isdigit() else 0, "url": "/" + quote(name),
+                "thumbnail": "/_site/thumb/" + quote(name) if kind.startswith("video/") else None}
+
+    @r.post("/_site/api/login")
+    def login(body: Login, request: Request):
+        # strikes count per IP *and* account, so logging in as yourself doesn't clear guesses at someone else
+        ip = client_ip(request) + "|" + body.username.strip().lower()
+        if limiter.blocked(ip):
+            return error(429, "Too many tries, wait a few minutes.")
+        try:
+            r1 = client.post("/api/auth/login", json={"username": body.username, "password": body.password})
+            if r1.status_code != 200:
+                if r1.status_code >= 500:
+                    return error(502, "Zipline isn't reachable right now.")
+                limiter.fail(ip)
+                return error(401, "Wrong username or password.")
+            match = SESSION_RE.search(r1.headers.get("set-cookie", ""))
+            if not match:
+                return error(502, "Zipline didn't start a session.")
+            r2 = client.get("/api/user/token", headers={"cookie": f"zipline_session={match.group(1)}"})
+        except httpx.HTTPError:
+            return error(502, "Zipline isn't reachable right now.")
+        finally:
+            client.cookies.clear()  # never let one user's Zipline session ride along on someone else's request
+        if r2.status_code != 200:
+            return error(502, "Zipline didn't hand out a token.")
+        limiter.reset(ip)
+        resp = JSONResponse({"username": r1.json()["user"]["username"]})
+        resp.set_cookie(COOKIE, r2.json()["token"], max_age=COOKIE_AGE, path="/",
+                        httponly=True, secure=True, samesite="lax")
+        return resp
+
+    @r.post("/_site/api/logout")
+    def logout():
+        resp = JSONResponse({"ok": True})
+        resp.delete_cookie(COOKIE, path="/", secure=True, httponly=True, samesite="lax")
+        return resp
+
+    @r.get("/_site/api/me")
+    def me(request: Request):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        result = relay(call("GET", "/api/user", token), lambda d: {"username": d["user"]["username"]})
+        if not isinstance(result, dict):
+            return result
+        stats = call("GET", "/api/user/stats", token)
+        ok = stats is not None and stats.status_code == 200
+        return {**result, "storageUsed": stats.json().get("storageUsed", 0) if ok else 0}
+
+    @r.get("/_site/api/files")
+    def files(request: Request, page: int = 1):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        params = {"page": max(page, 1), "perpage": PER_PAGE, "sortBy": "createdAt", "order": "desc"}
+        return relay(call("GET", "/api/user/files", token, params=params), lambda d: {
+            "files": [file_view(f) for f in d.get("page", [])],
+            "total": d.get("total", 0), "pages": d.get("pages", 1),
+        })
+
+    @r.get("/_site/api/files/{ident:path}")
+    def file(ident: str, request: Request):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        if bad_ident(ident):
+            return no_clip()
+        return relay(call("GET", file_path(ident), token), file_view)
+
+    @r.patch("/_site/api/files/{ident:path}")
+    def rename(ident: str, body: Rename, request: Request):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        if bad_ident(ident):
+            return no_clip()
+        new = clean_name(body.name)
+        try:
+            if exists(new):
+                return error(409, "That name is already used.")
+        except httpx.HTTPError:
+            return error(502, "Zipline isn't reachable right now.")
+        before = call("GET", file_path(ident), token)
+        resp = call("PATCH", file_path(ident), token, json={"name": new})
+        if before is not None and before.status_code == 200 and resp is not None and resp.status_code == 200:
+            was = before.json()
+            previews.renamed(was["name"], resp.json().get("name", new), int(was.get("size") or 0))
+        return relay(resp, lambda d: {"name": d.get("name", new)})
+
+    @r.delete("/_site/api/files/{ident:path}")
+    def delete(ident: str, request: Request):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        if bad_ident(ident):
+            return no_clip()
+        return relay(call("DELETE", file_path(ident), token), lambda d: {"ok": True})
+
+    @r.post("/_site/api/upload-ticket")
+    def upload_ticket(request: Request):
+        token = request.cookies.get(COOKIE)
+        if not token:
+            return logged_out()
+        return {"ticket": tickets.issue(token), "uploadUrl": settings.upload_url}
+
+    return r

intake/app/site_auth.py

@@ -0,0 +1,64 @@
+"""Login rate limit and one-time upload tickets for the clips site."""
+import secrets
+import time
+from collections import defaultdict, deque
+from collections.abc import Callable
+
+
+class LoginLimiter:
+    """At most `limit` failed logins per key (an IP) within `window` seconds."""
+
+    def __init__(self, limit: int = 5, window: float = 600, now: Callable[[], float] = time.monotonic):
+        self.limit, self.window, self.now = limit, window, now
+        self.fails: dict[str, deque] = defaultdict(deque)
+        self.swept = now()
+
+    def _recent(self, key: str) -> deque:
+        q, cutoff = self.fails[key], self.now() - self.window
+        while q and q[0] <= cutoff:
+            q.popleft()
+        return q
+
+    def blocked(self, key: str) -> bool:
+        if key not in self.fails:  # don't grow the table for every key that's merely checked
+            return False
+        return len(self._recent(key)) >= self.limit
+
+    def fail(self, key: str) -> None:
+        self._sweep()
+        self._recent(key).append(self.now())
+
+    def _sweep(self) -> None:
+        # forget keys whose strikes all expired, so guesses with made-up usernames can't grow the table forever
+        if self.now() - self.swept < self.window:
+            return
+        self.swept = self.now()
+        for key in [k for k in self.fails if not self._recent(k)]:
+            del self.fails[key]
+
+    def reset(self, key: str) -> None:
+        self.fails.pop(key, None)
+
+
+class TicketStore:
+    """One-time tickets that stand in for a Zipline token on /upload, valid `ttl` seconds."""
+
+    def __init__(self, ttl: float = 3600, now: Callable[[], float] = time.monotonic):
+        self.ttl, self.now = ttl, now
+        self.tickets: dict[str, tuple[str, float]] = {}
+
+    def _prune(self) -> None:
+        now = self.now()
+        for ticket in [t for t, (_, expires) in self.tickets.items() if expires <= now]:
+            del self.tickets[ticket]
+
+    def issue(self, token: str) -> str:
+        self._prune()
+        ticket = secrets.token_urlsafe(32)
+        self.tickets[ticket] = (token, self.now() + self.ttl)
+        return ticket
+
+    def redeem(self, ticket: str) -> str | None:
+        self._prune()
+        entry = self.tickets.pop(ticket, None)
+        return entry[0] if entry else None