kaledonya clips: a Discord-friendly clip site on top of Zipline

Co-Authored-By: Claude Opus 5.5 <[email protected]>

intake/app/server.py

@@ -0,0 +1,178 @@
+"""intake: the upload door in front of Zipline (rename, remux .mkv, pick a free name, forward)."""
+import asyncio
+import http.cookiejar
+import mimetypes
+import os
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import quote
+
+import httpx
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from starlette.datastructures import UploadFile
+
+from .errors import error, zipline_message
+from .names import clean_name, next_free_name, split_name
+from .previews import Previews
+from .remux import remux_to_mp4
+from .site_api import SITE_DIR, build_site_router
+from .site_auth import LoginLimiter, TicketStore
+
+FORM_SLACK = 1024 * 1024  # room for the multipart headers around the file
+UPLOAD_TRIES = 3
+
+
+@dataclass
+class Settings:
+    zipline_url: str = "http://zipline:3000"
+    max_bytes: int = 5 * 1024**3
+    deletes_at: str = "90d"
+    work_dir: Path = field(default_factory=lambda: Path(tempfile.gettempdir()))
+    site_origin: str = "http://localhost:3020"
+    upload_url: str = "http://localhost:3020"
+    site_name: str = "kaledonya clips"  # its first word is the big logo word on the login screen
+    server_name: str = "the server"  # shown as "Sending to <server_name>"
+    upload_hint: str = "Can't reach the upload server."  # shown when an upload can't connect at all
+    preview_over: int = 25 * 1000 * 1000  # bigger clips get a small copy for Discord (see previews.py)
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        return cls(
+            zipline_url=os.environ.get("ZIPLINE_URL", cls.zipline_url),
+            max_bytes=int(os.environ.get("MAX_BYTES", cls.max_bytes)),
+            deletes_at=os.environ.get("DELETES_AT", cls.deletes_at),
+            work_dir=Path(os.environ.get("WORK_DIR", tempfile.gettempdir())),
+            site_origin=os.environ.get("SITE_ORIGIN", cls.site_origin),
+            upload_url=os.environ.get("UPLOAD_URL", cls.upload_url),
+            site_name=os.environ.get("SITE_NAME", cls.site_name),
+            server_name=os.environ.get("SERVER_NAME", cls.server_name),
+            upload_hint=os.environ.get("UPLOAD_HINT", cls.upload_hint),
+            preview_over=int(os.environ.get("PREVIEW_OVER", cls.preview_over)),
+        )
+
+
+def create_app(settings: Settings | None = None, transport: httpx.BaseTransport | None = None) -> FastAPI:
+    settings = settings or Settings.from_env()
+    app = FastAPI(title="zipline-intake")
+    # the site (SITE_ORIGIN) uploads straight to this service at UPLOAD_URL, which may be another hostname
+    app.add_middleware(CORSMiddleware, allow_origins=[settings.site_origin],
+                       allow_methods=["POST"], allow_headers=["x-upload-ticket"])
+    client = httpx.Client(
+        base_url=settings.zipline_url,
+        transport=transport,
+        timeout=httpx.Timeout(60.0, read=900.0, write=None),
+    )
+    # never keep cookies: a Zipline login session must not ride along on anyone else's request
+    client.cookies.jar.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[]))
+    too_big = lambda: error(413, f"file is larger than the {settings.max_bytes / 1024**3:g} GB limit")
+
+    def exists(name: str) -> bool:
+        # 302: Zipline sends text and password-protected files to /view/<name>, but the name is taken
+        return client.head("/" + quote(name)).status_code in (200, 302)
+
+    tickets, limiter = TicketStore(), LoginLimiter()
+    previews = Previews(settings.work_dir / "previews", settings.zipline_url, settings.preview_over)
+    app.state.tickets, app.state.limiter, app.state.previews = tickets, limiter, previews
+    app.include_router(build_site_router(client, exists, tickets, limiter, settings, previews))
+    app.mount("/_site/static", StaticFiles(directory=SITE_DIR), name="site-static")
+
+    @app.middleware("http")
+    async def no_store_site_api(request: Request, call_next):
+        # Cloudflare caches anything ending in .mp4, even /_site/api/files/clip.mp4: never let it keep these
+        response = await call_next(request)
+        if request.url.path.startswith("/_site/api/"):
+            response.headers["cache-control"] = "no-store, private"
+        return response
+
+    def forward(body, stem: str, ext: str, token: str, content_type: str, remuxed: bool) -> JSONResponse:
+        taken: set[str] = set()
+        for _ in range(UPLOAD_TRIES):
+            try:
+                name = next_free_name(stem, ext, lambda n: n in taken or exists(n))
+            except httpx.HTTPError as e:
+                return error(502, f"could not reach Zipline: {e}")
+            except RuntimeError:
+                return error(409, "could not get a free filename, try again")
+            headers = {
+                "authorization": token,
+                "x-zipline-format": "name",
+                "x-zipline-filename": quote(name[: len(name) - len(ext)] if ext else name),
+                "x-zipline-original-name": "true",
+                "x-zipline-deletes-at": settings.deletes_at,
+            }
+            if ext:
+                headers["x-zipline-file-extension"] = ext
+            body.seek(0)
+            try:
+                r = client.post("/api/upload", headers=headers, files={"file": (name, body, content_type)})
+            except httpx.HTTPError as e:
+                return error(502, f"could not reach Zipline: {e}")
+            if r.status_code == 200:
+                # build the link ourselves: a '#' or '?' left raw in Zipline's url would cut it off
+                url = settings.site_origin + "/" + quote(name)
+                if content_type.startswith("video/"):
+                    previews.request(name, body.seek(0, os.SEEK_END))
+                return JSONResponse({"url": url, "name": name, "remuxed": remuxed})
+            if "already exists" in r.text:
+                taken.add(name)
+                continue
+            return error(r.status_code if r.status_code < 500 else 502, zipline_message(r))
+        return error(409, "could not get a free filename, try again")
+
+    def process(up: UploadFile, token: str) -> JSONResponse:
+        stem, ext = split_name(clean_name(up.filename or "upload"))
+        content_type = up.content_type or "application/octet-stream"
+        if content_type == "application/octet-stream":
+            # curl/ShareX don't say what the file is; Discord only plays video/mp4 etc. inline
+            content_type = mimetypes.guess_type(stem + ext)[0] or content_type
+        with tempfile.TemporaryDirectory(dir=settings.work_dir) as tmp:
+            body, remuxed = up.file, False
+            if ext.lower() == ".mkv":
+                src, dst = Path(tmp) / "in.mkv", Path(tmp) / "out.mp4"
+                up.file.seek(0)
+                with src.open("wb") as f:
+                    shutil.copyfileobj(up.file, f, 1024 * 1024)
+                if remux_to_mp4(src, dst):
+                    src.unlink()
+                    body, ext, content_type, remuxed = dst.open("rb"), ".mp4", "video/mp4", True
+            try:
+                return forward(body, stem, ext, token, content_type, remuxed)
+            finally:
+                if remuxed:
+                    body.close()
+
+    @app.get("/health")
+    def health():
+        return {"ok": True}
+
+    @app.post("/upload")
+    async def upload(request: Request):
+        token = request.headers.get("authorization")
+        if not token:
+            ticket = request.headers.get("x-upload-ticket")
+            if ticket:
+                token = tickets.redeem(ticket)
+                if not token:
+                    return error(401, "upload ticket expired, try again")
+        if not token:
+            return error(401, "missing Authorization header (your Zipline token)")
+        length = request.headers.get("content-length", "")
+        if length.isdigit() and int(length) > settings.max_bytes + FORM_SLACK:
+            return too_big()
+        form = await request.form(max_files=1, max_fields=10)
+        try:
+            up = form.get("file")
+            if not isinstance(up, UploadFile):
+                return error(400, "send the file in a form field named 'file'")
+            if up.size is not None and up.size > settings.max_bytes:
+                return too_big()
+            return await asyncio.to_thread(process, up, token)
+        finally:
+            await form.close()
+
+    return app