"""intake: the upload door in front of Zipline (rename, remux .mkv, pick a free name, forward).""" import asyncio import http.cookiejar import mimetypes import os import shutil import tempfile from dataclasses import dataclass, field from pathlib import Path from urllib.parse import quote import httpx from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from fastapi.staticfiles import StaticFiles from starlette.datastructures import UploadFile from .errors import error, zipline_message from .names import clean_name, next_free_name, split_name from .previews import Previews from .remux import remux_to_mp4 from .site_api import SITE_DIR, build_site_router from .site_auth import LoginLimiter, TicketStore FORM_SLACK = 1024 * 1024 # room for the multipart headers around the file UPLOAD_TRIES = 3 @dataclass class Settings: zipline_url: str = "http://zipline:3000" max_bytes: int = 5 * 1024**3 deletes_at: str = "90d" work_dir: Path = field(default_factory=lambda: Path(tempfile.gettempdir())) site_origin: str = "http://localhost:3020" upload_url: str = "http://localhost:3020" site_name: str = "kaledonya clips" # its first word is the big logo word on the login screen server_name: str = "the server" # shown as "Sending to " upload_hint: str = "Can't reach the upload server." # shown when an upload can't connect at all preview_over: int = 25 * 1000 * 1000 # bigger clips get a small copy for Discord (see previews.py) @classmethod def from_env(cls) -> "Settings": return cls( zipline_url=os.environ.get("ZIPLINE_URL", cls.zipline_url), max_bytes=int(os.environ.get("MAX_BYTES", cls.max_bytes)), deletes_at=os.environ.get("DELETES_AT", cls.deletes_at), work_dir=Path(os.environ.get("WORK_DIR", tempfile.gettempdir())), site_origin=os.environ.get("SITE_ORIGIN", cls.site_origin), upload_url=os.environ.get("UPLOAD_URL", cls.upload_url), site_name=os.environ.get("SITE_NAME", cls.site_name), server_name=os.environ.get("SERVER_NAME", cls.server_name), upload_hint=os.environ.get("UPLOAD_HINT", cls.upload_hint), preview_over=int(os.environ.get("PREVIEW_OVER", cls.preview_over)), ) def create_app(settings: Settings | None = None, transport: httpx.BaseTransport | None = None) -> FastAPI: settings = settings or Settings.from_env() app = FastAPI(title="zipline-intake") # the site (SITE_ORIGIN) uploads straight to this service at UPLOAD_URL, which may be another hostname app.add_middleware(CORSMiddleware, allow_origins=[settings.site_origin], allow_methods=["POST"], allow_headers=["x-upload-ticket"]) client = httpx.Client( base_url=settings.zipline_url, transport=transport, timeout=httpx.Timeout(60.0, read=900.0, write=None), ) # never keep cookies: a Zipline login session must not ride along on anyone else's request client.cookies.jar.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[])) too_big = lambda: error(413, f"file is larger than the {settings.max_bytes / 1024**3:g} GB limit") def exists(name: str) -> bool: # 302: Zipline sends text and password-protected files to /view/, but the name is taken return client.head("/" + quote(name)).status_code in (200, 302) tickets, limiter = TicketStore(), LoginLimiter() previews = Previews(settings.work_dir / "previews", settings.zipline_url, settings.preview_over) app.state.tickets, app.state.limiter, app.state.previews = tickets, limiter, previews app.include_router(build_site_router(client, exists, tickets, limiter, settings, previews)) app.mount("/_site/static", StaticFiles(directory=SITE_DIR), name="site-static") @app.middleware("http") async def no_store_site_api(request: Request, call_next): # Cloudflare caches anything ending in .mp4, even /_site/api/files/clip.mp4: never let it keep these response = await call_next(request) if request.url.path.startswith("/_site/api/"): response.headers["cache-control"] = "no-store, private" return response def forward(body, stem: str, ext: str, token: str, content_type: str, remuxed: bool) -> JSONResponse: taken: set[str] = set() for _ in range(UPLOAD_TRIES): try: name = next_free_name(stem, ext, lambda n: n in taken or exists(n)) except httpx.HTTPError as e: return error(502, f"could not reach Zipline: {e}") except RuntimeError: return error(409, "could not get a free filename, try again") headers = { "authorization": token, "x-zipline-format": "name", "x-zipline-filename": quote(name[: len(name) - len(ext)] if ext else name), "x-zipline-original-name": "true", "x-zipline-deletes-at": settings.deletes_at, } if ext: headers["x-zipline-file-extension"] = ext body.seek(0) try: r = client.post("/api/upload", headers=headers, files={"file": (name, body, content_type)}) except httpx.HTTPError as e: return error(502, f"could not reach Zipline: {e}") if r.status_code == 200: # build the link ourselves: a '#' or '?' left raw in Zipline's url would cut it off url = settings.site_origin + "/" + quote(name) if content_type.startswith("video/"): previews.request(name, body.seek(0, os.SEEK_END)) return JSONResponse({"url": url, "name": name, "remuxed": remuxed}) if "already exists" in r.text: taken.add(name) continue return error(r.status_code if r.status_code < 500 else 502, zipline_message(r)) return error(409, "could not get a free filename, try again") def process(up: UploadFile, token: str) -> JSONResponse: stem, ext = split_name(clean_name(up.filename or "upload")) content_type = up.content_type or "application/octet-stream" if content_type == "application/octet-stream": # curl/ShareX don't say what the file is; Discord only plays video/mp4 etc. inline content_type = mimetypes.guess_type(stem + ext)[0] or content_type with tempfile.TemporaryDirectory(dir=settings.work_dir) as tmp: body, remuxed = up.file, False if ext.lower() == ".mkv": src, dst = Path(tmp) / "in.mkv", Path(tmp) / "out.mp4" up.file.seek(0) with src.open("wb") as f: shutil.copyfileobj(up.file, f, 1024 * 1024) if remux_to_mp4(src, dst): src.unlink() body, ext, content_type, remuxed = dst.open("rb"), ".mp4", "video/mp4", True try: return forward(body, stem, ext, token, content_type, remuxed) finally: if remuxed: body.close() @app.get("/health") def health(): return {"ok": True} @app.post("/upload") async def upload(request: Request): token = request.headers.get("authorization") if not token: ticket = request.headers.get("x-upload-ticket") if ticket: token = tickets.redeem(ticket) if not token: return error(401, "upload ticket expired, try again") if not token: return error(401, "missing Authorization header (your Zipline token)") length = request.headers.get("content-length", "") if length.isdigit() and int(length) > settings.max_bytes + FORM_SLACK: return too_big() form = await request.form(max_files=1, max_fields=10) try: up = form.get("file") if not isinstance(up, UploadFile): return error(400, "send the file in a form field named 'file'") if up.size is not None and up.size > settings.max_bytes: return too_big() return await asyncio.to_thread(process, up, token) finally: await form.close() return app