179 lines
8.3 KiB
Python
179 lines
8.3 KiB
Python
"""intake: the upload door in front of Zipline (rename, remux .mkv, pick a free name, forward)."""
|
|
import asyncio
|
|
import http.cookiejar
|
|
import mimetypes
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from fastapi.responses import JSONResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from starlette.datastructures import UploadFile
|
|
|
|
from .errors import error, zipline_message
|
|
from .names import clean_name, next_free_name, split_name
|
|
from .previews import Previews
|
|
from .remux import remux_to_mp4
|
|
from .site_api import SITE_DIR, build_site_router
|
|
from .site_auth import LoginLimiter, TicketStore
|
|
|
|
FORM_SLACK = 1024 * 1024 # room for the multipart headers around the file
|
|
UPLOAD_TRIES = 3
|
|
|
|
|
|
@dataclass
|
|
class Settings:
|
|
zipline_url: str = "http://zipline:3000"
|
|
max_bytes: int = 5 * 1024**3
|
|
deletes_at: str = "90d"
|
|
work_dir: Path = field(default_factory=lambda: Path(tempfile.gettempdir()))
|
|
site_origin: str = "http://localhost:3020"
|
|
upload_url: str = "http://localhost:3020"
|
|
site_name: str = "kaledonya clips" # its first word is the big logo word on the login screen
|
|
server_name: str = "the server" # shown as "Sending to <server_name>"
|
|
upload_hint: str = "Can't reach the upload server." # shown when an upload can't connect at all
|
|
preview_over: int = 25 * 1000 * 1000 # bigger clips get a small copy for Discord (see previews.py)
|
|
|
|
@classmethod
|
|
def from_env(cls) -> "Settings":
|
|
return cls(
|
|
zipline_url=os.environ.get("ZIPLINE_URL", cls.zipline_url),
|
|
max_bytes=int(os.environ.get("MAX_BYTES", cls.max_bytes)),
|
|
deletes_at=os.environ.get("DELETES_AT", cls.deletes_at),
|
|
work_dir=Path(os.environ.get("WORK_DIR", tempfile.gettempdir())),
|
|
site_origin=os.environ.get("SITE_ORIGIN", cls.site_origin),
|
|
upload_url=os.environ.get("UPLOAD_URL", cls.upload_url),
|
|
site_name=os.environ.get("SITE_NAME", cls.site_name),
|
|
server_name=os.environ.get("SERVER_NAME", cls.server_name),
|
|
upload_hint=os.environ.get("UPLOAD_HINT", cls.upload_hint),
|
|
preview_over=int(os.environ.get("PREVIEW_OVER", cls.preview_over)),
|
|
)
|
|
|
|
|
|
def create_app(settings: Settings | None = None, transport: httpx.BaseTransport | None = None) -> FastAPI:
|
|
settings = settings or Settings.from_env()
|
|
app = FastAPI(title="zipline-intake")
|
|
# the site (SITE_ORIGIN) uploads straight to this service at UPLOAD_URL, which may be another hostname
|
|
app.add_middleware(CORSMiddleware, allow_origins=[settings.site_origin],
|
|
allow_methods=["POST"], allow_headers=["x-upload-ticket"])
|
|
client = httpx.Client(
|
|
base_url=settings.zipline_url,
|
|
transport=transport,
|
|
timeout=httpx.Timeout(60.0, read=900.0, write=None),
|
|
)
|
|
# never keep cookies: a Zipline login session must not ride along on anyone else's request
|
|
client.cookies.jar.set_policy(http.cookiejar.DefaultCookiePolicy(allowed_domains=[]))
|
|
too_big = lambda: error(413, f"file is larger than the {settings.max_bytes / 1024**3:g} GB limit")
|
|
|
|
def exists(name: str) -> bool:
|
|
# 302: Zipline sends text and password-protected files to /view/<name>, but the name is taken
|
|
return client.head("/" + quote(name)).status_code in (200, 302)
|
|
|
|
tickets, limiter = TicketStore(), LoginLimiter()
|
|
previews = Previews(settings.work_dir / "previews", settings.zipline_url, settings.preview_over)
|
|
app.state.tickets, app.state.limiter, app.state.previews = tickets, limiter, previews
|
|
app.include_router(build_site_router(client, exists, tickets, limiter, settings, previews))
|
|
app.mount("/_site/static", StaticFiles(directory=SITE_DIR), name="site-static")
|
|
|
|
@app.middleware("http")
|
|
async def no_store_site_api(request: Request, call_next):
|
|
# Cloudflare caches anything ending in .mp4, even /_site/api/files/clip.mp4: never let it keep these
|
|
response = await call_next(request)
|
|
if request.url.path.startswith("/_site/api/"):
|
|
response.headers["cache-control"] = "no-store, private"
|
|
return response
|
|
|
|
def forward(body, stem: str, ext: str, token: str, content_type: str, remuxed: bool) -> JSONResponse:
|
|
taken: set[str] = set()
|
|
for _ in range(UPLOAD_TRIES):
|
|
try:
|
|
name = next_free_name(stem, ext, lambda n: n in taken or exists(n))
|
|
except httpx.HTTPError as e:
|
|
return error(502, f"could not reach Zipline: {e}")
|
|
except RuntimeError:
|
|
return error(409, "could not get a free filename, try again")
|
|
headers = {
|
|
"authorization": token,
|
|
"x-zipline-format": "name",
|
|
"x-zipline-filename": quote(name[: len(name) - len(ext)] if ext else name),
|
|
"x-zipline-original-name": "true",
|
|
"x-zipline-deletes-at": settings.deletes_at,
|
|
}
|
|
if ext:
|
|
headers["x-zipline-file-extension"] = ext
|
|
body.seek(0)
|
|
try:
|
|
r = client.post("/api/upload", headers=headers, files={"file": (name, body, content_type)})
|
|
except httpx.HTTPError as e:
|
|
return error(502, f"could not reach Zipline: {e}")
|
|
if r.status_code == 200:
|
|
# build the link ourselves: a '#' or '?' left raw in Zipline's url would cut it off
|
|
url = settings.site_origin + "/" + quote(name)
|
|
if content_type.startswith("video/"):
|
|
previews.request(name, body.seek(0, os.SEEK_END))
|
|
return JSONResponse({"url": url, "name": name, "remuxed": remuxed})
|
|
if "already exists" in r.text:
|
|
taken.add(name)
|
|
continue
|
|
return error(r.status_code if r.status_code < 500 else 502, zipline_message(r))
|
|
return error(409, "could not get a free filename, try again")
|
|
|
|
def process(up: UploadFile, token: str) -> JSONResponse:
|
|
stem, ext = split_name(clean_name(up.filename or "upload"))
|
|
content_type = up.content_type or "application/octet-stream"
|
|
if content_type == "application/octet-stream":
|
|
# curl/ShareX don't say what the file is; Discord only plays video/mp4 etc. inline
|
|
content_type = mimetypes.guess_type(stem + ext)[0] or content_type
|
|
with tempfile.TemporaryDirectory(dir=settings.work_dir) as tmp:
|
|
body, remuxed = up.file, False
|
|
if ext.lower() == ".mkv":
|
|
src, dst = Path(tmp) / "in.mkv", Path(tmp) / "out.mp4"
|
|
up.file.seek(0)
|
|
with src.open("wb") as f:
|
|
shutil.copyfileobj(up.file, f, 1024 * 1024)
|
|
if remux_to_mp4(src, dst):
|
|
src.unlink()
|
|
body, ext, content_type, remuxed = dst.open("rb"), ".mp4", "video/mp4", True
|
|
try:
|
|
return forward(body, stem, ext, token, content_type, remuxed)
|
|
finally:
|
|
if remuxed:
|
|
body.close()
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"ok": True}
|
|
|
|
@app.post("/upload")
|
|
async def upload(request: Request):
|
|
token = request.headers.get("authorization")
|
|
if not token:
|
|
ticket = request.headers.get("x-upload-ticket")
|
|
if ticket:
|
|
token = tickets.redeem(ticket)
|
|
if not token:
|
|
return error(401, "upload ticket expired, try again")
|
|
if not token:
|
|
return error(401, "missing Authorization header (your Zipline token)")
|
|
length = request.headers.get("content-length", "")
|
|
if length.isdigit() and int(length) > settings.max_bytes + FORM_SLACK:
|
|
return too_big()
|
|
form = await request.form(max_files=1, max_fields=10)
|
|
try:
|
|
up = form.get("file")
|
|
if not isinstance(up, UploadFile):
|
|
return error(400, "send the file in a form field named 'file'")
|
|
if up.size is not None and up.size > settings.max_bytes:
|
|
return too_big()
|
|
return await asyncio.to_thread(process, up, token)
|
|
finally:
|
|
await form.close()
|
|
|
|
return app
|