From 78c755a7a28f6561a030bc85474160d30fd223f5 Mon Sep 17 00:00:00 2001 From: esefxdz Date: Sat, 31 Oct 2026 13:36:11 +0300 Subject: [PATCH] tgbridge: support a self-hosted bot api server, and say when a file is too big The cloud api refuses getFile over 20 MB, so an oversize video or gif never reached ffmpeg and the bridge blamed the fetch. Six of the nine media failures in the last month were that; the other three were a dropped read. - config.local_api() turns TELEGRAM_API_BASE_URL into base_url/base_file_url/ local_mode. Both directions read it, or only one of them migrates. - in local mode getFile hands back a path on disk and ptb reads it directly, so there is no 20 MB cap and no http transfer to drop. - MAX_INGEST_BYTES keeps that from being a way to pull a gigabyte into the bot's memory; falls back to stat() when the server omits file_size. - explicit read_timeout: ptb defaults to 5s, which is what "Timed out" was. one retry covers a dropped read. BadRequest subclasses NetworkError, so it is re-raised first or an oversize rejection gets re-sent. - the note now distinguishes too-large from could-not-fetch, and the warning records which attachment and which message. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 + tgbridge/config.py | 15 +++++++ tgbridge/discord_to_telegram.py | 4 +- tgbridge/forwarder.py | 79 ++++++++++++++++++++++++++++----- 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 0267c04..7745c97 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ _check_students.py _check_names.py _check_banner_api.py +credentials_test.txt diff --git a/tgbridge/config.py b/tgbridge/config.py index c140b1b..062c216 100644 --- a/tgbridge/config.py +++ b/tgbridge/config.py @@ -117,3 +117,18 @@ def load_routes() -> list[Route]: def avatar_channel_id() -> int | None: return _int_or_none(os.getenv("AVATAR_CACHE_CHANNEL_ID")) + + +def local_api() -> dict: + """-> bot kwargs for a self-hosted bot api server, {} for the cloud one. + + A local server lifts telegram's 20 MB getFile cap and hands back a path on + disk instead of a url, which ptb reads for itself once local_mode is on. + Both directions read this or only one of them migrates. + """ + base = os.getenv("TELEGRAM_API_BASE_URL", "").strip().rstrip("/") + if not base: + return {} + return {"base_url": f"{base}/bot", + "base_file_url": f"{base}/file/bot", + "local_mode": True} diff --git a/tgbridge/discord_to_telegram.py b/tgbridge/discord_to_telegram.py index 08654bd..fec8ded 100644 --- a/tgbridge/discord_to_telegram.py +++ b/tgbridge/discord_to_telegram.py @@ -15,7 +15,7 @@ from telegram import (Bot, InputMediaAudio, InputMediaDocument, InputMediaPhoto, InputMediaVideo, ReplyParameters) from telegram.error import TelegramError -from . import entities, store +from . import config, entities, store log = logging.getLogger(__name__) @@ -29,7 +29,7 @@ class DiscordToTelegramForwarder(commands.Cog): def __init__(self, bot: commands.Bot, tg_token: str, routes=None): self.bot = bot - self._tg_bot = Bot(token=tg_token) + self._tg_bot = Bot(token=tg_token, **config.local_api()) # discord_channel_id -> telegram_chat_id self.routes: dict[int, int] = {} for route in routes or []: diff --git a/tgbridge/forwarder.py b/tgbridge/forwarder.py index 35ca4c3..d5f3db4 100644 --- a/tgbridge/forwarder.py +++ b/tgbridge/forwarder.py @@ -4,8 +4,10 @@ import asyncio import io import logging +import os import time +from telegram.error import BadRequest, NetworkError from telegram.ext import Application, MessageHandler, filters from . import config, entities, media, store @@ -18,9 +20,24 @@ log = logging.getLogger(__name__) ALBUM_DEBOUNCE = 1.5 ALBUM_MAX_FILES = 10 DOWNLOAD_TIMEOUT = 90.0 +# ptb defaults read_timeout to 5s, which a slow transfer trips long before +# the ceilings above ever matter. kept under 24s so pool+connect+read stays +# inside the 30s get_file ceiling and ptb stays the one reporting the failure +READ_TIMEOUT = 20.0 +# a local api server lifts telegram's 20 MB cap, but _download still buffers +# the whole file in memory — past this, post the note instead of eating the box +MAX_INGEST_BYTES = 200 * 1024 * 1024 REPLY_SNIPPET = 120 +class TooLarge(Exception): + """An attachment we will not pull into memory.""" + + def __init__(self, megabytes: float | None = None): + super().__init__("attachment too large to bridge") + self.megabytes = megabytes + + class TelegramForwarder: def __init__(self, token, discord_bot=None, avatar_channel_id=None, routes=None, drop_pending=True): @@ -67,7 +84,10 @@ class TelegramForwarder: if removed: log.info("pruned %d expired message mappings", removed) - self._app = Application.builder().token(self.token).build() + builder = Application.builder().token(self.token) + for name, value in config.local_api().items(): + builder = getattr(builder, name)(value) + self._app = builder.build() self._app.add_handler(MessageHandler(filters.ALL, self._on_update)) self._app.add_error_handler(self._on_error) @@ -125,14 +145,45 @@ class TelegramForwarder: # ── downloads ─────────────────────────────────────────────────────────── async def _download(self, file_id: str): - """-> (bytes, filename). Raises on failure.""" - tg_file = await asyncio.wait_for( - self._app.bot.get_file(file_id), timeout=30.0) - buf = io.BytesIO() - await asyncio.wait_for( - tg_file.download_to_memory(buf), timeout=DOWNLOAD_TIMEOUT) - path = tg_file.file_path or "" - return buf.getvalue(), (path.split("/")[-1] or "file") + """-> (bytes, filename). Raises on failure. + + Retried once: a dropped read mid-transfer is the one failure here + nothing upstream recovers from. BadRequest subclasses NetworkError, + so it has to be ruled out first or a rejection gets re-sent. + """ + for attempt in (1, 2): + try: + tg_file = await asyncio.wait_for( + self._app.bot.get_file(file_id, read_timeout=READ_TIMEOUT), + timeout=30.0) + size = tg_file.file_size or 0 + if not size and os.path.isabs(tg_file.file_path or ""): + # local mode hands back a path, not a url — if the + # server left file_size out, ask the filesystem + try: + size = os.path.getsize(tg_file.file_path) + except OSError: + size = 0 + if size > MAX_INGEST_BYTES: + raise TooLarge(size / 1048576) + buf = io.BytesIO() + await asyncio.wait_for( + tg_file.download_to_memory(buf, read_timeout=READ_TIMEOUT), + timeout=DOWNLOAD_TIMEOUT) + except BadRequest as exc: + # only the cloud api refuses outright; a local server will + # hand the file over and MAX_INGEST_BYTES becomes the limit + if "too big" in str(exc).lower(): + raise TooLarge() from exc + raise + except NetworkError as exc: + if attempt == 2: + raise + log.info("download retry after %s: %s", + type(exc).__name__, exc) + continue + path = tg_file.file_path or "" + return buf.getvalue(), (path.split("/")[-1] or "file") # ── context lines ─────────────────────────────────────────────────────── @@ -347,8 +398,16 @@ class TelegramForwarder: for message in messages: try: got, note = await self._render_media(message) + except TooLarge as exc: + what = _describe_media(message) or "attachment" + size = f" — {exc.megabytes:.0f} MB" if exc.megabytes else "" + log.info("%s in message %s too large to bridge%s", + what, message.message_id, size) + got, note = [], f"*(attachment too large to bridge{size})*" except Exception as exc: - log.warning("media render failed: %s", exc) + log.warning("media render failed for %s in message %s: %s: %s", + _describe_media(message) or "attachment", + message.message_id, type(exc).__name__, exc) got, note = [], "*(attachment could not be fetched)*" files.extend(got) if note: