tgbridge: support a self-hosted bot api server, and say when a file is too big

The cloud api refuses getFile over 20 MB, so an oversize video or gif never
reached ffmpeg and the bridge blamed the fetch. Six of the nine media
failures in the last month were that; the other three were a dropped read.

- config.local_api() turns TELEGRAM_API_BASE_URL into base_url/base_file_url/
  local_mode. Both directions read it, or only one of them migrates.
- in local mode getFile hands back a path on disk and ptb reads it directly,
  so there is no 20 MB cap and no http transfer to drop.
- MAX_INGEST_BYTES keeps that from being a way to pull a gigabyte into the
  bot's memory; falls back to stat() when the server omits file_size.
- explicit read_timeout: ptb defaults to 5s, which is what "Timed out" was.
  one retry covers a dropped read. BadRequest subclasses NetworkError, so it
  is re-raised first or an oversize rejection gets re-sent.
- the note now distinguishes too-large from could-not-fetch, and the warning
  records which attachment and which message.

Co-Authored-By: Claude Opus 5 <[email protected]>

.gitignore

@@ -28,3 +28,4 @@ _check_students.py
 _check_names.py
 _check_banner_api.py
 
+credentials_test.txt

tgbridge/config.py

@@ -117,3 +117,18 @@ def load_routes() -> list[Route]:
 
 def avatar_channel_id() -> int | None:
     return _int_or_none(os.getenv("AVATAR_CACHE_CHANNEL_ID"))
+
+
+def local_api() -> dict:
+    """-> bot kwargs for a self-hosted bot api server, {} for the cloud one.
+
+    A local server lifts telegram's 20 MB getFile cap and hands back a path on
+    disk instead of a url, which ptb reads for itself once local_mode is on.
+    Both directions read this or only one of them migrates.
+    """
+    base = os.getenv("TELEGRAM_API_BASE_URL", "").strip().rstrip("/")
+    if not base:
+        return {}
+    return {"base_url": f"{base}/bot",
+            "base_file_url": f"{base}/file/bot",
+            "local_mode": True}

tgbridge/discord_to_telegram.py

@@ -15,7 +15,7 @@ from telegram import (Bot, InputMediaAudio, InputMediaDocument,
                       InputMediaPhoto, InputMediaVideo, ReplyParameters)
 from telegram.error import TelegramError
 
-from . import entities, store
+from . import config, entities, store
 
 log = logging.getLogger(__name__)
 
@@ -29,7 +29,7 @@ class DiscordToTelegramForwarder(commands.Cog):
 
     def __init__(self, bot: commands.Bot, tg_token: str, routes=None):
         self.bot = bot
-        self._tg_bot = Bot(token=tg_token)
+        self._tg_bot = Bot(token=tg_token, **config.local_api())
         # discord_channel_id -> telegram_chat_id
         self.routes: dict[int, int] = {}
         for route in routes or []:

tgbridge/forwarder.py

@@ -4,8 +4,10 @@
 import asyncio
 import io
 import logging
+import os
 import time
 
+from telegram.error import BadRequest, NetworkError
 from telegram.ext import Application, MessageHandler, filters
 
 from . import config, entities, media, store
@@ -18,9 +20,24 @@ log = logging.getLogger(__name__)
 ALBUM_DEBOUNCE = 1.5
 ALBUM_MAX_FILES = 10
 DOWNLOAD_TIMEOUT = 90.0
+# ptb defaults read_timeout to 5s, which a slow transfer trips long before
+# the ceilings above ever matter. kept under 24s so pool+connect+read stays
+# inside the 30s get_file ceiling and ptb stays the one reporting the failure
+READ_TIMEOUT = 20.0
+# a local api server lifts telegram's 20 MB cap, but _download still buffers
+# the whole file in memory — past this, post the note instead of eating the box
+MAX_INGEST_BYTES = 200 * 1024 * 1024
 REPLY_SNIPPET = 120
 
 
+class TooLarge(Exception):
+    """An attachment we will not pull into memory."""
+
+    def __init__(self, megabytes: float | None = None):
+        super().__init__("attachment too large to bridge")
+        self.megabytes = megabytes
+
+
 class TelegramForwarder:
     def __init__(self, token, discord_bot=None, avatar_channel_id=None,
                  routes=None, drop_pending=True):
@@ -67,7 +84,10 @@ class TelegramForwarder:
             if removed:
                 log.info("pruned %d expired message mappings", removed)
 
-            self._app = Application.builder().token(self.token).build()
+            builder = Application.builder().token(self.token)
+            for name, value in config.local_api().items():
+                builder = getattr(builder, name)(value)
+            self._app = builder.build()
             self._app.add_handler(MessageHandler(filters.ALL, self._on_update))
             self._app.add_error_handler(self._on_error)
 
@@ -125,12 +145,43 @@ class TelegramForwarder:
     # ── downloads ───────────────────────────────────────────────────────────
 
     async def _download(self, file_id: str):
-        """-> (bytes, filename). Raises on failure."""
+        """-> (bytes, filename). Raises on failure.
+
+        Retried once: a dropped read mid-transfer is the one failure here
+        nothing upstream recovers from. BadRequest subclasses NetworkError,
+        so it has to be ruled out first or a rejection gets re-sent.
+        """
+        for attempt in (1, 2):
+            try:
                 tg_file = await asyncio.wait_for(
-            self._app.bot.get_file(file_id), timeout=30.0)
+                    self._app.bot.get_file(file_id, read_timeout=READ_TIMEOUT),
+                    timeout=30.0)
+                size = tg_file.file_size or 0
+                if not size and os.path.isabs(tg_file.file_path or ""):
+                    # local mode hands back a path, not a url — if the
+                    # server left file_size out, ask the filesystem
+                    try:
+                        size = os.path.getsize(tg_file.file_path)
+                    except OSError:
+                        size = 0
+                if size > MAX_INGEST_BYTES:
+                    raise TooLarge(size / 1048576)
                 buf = io.BytesIO()
                 await asyncio.wait_for(
-            tg_file.download_to_memory(buf), timeout=DOWNLOAD_TIMEOUT)
+                    tg_file.download_to_memory(buf, read_timeout=READ_TIMEOUT),
+                    timeout=DOWNLOAD_TIMEOUT)
+            except BadRequest as exc:
+                # only the cloud api refuses outright; a local server will
+                # hand the file over and MAX_INGEST_BYTES becomes the limit
+                if "too big" in str(exc).lower():
+                    raise TooLarge() from exc
+                raise
+            except NetworkError as exc:
+                if attempt == 2:
+                    raise
+                log.info("download retry after %s: %s",
+                         type(exc).__name__, exc)
+                continue
             path = tg_file.file_path or ""
             return buf.getvalue(), (path.split("/")[-1] or "file")
 
@@ -347,8 +398,16 @@ class TelegramForwarder:
         for message in messages:
             try:
                 got, note = await self._render_media(message)
+            except TooLarge as exc:
+                what = _describe_media(message) or "attachment"
+                size = f" — {exc.megabytes:.0f} MB" if exc.megabytes else ""
+                log.info("%s in message %s too large to bridge%s",
+                         what, message.message_id, size)
+                got, note = [], f"*(attachment too large to bridge{size})*"
             except Exception as exc:
-                log.warning("media render failed: %s", exc)
+                log.warning("media render failed for %s in message %s: %s: %s",
+                            _describe_media(message) or "attachment",
+                            message.message_id, type(exc).__name__, exc)
                 got, note = [], "*(attachment could not be fetched)*"
             files.extend(got)
             if note: