esefandClaude Opus 5.5
f8b3501637
The RSSHub feed isn't strictly chronological, so 'post everything above the last guid' could skip tweets. Now a single poller fans out to both webhooks, each remembering posted guids (data/twitter_seen.json, atomic writes): new tweets anywhere above the oldest known one are posted oldest-first, old tweets sliding in after a deletion are ignored, failed posts retry next poll and Discord 429s are honoured. Links go out as fxtwitter.com for real embeds. The owner gets a DM after 90 min of broken feed and again on recovery. Night polling is now 1-2 h. Co-Authored-By: Claude Opus 5.5 <[email protected]>
276 lines
11 KiB
Python
276 lines
11 KiB
Python
# Twitter/X RSS-to-Discord forwarding (feed: self-hosted RSSHub, see esefrss).
|
|
# One poller fetches the feed and fans new entries out to every webhook.
|
|
# Each webhook remembers which tweets it has already posted, so a reordered,
|
|
# partial or briefly broken feed never skips or reposts anything.
|
|
import os
|
|
import json
|
|
import time
|
|
import random
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime
|
|
from zoneinfo import ZoneInfo
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
import aiohttp
|
|
import feedparser
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# The feed comes from RSSHub on yuuka using a real X account's cookie, so
|
|
# polls are jittered and much slower at night to avoid a robotic 24/7 pattern.
|
|
POLL_INTERVAL = (240, 480) # daytime: random 4-8 minutes
|
|
NIGHT_POLL_INTERVAL = (3600, 7200) # night: random 1-2 hours
|
|
NIGHT_HOURS = range(1, 6) # 01:00-05:59 Istanbul; the account posts ~06:00-08:00
|
|
LOCAL_TZ = ZoneInfo('Europe/Istanbul')
|
|
REQUEST_TIMEOUT = 30 # seconds per HTTP call
|
|
SEEN_LIMIT = 500 # remembered guids per webhook (the feed holds 20)
|
|
POST_GAP = 1.5 # seconds between webhook posts
|
|
POST_ATTEMPTS = 3
|
|
ALERT_AFTER = 90 * 60 # feed broken this long -> DM the owner
|
|
|
|
EMBED_HOST = 'fxtwitter.com' # proper Discord embeds; X never sees these
|
|
|
|
|
|
def embed_link(link: str) -> str:
|
|
"""Point a tweet link at fxtwitter so Discord shows a real embed."""
|
|
parts = urlsplit(link.replace('#m', ''))
|
|
if parts.netloc in ('x.com', 'twitter.com', 'www.twitter.com', 'nitter.net'):
|
|
parts = parts._replace(netloc=EMBED_HOST)
|
|
return urlunsplit(parts)
|
|
|
|
|
|
def _guid(entry) -> str:
|
|
return entry.get('id') or entry.get('link', '')
|
|
|
|
|
|
class TwitterRSSForwarder:
|
|
"""Polls the Twitter RSS feed and forwards new items to Discord webhooks.
|
|
|
|
Reads TWITTER_RSS_URL and optional comma-separated TWITTER_RSS_FALLBACKS.
|
|
`webhooks` maps a name (used in the state file) to a webhook URL.
|
|
`alert` is an optional coroutine taking a message, called when the feed
|
|
has been broken for ALERT_AFTER and again when it recovers.
|
|
"""
|
|
|
|
def __init__(self, webhooks: dict[str, str], state_file: str = 'data/twitter_seen.json', alert=None):
|
|
primary = os.getenv('TWITTER_RSS_URL', '')
|
|
fallbacks_raw = os.getenv('TWITTER_RSS_FALLBACKS', '')
|
|
|
|
urls = [primary] if primary else []
|
|
if fallbacks_raw:
|
|
urls.extend(u.strip() for u in fallbacks_raw.split(',') if u.strip())
|
|
|
|
if not urls:
|
|
raise ValueError('TWITTER_RSS_URL is not set')
|
|
|
|
self.rss_urls = urls
|
|
self.webhooks = {name: url for name, url in webhooks.items() if url}
|
|
self.state_file = state_file
|
|
self.alert = alert
|
|
|
|
self.seen: dict[str, list[str]] = {}
|
|
self._running = False
|
|
self._session: aiohttp.ClientSession | None = None
|
|
|
|
self._bad_since: float | None = None
|
|
self._alerted = False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public lifecycle
|
|
# ------------------------------------------------------------------
|
|
|
|
async def start(self):
|
|
self._running = True
|
|
self._load_state()
|
|
|
|
timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT)
|
|
self._session = aiohttp.ClientSession(timeout=timeout)
|
|
|
|
logger.info('twitter rss forwarder started — %s -> %s',
|
|
self.rss_urls[0], ', '.join(self.webhooks) or 'no webhooks')
|
|
|
|
while self._running:
|
|
try:
|
|
await self._poll_cycle()
|
|
except asyncio.CancelledError:
|
|
logger.info('twitter rss forwarder cancelled during poll')
|
|
break
|
|
except Exception:
|
|
logger.exception('unhandled error in twitter rss poll loop — will retry next cycle')
|
|
|
|
try:
|
|
await asyncio.sleep(self._next_delay())
|
|
except asyncio.CancelledError:
|
|
self._running = False
|
|
raise
|
|
|
|
@staticmethod
|
|
def _next_delay() -> float:
|
|
night = datetime.now(LOCAL_TZ).hour in NIGHT_HOURS
|
|
return random.uniform(*(NIGHT_POLL_INTERVAL if night else POLL_INTERVAL))
|
|
|
|
def stop(self):
|
|
self._running = False
|
|
logger.info('twitter rss forwarder stop requested')
|
|
|
|
async def close(self):
|
|
if self._session is not None:
|
|
await self._session.close()
|
|
self._session = None
|
|
logger.debug('twitter rss http session closed')
|
|
|
|
# ------------------------------------------------------------------
|
|
# Poll cycle
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _poll_cycle(self):
|
|
entries, problem = await self._fetch_any()
|
|
if not entries:
|
|
await self._note_bad(problem or 'feed returned no items')
|
|
return
|
|
await self._note_good()
|
|
|
|
for name, webhook in self.webhooks.items():
|
|
await self._forward(name, webhook, entries)
|
|
|
|
async def _fetch_any(self) -> tuple[list[dict], str | None]:
|
|
problems = []
|
|
for url in self.rss_urls:
|
|
try:
|
|
async with self._session.get(url) as resp:
|
|
resp.raise_for_status()
|
|
text = await resp.text()
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
problems.append(f'{url}: {exc}')
|
|
continue
|
|
|
|
feed = feedparser.parse(text)
|
|
if feed.entries:
|
|
return feed.entries, None
|
|
problems.append(f'{url}: no items' + (f' ({feed.bozo_exception})' if feed.bozo else ''))
|
|
return [], '; '.join(problems)
|
|
|
|
async def _forward(self, name: str, webhook: str, entries: list[dict]):
|
|
ids = [_guid(e) for e in entries]
|
|
seen = self.seen.get(name)
|
|
|
|
if seen is None:
|
|
# first run for this webhook: remember the current feed, post nothing
|
|
self.seen[name] = ids[::-1]
|
|
self._save_state()
|
|
logger.info('twitter rss [%s] first run — remembered %d tweets', name, len(ids))
|
|
return
|
|
|
|
seen_set = set(seen)
|
|
known = [i for i, g in enumerate(ids) if g in seen_set]
|
|
if not known:
|
|
# nothing in the feed is familiar (long outage or a feed change):
|
|
# re-baseline instead of flooding the channel with the whole page
|
|
logger.warning('twitter rss [%s] no known tweets in feed — re-baselining without posting', name)
|
|
seen.extend(ids[::-1])
|
|
del seen[:-SEEN_LIMIT]
|
|
self._save_state()
|
|
return
|
|
|
|
# Unseen entries above the lowest known one are new (the feed isn't strictly
|
|
# chronological, so they can sit below newer known tweets). Unseen entries
|
|
# below every known one are old tweets that slid in when something above
|
|
# was deleted, so they are only remembered.
|
|
lowest_known = known[-1]
|
|
new = [e for e in entries[:lowest_known] if _guid(e) not in seen_set]
|
|
tail = [g for g in ids[lowest_known:] if g not in seen_set]
|
|
if tail:
|
|
seen.extend(tail)
|
|
del seen[:-SEEN_LIMIT]
|
|
self._save_state()
|
|
|
|
for entry in reversed(new): # oldest first
|
|
if not await self._post(name, webhook, entry):
|
|
break # leave the rest unseen; retried next poll
|
|
seen.append(_guid(entry))
|
|
del seen[:-SEEN_LIMIT]
|
|
self._save_state()
|
|
await asyncio.sleep(POST_GAP)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Discord webhook
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _post(self, name: str, webhook: str, entry) -> bool:
|
|
link = embed_link(entry.get('link', ''))
|
|
payload = {'username': '@CalabiyauLeaks', 'content': link}
|
|
|
|
for _ in range(POST_ATTEMPTS):
|
|
try:
|
|
async with self._session.post(webhook, json=payload) as resp:
|
|
if resp.status in (200, 204):
|
|
logger.info('twitter rss [%s] posted: %s', name, link)
|
|
return True
|
|
if resp.status == 429:
|
|
data = await resp.json(content_type=None)
|
|
wait = float(data.get('retry_after', 2))
|
|
logger.info('twitter rss [%s] rate limited, waiting %.1fs', name, wait)
|
|
await asyncio.sleep(min(wait, 60))
|
|
continue
|
|
body = await resp.text()
|
|
logger.warning('twitter rss [%s] webhook post failed: %s %s', name, resp.status, body[:200])
|
|
if resp.status < 500:
|
|
return False
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning('twitter rss [%s] webhook post error: %s', name, type(exc).__name__)
|
|
await asyncio.sleep(2)
|
|
return False
|
|
|
|
# ------------------------------------------------------------------
|
|
# Outage alerts
|
|
# ------------------------------------------------------------------
|
|
|
|
async def _note_bad(self, problem: str):
|
|
now = time.monotonic()
|
|
if self._bad_since is None:
|
|
self._bad_since = now
|
|
logger.warning('twitter rss feed problem: %s', problem)
|
|
|
|
broken_for = now - self._bad_since
|
|
if not self._alerted and broken_for >= ALERT_AFTER:
|
|
self._alerted = True
|
|
await self._send_alert(
|
|
f'⚠️ esefrss: the @CalabiyauLeaks feed has been broken for {broken_for / 60:.0f} min '
|
|
f'({problem[:300]}).\nIf X locked @esefos2: log in on arona, pass the check, then put a fresh '
|
|
f'auth_token in ~/esefrss/.env and `docker compose up -d --force-recreate` (see esefrss README).'
|
|
)
|
|
|
|
async def _note_good(self):
|
|
if self._alerted:
|
|
await self._send_alert('✅ esefrss: the @CalabiyauLeaks feed is working again.')
|
|
self._bad_since = None
|
|
self._alerted = False
|
|
|
|
async def _send_alert(self, text: str):
|
|
if self.alert is None:
|
|
return
|
|
try:
|
|
await self.alert(text)
|
|
except Exception:
|
|
logger.exception('twitter rss alert failed')
|
|
|
|
# ------------------------------------------------------------------
|
|
# Persistence
|
|
# ------------------------------------------------------------------
|
|
|
|
def _load_state(self):
|
|
if os.path.exists(self.state_file):
|
|
with open(self.state_file, 'r') as f:
|
|
self.seen = {k: list(v) for k, v in json.load(f).items()}
|
|
|
|
def _save_state(self):
|
|
os.makedirs(os.path.dirname(self.state_file) or '.', exist_ok=True)
|
|
tmp = self.state_file + '.tmp'
|
|
with open(tmp, 'w') as f:
|
|
json.dump(self.seen, f)
|
|
os.replace(tmp, self.state_file)
|