Make roster refresh rebuild from SchaleDB base and fail-safe

blue_archive/data.py

@@ -38,7 +38,17 @@ class StudentDB:
         if self._loaded:
             return
         with open(BUNDLED_DB, "r", encoding="utf-8") as f:
-            self.students = json.load(f)
+            students = json.load(f)
+        self._index(students)
+
+    def load_data(self, students: list[dict]) -> None:
+        """Index an already-parsed student list (used by the roster refresher)."""
+        if self._loaded:
+            return
+        self._index(list(students))
+
+    def _index(self, students: list[dict]) -> None:
+        self.students = students
 
         excluded = 0
         for s in self.students:
@@ -81,15 +91,7 @@ class StudentDB:
         )
 
     async def merge_ennead(self, banner_cache: dict) -> int:
-        """Merge characters from the ennead.cc API that aren't in the SchaleDB.
-
-        Fetches the full character list, maps fields to SchaleDB format, and
-        derives IsLimited from banner history.  Only characters with IDs not
-        already present are added — existing SchaleDB entries are never touched.
-
-        Must be called after load() and after banners have been fetched.
-        Returns the number of new characters merged.
-        """
+        """Fetch the ennead.cc character API and merge any new characters."""
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.get(CHARACTER_API, timeout=aiohttp.ClientTimeout(total=15)) as resp:
@@ -101,9 +103,16 @@ class StudentDB:
             log.exception("Failed to fetch ennead character API")
             return 0
 
-        # Build IsLimited from all known banners (0=standard, 1=limited)
-        is_limited = self._build_is_limited(banner_cache)
+        return self.merge_chars(chars, banner_cache)
 
+    def merge_chars(self, chars: list[dict], banner_cache: dict) -> int:
+        """Merge already-fetched ennead characters that aren't in the roster.
+
+        Maps fields to SchaleDB format and derives IsLimited from banner
+        history. Only characters with IDs not already present are added —
+        existing entries are never touched. Returns the number merged.
+        """
+        is_limited = self._build_is_limited(banner_cache)
         try:
             added = self._merge_chars(chars, is_limited)
             if added:
@@ -116,6 +125,7 @@ class StudentDB:
                 )
         except Exception:
             log.exception("Failed to merge ennead characters")
+            return 0
         return added
 
     def _merge_chars(self, chars: list[dict], is_limited: dict[str, int]) -> int:

blue_archive/refresh_roster.py

@@ -1,28 +1,51 @@
-"""Refresh the bundled Blue Archive roster from the live ennead.cc API.
+"""Rebuild the bundled Blue Archive roster from upstream sources.
 
-Loads the current ba_students.json, merges any characters that have been
-released since the last run, and atomically writes the result back. Run weekly
-via the discordbot-refresh systemd timer so fresh clones stay current without
-depending on the live API at startup.
+Downloads the latest SchaleDB export (full fields including portrait data),
+then merges the live ennead.cc character feed for anything SchaleDB doesn't
+cover (variants, welfare, collabs). Writes the result atomically.
+
+Fails WITHOUT writing if any required upstream fetch fails, so a transient
+network error can never shrink the existing roster.
+
+Run weekly via the discordbot-refresh systemd timer.
 """
 import asyncio
 import json
 from pathlib import Path
 
-from blue_archive.data import db, fetch_banners
+import aiohttp
 
+from blue_archive.constants import CHARACTER_API
+from blue_archive.data import StudentDB, fetch_banners
+
+SCHALEDB_URL = "https://raw.githubusercontent.com/SchaleDB/SchaleDB/main/data/en/students.min.json"
 OUT = Path(__file__).with_name("ba_students.json")
 
 
+async def fetch_json(session: aiohttp.ClientSession, url: str, timeout: int = 30) -> list[dict]:
+    async with session.get(url, timeout=aiohttp.ClientTimeout(total=timeout)) as resp:
+        resp.raise_for_status()
+        return json.loads(await resp.text())
+
+
 async def main() -> None:
-    db.load()
-    banners = await fetch_banners()
-    added = await db.merge_ennead(banners)
+    async with aiohttp.ClientSession() as session:
+        base = await fetch_json(session, SCHALEDB_URL)
+        chars = await fetch_json(session, CHARACTER_API)
+        banners = await fetch_banners(session)
+
+    if not chars:
+        raise RuntimeError("ennead character API returned an empty list")
+
+    db = StudentDB()
+    db.load_data(base)
+    added = db.merge_chars(chars, banners)
 
     tmp = OUT.with_suffix(".json.tmp")
     tmp.write_text(json.dumps(db.students, ensure_ascii=False), encoding="utf-8")
     tmp.replace(OUT)  # atomic swap so a reader never sees a partial file
-    print(f"refreshed {OUT.name}: {len(db.students)} students (+{added} new)")
+    print(f"refreshed {OUT.name}: {len(db.students)} students "
+          f"(SchaleDB {len(base)} + {added} ennead)")
 
 
 if __name__ == "__main__":