#!/bin/sh
# Hang watchdog (cron every 5 min). Docker already restarts the server when it crashes; this covers
# the other case: container "running" but the server not answering. Restarts it after 3 failed
# checks in a row (~15 min). Does nothing if the container was stopped on purpose.
# DRYRUN=1 logs without restarting; PORT=... overrides the port (testing).
. "$(dirname "$0")/env.sh"
STATE=logs/.watchdog-fails
LOG=logs/watchdog.log
PORT=${PORT:-$QPORT}
[ "$(docker inspect -f '{{.State.Status}}' "$NAME" 2>/dev/null)" = running ] || { rm -f $STATE; exit 0; }
if python3 tools/qlq.py 127.0.0.1 "$PORT" >/dev/null 2>&1; then
  rm -f $STATE
else
  n=$(( $(cat $STATE 2>/dev/null || echo 0) + 1 ))
  echo $n > $STATE
  echo "$(date '+%F %T') no answer ($n/3)" >> $LOG
  if [ $n -ge 3 ]; then
    echo "$(date '+%F %T') restarting $NAME" >> $LOG
    [ -n "$DRYRUN" ] || docker restart "$NAME" >/dev/null 2>&1
    rm -f $STATE
  fi
fi