#!/bin/sh
# Hang watchdog (cron every 5 min, see install-cron.sh). Docker already restarts the server when
# it crashes; this covers the other case: container "running" but the server not answering.
# Restarts it after 3 failed checks in a row (~15 min). Skips a container stopped on purpose.
. "$(dirname "$0")/env.sh"
cd "$D"
STATE=.watchdog-fails
LOG=watchdog.log
[ "$(docker inspect -f '{{.State.Status}}' "$NAME" 2>/dev/null)" = running ] || { rm -f $STATE; exit 0; }
if python3 tools/hlq.py info 127.0.0.1 "$PORT" >/dev/null 2>&1 || \
   { sleep 5; python3 tools/hlq.py info 127.0.0.1 "$PORT" >/dev/null 2>&1; }; then
  rm -f $STATE
else
  n=$(( $(cat $STATE 2>/dev/null || echo 0) + 1 ))
  echo $n > $STATE
  echo "$(date '+%F %T') no answer ($n/3)" >> $LOG
  if [ $n -ge 3 ]; then
    echo "$(date '+%F %T') restarting $NAME" >> $LOG
    docker restart "$NAME" >/dev/null 2>&1
    rm -f $STATE
  fi
fi