#!/bin/bash
# Restart a seat whose Xvfb or relay died out from under it.
# lab-live.sh waits on the relay pid only; a dead Xvfb leaves the unit
# "active" while every session is broken (observed three times on the
# AWS lab). Runs from a systemd timer as root.
set -u

now=$(awk '{printf "%d", $1 * 1000000}' /proc/uptime)

for SEAT in 1 2 3; do
  UNIT="w3cs-live@$SEAT.service"
  systemctl is-active --quiet "$UNIT" || continue

  # Give a freshly (re)started seat time to bring Xvfb and the relay up.
  started=$(systemctl show -p ActiveEnterTimestampMonotonic --value "$UNIT")
  [ -n "$started" ] && [ "$((now - started))" -lt 60000000 ] && continue

  missing=""
  pgrep -f "Xvfb :1$SEAT " >/dev/null || missing="Xvfb"
  pgrep -f "w3cs-webrtc-relay.*--display :1$SEAT" >/dev/null || \
    missing="$missing relay"

  if [ -n "$missing" ]; then
    echo "w3cs-watchdog: seat $SEAT missing:$missing -> restarting $UNIT"
    systemctl restart "$UNIT"
  fi
done