#!/bin/bash
# hermes-standby-watchdog.sh — Periodic health check for warm standby
# Runs every 10 min via cron on app1-bu.
# If the live netcup box doesn't respond for ~3.5 minutes, takes over.
# Notifies via Telegram + email before failover.
set -euo pipefail

LIVE_HOST="152.53.192.33"
LIVE_NAME="app1 (netcup)"
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
ENDPOINT="https://s3.us-east-1.wasabisys.com"
BUCKET="hermes-vps-backups"
SRC_PREFIX="live"
LOG="/var/log/hermes-standby-watchdog.log"
TELEGRAM_BOT_TOKEN="8359374835:AAFbySqy5s_qGN1bBZ2kPyQErdXGFB_IFN4"
TELEGRAM_CHAT_ID="5813481339"
MAIL_FROM="g@germainebrown.com"
MAIL_TO="g@germainebrown.com"
SMTP_HOST="mail.germainebrown.com"
SMTP_PORT=587
MAIL_PASS="LoveMyBoys1520!"

# Source AWS CLI
if [ -f /opt/awscli-venv/bin/activate ]; then
    source /opt/awscli-venv/bin/activate
fi

log() {
    echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"
}

notify_telegram() {
    local msg="$1"
    curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
        -d "chat_id=${TELEGRAM_CHAT_ID}" \
        -d "text=${msg}" \
        -d "parse_mode=Markdown" >/dev/null 2>&1
}

notify_email() {
    local subject="$1"
    local body="$2"
    python3 -c "
import smtplib
from email.mime.text import MIMEText
msg = MIMEText('''${body}''')
msg['From'] = '${MAIL_FROM}'
msg['To'] = '${MAIL_TO}'
msg['Subject'] = '${subject}'
with smtplib.SMTP('${SMTP_HOST}', ${SMTP_PORT}) as s:
    s.starttls()
    s.login('${MAIL_FROM}', '${MAIL_PASS}')
    s.send_message(msg)
" 2>/dev/null
}

# If Hermes is already running on THIS box, nothing to do
if hermes gateway status >/dev/null 2>&1 || pgrep -f "hermes gateway" >/dev/null 2>&1; then
    exit 0
fi

# If live box is reachable, stay dormant
if ping -c 3 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
    exit 0
fi

# Live box seems down. Confirm over ~3.5 minutes before declaring it dead
log "=== Watchdog: live box ($LIVE_HOST) not responding, confirming over 4 cycles (~3.5 min)... ==="
FAILED=0
for i in 1 2 3 4; do
    sleep 60
    if ! ping -c 2 -W 5 "$LIVE_HOST" >/dev/null 2>&1; then
        FAILED=$((FAILED + 1))
        log "  Confirm cycle $i/4: FAILED (${FAILED} so far)"
    else
        log "  Confirm cycle $i/4: OK — live box recovered"
        exit 0
    fi
done

# All 3 pings failed — live box is dead
log "=== FAILOVER: Live box confirmed down, taking over ==="

TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
ALERT_SUBJECT="🚨 Hermes Failover — $LIVE_NAME is offline"
ALERT_TG="🚨 *Hermes Failover* — $LIVE_NAME is offline

App1 (netcup) stopped responding to pings. Taking over on app1-bu (Hetzner).

Timestamp: $TIMESTAMP
Action: S3 sync → Hermes gateway start

Check $MAIL_TO for full details."
ALERT_EMAIL_BODY="Hermes Failover Alert

The primary Hermes server ($LIVE_NAME — $LIVE_HOST) is not responding to pings after ~3.5 minutes of confirmation checks (4 cycles at 60s intervals).

Time: $TIMESTAMP
Standby: app1-bu.itpropartner.com (5.161.114.8)

Actions taken:
1. Synced latest state from S3 (s3://hermes-vps-backups/live/)
2. Started Hermes gateway on standby
3. This notification sent via Telegram + email

Next steps:
- Verify Hermes is running: systemctl status hermes-gateway
- Check logs: journalctl -u hermes-gateway -n 50
- If app1 netcup box comes back: power it off to avoid split-brain
- Investigate root cause of outage"

# Send Telegram alert
log "Sending Telegram alert..."
notify_telegram "$ALERT_TG"
log "Telegram alert sent"

# Send email alert
log "Sending email alert..."
notify_email "$ALERT_SUBJECT" "$ALERT_EMAIL_BODY"
log "Email alert sent"

# Sync latest state from S3
log "Syncing from s3://$BUCKET/$SRC_PREFIX/..."
aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
    --endpoint-url "$ENDPOINT" \
    --exclude "audio_cache/*" \
    --exclude "image_cache/*" \
    --exclude "cache/*" \
    --exclude "sandboxes/*" \
    --exclude "kanban/*" \
    --exclude "node/*" \
    --exclude "bin/*" \
    --exclude "logs/*" \
    --exclude "*.lock" \
    --exclude "state.db-shm" \
    --exclude "state.db-wal" \
    --no-progress 2>&1 >> "$LOG"

chown -R root:root "$HERMES_HOME" 2>/dev/null || true

# Start Hermes
log "Starting Hermes gateway..."
hermes gateway start 2>&1 >> "$LOG"
log "=== Standby failover complete ==="