#!/bin/bash
# hermes-standby-restore.sh — Warm standby restore from S3
# Install on the old Hetzner server. On boot:
#   1. Waits for network
#   2. Pings the live netcup box — if alive, stays dormant (clean exit)
#   3. If live box is dead, pulls latest state from S3
#   4. Starts Hermes gateway in its place
#
# Designed as a failover: never runs Hermes alongside the live instance.
set -euo pipefail

ENDPOINT="https://s3.us-east-1.wasabisys.com"
BUCKET="hermes-vps-backups"
SRC_PREFIX="live"
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
LOG="/var/log/hermes-standby-restore.log"
LIVE_HOST="152.53.192.33"
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"

log() {
    echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
}

log "=== Hermes standby restore starting ==="

# Step 1: Wait for network
log "Waiting for network..."
for i in $(seq 1 30); do
    if ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1; then
        log "Network up after ${i}s"
        break
    fi
    if [ "$i" -eq 30 ]; then
        log "ERROR: No network after 30s — cannot check live status, exiting"
        exit 1
    fi
    sleep 1
done

# Step 2: Health check — is the live netcup box alive?
log "Checking if live Hermes box ($LIVE_HOST) is reachable..."
if ping -c 2 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
    log "Live box is REACHABLE — staying dormant (clean exit)"
    log "=== Standby skipped: live box healthy ==="
    # Don't start Hermes. The old box stays cold.
    # The restore script + state are on S3 for when we actually need them.
    exit 0
fi

log "Live box is UNREACHABLE — proceeding with failover..."

# Step 3: Load AWS CLI
log "Loading AWS CLI..."
if [ -f /opt/awscli-venv/bin/activate ]; then
    source /opt/awscli-venv/bin/activate
elif command -v aws &>/dev/null; then
    log "Using system aws CLI"
else
    log "ERROR: AWS CLI not found"
    exit 1
fi

# Step 4: Sync latest state from S3
log "Syncing from s3://$BUCKET/$SRC_PREFIX/ to $HERMES_HOME/..."
if aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
    --endpoint-url "$ENDPOINT" \
    --exclude "audio_cache/*" \
    --exclude "image_cache/*" \
    --exclude "cache/*" \
    --exclude "sandboxes/*" \
    --exclude "kanban/*" \
    --exclude "node/*" \
    --exclude "bin/*" \
    --exclude "logs/*" \
    --exclude "*.lock" \
    --exclude "state.db-shm" \
    --exclude "state.db-wal" \
    --no-progress 2>&1 | tee -a "$LOG"; then
    
    log "Sync completed successfully"
else
    log "ERROR: Sync failed — Hermes may start with stale data"
fi

# Step 5: Fix permissions
chown -R root:root "$HERMES_HOME" 2>/dev/null || true

# Step 6: Start Hermes (only reached if live box is dead)
log "=== FAILOVER: Starting Hermes gateway ==="
hermes gateway start 2>&1 | tee -a "$LOG"

log "=== Standby failover complete ==="