93 lines
2.7 KiB
Bash
93 lines
2.7 KiB
Bash
#!/bin/bash
|
|
# hermes-standby-restore.sh — Warm standby restore from S3
|
|
# Install on the old Hetzner server. On boot:
|
|
# 1. Waits for network
|
|
# 2. Pings the live netcup box — if alive, stays dormant (clean exit)
|
|
# 3. If live box is dead, pulls latest state from S3
|
|
# 4. Starts Hermes gateway in its place
|
|
#
|
|
# Designed as a failover: never runs Hermes alongside the live instance.
|
|
set -euo pipefail
|
|
|
|
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
|
BUCKET="hermes-vps-backups"
|
|
SRC_PREFIX="live"
|
|
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
|
LOG="/var/log/hermes-standby-restore.log"
|
|
LIVE_HOST="152.53.192.33"
|
|
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
|
|
|
log() {
|
|
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
|
|
}
|
|
|
|
log "=== Hermes standby restore starting ==="
|
|
|
|
# Step 1: Wait for network
|
|
log "Waiting for network..."
|
|
for i in $(seq 1 30); do
|
|
if ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1; then
|
|
log "Network up after ${i}s"
|
|
break
|
|
fi
|
|
if [ "$i" -eq 30 ]; then
|
|
log "ERROR: No network after 30s — cannot check live status, exiting"
|
|
exit 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
# Step 2: Health check — is the live netcup box alive?
|
|
log "Checking if live Hermes box ($LIVE_HOST) is reachable..."
|
|
if ping -c 2 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
|
|
log "Live box is REACHABLE — staying dormant (clean exit)"
|
|
log "=== Standby skipped: live box healthy ==="
|
|
# Don't start Hermes. The old box stays cold.
|
|
# The restore script + state are on S3 for when we actually need them.
|
|
exit 0
|
|
fi
|
|
|
|
log "Live box is UNREACHABLE — proceeding with failover..."
|
|
|
|
# Step 3: Load AWS CLI
|
|
log "Loading AWS CLI..."
|
|
if [ -f /opt/awscli-venv/bin/activate ]; then
|
|
source /opt/awscli-venv/bin/activate
|
|
elif command -v aws &>/dev/null; then
|
|
log "Using system aws CLI"
|
|
else
|
|
log "ERROR: AWS CLI not found"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 4: Sync latest state from S3
|
|
log "Syncing from s3://$BUCKET/$SRC_PREFIX/ to $HERMES_HOME/..."
|
|
if aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
|
|
--endpoint-url "$ENDPOINT" \
|
|
--exclude "audio_cache/*" \
|
|
--exclude "image_cache/*" \
|
|
--exclude "cache/*" \
|
|
--exclude "sandboxes/*" \
|
|
--exclude "kanban/*" \
|
|
--exclude "node/*" \
|
|
--exclude "bin/*" \
|
|
--exclude "logs/*" \
|
|
--exclude "*.lock" \
|
|
--exclude "state.db-shm" \
|
|
--exclude "state.db-wal" \
|
|
--no-progress 2>&1 | tee -a "$LOG"; then
|
|
|
|
log "Sync completed successfully"
|
|
else
|
|
log "ERROR: Sync failed — Hermes may start with stale data"
|
|
fi
|
|
|
|
# Step 5: Fix permissions
|
|
chown -R root:root "$HERMES_HOME" 2>/dev/null || true
|
|
|
|
# Step 6: Start Hermes (only reached if live box is dead)
|
|
log "=== FAILOVER: Starting Hermes gateway ==="
|
|
hermes gateway start 2>&1 | tee -a "$LOG"
|
|
|
|
log "=== Standby failover complete ==="
|