Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README
This commit is contained in:
root
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# hermes-standby-watchdog.sh — Periodic health check for warm standby
|
||||
# Runs every 10 min via cron on app1-bu.
|
||||
# If the live netcup box doesn't respond for ~3.5 minutes, takes over.
|
||||
# Notifies via Telegram + email before failover.
|
||||
set -euo pipefail
|
||||
|
||||
LIVE_HOST="152.53.192.33"
|
||||
LIVE_NAME="app1 (netcup)"
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
SRC_PREFIX="live"
|
||||
LOG="/var/log/hermes-standby-watchdog.log"
|
||||
TELEGRAM_BOT_TOKEN="8359374835:AAFbySqy5s_qGN1bBZ2kPyQErdXGFB_IFN4"
|
||||
TELEGRAM_CHAT_ID="5813481339"
|
||||
MAIL_FROM="g@germainebrown.com"
|
||||
MAIL_TO="g@germainebrown.com"
|
||||
SMTP_HOST="mail.germainebrown.com"
|
||||
SMTP_PORT=587
|
||||
MAIL_PASS="LoveMyBoys1520!"
|
||||
|
||||
# Source AWS CLI
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"
|
||||
}
|
||||
|
||||
notify_telegram() {
|
||||
local msg="$1"
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-d "chat_id=${TELEGRAM_CHAT_ID}" \
|
||||
-d "text=${msg}" \
|
||||
-d "parse_mode=Markdown" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
notify_email() {
|
||||
local subject="$1"
|
||||
local body="$2"
|
||||
python3 -c "
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
msg = MIMEText('''${body}''')
|
||||
msg['From'] = '${MAIL_FROM}'
|
||||
msg['To'] = '${MAIL_TO}'
|
||||
msg['Subject'] = '${subject}'
|
||||
with smtplib.SMTP('${SMTP_HOST}', ${SMTP_PORT}) as s:
|
||||
s.starttls()
|
||||
s.login('${MAIL_FROM}', '${MAIL_PASS}')
|
||||
s.send_message(msg)
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
# If Hermes is already running on THIS box, nothing to do
|
||||
if hermes gateway status >/dev/null 2>&1 || pgrep -f "hermes gateway" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If live box is reachable, stay dormant
|
||||
if ping -c 3 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Live box seems down. Confirm over ~3.5 minutes before declaring it dead
|
||||
log "=== Watchdog: live box ($LIVE_HOST) not responding, confirming over 4 cycles (~3.5 min)... ==="
|
||||
FAILED=0
|
||||
for i in 1 2 3 4; do
|
||||
sleep 60
|
||||
if ! ping -c 2 -W 5 "$LIVE_HOST" >/dev/null 2>&1; then
|
||||
FAILED=$((FAILED + 1))
|
||||
log " Confirm cycle $i/4: FAILED (${FAILED} so far)"
|
||||
else
|
||||
log " Confirm cycle $i/4: OK — live box recovered"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# All 3 pings failed — live box is dead
|
||||
log "=== FAILOVER: Live box confirmed down, taking over ==="
|
||||
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
ALERT_SUBJECT="🚨 Hermes Failover — $LIVE_NAME is offline"
|
||||
ALERT_TG="🚨 *Hermes Failover* — $LIVE_NAME is offline
|
||||
|
||||
App1 (netcup) stopped responding to pings. Taking over on app1-bu (Hetzner).
|
||||
|
||||
Timestamp: $TIMESTAMP
|
||||
Action: S3 sync → Hermes gateway start
|
||||
|
||||
Check $MAIL_TO for full details."
|
||||
ALERT_EMAIL_BODY="Hermes Failover Alert
|
||||
|
||||
The primary Hermes server ($LIVE_NAME — $LIVE_HOST) is not responding to pings after ~3.5 minutes of confirmation checks (4 cycles at 60s intervals).
|
||||
|
||||
Time: $TIMESTAMP
|
||||
Standby: app1-bu.itpropartner.com (5.161.114.8)
|
||||
|
||||
Actions taken:
|
||||
1. Synced latest state from S3 (s3://hermes-vps-backups/live/)
|
||||
2. Started Hermes gateway on standby
|
||||
3. This notification sent via Telegram + email
|
||||
|
||||
Next steps:
|
||||
- Verify Hermes is running: systemctl status hermes-gateway
|
||||
- Check logs: journalctl -u hermes-gateway -n 50
|
||||
- If app1 netcup box comes back: power it off to avoid split-brain
|
||||
- Investigate root cause of outage"
|
||||
|
||||
# Send Telegram alert
|
||||
log "Sending Telegram alert..."
|
||||
notify_telegram "$ALERT_TG"
|
||||
log "Telegram alert sent"
|
||||
|
||||
# Send email alert
|
||||
log "Sending email alert..."
|
||||
notify_email "$ALERT_SUBJECT" "$ALERT_EMAIL_BODY"
|
||||
log "Email alert sent"
|
||||
|
||||
# Sync latest state from S3
|
||||
log "Syncing from s3://$BUCKET/$SRC_PREFIX/..."
|
||||
aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude "audio_cache/*" \
|
||||
--exclude "image_cache/*" \
|
||||
--exclude "cache/*" \
|
||||
--exclude "sandboxes/*" \
|
||||
--exclude "kanban/*" \
|
||||
--exclude "node/*" \
|
||||
--exclude "bin/*" \
|
||||
--exclude "logs/*" \
|
||||
--exclude "*.lock" \
|
||||
--exclude "state.db-shm" \
|
||||
--exclude "state.db-wal" \
|
||||
--no-progress 2>&1 >> "$LOG"
|
||||
|
||||
chown -R root:root "$HERMES_HOME" 2>/dev/null || true
|
||||
|
||||
# Start Hermes
|
||||
log "Starting Hermes gateway..."
|
||||
hermes gateway start 2>&1 >> "$LOG"
|
||||
log "=== Standby failover complete ==="
|
||||
Reference in New Issue
Block a user