Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README

2026-07-15 17:45:36 -04:00
commit ae056eaf83
197 changed files with 26127 additions and 0 deletions
@@ -0,0 +1,92 @@
+#!/bin/bash
+# hermes-standby-restore.sh — Warm standby restore from S3
+# Install on the old Hetzner server. On boot:
+#   1. Waits for network
+#   2. Pings the live netcup box — if alive, stays dormant (clean exit)
+#   3. If live box is dead, pulls latest state from S3
+#   4. Starts Hermes gateway in its place
+#
+# Designed as a failover: never runs Hermes alongside the live instance.
+set -euo pipefail
+
+ENDPOINT="https://s3.us-east-1.wasabisys.com"
+BUCKET="hermes-vps-backups"
+SRC_PREFIX="live"
+HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
+LOG="/var/log/hermes-standby-restore.log"
+LIVE_HOST="152.53.192.33"
+HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
+
+log() {
+    echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
+}
+
+log "=== Hermes standby restore starting ==="
+
+# Step 1: Wait for network
+log "Waiting for network..."
+for i in $(seq 1 30); do
+    if ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1; then
+        log "Network up after ${i}s"
+        break
+    fi
+    if [ "$i" -eq 30 ]; then
+        log "ERROR: No network after 30s — cannot check live status, exiting"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Step 2: Health check — is the live netcup box alive?
+log "Checking if live Hermes box ($LIVE_HOST) is reachable..."
+if ping -c 2 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
+    log "Live box is REACHABLE — staying dormant (clean exit)"
+    log "=== Standby skipped: live box healthy ==="
+    # Don't start Hermes. The old box stays cold.
+    # The restore script + state are on S3 for when we actually need them.
+    exit 0
+fi
+
+log "Live box is UNREACHABLE — proceeding with failover..."
+
+# Step 3: Load AWS CLI
+log "Loading AWS CLI..."
+if [ -f /opt/awscli-venv/bin/activate ]; then
+    source /opt/awscli-venv/bin/activate
+elif command -v aws &>/dev/null; then
+    log "Using system aws CLI"
+else
+    log "ERROR: AWS CLI not found"
+    exit 1
+fi
+
+# Step 4: Sync latest state from S3
+log "Syncing from s3://$BUCKET/$SRC_PREFIX/ to $HERMES_HOME/..."
+if aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
+    --endpoint-url "$ENDPOINT" \
+    --exclude "audio_cache/*" \
+    --exclude "image_cache/*" \
+    --exclude "cache/*" \
+    --exclude "sandboxes/*" \
+    --exclude "kanban/*" \
+    --exclude "node/*" \
+    --exclude "bin/*" \
+    --exclude "logs/*" \
+    --exclude "*.lock" \
+    --exclude "state.db-shm" \
+    --exclude "state.db-wal" \
+    --no-progress 2>&1 | tee -a "$LOG"; then
+    
+    log "Sync completed successfully"
+else
+    log "ERROR: Sync failed — Hermes may start with stale data"
+fi
+
+# Step 5: Fix permissions
+chown -R root:root "$HERMES_HOME" 2>/dev/null || true
+
+# Step 6: Start Hermes (only reached if live box is dead)
+log "=== FAILOVER: Starting Hermes gateway ==="
+hermes gateway start 2>&1 | tee -a "$LOG"
+
+log "=== Standby failover complete ==="