Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README

2026-07-15 17:45:36 -04:00
commit ae056eaf83
197 changed files with 26127 additions and 0 deletions
@@ -0,0 +1,50 @@
+#!/bin/bash
+# service-health-check.sh — Check all critical services and report failures
+# Silent on success. Reports only when something is down.
+
+CHECKS=()
+FAILED=0
+
+# 1. Core services
+systemctl is-active caddy > /dev/null 2>&1 || { CHECKS+=("Caddy (sign.itpropartner.com)"); FAILED=1; }
+systemctl is-active mysql-tunnel > /dev/null 2>&1 || { CHECKS+=("MySQL SSH tunnel (wphost02)"); FAILED=1; }
+
+# 2. Docker containers
+docker ps --format '{{.Names}}' 2>/dev/null | grep -q searxng || { CHECKS+=("SearXNG container"); FAILED=1; }
+docker ps --format '{{.Names}}' 2>/dev/null | grep -q docuseal || { CHECKS+=("DocuSeal container"); FAILED=1; }
+
+# 3. DocuSeal responding
+curl -sf -o /dev/null http://127.0.0.1:3000 2>/dev/null || { CHECKS+=("DocuSeal web UI"); FAILED=1; }
+
+# 5. SearXNG responding
+curl -sf "http://127.0.0.1:8888/search?q=test&format=json" -o /dev/null 2>/dev/null || { CHECKS+=("SearXNG search"); FAILED=1; }
+
+# 6. WireGuard tunnel to home router
+ping -c 1 -W 3 10.77.0.2 > /dev/null 2>&1 || { CHECKS+=("WireGuard tunnel (home router)"); FAILED=1; }
+
+# 7. MySQL through tunnel
+mysql -h 127.0.0.1 -P 33060 -u apextrackexperience_1781549652 -p"K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS" --skip-ssl -e "SELECT 1" > /dev/null 2>&1 || { CHECKS+=("MySQL database (apextrackexperience)"); FAILED=1; }
+
+# 8. wphost02 reachable
+ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/itpp-infra root@5.161.62.38 "hostname" > /dev/null 2>&1 || { CHECKS+=("wphost02 SSH"); FAILED=1; }
+
+# 9. Hermes gateway process
+ps aux | grep -v grep | grep -q "hermes.*gateway" || { CHECKS+=("Hermes gateway process"); FAILED=1; }
+
+# 10. Cloudflare API token valid
+CLOUD_TOKEN=${CLOUDFLARE_API_TOKEN:-$(grep CLOUDFLARE_API_TOKEN ~/.hermes/.env | cut -d= -f2-)}
+curl -sf -X GET "https://api.cloudflare.com/client/v4/user/tokens/verify" -H "Authorization: Bearer $CLOUD_TOKEN" | python3 -c "import sys,json; sys.exit(0 if json.load(sys.stdin).get('success') else 1)" 2>/dev/null || { CHECKS+=("Cloudflare API token"); FAILED=1; }
+
+# Report
+if [ $FAILED -eq 1 ]; then
+    echo "⚠️  Service Health Alert — $(date +'%b %d, %Y %I:%M %p')"
+    echo ""
+    for c in "${CHECKS[@]}"; do
+        echo "  ❌ $c"
+    done
+    echo ""
+    echo "Total: ${#CHECKS[@]} service(s) unhealthy"
+    exit 1
+fi
+
+exit 0