51 lines
2.3 KiB
Bash
Executable File
51 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# service-health-check.sh — Check all critical services and report failures
|
|
# Silent on success. Reports only when something is down.
|
|
|
|
CHECKS=()
|
|
FAILED=0
|
|
|
|
# 1. Core services
|
|
systemctl is-active caddy > /dev/null 2>&1 || { CHECKS+=("Caddy (sign.itpropartner.com)"); FAILED=1; }
|
|
systemctl is-active mysql-tunnel > /dev/null 2>&1 || { CHECKS+=("MySQL SSH tunnel (wphost02)"); FAILED=1; }
|
|
|
|
# 2. Docker containers
|
|
docker ps --format '{{.Names}}' 2>/dev/null | grep -q searxng || { CHECKS+=("SearXNG container"); FAILED=1; }
|
|
docker ps --format '{{.Names}}' 2>/dev/null | grep -q docuseal || { CHECKS+=("DocuSeal container"); FAILED=1; }
|
|
|
|
# 3. DocuSeal responding
|
|
curl -sf -o /dev/null http://127.0.0.1:3000 2>/dev/null || { CHECKS+=("DocuSeal web UI"); FAILED=1; }
|
|
|
|
# 5. SearXNG responding
|
|
curl -sf "http://127.0.0.1:8888/search?q=test&format=json" -o /dev/null 2>/dev/null || { CHECKS+=("SearXNG search"); FAILED=1; }
|
|
|
|
# 6. WireGuard tunnel to home router
|
|
ping -c 1 -W 3 10.77.0.2 > /dev/null 2>&1 || { CHECKS+=("WireGuard tunnel (home router)"); FAILED=1; }
|
|
|
|
# 7. MySQL through tunnel
|
|
mysql -h 127.0.0.1 -P 33060 -u apextrackexperience_1781549652 -p"K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS" --skip-ssl -e "SELECT 1" > /dev/null 2>&1 || { CHECKS+=("MySQL database (apextrackexperience)"); FAILED=1; }
|
|
|
|
# 8. wphost02 reachable
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/itpp-infra root@5.161.62.38 "hostname" > /dev/null 2>&1 || { CHECKS+=("wphost02 SSH"); FAILED=1; }
|
|
|
|
# 9. Hermes gateway process
|
|
ps aux | grep -v grep | grep -q "hermes.*gateway" || { CHECKS+=("Hermes gateway process"); FAILED=1; }
|
|
|
|
# 10. Cloudflare API token valid
|
|
CLOUD_TOKEN=${CLOUDFLARE_API_TOKEN:-$(grep CLOUDFLARE_API_TOKEN ~/.hermes/.env | cut -d= -f2-)}
|
|
curl -sf -X GET "https://api.cloudflare.com/client/v4/user/tokens/verify" -H "Authorization: Bearer $CLOUD_TOKEN" | python3 -c "import sys,json; sys.exit(0 if json.load(sys.stdin).get('success') else 1)" 2>/dev/null || { CHECKS+=("Cloudflare API token"); FAILED=1; }
|
|
|
|
# Report
|
|
if [ $FAILED -eq 1 ]; then
|
|
echo "⚠️ Service Health Alert — $(date +'%b %d, %Y %I:%M %p')"
|
|
echo ""
|
|
for c in "${CHECKS[@]}"; do
|
|
echo " ❌ $c"
|
|
done
|
|
echo ""
|
|
echo "Total: ${#CHECKS[@]} service(s) unhealthy"
|
|
exit 1
|
|
fi
|
|
|
|
exit 0
|