Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README

2026-07-15 17:45:36 -04:00
commit ae056eaf83
197 changed files with 26127 additions and 0 deletions
@@ -0,0 +1,58 @@
+#!/bin/bash
+# apex-mail-watchdog.sh — Check Apex WPForms email delivery every 5 min
+# Runs from Core via SSH to wphost02. Tests SMTP + checks debug events.
+# Exit codes: 0=OK, 1=SMTP FAIL, 2=DEBUG FAILURES
+
+WPHOST="root@5.161.62.38"
+ALERT_TO="g@germainebrown.com"
+LOG="/var/log/apex-mail-watchdog.log"
+SSH_KEY="/root/.ssh/itpp-infra"
+TIMEOUT=15
+
+log() {
+    echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"
+}
+
+log "Apex mail watchdog check starting..."
+
+# Step 1: Test SMTP connectivity (login only — no email sent)
+TEST_RESULT=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "$WPHOST" \
+  "python3 -c \"
+import smtplib, ssl
+try:
+    ctx = ssl.create_default_context()
+    with smtplib.SMTP('c1113726.sgvps.net', 2525, timeout=$TIMEOUT) as s:
+        s.starttls(context=ctx)
+        s.login('contact@apextrackexperience.com', 'apex.track!!')
+    print('OK')
+except Exception as e:
+    print(f'FAIL: {e}')
+\"" 2>&1)
+
+if echo "$TEST_RESULT" | grep -q "^OK"; then
+    log "SMTP test passed"
+else
+    log "SMTP FAILED: $TEST_RESULT"
+    echo "RESULT:FAIL|$TEST_RESULT"
+    exit 1
+fi
+
+# Step 2: Check WP Mail SMTP debug events for recent failures
+DEBUG_CHECK=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "$WPHOST" \
+  "mysql -u apextrackexperience_1781549652 -p'K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS' apextrackexperience_1781549652 -N -e \
+    \"SELECT COUNT(*) FROM wp_wpmailsmtp_debug_events WHERE event_type = 0 AND created_at >= NOW() - INTERVAL 10 MINUTE;\"" 2>&1)
+
+if echo "$DEBUG_CHECK" | grep -qE '^[0-9]+$'; then
+    FAILURES=$(echo "$DEBUG_CHECK" | tr -d ' ')
+    if [ "$FAILURES" -gt 0 ] 2>/dev/null; then
+        log "WARNING: $FAILURES email failures in last 10 min"
+        echo "RESULT:WARN|${FAILURES} failures detected"
+        exit 2
+    fi
+    log "No recent debug failures"
+else
+    log "DB check failed (non-fatal): $DEBUG_CHECK"
+fi
+
+log "All clear"
+exit 0