Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README
This commit is contained in:
root
@@ -0,0 +1 @@
|
||||
["1404da5b964ac5b16e6e20df75288618", "3e6395e846559d98c7a64f8cb819ec8d", "7274830df1c933aa0b52fcc0bf5df0e1", "74800981c1ef165e6eb624b4a4cd406e", "1f2b0f295cbf285d0e64c18b50a6a32a", "a15697e65e2a1b78a3dfd71ecd22cca6", "48802b788421f2d3062ec6a112e841e0", "f707b4979bd0e9295ae29d7e747d1df2"]
|
||||
@@ -0,0 +1 @@
|
||||
["4af3bda5dd62860ca3dd6971ae31b470", "f28e74ff09034fca01ed7ec76605a9fd"]
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch body of email sequence number 2 from shonuff inbox."""
|
||||
import imaplib, email
|
||||
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
USER = "shonuff@germainebrown.com"
|
||||
PW = "Catches.bullets1985"
|
||||
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(USER, PW)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.fetch("2", "(RFC822)")
|
||||
if status == "OK":
|
||||
msg = email.message_from_bytes(data[0][1])
|
||||
body = ""
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
elif ct == "text/html" and not body:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += "[HTML content omitted]\n" + payload.decode("utf-8", errors="replace")[:500]
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
body = payload.decode("utf-8", errors="replace")
|
||||
print(body[:3000])
|
||||
else:
|
||||
print(f"Failed: {status}")
|
||||
|
||||
conn.logout()
|
||||
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
nohup systemctl --user start hermes-gateway-anita > /tmp/anita-start.log 2>&1 &
|
||||
echo "Anita gateway restart dispatched"
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
"""apex-mail-watchdog.py — Check Apex WPForms email delivery every 5 min.
|
||||
|
||||
Connects to the Apex SMTP, sends a test, and checks debug events for
|
||||
recent failures. If mail is down, sends an alert via Hermes cron.
|
||||
"""
|
||||
import smtplib, ssl, json, sys, os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
import mysql.connector
|
||||
|
||||
LOG = "/var/log/apex-mail-watchdog.log"
|
||||
SMTP_HOST = "c1113726.sgvps.net"
|
||||
SMTP_PORT = 2525
|
||||
SMTP_USER = "contact@apextrackexperience.com"
|
||||
SMTP_PASS = "apex.track!!"
|
||||
ALERT_TO = "g@germainebrown.com"
|
||||
FROM = "contact@apextrackexperience.com"
|
||||
|
||||
def log(msg):
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
line = f"[{ts}] {msg}"
|
||||
print(line)
|
||||
with open(LOG, "a") as f:
|
||||
f.write(line + "\n")
|
||||
|
||||
def check_smtp():
|
||||
"""Try sending a test email to the admin address."""
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=15) as s:
|
||||
s.starttls(context=ctx)
|
||||
s.login(SMTP_USER, SMTP_PASS)
|
||||
msg = (
|
||||
f"From: {FROM}\n"
|
||||
f"To: {ALERT_TO}\n"
|
||||
f"Subject: Apex Mail Watchdog Test\n\n"
|
||||
f"Test email from apex-mail-watchdog.py — {datetime.now().isoformat()}"
|
||||
)
|
||||
s.sendmail(FROM, [ALERT_TO], msg)
|
||||
return True, None
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
def check_debug_events():
|
||||
"""Check WP Mail SMTP debug table for recent failures (> 0 in last 10 min)."""
|
||||
try:
|
||||
conn = mysql.connector.connect(
|
||||
host="localhost",
|
||||
user="apextrackexperience_1781549652",
|
||||
password="K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS",
|
||||
database="apextrackexperience_1781549652",
|
||||
)
|
||||
cursor = conn.cursor()
|
||||
# Count errors (event_type=0) in last 10 minutes
|
||||
cursor.execute(
|
||||
"SELECT COUNT(*) FROM wp_wpmailsmtp_debug_events "
|
||||
"WHERE event_type = 0 AND created_at >= NOW() - INTERVAL 10 MINUTE"
|
||||
)
|
||||
count = cursor.fetchone()[0]
|
||||
conn.close()
|
||||
return count, None
|
||||
except Exception as e:
|
||||
return None, str(e)
|
||||
|
||||
def main():
|
||||
log("Apex mail watchdog check starting...")
|
||||
|
||||
smtp_ok, smtp_err = check_smtp()
|
||||
errors, debug_err = check_debug_events()
|
||||
|
||||
if not smtp_ok:
|
||||
log(f"SMTP FAILED: {smtp_err}")
|
||||
print(f"RESULT:FAIL|{smtp_err}")
|
||||
sys.exit(1)
|
||||
|
||||
if errors and errors > 0:
|
||||
log(f"DEBUG EVENTS: {errors} failures in last 10 min")
|
||||
print(f"RESULT:WARN|{errors} email failures detected")
|
||||
sys.exit(2)
|
||||
|
||||
if debug_err:
|
||||
log(f"DB ERROR: {debug_err}")
|
||||
# Don't fail on DB errors — SMTP test passed
|
||||
|
||||
log("SMTP OK — all clear")
|
||||
print("RESULT:OK|Mail delivery healthy")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# apex-mail-watchdog.sh — Check Apex WPForms email delivery every 5 min
|
||||
# Runs from Core via SSH to wphost02. Tests SMTP + checks debug events.
|
||||
# Exit codes: 0=OK, 1=SMTP FAIL, 2=DEBUG FAILURES
|
||||
|
||||
WPHOST="root@5.161.62.38"
|
||||
ALERT_TO="g@germainebrown.com"
|
||||
LOG="/var/log/apex-mail-watchdog.log"
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
TIMEOUT=15
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"
|
||||
}
|
||||
|
||||
log "Apex mail watchdog check starting..."
|
||||
|
||||
# Step 1: Test SMTP connectivity (login only — no email sent)
|
||||
TEST_RESULT=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "$WPHOST" \
|
||||
"python3 -c \"
|
||||
import smtplib, ssl
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
with smtplib.SMTP('c1113726.sgvps.net', 2525, timeout=$TIMEOUT) as s:
|
||||
s.starttls(context=ctx)
|
||||
s.login('contact@apextrackexperience.com', 'apex.track!!')
|
||||
print('OK')
|
||||
except Exception as e:
|
||||
print(f'FAIL: {e}')
|
||||
\"" 2>&1)
|
||||
|
||||
if echo "$TEST_RESULT" | grep -q "^OK"; then
|
||||
log "SMTP test passed"
|
||||
else
|
||||
log "SMTP FAILED: $TEST_RESULT"
|
||||
echo "RESULT:FAIL|$TEST_RESULT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 2: Check WP Mail SMTP debug events for recent failures
|
||||
DEBUG_CHECK=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=5 "$WPHOST" \
|
||||
"mysql -u apextrackexperience_1781549652 -p'K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS' apextrackexperience_1781549652 -N -e \
|
||||
\"SELECT COUNT(*) FROM wp_wpmailsmtp_debug_events WHERE event_type = 0 AND created_at >= NOW() - INTERVAL 10 MINUTE;\"" 2>&1)
|
||||
|
||||
if echo "$DEBUG_CHECK" | grep -qE '^[0-9]+$'; then
|
||||
FAILURES=$(echo "$DEBUG_CHECK" | tr -d ' ')
|
||||
if [ "$FAILURES" -gt 0 ] 2>/dev/null; then
|
||||
log "WARNING: $FAILURES email failures in last 10 min"
|
||||
echo "RESULT:WARN|${FAILURES} failures detected"
|
||||
exit 2
|
||||
fi
|
||||
log "No recent debug failures"
|
||||
else
|
||||
log "DB check failed (non-fatal): $DEBUG_CHECK"
|
||||
fi
|
||||
|
||||
log "All clear"
|
||||
exit 0
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
# Comprehensive server audit script
|
||||
# Usage: ./audit-server.sh <host> [user]
|
||||
# Collects system info, Docker, services, ports, web servers, databases, packages, users, cron
|
||||
|
||||
HOST="$1"
|
||||
USER="${2:-root}"
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
SSH_CMD="ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 $USER@$HOST"
|
||||
|
||||
echo "=============================================="
|
||||
echo "AUDIT: $HOST (user: $USER)"
|
||||
echo "=============================================="
|
||||
|
||||
# 1. System info
|
||||
echo "--- SYSTEM INFO ---"
|
||||
$SSH_CMD "hostname; cat /etc/os-release 2>/dev/null | head -5; uname -a; uptime; echo 'CPU:'; nproc; lscpu 2>/dev/null | grep 'Model name' | head -1; echo 'RAM:'; free -h | head -2; echo 'DISK:'; df -h / | tail -1"
|
||||
|
||||
# 2. Docker containers
|
||||
echo "--- DOCKER CONTAINERS ---"
|
||||
$SSH_CMD "docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null || echo 'Docker not available'"
|
||||
|
||||
echo "--- DOCKER IMAGES ---"
|
||||
$SSH_CMD "docker images --format 'table {{.Repository}}\t{{.Tag}}\t{{.Size}}' 2>/dev/null || echo 'Docker not available'"
|
||||
|
||||
echo "--- DOCKER COMPOSE ---"
|
||||
$SSH_CMD "ls /root/docker-compose*.yml /root/*/docker-compose*.yml /root/docker/*.yml /opt/*/docker-compose*.yml 2>/dev/null || echo 'No docker-compose files found in common locations'"
|
||||
|
||||
# 3. Systemd services
|
||||
echo "--- RUNNING SYSTEMD SERVICES ---"
|
||||
$SSH_CMD "systemctl list-units --type=service --state=running --no-pager --no-legend 2>/dev/null | head -40"
|
||||
|
||||
echo "--- CUSTOM SYSTEMD SERVICES ---"
|
||||
$SSH_CMD "ls /etc/systemd/system/*.service 2>/dev/null | grep -v -E '(systemd-|getty|serial-getty|dbus|sshd|cron|networking|ufw|ssh)' | head -30"
|
||||
|
||||
# 4. Listening ports
|
||||
echo "--- TCP LISTENING PORTS ---"
|
||||
$SSH_CMD "ss -tlnp 2>/dev/null | head -30"
|
||||
|
||||
echo "--- UDP LISTENING PORTS ---"
|
||||
$SSH_CMD "ss -ulnp 2>/dev/null | head -20"
|
||||
|
||||
# 5. Web servers
|
||||
echo "--- WEB SERVERS ---"
|
||||
$SSH_CMD "which nginx apache2 caddy 2>/dev/null; echo '---'; ls /etc/nginx/sites-enabled/ 2>/dev/null; ls /etc/apache2/sites-enabled/ 2>/dev/null; cat /etc/caddy/Caddyfile 2>/dev/null | head -30"
|
||||
|
||||
# 6. Databases
|
||||
echo "--- DATABASES ---"
|
||||
$SSH_CMD "which mysql mysqld psql postgres sqlite3 mongod redis-server 2>/dev/null; echo '---'; systemctl is-active mysql mariadb postgresql mongod redis 2>/dev/null"
|
||||
|
||||
# 7. Notable packages
|
||||
echo "--- NOTABLE PACKAGES ---"
|
||||
$SSH_CMD "dpkg -l 2>/dev/null | grep -iE 'docker|nginx|apache|mysql|postgres|redis|mongodb|node|python|php|java|go|rust|ruby|caddy|traccar|unifi' | head -30"
|
||||
|
||||
# 8. Users with home directories
|
||||
echo "--- HOME DIRECTORIES ---"
|
||||
$SSH_CMD "ls /home/ 2>/dev/null; echo '---'; cat /etc/passwd 2>/dev/null | grep -E '/home/|/root' | cut -d: -f1"
|
||||
|
||||
# 9. Custom scripts and cron
|
||||
echo "--- CRONTAB (root) ---"
|
||||
$SSH_CMD "crontab -l 2>/dev/null || echo 'No crontab for root'"
|
||||
|
||||
echo "--- CUSTOM SCRIPTS ---"
|
||||
$SSH_CMD "ls /root/*.sh /root/*.py 2>/dev/null || echo 'No scripts found in /root'"
|
||||
$SSH_CMD "ls /usr/local/bin/*.sh /usr/local/bin/*.py 2>/dev/null || echo 'No scripts found in /usr/local/bin'"
|
||||
|
||||
echo "=============================================="
|
||||
echo "END AUDIT: $HOST"
|
||||
echo "=============================================="
|
||||
echo ""
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# backup-audit-check.sh — Daily check: was yesterday's full backup successful?
|
||||
# Runs ~1h after scheduled backup (2 AM UTC). Reports to Germaine.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
PREFIX="hermes-full-backup"
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
# Get the latest backup file
|
||||
latest=$(aws s3 ls "s3://$BUCKET/$PREFIX/" --endpoint-url "$ENDPOINT" 2>&1 | grep "\.tar\.gz$" | sort | tail -1)
|
||||
|
||||
if [ -z "$latest" ]; then
|
||||
echo "🔴 BACKUP AUDIT: No full backup archives found on S3!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
date_str=$(echo "$latest" | awk '{print $1}')
|
||||
size_bytes=$(echo "$latest" | awk '{print $3}')
|
||||
file_name=$(echo "$latest" | awk '{print $4}')
|
||||
|
||||
# Convert date to epoch
|
||||
backup_epoch=$(date -d "$date_str" +%s 2>/dev/null || date -j -f "%Y-%m-%d" "$date_str" +%s 2>/dev/null)
|
||||
now_epoch=$(date +%s)
|
||||
age_hours=$(( (now_epoch - backup_epoch) / 3600 ))
|
||||
|
||||
if [ "$age_hours" -lt 36 ]; then
|
||||
echo "✅ BACKUP OK — Last full backup: $file_name ($date_str, ${size_bytes}B, ${age_hours}h ago)"
|
||||
exit 0
|
||||
elif [ "$age_hours" -lt 60 ]; then
|
||||
echo "⚠️ BACKUP WARNING — Last backup $age_hours hours old: $file_name"
|
||||
exit 0
|
||||
else
|
||||
echo "🔴 BACKUP STALE — Last backup was $age_hours hours ago: $file_name"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
BACKUP_DIR="/tmp/unms-backup-$(date +%Y%m%d_%H%M%S)"
|
||||
BUCKET="itpropartner-backups"
|
||||
AWS_ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "=== UISP/UNMS Backup — $(date) ==="
|
||||
|
||||
# 1. Dump PostgreSQL database(s)
|
||||
echo "→ Dumping PostgreSQL databases..."
|
||||
mkdir -p "$BACKUP_DIR/postgres"
|
||||
|
||||
# Get list of databases from the running container
|
||||
DB_NAMES=$(docker exec unms-postgres psql -U postgres -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;" 2>/dev/null | grep -v '^\s*$' | tr -d ' ')
|
||||
|
||||
for DB in $DB_NAMES; do
|
||||
echo " Dumping: $DB"
|
||||
docker exec unms-postgres pg_dump -U postgres -d "$DB" --format=custom \
|
||||
2>/dev/null > "$BACKUP_DIR/postgres/${DB}.dump" || \
|
||||
docker exec unms-postgres pg_dump -U postgres -d "$DB" --format=custom \
|
||||
> "$BACKUP_DIR/postgres/${DB}.dump" 2>/dev/null || \
|
||||
echo " WARN: Could not dump $DB (may not exist anymore)"
|
||||
done
|
||||
|
||||
# Also dump the entire Postgres cluster for completeness
|
||||
echo " Dumping all globals..."
|
||||
docker exec unms-postgres pg_dumpall -U postgres -g 2>/dev/null > "$BACKUP_DIR/postgres/globals.sql"
|
||||
|
||||
# 2. Copy Docker compose and config
|
||||
echo "→ Copying Docker compose & config..."
|
||||
mkdir -p "$BACKUP_DIR/unms-app"
|
||||
cp -a /home/unms/app/docker-compose.yml "$BACKUP_DIR/unms-app/" 2>/dev/null || true
|
||||
cp -a /home/unms/app/unms.conf "$BACKUP_DIR/unms-app/" 2>/dev/null || true
|
||||
cp -a /home/unms/app/metadata "$BACKUP_DIR/unms-app/" 2>/dev/null || true
|
||||
|
||||
# 3. Copy data directory (postgres data etc.)
|
||||
echo "→ Copying data directory (this may take a while)..."
|
||||
mkdir -p "$BACKUP_DIR/data"
|
||||
rsync -a --relative /home/unms/data/./ "$BACKUP_DIR/data/" 2>/dev/null || \
|
||||
cp -a /home/unms/data "$BACKUP_DIR/" 2>/dev/null || true
|
||||
|
||||
# 4. Save Docker image info for version pinning
|
||||
echo "→ Saving container image versions..."
|
||||
docker ps --format '{{.Image}}' | sort -u > "$BACKUP_DIR/images.txt"
|
||||
|
||||
# 5. Tar it all up
|
||||
echo "→ Creating archive..."
|
||||
ARCHIVE="/tmp/unms-full-backup-${DATE}.tar.gz"
|
||||
cd /tmp
|
||||
tar czf "$ARCHIVE" -C "$BACKUP_DIR" . 2>/dev/null
|
||||
|
||||
# 6. Upload to Wasabi S3
|
||||
echo "→ Uploading to Wasabi S3..."
|
||||
aws s3 cp "$ARCHIVE" "s3://${BUCKET}/uisp/${DATE}/" \
|
||||
--endpoint-url "$AWS_ENDPOINT" 2>&1
|
||||
|
||||
# 7. Cleanup
|
||||
echo "→ Cleaning up..."
|
||||
rm -rf "$BACKUP_DIR"
|
||||
rm -f "$ARCHIVE"
|
||||
|
||||
echo "=== Backup complete ==="
|
||||
echo "Bucket: ${BUCKET}/uisp/${DATE}/"
|
||||
Executable
+461
@@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IT Pro Partner Portal — Backup & Disaster Recovery
|
||||
|
||||
Backs up the portal database + config + uploaded files to S3.
|
||||
Supports two targets: primary (local region) and cross-region (DR).
|
||||
|
||||
Usage:
|
||||
python3 backup_portal.py # Full backup to configured targets
|
||||
python3 backup_portal.py --target primary # Primary only
|
||||
python3 backup_portal.py --target cross_region # DR only
|
||||
python3 backup_portal.py --restore backup_2026-07-03_040001.tar.gz # Restore from file
|
||||
|
||||
Required env vars:
|
||||
PORTAL_DB_URL — sqlite:///path or postgresql://...
|
||||
PORTAL_FILES_DIR — /path/to/uploads
|
||||
AWS_ACCESS_KEY_ID — Wasabi or AWS IAM key
|
||||
AWS_SECRET_ACCESS_KEY — Wasabi or AWS IAM secret
|
||||
|
||||
Backup targets (configured in backup_config table or env):
|
||||
S3_BUCKET_PRIMARY — itpropartner-backups-prod
|
||||
S3_BUCKET_DR — itpropartner-backups-dr (cross-region)
|
||||
S3_ENDPOINT — https://s3.us-east-1.wasabisys.com (optional)
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# ─── Configuration ────────────────────────────────────────────────
|
||||
|
||||
BACKUP_DIR = Path("/tmp/itpropartner-backups")
|
||||
RETENTION_DAYS = int(os.environ.get("BACKUP_RETENTION_DAYS", "30"))
|
||||
TIMESTAMP = datetime.now(timezone.utc).strftime("%Y-%m-%d_%H%M%S")
|
||||
BACKUP_FILENAME = f"itpropartner_backup_{TIMESTAMP}.tar.gz"
|
||||
CHECKSUM_FILENAME = f"{BACKUP_FILENAME}.sha256"
|
||||
|
||||
# AWS/S3 config
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
|
||||
BACKUP_TARGETS = {}
|
||||
|
||||
primary_bucket = os.environ.get("S3_BUCKET_PRIMARY")
|
||||
if primary_bucket:
|
||||
BACKUP_TARGETS["primary"] = {
|
||||
"bucket": primary_bucket,
|
||||
"region": S3_REGION,
|
||||
"endpoint": S3_ENDPOINT,
|
||||
"prefix": "portal-backup/",
|
||||
}
|
||||
|
||||
dr_bucket = os.environ.get("S3_BUCKET_DR")
|
||||
if dr_bucket:
|
||||
BACKUP_TARGETS["cross_region"] = {
|
||||
"bucket": dr_bucket,
|
||||
"region": os.environ.get("S3_REGION_DR", "us-west-2"),
|
||||
"endpoint": os.environ.get("S3_ENDPOINT_DR", ""),
|
||||
"prefix": "portal-backup/",
|
||||
}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
# ─── Database ─────────────────────────────────────────────────────
|
||||
|
||||
DB_URL = os.environ.get("PORTAL_DB_URL", "")
|
||||
FILES_DIR = os.environ.get("PORTAL_FILES_DIR", "")
|
||||
|
||||
|
||||
class BackupRunner:
|
||||
"""Manages the full backup lifecycle for a single target."""
|
||||
|
||||
def __init__(self, target_name: str, target_config: dict):
|
||||
self.target_name = target_name
|
||||
self.bucket = target_config["bucket"]
|
||||
self.region = target_config["region"]
|
||||
self.endpoint = target_config["endpoint"]
|
||||
self.prefix = target_config["prefix"]
|
||||
self.run_id = str(uuid.uuid4())
|
||||
self.start_time = time.time()
|
||||
self.status = "running"
|
||||
self.error = None
|
||||
self.db_size = 0
|
||||
self.files_size = 0
|
||||
self.archive_size = 0
|
||||
self.checksum = ""
|
||||
|
||||
def run(self) -> dict:
|
||||
"""Execute backup: dump DB, collect files, archive, upload."""
|
||||
try:
|
||||
BACKUP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
work_dir = BACKUP_DIR / self.target_name
|
||||
if work_dir.exists():
|
||||
shutil.rmtree(work_dir)
|
||||
work_dir.mkdir(parents=True)
|
||||
|
||||
# 1. Dump database
|
||||
db_path = self._dump_database(work_dir)
|
||||
if db_path:
|
||||
self.db_size = os.path.getsize(db_path)
|
||||
|
||||
# 2. Collect files (config, uploads, scripts)
|
||||
files_path = self._collect_files(work_dir)
|
||||
|
||||
# 3. Write metadata
|
||||
self._write_metadata(work_dir)
|
||||
|
||||
# 4. Create compressed archive
|
||||
archive_path, self.archive_size, self.checksum = self._archive(work_dir)
|
||||
|
||||
# 5. Upload to S3
|
||||
self._upload(archive_path)
|
||||
|
||||
# 6. Cleanup old backups
|
||||
self._cleanup_old()
|
||||
|
||||
# 7. Cleanup temp
|
||||
shutil.rmtree(work_dir)
|
||||
|
||||
self.status = "success"
|
||||
except Exception as e:
|
||||
self.status = "failed"
|
||||
self.error = str(e)
|
||||
raise
|
||||
|
||||
finally:
|
||||
self.duration = time.time() - self.start_time
|
||||
|
||||
return {
|
||||
"target": self.target_name,
|
||||
"bucket": self.bucket,
|
||||
"status": self.status,
|
||||
"archive_key": f"{self.prefix}{BACKUP_FILENAME}",
|
||||
"checksum": self.checksum,
|
||||
"size_bytes": self.archive_size,
|
||||
"duration_seconds": round(self.duration, 2),
|
||||
"db_backup_size": self.db_size,
|
||||
"files_backup_size": self.files_size,
|
||||
}
|
||||
|
||||
def _dump_database(self, work_dir: Path) -> Path | None:
|
||||
"""Dump the database to a compressed SQL file."""
|
||||
if not DB_URL:
|
||||
return None
|
||||
|
||||
db_file = work_dir / "portal-database.sql"
|
||||
db_gz_file = work_dir / "portal-database.sql.gz"
|
||||
|
||||
print(f" 📦 Dumping database: {DB_URL[:60]}...")
|
||||
|
||||
if DB_URL.startswith("sqlite://"):
|
||||
# SQLite — just copy the file
|
||||
sqlite_path = DB_URL.replace("sqlite:///", "")
|
||||
if not sqlite_path.startswith("/"):
|
||||
sqlite_path = os.path.expanduser(f"~/{sqlite_path}")
|
||||
if not os.path.exists(sqlite_path):
|
||||
print(f" ⚠️ SQLite file not found: {sqlite_path}")
|
||||
return None
|
||||
|
||||
# Dump to SQL
|
||||
with open(sqlite_path) as src:
|
||||
data = src.read()
|
||||
with gzip.open(db_gz_file, "wt") as dst:
|
||||
dst.write(f"-- SQLite dump from {sqlite_path}\n-- Generated: {now_iso()}\n\n")
|
||||
dst.write(data)
|
||||
return db_gz_file
|
||||
|
||||
elif DB_URL.startswith("postgresql"):
|
||||
# PostgreSQL — use pg_dump
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pg_dump", "--no-owner", "--no-acl", "--compress=9", "--file", str(db_gz_file), DB_URL],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" ⚠️ pg_dump failed: {result.stderr[:200]}")
|
||||
return None
|
||||
return db_gz_file
|
||||
except FileNotFoundError:
|
||||
print(" ⚠️ pg_dump not installed")
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
print(" ⚠️ pg_dump timed out")
|
||||
return None
|
||||
|
||||
print(f" ⚠️ Unsupported DB URL: {DB_URL.split('://')[0]}://...")
|
||||
return None
|
||||
|
||||
def _collect_files(self, work_dir: Path) -> Path:
|
||||
"""Collect config files, uploads, and scripts."""
|
||||
files_dir = work_dir / "files"
|
||||
files_dir.mkdir()
|
||||
|
||||
# Portal config directory
|
||||
config_dirs = [
|
||||
FILES_DIR if FILES_DIR and os.path.exists(FILES_DIR) else None,
|
||||
Path(os.path.expanduser("~/.hermes")),
|
||||
]
|
||||
|
||||
for src_dir in config_dirs:
|
||||
if not src_dir or not Path(src_dir).exists():
|
||||
continue
|
||||
src_path = Path(src_dir)
|
||||
dst_path = files_dir / src_path.name
|
||||
try:
|
||||
shutil.copytree(src_path, dst_path, ignore=shutil.ignore_patterns("*.pyc", "__pycache__", ".git"))
|
||||
size = sum(f.stat().st_size for f in dst_path.rglob("*") if f.is_file())
|
||||
print(f" 📁 Collected {src_path.name}: {size:,} bytes")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Could not copy {src_path}: {e}")
|
||||
|
||||
# Write file manifest
|
||||
manifest = []
|
||||
for f in files_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
rel = f.relative_to(files_dir)
|
||||
manifest.append({"path": str(rel), "size": f.stat().st_size})
|
||||
self.files_size = sum(m["size"] for m in manifest)
|
||||
(files_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
|
||||
|
||||
return files_dir
|
||||
|
||||
def _write_metadata(self, work_dir: Path) -> None:
|
||||
"""Write backup metadata JSON."""
|
||||
metadata = {
|
||||
"backup_version": "1.0",
|
||||
"created_at": now_iso(),
|
||||
"hostname": os.uname().nodename,
|
||||
"target": self.target_name,
|
||||
"bucket": self.bucket,
|
||||
"components": {
|
||||
"database": bool(DB_URL),
|
||||
"files": bool(FILES_DIR),
|
||||
},
|
||||
"config": {
|
||||
"db_url_prefix": DB_URL.split("://")[0] if DB_URL else None,
|
||||
"files_dir": FILES_DIR or None,
|
||||
"retention_days": RETENTION_DAYS,
|
||||
},
|
||||
}
|
||||
(work_dir / "metadata.json").write_text(json.dumps(metadata, indent=2))
|
||||
|
||||
def _archive(self, work_dir: Path) -> tuple[Path, int, str]:
|
||||
"""Create compressed tar archive and compute checksum."""
|
||||
archive_path = BACKUP_DIR / BACKUP_FILENAME
|
||||
|
||||
with tarfile.open(archive_path, "w:gz") as tar:
|
||||
tar.add(work_dir, arcname=self.target_name)
|
||||
|
||||
# Checksum
|
||||
sha256 = hashlib.sha256()
|
||||
with open(archive_path, "rb") as f:
|
||||
while chunk := f.read(65536):
|
||||
sha256.update(chunk)
|
||||
checksum = sha256.hexdigest()
|
||||
|
||||
# Write checksum file alongside archive
|
||||
checksum_path = BACKUP_DIR / CHECKSUM_FILENAME
|
||||
checksum_path.write_text(f"{checksum} {BACKUP_FILENAME}\n")
|
||||
|
||||
size = os.path.getsize(archive_path)
|
||||
print(f" 📦 Archive: {BACKUP_FILENAME} ({size:,} bytes, sha256: {checksum[:16]}...)")
|
||||
|
||||
return archive_path, size, checksum
|
||||
|
||||
def _upload(self, archive_path: Path) -> None:
|
||||
"""Upload archive and checksum to S3 via aws-cli."""
|
||||
key = f"{self.prefix}{BACKUP_FILENAME}"
|
||||
checksum_key = f"{self.prefix}{CHECKSUM_FILENAME}"
|
||||
|
||||
endpoint_flag = []
|
||||
if self.endpoint:
|
||||
endpoint_flag = ["--endpoint-url", self.endpoint]
|
||||
|
||||
region_flag = ["--region", self.region]
|
||||
|
||||
# Upload archive
|
||||
print(f" ☁️ Uploading to s3://{self.bucket}/{key}")
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "cp", str(archive_path), f"s3://{self.bucket}/{key}"] + endpoint_flag + region_flag,
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"S3 upload failed: {result.stderr}")
|
||||
|
||||
# Upload checksum
|
||||
checksum_path = BACKUP_DIR / CHECKSUM_FILENAME
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "cp", str(checksum_path), f"s3://{self.bucket}/{checksum_key}"] + endpoint_flag + region_flag,
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"S3 checksum upload failed: {result.stderr}")
|
||||
|
||||
print(f" ✅ Uploaded to s3://{self.bucket}/{key}")
|
||||
|
||||
def _cleanup_old(self) -> None:
|
||||
"""Remove local temporary files."""
|
||||
for f in BACKUP_DIR.glob("*.tar.gz"):
|
||||
if f.name != BACKUP_FILENAME:
|
||||
f.unlink()
|
||||
for f in BACKUP_DIR.glob("*.sha256"):
|
||||
if f.name != CHECKSUM_FILENAME:
|
||||
f.unlink()
|
||||
|
||||
# Remote cleanup via S3 lifecycle is preferred, but we can also
|
||||
# list and delete old backups here if needed.
|
||||
print(f" 🧹 Retention: {RETENTION_DAYS} days")
|
||||
|
||||
|
||||
def restore_backup(archive_name: str) -> None:
|
||||
"""Restore portal from a backup archive stored in S3."""
|
||||
print(f"🔄 Restoring from {archive_name}")
|
||||
|
||||
# Try primary bucket first, then DR
|
||||
for target_name, target in BACKUP_TARGETS.items():
|
||||
key = f"{target['prefix']}{archive_name}"
|
||||
endpoint_flag = ["--endpoint-url", target["endpoint"]] if target["endpoint"] else []
|
||||
|
||||
print(f" Looking in s3://{target['bucket']}/{key}...")
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "ls", f"s3://{target['bucket']}/{key}"] + endpoint_flag,
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
# Download
|
||||
local_path = BACKUP_DIR / archive_name
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "cp", f"s3://{target['bucket']}/{key}", str(local_path)] + endpoint_flag,
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" ❌ Download failed: {result.stderr}")
|
||||
continue
|
||||
|
||||
# Verify checksum
|
||||
checksum_key = f"{target['prefix']}{archive_name}.sha256"
|
||||
checksum_local = BACKUP_DIR / f"{archive_name}.sha256"
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "cp", f"s3://{target['bucket']}/{checksum_key}", str(checksum_local)] + endpoint_flag,
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
import hashlib
|
||||
sha256 = hashlib.sha256()
|
||||
with open(local_path, "rb") as f:
|
||||
while chunk := f.read(65536):
|
||||
sha256.update(chunk)
|
||||
expected = checksum_local.read_text().strip().split()[0]
|
||||
if sha256.hexdigest() != expected:
|
||||
print(f" ⚠️ Checksum mismatch! Expected {expected}, got {sha256.hexdigest()}")
|
||||
|
||||
# Extract
|
||||
print(f" 📂 Extracting {local_path}...")
|
||||
with tarfile.open(local_path, "r:gz") as tar:
|
||||
tar.extractall(BACKUP_DIR)
|
||||
|
||||
print(f"\n✅ Restored to {BACKUP_DIR}/{target_name}/")
|
||||
print(f" To restore database:")
|
||||
if DB_URL:
|
||||
print(f" cat {BACKUP_DIR}/{target_name}/portal-database.sql.gz | gunzip | sqlite3 portal.db")
|
||||
print(f" To restore files:")
|
||||
print(f" cp -r {BACKUP_DIR}/{target_name}/files/* /path/to/portal/")
|
||||
return
|
||||
|
||||
print(f"❌ Could not find {archive_name} in any backup target")
|
||||
|
||||
|
||||
# ─── Main ──────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="IT Pro Partner Portal Backup")
|
||||
parser.add_argument("--target", choices=["primary", "cross_region", "all"], default="all",
|
||||
help="Backup target (default: all configured)")
|
||||
parser.add_argument("--restore", metavar="ARCHIVE.tar.gz",
|
||||
help="Restore from a backup archive")
|
||||
parser.add_argument("--list", action="store_true",
|
||||
help="List available backups in S3")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not BACKUP_TARGETS:
|
||||
print("❌ No backup targets configured. Set S3_BUCKET_PRIMARY or S3_BUCKET_DR.")
|
||||
sys.exit(1)
|
||||
|
||||
if args.restore:
|
||||
restore_backup(args.restore)
|
||||
return
|
||||
|
||||
if args.list:
|
||||
for target_name, target in BACKUP_TARGETS.items():
|
||||
endpoint_flag = ["--endpoint-url", target["endpoint"]] if target["endpoint"] else []
|
||||
print(f"📋 Backups in {target['bucket']} ({target_name}):")
|
||||
result = subprocess.run(
|
||||
["aws", "s3", "ls", f"s3://{target['bucket']}/{target['prefix']}"] + endpoint_flag,
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
if line:
|
||||
print(f" {line}")
|
||||
else:
|
||||
print(f" {result.stderr}")
|
||||
return
|
||||
|
||||
# Run backups
|
||||
results = []
|
||||
for target_name, target_config in BACKUP_TARGETS.items():
|
||||
if args.target != "all" and args.target != target_name:
|
||||
continue
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Backing up to {target_name}: {target_config['bucket']}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
runner = BackupRunner(target_name, target_config)
|
||||
try:
|
||||
result = runner.run()
|
||||
results.append(result)
|
||||
print(f" ✅ Success: {result['size_bytes']:,} bytes in {result['duration_seconds']:.1f}s")
|
||||
except Exception as e:
|
||||
print(f" ❌ Failed: {e}")
|
||||
results.append({"target": target_name, "status": "failed", "error": str(e)})
|
||||
|
||||
# Print summary
|
||||
print(f"\n{'='*60}")
|
||||
print("Backup Summary")
|
||||
print(f"{'='*60}")
|
||||
for r in results:
|
||||
status_icon = "✅" if r["status"] == "success" else "❌"
|
||||
print(f" {status_icon} {r['target']}: {r['status']}")
|
||||
if r["status"] == "success":
|
||||
print(f" Archive: {r['archive_key']}")
|
||||
print(f" Size: {r['size_bytes']:,} bytes")
|
||||
print(f" Duration: {r['duration_seconds']:.1f}s")
|
||||
else:
|
||||
print(f" Error: {r.get('error', 'unknown')}")
|
||||
|
||||
if DRY_RUN:
|
||||
print(f"\n🔍 DRY RUN — No backups were uploaded")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Check for DRY_RUN
|
||||
DRY_RUN = os.environ.get("DRY_RUN", "").lower() in ("1", "true", "yes")
|
||||
main()
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check IMAP inbox for bounce-back / delivery-failure emails and alert if found.
|
||||
|
||||
Silent (exit 0 with no output) when nothing to report.
|
||||
Outputs summary when bounces are found.
|
||||
"""
|
||||
|
||||
import imaplib
|
||||
import email
|
||||
from email.utils import parsedate_to_datetime
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
ACCOUNTS = [
|
||||
{"user": "g@germainebrown.com", "pw_file": "/root/.config/himalaya/g-germainebrown.pass", "label": "Germaine"},
|
||||
{"user": "shonuff@germainebrown.com", "pw": "Catches.bullets1985", "label": "Sho'Nuff"},
|
||||
]
|
||||
CHECK_HOURS = 24
|
||||
|
||||
BOUNCE_KEYWORDS = [
|
||||
"mail delivery failed", "delivery status notification", "undelivered",
|
||||
"returned mail", "mail delivery failure", "non-delivery",
|
||||
"delivery failed", "undeliverable", "delivery report",
|
||||
"returned to sender", "message bounced",
|
||||
]
|
||||
|
||||
BOUNCE_SENDERS = [
|
||||
"mailer-daemon", "postmaster", "mail delivery system",
|
||||
]
|
||||
|
||||
|
||||
def safe_str(val):
|
||||
if val is None:
|
||||
return ""
|
||||
if isinstance(val, str):
|
||||
return val
|
||||
try:
|
||||
return str(val)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def is_bounce(msg):
|
||||
subject = safe_str(msg.get("Subject", "")).lower()
|
||||
sender = safe_str(msg.get("From", "")).lower()
|
||||
|
||||
for kw in BOUNCE_KEYWORDS:
|
||||
if kw in subject:
|
||||
return True
|
||||
|
||||
for s in BOUNCE_SENDERS:
|
||||
if s in sender:
|
||||
return True
|
||||
|
||||
if msg.get_content_type() == "text/plain":
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
body = safe_str(payload).lower()
|
||||
if "permanent error" in body or "could not be delivered" in body:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
all_bounces = []
|
||||
|
||||
since = (datetime.now(timezone.utc) - timedelta(hours=CHECK_HOURS)).strftime("%d-%b-%Y")
|
||||
|
||||
for acct in ACCOUNTS:
|
||||
try:
|
||||
if "pw" in acct:
|
||||
pw = acct["pw"]
|
||||
else:
|
||||
pw = open(acct["pw_file"]).read().strip()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(acct["user"], pw)
|
||||
conn.select("INBOX")
|
||||
except Exception as e:
|
||||
print(f"ERROR: {acct['label']} IMAP failed: {e}")
|
||||
continue
|
||||
|
||||
status, data = conn.search(None, f'(SINCE {since})')
|
||||
if status != "OK":
|
||||
conn.logout()
|
||||
continue
|
||||
|
||||
for num in data[0].split():
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
if is_bounce(raw):
|
||||
all_bounces.append({
|
||||
"account": acct["label"],
|
||||
"subject": raw.get("Subject", "(no subject)"),
|
||||
"from": raw.get("From", "(unknown)"),
|
||||
"date": raw.get("Date", ""),
|
||||
})
|
||||
|
||||
conn.logout()
|
||||
|
||||
if not all_bounces:
|
||||
return
|
||||
|
||||
print(f"📬 {len(all_bounces)} bounce-back(s) detected in the last {CHECK_HOURS}h:\n")
|
||||
for b in all_bounces:
|
||||
print(f" [{b['account']}] {b['subject']}")
|
||||
print(f" From: {b['from']}")
|
||||
print(f" Date: {b['date']}")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
"""BoxPilot Logistics email triage — detect and move solicitations.
|
||||
|
||||
Heuristic: if sender is from a free/personal email domain (gmail, yahoo, etc.)
|
||||
AND the email body or subject references a different business domain,
|
||||
it's likely solicitation. Move to INBOX.Solicitation.
|
||||
"""
|
||||
import imaplib, email, re, os, json, hashlib
|
||||
from email.utils import parsedate_to_datetime
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
HOST = "mail.boxpilotlogistics.com"
|
||||
PORT = 993
|
||||
USER = "hello@boxpilotlogistics.com"
|
||||
PW = "1New.opportunity"
|
||||
SOLICIT_FOLDER = "Solicitation"
|
||||
CHECK_HOURS = 48
|
||||
|
||||
# Free/personal email domains — if sender is from these AND claims another company, it's sus
|
||||
FREE_DOMAINS = {
|
||||
"gmail.com", "yahoo.com", "yahoo.co.uk", "ymail.com", "outlook.com",
|
||||
"hotmail.com", "live.com", "msn.com", "aol.com", "icloud.com",
|
||||
"me.com", "mac.com", "protonmail.com", "proton.me", "pm.me",
|
||||
"mail.com", "inbox.com", "zoho.com", "yandex.com", "gmx.com",
|
||||
"fastmail.com", "tutanota.com", "rediffmail.com", "libero.it",
|
||||
"web.de", "gmx.de", "t-online.de", "online.de",
|
||||
}
|
||||
|
||||
def safe_str(val):
|
||||
if val is None: return ""
|
||||
if isinstance(val, str): return val
|
||||
try: return str(val)
|
||||
except: return ""
|
||||
|
||||
def get_body_text(msg):
|
||||
"""Extract plain text body."""
|
||||
body = ""
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
elif ct == "text/html" and not body:
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
body = payload.decode("utf-8", errors="replace")
|
||||
return body
|
||||
|
||||
def extract_domains(text):
|
||||
"""Extract all domain names from text (from URLs and email addresses)."""
|
||||
domains = set()
|
||||
# URLs
|
||||
for m in re.finditer(r'https?://(?:www\.)?([a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.[a-z]{2,}(?:\.[a-z]{2,})?)', text, re.IGNORECASE):
|
||||
d = m.group(1).lower()
|
||||
# Skip common non-business domains
|
||||
if d not in ('google.com', 'youtube.com', 'facebook.com', 'linkedin.com', 'twitter.com', 'x.com', 'instagram.com'):
|
||||
domains.add(d)
|
||||
# Email addresses in body
|
||||
for m in re.finditer(r'[a-z0-9._%+-]+@([a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.[a-z]{2,}(?:\.[a-z]{2,})?)', text, re.IGNORECASE):
|
||||
d = m.group(1).lower()
|
||||
if d not in FREE_DOMAINS:
|
||||
domains.add(d)
|
||||
return domains
|
||||
|
||||
def is_solicitation(msg):
|
||||
"""Determine if a message is likely solicitation."""
|
||||
sender = safe_str(msg.get("From", "")).lower()
|
||||
subject = safe_str(msg.get("Subject", "")).lower()
|
||||
body = get_body_text(msg).lower()
|
||||
full_text = subject + " " + body
|
||||
|
||||
# Extract sender's domain
|
||||
sender_domain = ""
|
||||
m = re.search(r'@([a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.[a-z]{2,}(?:\.[a-z]{2,})?)', sender)
|
||||
if m:
|
||||
sender_domain = m.group(1).lower()
|
||||
|
||||
# If sender is from a free domain, check if they claim another company
|
||||
if sender_domain in FREE_DOMAINS:
|
||||
# Extract business domains mentioned in the email
|
||||
biz_domains = extract_domains(full_text)
|
||||
# If they reference a business domain different from their sender domain
|
||||
# AND that business domain is a real company domain (not another free domain)
|
||||
legit_biz = [d for d in biz_domains if d not in FREE_DOMAINS]
|
||||
if legit_biz:
|
||||
return True, f"Free domain ({sender_domain}) referencing business domain(s): {', '.join(list(legit_biz)[:3])}"
|
||||
|
||||
# Also flag common solicitation subject patterns from free domains
|
||||
if sender_domain in FREE_DOMAINS:
|
||||
spam_kw = ["quote", "insurance", "factoring", "fuel", "dispatch", "load board",
|
||||
"freight", "logistics", "carrier", "trucking", "broker", "authority",
|
||||
"mc authority", "dot authority", "compliance", "ifta", "irp",
|
||||
"elog", "eld", "safety", "audit", "lease", "warranty", "extended warranty"]
|
||||
found = [kw for kw in spam_kw if kw in subject]
|
||||
if found:
|
||||
return True, f"Free domain ({sender_domain}) with solicitation subject keywords: {', '.join(found)}"
|
||||
|
||||
return False, ""
|
||||
|
||||
def main():
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(USER, PW)
|
||||
except Exception as e:
|
||||
print(f"ERROR: Connection failed: {e}")
|
||||
return
|
||||
|
||||
# Select inbox
|
||||
conn.select("INBOX")
|
||||
|
||||
# Search for recent unread messages
|
||||
since = (datetime.now(timezone.utc) - timedelta(hours=CHECK_HOURS)).strftime("%d-%b-%Y")
|
||||
status, data = conn.search(None, f'(UNSEEN SINCE {since})')
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
return # silent
|
||||
|
||||
moved = 0
|
||||
for num in data[0].split():
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
result, reason = is_solicitation(raw)
|
||||
if result:
|
||||
# Copy to Solicitation folder, then delete from inbox
|
||||
copy_status = conn.copy(num, SOLICIT_FOLDER)
|
||||
if copy_status[0] == "OK":
|
||||
conn.store(num, "+FLAGS", "\\Deleted")
|
||||
moved += 1
|
||||
subj = safe_str(raw.get("Subject", "(no subject)"))
|
||||
sender = safe_str(raw.get("From", "(unknown)"))
|
||||
|
||||
# Expunge deleted messages
|
||||
if moved > 0:
|
||||
conn.expunge()
|
||||
|
||||
conn.logout()
|
||||
|
||||
if moved > 0:
|
||||
print(f"📬 BoxPilot — Moved {moved} solicitation(s) to Solicitation folder")
|
||||
# silent if nothing moved
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+410
@@ -0,0 +1,410 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Boys' Email Monitor — polls garrison@iamgmb.com and greyson@iamgmb.com inboxes,
|
||||
logs new emails, tracks sender frequency, and sends daily summary emails.
|
||||
|
||||
Usage:
|
||||
python3 boys-mail-monitor.py collect
|
||||
python3 boys-mail-monitor.py summary
|
||||
python3 boys-mail-monitor.py test-summary
|
||||
"""
|
||||
import imaplib
|
||||
import email
|
||||
import json
|
||||
import os
|
||||
import smtplib
|
||||
import sys
|
||||
import logging
|
||||
import random
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html import escape
|
||||
from pathlib import Path
|
||||
|
||||
# ── Sho'Nuff's signature ──────────────────────────
|
||||
SHONUFF_TITLES = [
|
||||
"Germaine's AI Ops Engineer",
|
||||
"Germaine's Baddest AI Mofo Low Down Around This Town",
|
||||
"Germaine's One-and-Only Digital Master",
|
||||
"Germaine's Converse-Kicking Assistant",
|
||||
"Keeper of Germaine's Teeth (Catches Bullets)",
|
||||
"Germaine's AI Problem Child",
|
||||
"Germaine's 24/7 Co-Pilot",
|
||||
"Germaine's Digital Henchman",
|
||||
"Germaine's Glow Up Coordinator",
|
||||
"Germaine's Digital Sidekick",
|
||||
"Germaine's AI Bodyguard",
|
||||
"Germaine's Chaos Coordinator",
|
||||
"Germaine's Full-Time Menace",
|
||||
]
|
||||
SHONUFF_CLOSINGS = [
|
||||
"Who's the Master?",
|
||||
"Kiss my Converse!",
|
||||
"Am I the meanest?",
|
||||
"Am I the prettiest?",
|
||||
"Am I the baddest mofo low down around this town?",
|
||||
"All right, Leroy. Who's the one-and-only Master?",
|
||||
"YOU'LL... NEVER... USE... THIS... FOOT... AGAIN!",
|
||||
"Catches bullets with his teeth?",
|
||||
]
|
||||
|
||||
BASE_DIR = "/root/.hermes"
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
LOG_FILE = "/var/log/boys-mail-monitor.log"
|
||||
MAIL_LOG = os.path.join(DATA_DIR, "boys-mail.json")
|
||||
SENDERS_DB = os.path.join(DATA_DIR, "boys-senders.json")
|
||||
PROCESSED_DB = os.path.join(DATA_DIR, "boys-processed.json")
|
||||
|
||||
BOYS_ACCOUNTS = {
|
||||
"Garrison": {
|
||||
"host": "heracles.mxrouting.net",
|
||||
"port": 993,
|
||||
"user": "garrison@iamgmb.com",
|
||||
"pw_file": "/root/.config/himalaya/garrison-iamgmb.pass",
|
||||
},
|
||||
"Greyson": {
|
||||
"host": "heracles.mxrouting.net",
|
||||
"port": 993,
|
||||
"user": "greyson@iamgmb.com",
|
||||
"pw_file": "/root/.config/himalaya/greyson-iamgmb.pass",
|
||||
},
|
||||
}
|
||||
|
||||
SMTP_HOST = "mail.germainebrown.com"
|
||||
SMTP_PORT = 2525
|
||||
SMTP_USER = "shonuff@germainebrown.com"
|
||||
SMTP_PASS_FILE = "/root/.config/himalaya/shonuff.pass"
|
||||
SMTP_FROM = "shonuff@germainebrown.com"
|
||||
SMTP_FROM_NAME = "Sho'Nuff Brown"
|
||||
BCC_ADDR = "g@germainebrown.com"
|
||||
|
||||
RECIPIENTS = {
|
||||
"Garrison": "tinamichelle1008@gmail.com",
|
||||
"Greyson": "katherineeubank@gmail.com",
|
||||
}
|
||||
MOM_NAMES = {"Garrison": "Tina", "Greyson": "Kate"}
|
||||
HIGH_VOLUME_THRESHOLD = 3
|
||||
TRUSTED_IGNORE_SENDERS = {"shonuff@germainebrown.com"}
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[logging.FileHandler(LOG_FILE), logging.StreamHandler()],
|
||||
)
|
||||
log = logging.getLogger("boys-mail")
|
||||
|
||||
|
||||
def read_secret(path):
|
||||
return Path(path).read_text().strip()
|
||||
|
||||
|
||||
def load_json(path, default):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
log.warning("Corrupt JSON %s, reinitializing", path)
|
||||
return default
|
||||
|
||||
|
||||
def save_json_atomic(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
tmp = f"{path}.tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
def safe_str(val):
|
||||
if val is None:
|
||||
return ""
|
||||
try:
|
||||
return str(val)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def extract_sender_name(from_header):
|
||||
if not from_header:
|
||||
return "Unknown Sender"
|
||||
name, addr = email.utils.parseaddr(from_header)
|
||||
return name if name else addr or "Unknown Sender"
|
||||
|
||||
|
||||
def extract_sender_email(from_header):
|
||||
if not from_header:
|
||||
return "unknown@unknown"
|
||||
_, addr = email.utils.parseaddr(from_header)
|
||||
return (addr or from_header).lower().strip()
|
||||
|
||||
|
||||
def parse_email_date(date_str):
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def format_time_for_summary(email_date):
|
||||
if isinstance(email_date, str):
|
||||
email_date = parse_email_date(email_date)
|
||||
et = email_date.astimezone(timezone(timedelta(hours=-4)))
|
||||
return et.strftime("%I:%M %p").lstrip("0")
|
||||
|
||||
|
||||
def format_date_header(date_obj):
|
||||
et = date_obj.astimezone(timezone(timedelta(hours=-4)))
|
||||
return et.strftime("%A, %B %d, %Y")
|
||||
|
||||
|
||||
def message_uid_key(child_name, uid):
|
||||
return f"{child_name}:{uid}"
|
||||
|
||||
|
||||
def message_id_key(child_name, msg_id):
|
||||
msg_id = (msg_id or "").strip()
|
||||
return f"{child_name}:msgid:{msg_id}" if msg_id else ""
|
||||
|
||||
|
||||
def poll_inbox(child_name, config, mail_log_data, senders_data, processed):
|
||||
new_count = 0
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(config["host"], config["port"])
|
||||
conn.login(config["user"], read_secret(config["pw_file"]))
|
||||
conn.select("INBOX")
|
||||
status, data = conn.uid("SEARCH", None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
log.info("%s: No UNSEEN messages", child_name)
|
||||
return 0
|
||||
|
||||
for uid_b in data[0].split():
|
||||
uid = uid_b.decode()
|
||||
uid_key = message_uid_key(child_name, uid)
|
||||
if uid_key in processed:
|
||||
continue
|
||||
try:
|
||||
status, msg_data = conn.uid("FETCH", uid, "(BODY.PEEK[])")
|
||||
if status != "OK" or not msg_data or not isinstance(msg_data[0], tuple):
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
from_hdr = safe_str(raw.get("From", "")).strip()
|
||||
sender_email = extract_sender_email(from_hdr)
|
||||
msg_id = safe_str(raw.get("Message-ID", ""))
|
||||
mid_key = message_id_key(child_name, msg_id)
|
||||
if mid_key and mid_key in processed:
|
||||
processed.add(uid_key)
|
||||
continue
|
||||
|
||||
if sender_email in TRUSTED_IGNORE_SENDERS:
|
||||
processed.add(uid_key)
|
||||
if mid_key:
|
||||
processed.add(mid_key)
|
||||
log.info("%s: Ignored trusted sender %s uid=%s", child_name, sender_email, uid)
|
||||
continue
|
||||
|
||||
subject = safe_str(raw.get("Subject", "(no subject)"))
|
||||
date_str = safe_str(raw.get("Date", ""))
|
||||
sender_name = extract_sender_name(from_hdr)
|
||||
entry = {
|
||||
"id": uid_key,
|
||||
"child": child_name,
|
||||
"from": from_hdr,
|
||||
"from_name": sender_name,
|
||||
"from_email": sender_email,
|
||||
"subject": subject,
|
||||
"date": date_str,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"uid": uid,
|
||||
"message_id": msg_id,
|
||||
}
|
||||
mail_log_data.append(entry)
|
||||
processed.add(uid_key)
|
||||
if mid_key:
|
||||
processed.add(mid_key)
|
||||
|
||||
if sender_email not in senders_data:
|
||||
senders_data[sender_email] = {"name": sender_name, "first_seen": entry["timestamp"], "last_seen": entry["timestamp"], "count": 0, "child": child_name}
|
||||
senders_data[sender_email]["last_seen"] = entry["timestamp"]
|
||||
senders_data[sender_email]["count"] += 1
|
||||
if sender_name != "Unknown Sender" and sender_name:
|
||||
senders_data[sender_email]["name"] = sender_name
|
||||
new_count += 1
|
||||
log.info("%s: New email uid=%s from %s — %s", child_name, uid, sender_email, subject[:60])
|
||||
except Exception as e:
|
||||
log.error("%s: Error processing UID %s: %s", child_name, uid, e)
|
||||
conn.logout()
|
||||
except Exception as e:
|
||||
log.error("%s: IMAP connection error: %s", child_name, e)
|
||||
return new_count
|
||||
|
||||
|
||||
def do_collect():
|
||||
log.info("=" * 50)
|
||||
log.info("Boys' Mail Collection Run")
|
||||
log.info("=" * 50)
|
||||
mail_log = load_json(MAIL_LOG, [])
|
||||
senders = load_json(SENDERS_DB, {})
|
||||
processed = set(load_json(PROCESSED_DB, []))
|
||||
total = 0
|
||||
for child_name, config in BOYS_ACCOUNTS.items():
|
||||
total += poll_inbox(child_name, config, mail_log, senders, processed)
|
||||
cutoff_ts = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
|
||||
mail_log = [e for e in mail_log if e.get("timestamp", "") >= cutoff_ts]
|
||||
for info in senders.values():
|
||||
if info.get("count", 0) >= HIGH_VOLUME_THRESHOLD and not info.get("flagged"):
|
||||
info["flagged"] = True
|
||||
save_json_atomic(MAIL_LOG, mail_log)
|
||||
save_json_atomic(SENDERS_DB, senders)
|
||||
save_json_atomic(PROCESSED_DB, sorted(processed))
|
||||
log.info("Total new emails collected: %d", total)
|
||||
log.info("Total tracked senders: %d", len(senders))
|
||||
return total
|
||||
|
||||
|
||||
def shonuff_title():
|
||||
return random.choice(SHONUFF_TITLES)
|
||||
|
||||
|
||||
def shonuff_closing():
|
||||
return random.choice(SHONUFF_CLOSINGS)
|
||||
|
||||
|
||||
def build_summary_html(child_name, today_emails, senders_for_child):
|
||||
mom_name = MOM_NAMES.get(child_name, "Mom")
|
||||
today = datetime.now(timezone.utc)
|
||||
date_str = format_date_header(today)
|
||||
if not today_emails:
|
||||
emails_section = '<div style="background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;padding:16px;margin:16px 0;text-align:center;"><p style="font-size:14px;color:#166534;margin:0;">✅ No new emails today</p></div>'
|
||||
else:
|
||||
rows = ""
|
||||
for em in today_emails:
|
||||
sender_display = em.get("from_name", em.get("from", "Unknown"))
|
||||
subj = escape(em.get("subject", "(no subject)"))
|
||||
t = format_time_for_summary(em.get("date", ""))
|
||||
rows += f"<tr><td style='padding:8px 10px;border:1px solid #e5e7eb;'>{escape(sender_display)}</td><td style='padding:8px 10px;border:1px solid #e5e7eb;'>{subj}</td><td style='padding:8px 10px;border:1px solid #e5e7eb;white-space:nowrap;'>{t}</td></tr>"
|
||||
emails_section = f"<h3 style='font-size:14px;font-weight:600;margin:20px 0 8px;'>📨 New Emails</h3><table style='width:100%;border-collapse:collapse;font-size:13px;'><tr style='background:#f3f4f6;'><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:left;'>From</th><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:left;'>Subject</th><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:left;'>Time</th></tr>{rows}</table>"
|
||||
|
||||
frequency_section = ""
|
||||
if senders_for_child:
|
||||
freq_rows = ""
|
||||
for sender_email, info in sorted(senders_for_child.items(), key=lambda x: x[1].get("count", 0), reverse=True)[:10]:
|
||||
count = info.get("count", 0)
|
||||
note = "⚠️ High volume — potential subscription" if count >= HIGH_VOLUME_THRESHOLD else ""
|
||||
bg = ' style="background:#fef3c7;"' if note else ""
|
||||
freq_rows += f"<tr{bg}><td style='padding:8px 10px;border:1px solid #e5e7eb;'>{escape(info.get('name', sender_email))}<br><span style='font-size:11px;color:#999;'>{escape(sender_email)}</span></td><td style='padding:8px 10px;border:1px solid #e5e7eb;text-align:center;font-weight:600;'>{count}</td><td style='padding:8px 10px;border:1px solid #e5e7eb;font-size:12px;'>{note}</td></tr>"
|
||||
frequency_section = f"<h3 style='font-size:14px;font-weight:600;margin:20px 0 8px;'>📊 This Week's Top Senders</h3><table style='width:100%;border-collapse:collapse;font-size:13px;'><tr style='background:#f3f4f6;'><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:left;'>Sender</th><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:center;'>Count</th><th style='padding:8px 10px;border:1px solid #e5e7eb;text-align:left;'>Note</th></tr>{freq_rows}</table>"
|
||||
|
||||
return f"""<!DOCTYPE html><html><head><meta charset='utf-8'></head><body style="font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;color:#1a1a1a;max-width:560px;margin:0 auto;padding:20px;"><div style='border-bottom:2px solid #dc2626;padding-bottom:12px;margin-bottom:20px;'><h2 style='margin:0;font-size:18px;font-weight:600;'>📬 {escape(child_name)}'s Email Summary</h2><p style='color:#666;font-size:12px;margin:4px 0 0;'>{escape(date_str)}</p></div><p style='font-size:14px;line-height:1.6;color:#333;'>Hey {escape(mom_name)}!</p><p style='font-size:14px;line-height:1.6;'>Here's what came through today:</p>{emails_section}{frequency_section}<p style='font-size:12px;color:#888;font-style:italic;margin:18px 0 12px;'><em>{shonuff_closing()}</em></p><hr style='border:none;height:2px;width:40px;background:#cc0000;margin:0 0 12px 0;border-radius:2px;'><table cellpadding='0' cellspacing='0' border='0' style="font-family:'Inter',-apple-system,sans-serif;color:#2c2c2c;font-size:13px;line-height:1.6;"><tr><td style='padding-right:14px;vertical-align:middle;width:80px;'><img src='https://core.itpropartner.com/shonuff-signature.png' alt='SB' width='72' height='72' style='border-radius:50%;display:block;max-width:72px;'></td><td style='vertical-align:middle;'><div style='font-size:15px;font-weight:700;color:#1a1a1a;'>Sho'Nuff Brown</div><div style='font-size:12px;color:#888;margin-bottom:1px;'>{shonuff_title()}</div><div style='font-size:12px;color:#aaa;margin-bottom:6px;'>IT Pro Partner</div><div style='font-size:12px;color:#999;'><a href='mailto:shonuff@germainebrown.com' style='color:#555;text-decoration:none;border-bottom:1px dotted #ccc;'>shonuff@germainebrown.com</a></div></td></tr></table><p style='font-size:11px;color:#aaa;'>Reply to adjust how often you get these summaries, request {escape(child_name)}'s login credentials for your phone, or flag senders to unsubscribe.</p><p style='font-size:11px;color:#aaa;'>Webmail: <a href='http://webmail.iamgmb.com' style='color:#1d4ed8;'>webmail.iamgmb.com</a></p></body></html>"""
|
||||
|
||||
|
||||
def build_plain_text(child_name, today_emails, senders_for_child):
|
||||
mom_name = MOM_NAMES.get(child_name, "Mom")
|
||||
lines = [f"{child_name}'s Email Summary", "=" * 40, "", f"Hey {mom_name},", "", "Here's what came through today:", ""]
|
||||
if not today_emails:
|
||||
lines.append("✅ No new emails today")
|
||||
else:
|
||||
lines.append("📨 New Emails:")
|
||||
for em in today_emails:
|
||||
lines.append(f" From: {em.get('from_name', em.get('from', 'Unknown'))}")
|
||||
lines.append(f" Re: {em.get('subject', '(no subject)')}")
|
||||
lines.append(f" Time: {format_time_for_summary(em.get('date', ''))}")
|
||||
lines.append("")
|
||||
if senders_for_child:
|
||||
lines.append("📊 Top Senders This Week:")
|
||||
for sender_email, info in sorted(senders_for_child.items(), key=lambda x: x[1].get("count", 0), reverse=True)[:10]:
|
||||
note = " ⚠️ HIGH VOLUME" if info.get("count", 0) >= HIGH_VOLUME_THRESHOLD else ""
|
||||
lines.append(f" {info.get('name', sender_email)} ({sender_email}): {info.get('count', 0)}{note}")
|
||||
lines += ["", "--", "Sho'Nuff Brown", "shonuff@germainebrown.com", "", f"Reply to adjust frequency, request {child_name}'s login credentials for your phone, or flag senders to unsubscribe.", "Webmail: http://webmail.iamgmb.com"]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def send_email(child_name, today_emails, senders_for_child, to_addr, test=False):
|
||||
html = build_summary_html(child_name, today_emails, senders_for_child)
|
||||
plain = build_plain_text(child_name, today_emails, senders_for_child)
|
||||
subject = f"{'TEST: ' if test else ''}📬 {child_name}'s Email Summary — {datetime.now(timezone.utc).strftime('%b %d')}"
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"{SMTP_FROM_NAME} <{SMTP_FROM}>"
|
||||
msg["To"] = to_addr
|
||||
if to_addr.lower() != BCC_ADDR.lower():
|
||||
msg["Bcc"] = BCC_ADDR
|
||||
msg["Subject"] = subject
|
||||
msg.attach(MIMEText(plain, "plain"))
|
||||
msg.attach(MIMEText(html, "html"))
|
||||
with smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=20) as s:
|
||||
s.starttls()
|
||||
s.login(SMTP_USER, read_secret(SMTP_PASS_FILE))
|
||||
s.send_message(msg)
|
||||
log.info("Summary sent for %s to %s", child_name, to_addr)
|
||||
return True
|
||||
|
||||
|
||||
def filter_today(mail_log, child_name):
|
||||
today = datetime.now(timezone.utc).astimezone(timezone(timedelta(hours=-4))).strftime("%Y-%m-%d")
|
||||
out = []
|
||||
for e in mail_log:
|
||||
if e.get("child") != child_name:
|
||||
continue
|
||||
if parse_email_date(e.get("date", "")).astimezone(timezone(timedelta(hours=-4))).strftime("%Y-%m-%d") == today:
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def do_summary(test_to=None):
|
||||
log.info("=" * 50)
|
||||
log.info("Boys' Daily Summary Run")
|
||||
log.info("=" * 50)
|
||||
mail_log = load_json(MAIL_LOG, [])
|
||||
senders = load_json(SENDERS_DB, {})
|
||||
results = []
|
||||
for child_name in BOYS_ACCOUNTS:
|
||||
to_addr = test_to or RECIPIENTS.get(child_name)
|
||||
today_emails = filter_today(mail_log, child_name)
|
||||
senders_for_child = {email_addr: info for email_addr, info in senders.items() if info.get("child") == child_name and email_addr not in TRUSTED_IGNORE_SENDERS}
|
||||
if to_addr and "@" in to_addr and "example.com" not in to_addr:
|
||||
try:
|
||||
send_email(child_name, today_emails, senders_for_child, to_addr, test=bool(test_to))
|
||||
results.append((child_name, to_addr, True, len(today_emails)))
|
||||
except Exception as e:
|
||||
log.error("Failed to send summary for %s: %s", child_name, e)
|
||||
results.append((child_name, to_addr, False, len(today_emails)))
|
||||
else:
|
||||
log.warning("%s: No valid recipient email configured (%s)", child_name, to_addr)
|
||||
results.append((child_name, to_addr, False, len(today_emails)))
|
||||
log.info("Summary run complete: %s", results)
|
||||
return results
|
||||
|
||||
|
||||
def do_test_summary():
|
||||
do_summary(test_to=SMTP_FROM)
|
||||
print("✅ Test summaries sent to shonuff@germainebrown.com — check your inbox.")
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
if len(sys.argv) < 2 or sys.argv[1] in {"--help", "-h", "help"}:
|
||||
print("Usage: boys-mail-monitor.py [collect|summary|test-summary]")
|
||||
print(" collect — Poll inboxes for new emails")
|
||||
print(" summary — Send daily summary emails")
|
||||
print(" test-summary — Send test summaries to shonuff inbox")
|
||||
return 0
|
||||
command = sys.argv[1]
|
||||
if command == "collect":
|
||||
do_collect()
|
||||
elif command == "summary":
|
||||
do_summary()
|
||||
elif command == "test-summary":
|
||||
do_test_summary()
|
||||
else:
|
||||
print(f"Unknown command: {command}")
|
||||
return 2
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+674
@@ -0,0 +1,674 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Boys' Email Monitor — polls garrison@iamgmb.com and greyson@iamgmb.com inboxes,
|
||||
logs new emails, tracks sender frequency, and sends daily summary emails.
|
||||
|
||||
Usage:
|
||||
# Collect new emails (run every 60 min via cron)
|
||||
python3 boys-mail-monitor.py collect
|
||||
|
||||
# Send daily summary (run at 7 PM ET via cron)
|
||||
python3 boys-mail-monitor.py summary
|
||||
|
||||
# Test send (dry run summary to sho'nuff)
|
||||
python3 boys-mail-monitor.py test-summary
|
||||
"""
|
||||
import imaplib
|
||||
import email
|
||||
import json
|
||||
import os
|
||||
import smtplib
|
||||
import sys
|
||||
import logging
|
||||
import re
|
||||
import random
|
||||
|
||||
# ── Sho'Nuff's signature ──────────────────────────
|
||||
SHONUFF_TITLES = [
|
||||
"Germaine's AI Ops Engineer",
|
||||
"Germaine's Baddest AI Mofo Low Down Around This Town",
|
||||
"Germaine's One-and-Only Digital Master",
|
||||
"Germaine's Converse-Kicking Assistant",
|
||||
"Keeper of Germaine's Teeth (Catches Bullets)",
|
||||
"Germaine's AI Problem Child",
|
||||
"Germaine's 24/7 Co-Pilot",
|
||||
"Germaine's Digital Henchman",
|
||||
"Germaine's Glow Up Coordinator",
|
||||
"Germaine's Digital Sidekick",
|
||||
"Germaine's AI Bodyguard",
|
||||
"Germaine's Chaos Coordinator",
|
||||
"Germaine's Full-Time Menace",
|
||||
]
|
||||
SHONUFF_CLOSINGS = [
|
||||
"Who's the Master?",
|
||||
"Kiss my Converse!",
|
||||
"Am I the meanest?",
|
||||
"Am I the prettiest?",
|
||||
"Am I the baddest mofo low down around this town?",
|
||||
"All right, Leroy. Who's the one-and-only Master?",
|
||||
"YOU'LL... NEVER... USE... THIS... FOOT... AGAIN!",
|
||||
"Catches bullets with his teeth?",
|
||||
]
|
||||
|
||||
def shonuff_title():
|
||||
return random.choice(SHONUFF_TITLES)
|
||||
|
||||
def shonuff_closing():
|
||||
return random.choice(SHONUFF_CLOSINGS)
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.utils import parsedate_to_datetime, formataddr
|
||||
from html import escape
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────
|
||||
|
||||
BASE_DIR = "/root/.hermes"
|
||||
DATA_DIR = os.path.join(BASE_DIR, "data")
|
||||
SCRIPTS_DIR = os.path.join(BASE_DIR, "scripts")
|
||||
LOG_FILE = "/var/log/boys-mail-monitor.log"
|
||||
MAIL_LOG = os.path.join(DATA_DIR, "boys-mail.json")
|
||||
SENDERS_DB = os.path.join(DATA_DIR, "boys-senders.json")
|
||||
|
||||
# IMAP — Boys' accounts
|
||||
BOYS_ACCOUNTS = {
|
||||
"Garrison": {
|
||||
"host": "heracles.mxrouting.net",
|
||||
"port": 993,
|
||||
"user": "garrison@iamgmb.com",
|
||||
"pw": "Baby.g2015!",
|
||||
},
|
||||
"Greyson": {
|
||||
"host": "heracles.mxrouting.net",
|
||||
"port": 993,
|
||||
"user": "greyson@iamgmb.com",
|
||||
"pw": "Boogie.woogie20",
|
||||
},
|
||||
}
|
||||
|
||||
# SMTP — Sho'Nuff's outbound
|
||||
SMTP_HOST = "mail.germainebrown.com"
|
||||
SMTP_PORT = 2525
|
||||
SMTP_USER = "shonuff@germainebrown.com"
|
||||
SMTP_PASS = "Catches.bullets1985"
|
||||
SMTP_FROM = "shonuff@germainebrown.com"
|
||||
SMTP_FROM_NAME = "Sho'Nuff Brown"
|
||||
BCC_ADDR = "g@germainebrown.com"
|
||||
|
||||
# Recipient email addresses — PLACEHOLDERS until Germaine provides them
|
||||
# Tina gets Garrison's summaries, Kate gets Greyson's summaries
|
||||
RECIPIENTS = {
|
||||
"Garrison": "tinamichelle1008@gmail.com",
|
||||
"Greyson": "katherineeubank@gmail.com",
|
||||
}
|
||||
|
||||
MOM_NAMES = {
|
||||
"Garrison": "Tina",
|
||||
"Greyson": "Kate",
|
||||
}
|
||||
|
||||
HIGH_VOLUME_THRESHOLD = 3
|
||||
|
||||
# ── Logging ────────────────────────────────────────────────────────────────
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler(LOG_FILE),
|
||||
logging.StreamHandler(),
|
||||
],
|
||||
)
|
||||
log = logging.getLogger("boys-mail")
|
||||
|
||||
|
||||
# ── Data helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
def load_json(path, default):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
log.warning("Corrupt JSON %s, reinitializing", path)
|
||||
return default
|
||||
|
||||
|
||||
def save_json(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
|
||||
def safe_str(val):
|
||||
if val is None:
|
||||
return ""
|
||||
if isinstance(val, str):
|
||||
return val
|
||||
try:
|
||||
return str(val)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def extract_sender_name(from_header):
|
||||
"""Extract a display name from a From header like 'John Doe <john@example.com>'."""
|
||||
if not from_header:
|
||||
return "Unknown Sender"
|
||||
name, addr = email.utils.parseaddr(from_header)
|
||||
return name if name else addr
|
||||
|
||||
|
||||
def extract_sender_email(from_header):
|
||||
"""Extract just the email address from a From header."""
|
||||
if not from_header:
|
||||
return "unknown@unknown"
|
||||
name, addr = email.utils.parseaddr(from_header)
|
||||
return addr.lower() if addr else from_header.lower().strip()
|
||||
|
||||
|
||||
def parse_email_date(date_str):
|
||||
"""Try to parse an email date string."""
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def format_time_for_summary(email_date):
|
||||
"""Format a datetime to something like '2:30 PM'."""
|
||||
if isinstance(email_date, str):
|
||||
email_date = parse_email_date(email_date)
|
||||
# Convert to ET
|
||||
et = email_date - timedelta(hours=4) # EDT roughly
|
||||
return et.strftime("%I:%M %p").lstrip("0")
|
||||
|
||||
|
||||
def format_date_header(date_obj):
|
||||
"""Format a date like 'Wednesday, July 8, 2026'."""
|
||||
if isinstance(date_obj, str):
|
||||
date_obj = datetime.now(timezone.utc)
|
||||
# Convert to ET
|
||||
et = date_obj
|
||||
return et.strftime("%A, %B %d, %Y")
|
||||
|
||||
|
||||
def get_day_name(date_obj):
|
||||
"""Get day name like 'Wednesday'."""
|
||||
if isinstance(date_obj, str):
|
||||
date_obj = datetime.now(timezone.utc)
|
||||
return date_obj.strftime("%A")
|
||||
|
||||
|
||||
# ── IMAP Polling ───────────────────────────────────────────────────────────
|
||||
|
||||
def poll_inbox(child_name, config, mail_log_data, senders_data):
|
||||
"""Poll a single child's inbox for UNSEEN messages. Logs new ones."""
|
||||
child_lower = child_name.lower()
|
||||
new_count = 0
|
||||
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(config["host"], config["port"])
|
||||
conn.login(config["user"], config["pw"])
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
log.info("%s: No UNSEEN messages", child_name)
|
||||
return 0
|
||||
|
||||
for num in data[0].split():
|
||||
try:
|
||||
# BODY.PEEK so we don't mark as seen — boys leave UNSEEN
|
||||
status, msg_data = conn.fetch(num, "(BODY.PEEK[] BODY.PEEK[HEADER])")
|
||||
if status != "OK":
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
|
||||
from_hdr = safe_str(raw.get("From", "")).strip()
|
||||
subject = safe_str(raw.get("Subject", "(no subject)"))
|
||||
date_str = safe_str(raw.get("Date", ""))
|
||||
msg_id = safe_str(raw.get("Message-ID", str(num)))
|
||||
|
||||
sender_email = extract_sender_email(from_hdr)
|
||||
sender_name = extract_sender_name(from_hdr)
|
||||
|
||||
# Build entry
|
||||
entry = {
|
||||
"child": child_name,
|
||||
"from": from_hdr,
|
||||
"from_name": sender_name,
|
||||
"from_email": sender_email,
|
||||
"subject": subject,
|
||||
"date": date_str,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"uid": num.decode(),
|
||||
}
|
||||
|
||||
# Add to mail log
|
||||
mail_log_data.append(entry)
|
||||
|
||||
# Update sender tracking
|
||||
if sender_email not in senders_data:
|
||||
senders_data[sender_email] = {
|
||||
"name": sender_name,
|
||||
"first_seen": entry["timestamp"],
|
||||
"last_seen": entry["timestamp"],
|
||||
"count": 0,
|
||||
"child": child_name,
|
||||
}
|
||||
senders_data[sender_email]["last_seen"] = entry["timestamp"]
|
||||
senders_data[sender_email]["count"] += 1
|
||||
# Update name if we now have a better one
|
||||
if sender_name != "Unknown Sender" and sender_name:
|
||||
senders_data[sender_email]["name"] = sender_name
|
||||
|
||||
new_count += 1
|
||||
log.info(
|
||||
"%s: New email from %s — %s",
|
||||
child_name, sender_email, subject[:60],
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
log.error("%s: Error processing message %s: %s", child_name, num, e)
|
||||
continue
|
||||
|
||||
conn.logout()
|
||||
|
||||
except Exception as e:
|
||||
log.error("%s: IMAP connection error: %s", child_name, e)
|
||||
return 0
|
||||
|
||||
return new_count
|
||||
|
||||
|
||||
def do_collect():
|
||||
"""Collect new emails from both inboxes."""
|
||||
log.info("=" * 50)
|
||||
log.info("Boys' Mail Collection Run")
|
||||
log.info("=" * 50)
|
||||
|
||||
mail_log = load_json(MAIL_LOG, [])
|
||||
senders = load_json(SENDERS_DB, {})
|
||||
total = 0
|
||||
|
||||
for child_name, config in BOYS_ACCOUNTS.items():
|
||||
try:
|
||||
count = poll_inbox(child_name, config, mail_log, senders)
|
||||
total += count
|
||||
except Exception as e:
|
||||
log.error("Failed to poll %s: %s", child_name, e)
|
||||
|
||||
# Trim mail log — keep entries from the last 30 days using the script's timestamp
|
||||
cutoff_ts = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
|
||||
mail_log = [e for e in mail_log if e.get("timestamp", "") >= cutoff_ts]
|
||||
|
||||
save_json(MAIL_LOG, mail_log)
|
||||
save_json(SENDERS_DB, senders)
|
||||
|
||||
log.info("Total new emails collected: %d", total)
|
||||
log.info("Total tracked senders: %d", len(senders))
|
||||
|
||||
# Flag high-volume senders
|
||||
for email_addr, info in senders.items():
|
||||
if info["count"] >= HIGH_VOLUME_THRESHOLD and not info.get("flagged"):
|
||||
info["flagged"] = True
|
||||
log.warning(
|
||||
"HIGH VOLUME: %s (%s) — %d emails for %s",
|
||||
info["name"], email_addr, info["count"], info["child"],
|
||||
)
|
||||
save_json(SENDERS_DB, senders)
|
||||
|
||||
return total
|
||||
|
||||
|
||||
# ── Email Builder ──────────────────────────────────────────────────────────
|
||||
|
||||
def build_summary_html(child_name, today_emails, senders_for_child):
|
||||
"""Build the full HTML summary email body manually (no send-shonuff.py)."""
|
||||
mom_name = MOM_NAMES.get(child_name, "Mom")
|
||||
today = datetime.now(timezone.utc)
|
||||
date_str = format_date_header(today)
|
||||
|
||||
# ── New Emails Table ───────────────────────────────────────────────
|
||||
if not today_emails:
|
||||
emails_section = (
|
||||
'<div style="background:#f0fdf4;border:1px solid #bbf7d0;border-radius:8px;'
|
||||
'padding:16px;margin:16px 0;text-align:center;">'
|
||||
'<p style="font-size:14px;color:#166534;margin:0;">✅ No new emails today</p>'
|
||||
'</div>'
|
||||
)
|
||||
else:
|
||||
rows = ""
|
||||
for em in today_emails:
|
||||
sender_display = em.get("from_name", em.get("from", "Unknown"))
|
||||
subj = escape(em.get("subject", "(no subject)"))
|
||||
t = format_time_for_summary(em.get("date", ""))
|
||||
rows += (
|
||||
"<tr>"
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;">{escape(sender_display)}</td>'
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;">{subj}</td>'
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;white-space:nowrap;">{t}</td>'
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
emails_section = f"""
|
||||
<h3 style="font-size:14px;font-weight:600;margin:20px 0 8px;">📨 New Emails</h3>
|
||||
<table style="width:100%;border-collapse:collapse;font-size:13px;">
|
||||
<tr style="background:#f3f4f6;">
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:left;font-weight:600;">From</th>
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:left;font-weight:600;">Subject</th>
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:left;font-weight:600;">Time</th>
|
||||
</tr>
|
||||
{rows}
|
||||
</table>
|
||||
"""
|
||||
|
||||
# ── Sender Frequency Section ───────────────────────────────────────
|
||||
if not senders_for_child:
|
||||
frequency_section = ""
|
||||
else:
|
||||
# Sort by count descending
|
||||
sorted_senders = sorted(
|
||||
senders_for_child.items(), key=lambda x: x[1]["count"], reverse=True
|
||||
)
|
||||
freq_rows = ""
|
||||
for sender_email, info in sorted_senders:
|
||||
name = info.get("name", sender_email)
|
||||
count = info["count"]
|
||||
note = ""
|
||||
bg = ""
|
||||
if count >= HIGH_VOLUME_THRESHOLD:
|
||||
note = "⚠️ High volume — potential subscription"
|
||||
bg = ' style="background:#fef3c7;"'
|
||||
|
||||
freq_rows += (
|
||||
f"<tr{bg}>"
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;">{escape(name)}<br><span style="font-size:11px;color:#999;">{escape(sender_email)}</span></td>'
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;text-align:center;font-weight:600;">{count}</td>'
|
||||
f'<td style="padding:8px 10px;border:1px solid #e5e7eb;font-size:12px;">{note}</td>'
|
||||
"</tr>"
|
||||
)
|
||||
|
||||
frequency_section = f"""
|
||||
<h3 style="font-size:14px;font-weight:600;margin:20px 0 8px;">📊 This Week's Top Senders</h3>
|
||||
<table style="width:100%;border-collapse:collapse;font-size:13px;">
|
||||
<tr style="background:#f3f4f6;">
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:left;font-weight:600;">Sender</th>
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:center;font-weight:600;">Count</th>
|
||||
<th style="padding:8px 10px;border:1px solid #e5e7eb;text-align:left;font-weight:600;">Note</th>
|
||||
</tr>
|
||||
{freq_rows}
|
||||
</table>
|
||||
"""
|
||||
|
||||
html = f"""<!DOCTYPE html>
|
||||
<html>
|
||||
<head><meta charset="utf-8"></head>
|
||||
<body style="font-family:'Inter',-apple-system,BlinkMacSystemFont,sans-serif;color:#1a1a1a;max-width:560px;margin:0 auto;padding:20px;">
|
||||
|
||||
<!-- Red divider title bar -->
|
||||
<div style="border-bottom:2px solid #dc2626;padding-bottom:12px;margin-bottom:20px;">
|
||||
<h2 style="margin:0;font-size:18px;font-weight:600;">📬 {escape(child_name)}'s Email Summary</h2>
|
||||
<p style="color:#666;font-size:12px;margin:4px 0 0;">{escape(date_str)}</p>
|
||||
</div>
|
||||
|
||||
<!-- Body content -->
|
||||
<p style="font-size:14px;line-height:1.6;color:#333;">Hey {escape(mom_name)}!</p>
|
||||
|
||||
<p style="font-size:14px;line-height:1.6;">Here's what came through today:</p>
|
||||
|
||||
{emails_section}
|
||||
|
||||
{frequency_section}
|
||||
|
||||
<!-- Circular signature -->
|
||||
<p style="font-size:12px;color:#888;font-style:italic;margin:0 0 12px;">_{shonuff_closing()}_</p>
|
||||
<hr style="border:none;height:2px;width:40px;background:#cc0000;margin:0 0 12px 0;border-radius:2px;">
|
||||
<table cellpadding="0" cellspacing="0" border="0" style="font-family:'Inter',-apple-system,sans-serif;color:#2c2c2c;font-size:13px;line-height:1.6;">
|
||||
<tr>
|
||||
<td style="padding-right:14px;vertical-align:middle;width:80px;">
|
||||
<img src="https://core.itpropartner.com/shonuff-signature.png" alt="SB" width="72" height="auto" style="border-radius:50%;display:block;max-width:72px;">
|
||||
</td>
|
||||
<td style="vertical-align:middle;">
|
||||
<div style="font-size:15px;font-weight:700;color:#1a1a1a;letter-spacing:-0.2px;">Sho'Nuff Brown</div>
|
||||
<div style="font-size:12px;color:#888;margin-bottom:1px;">{shonuff_title()}</div>
|
||||
<div style="font-size:12px;color:#aaa;margin-bottom:6px;">iAmGMB</div>
|
||||
<table cellpadding="0" cellspacing="0" border="0" style="font-size:12px;color:#999;line-height:1.7;">
|
||||
<tr><td style="padding-right:4px;vertical-align:top;white-space:nowrap;color:#bbb;">@</td><td><a href="mailto:shonuff@germainebrown.com" style="color:#555;text-decoration:none;border-bottom:1px dotted #ccc;">shonuff@germainebrown.com</a></td></tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
<p style="font-size:11px;color:#aaa;">Reply to adjust how often you get these summaries, request {escape(child_name)}'s login credentials for your phone, or flag senders to unsubscribe.</p>
|
||||
<p style="font-size:11px;color:#aaa;">Webmail: <a href="http://webmail.iamgmb.com" style="color:#1d4ed8;">webmail.iamgmb.com</a></p>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
return html
|
||||
|
||||
|
||||
def build_plain_text(child_name, today_emails, senders_for_child):
|
||||
"""Build a plain-text fallback version."""
|
||||
mom_name = MOM_NAMES.get(child_name, "Mom")
|
||||
lines = [f"{child_name}'s Email Summary", "=" * 40, ""]
|
||||
lines.append(f"Hey {mom_name},")
|
||||
lines.append("")
|
||||
lines.append("Here's what came through today:")
|
||||
lines.append("")
|
||||
|
||||
if not today_emails:
|
||||
lines.append("✅ No new emails today")
|
||||
else:
|
||||
lines.append("📨 New Emails:")
|
||||
lines.append("-" * 40)
|
||||
for em in today_emails:
|
||||
sender = em.get("from_name", em.get("from", "Unknown"))
|
||||
subj = em.get("subject", "(no subject)")
|
||||
t = format_time_for_summary(em.get("date", ""))
|
||||
lines.append(f" From: {sender}")
|
||||
lines.append(f" Re: {subj}")
|
||||
lines.append(f" Time: {t}")
|
||||
lines.append("")
|
||||
|
||||
if senders_for_child:
|
||||
lines.append("📊 Top Senders This Week:")
|
||||
lines.append("-" * 40)
|
||||
sorted_senders = sorted(
|
||||
senders_for_child.items(), key=lambda x: x[1]["count"], reverse=True
|
||||
)
|
||||
for sender_email, info in sorted_senders:
|
||||
name = info.get("name", sender_email)
|
||||
count = info["count"]
|
||||
note = " ⚠️ HIGH VOLUME" if count >= HIGH_VOLUME_THRESHOLD else ""
|
||||
lines.append(f" {name} ({sender_email}): {count}{note}")
|
||||
|
||||
lines.append("")
|
||||
lines.append("--")
|
||||
lines.append("Sho'Nuff Brown — Operations Engineer")
|
||||
lines.append("shonuff@germainebrown.com")
|
||||
lines.append("")
|
||||
lines.append(f"Reply to adjust frequency, request {child_name}'s login credentials for your phone, or flag senders to unsubscribe.")
|
||||
lines.append("Webmail: http://webmail.iamgmb.com")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def send_email(child_name, today_emails, senders_for_child, to_addr):
|
||||
"""
|
||||
Send a daily summary email for a child.
|
||||
Builds HTML manually (no send-shonuff.py wrapper).
|
||||
"""
|
||||
html = build_summary_html(child_name, today_emails, senders_for_child)
|
||||
plain = build_plain_text(child_name, today_emails, senders_for_child)
|
||||
|
||||
subject = f"📬 {child_name}'s Email Summary — {datetime.now(timezone.utc).strftime('%b %d')}"
|
||||
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"{SMTP_FROM_NAME} <{SMTP_FROM}>"
|
||||
msg["To"] = to_addr
|
||||
msg["Bcc"] = BCC_ADDR
|
||||
msg["Subject"] = subject
|
||||
|
||||
msg.attach(MIMEText(plain, "plain"))
|
||||
msg.attach(MIMEText(html, "html"))
|
||||
|
||||
try:
|
||||
with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as s:
|
||||
s.starttls()
|
||||
s.login(SMTP_USER, SMTP_PASS)
|
||||
s.send_message(msg)
|
||||
log.info(
|
||||
"Summary sent for %s to %s (BCC: %s)",
|
||||
child_name, to_addr, BCC_ADDR,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
log.error("Failed to send summary for %s: %s", child_name, e)
|
||||
return False
|
||||
|
||||
|
||||
# ── Summary Generation ─────────────────────────────────────────────────────
|
||||
|
||||
def do_summary():
|
||||
"""Generate and send daily email summaries for both children."""
|
||||
log.info("=" * 50)
|
||||
log.info("Boys' Daily Summary Run")
|
||||
log.info("=" * 50)
|
||||
|
||||
mail_log = load_json(MAIL_LOG, [])
|
||||
senders = load_json(SENDERS_DB, {})
|
||||
|
||||
# Get today's date in ET
|
||||
today = datetime.now(timezone.utc) - timedelta(hours=4) # EDT
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
log.info("Processing summaries for date: %s", today_str)
|
||||
log.info("Total mail log entries: %d", len(mail_log))
|
||||
log.info("Total tracked senders: %d", len(senders))
|
||||
|
||||
results = []
|
||||
|
||||
for child_name in BOYS_ACCOUNTS:
|
||||
mom_name = MOM_NAMES.get(child_name, "Mom")
|
||||
to_addr = RECIPIENTS.get(child_name)
|
||||
|
||||
# Filter today's emails for this child
|
||||
today_emails = [
|
||||
e for e in mail_log
|
||||
if e.get("child") == child_name
|
||||
and parse_email_date(e.get("date", "")).strftime("%Y-%m-%d") == today_str
|
||||
]
|
||||
|
||||
# Filter senders for this child
|
||||
senders_for_child = {
|
||||
email_addr: info
|
||||
for email_addr, info in senders.items()
|
||||
if info.get("child") == child_name
|
||||
}
|
||||
|
||||
log.info(
|
||||
"%s: %d emails today, %d tracked senders, sending to %s",
|
||||
child_name, len(today_emails), len(senders_for_child), to_addr,
|
||||
)
|
||||
|
||||
if to_addr and "@" in to_addr and "example.com" not in to_addr:
|
||||
success = send_email(child_name, today_emails, senders_for_child, to_addr)
|
||||
results.append((child_name, to_addr, success, len(today_emails)))
|
||||
else:
|
||||
log.warning(
|
||||
"%s: No valid recipient email configured for %s (%s). "
|
||||
"Germaine needs to provide it!",
|
||||
child_name, mom_name, to_addr,
|
||||
)
|
||||
# Print the summary to log anyway for verification
|
||||
log.info(
|
||||
"Would have sent to %s:\n%s",
|
||||
to_addr,
|
||||
build_summary_html(child_name, today_emails, senders_for_child),
|
||||
)
|
||||
results.append((child_name, to_addr, False, len(today_emails)))
|
||||
|
||||
log.info("Summary run complete: %s", results)
|
||||
return results
|
||||
|
||||
|
||||
def do_test_summary():
|
||||
"""Send a test summary to Sho'Nuff himself to verify formatting."""
|
||||
log.info("=" * 50)
|
||||
log.info("Boys' Mail — Test Summary (sent to shonuff inbox)")
|
||||
log.info("=" * 50)
|
||||
|
||||
mail_log = load_json(MAIL_LOG, [])
|
||||
senders = load_json(SENDERS_DB, {})
|
||||
|
||||
today = datetime.now(timezone.utc) - timedelta(hours=4)
|
||||
today_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
for child_name in BOYS_ACCOUNTS:
|
||||
today_emails = [
|
||||
e for e in mail_log
|
||||
if e.get("child") == child_name
|
||||
and parse_email_date(e.get("date", "")).strftime("%Y-%m-%d") == today_str
|
||||
]
|
||||
senders_for_child = {
|
||||
email_addr: info
|
||||
for email_addr, info in senders.items()
|
||||
if info.get("child") == child_name
|
||||
}
|
||||
|
||||
# Send test to sho'nuff
|
||||
subject = f"📬 TEST: {child_name}'s Email Summary — {today.strftime('%b %d')}"
|
||||
html = build_summary_html(child_name, today_emails, senders_for_child)
|
||||
plain = build_plain_text(child_name, today_emails, senders_for_child)
|
||||
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"{SMTP_FROM_NAME} <{SMTP_FROM}>"
|
||||
msg["To"] = SMTP_FROM
|
||||
msg["Subject"] = subject
|
||||
|
||||
msg.attach(MIMEText(plain, "plain"))
|
||||
msg.attach(MIMEText(html, "html"))
|
||||
|
||||
with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as s:
|
||||
s.starttls()
|
||||
s.login(SMTP_USER, SMTP_PASS)
|
||||
s.send_message(msg)
|
||||
|
||||
log.info("Test summary for %s sent to %s", child_name, SMTP_FROM)
|
||||
|
||||
print("✅ Test summaries sent to shonuff@germainebrown.com — check your inbox.")
|
||||
|
||||
|
||||
# ── Main CLI ───────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: boys-mail-monitor.py [collect|summary|test-summary]")
|
||||
print(" collect — Poll inboxes for new emails")
|
||||
print(" summary — Send daily summary emails")
|
||||
print(" test-summary — Send test summaries to shonuff inbox")
|
||||
return
|
||||
|
||||
command = sys.argv[1]
|
||||
|
||||
if command == "collect":
|
||||
do_collect()
|
||||
elif command == "summary":
|
||||
do_summary()
|
||||
elif command == "test-summary":
|
||||
do_test_summary()
|
||||
else:
|
||||
print(f"Unknown command: {command}")
|
||||
print("Usage: boys-mail-monitor.py [collect|summary|test-summary]")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the full Hermes recovery bundle markdown file."""
|
||||
import sqlite3, json, datetime, os, subprocess, sys
|
||||
|
||||
db = sqlite3.connect('/root/.hermes/state.db')
|
||||
c = db.cursor()
|
||||
|
||||
# ===== 1. Get today's user conversations =====
|
||||
sessions_data = []
|
||||
c.execute("""
|
||||
SELECT id, title, started_at, ended_at, end_reason, message_count
|
||||
FROM sessions
|
||||
WHERE started_at >= 1783180800 AND id LIKE '2026%'
|
||||
ORDER BY started_at
|
||||
""")
|
||||
for row in c.fetchall():
|
||||
sessions_data.append({
|
||||
'id': row[0],
|
||||
'title': row[1] or '(no title)',
|
||||
'started': datetime.datetime.fromtimestamp(row[2]).strftime('%H:%M:%S'),
|
||||
'ended': datetime.datetime.fromtimestamp(row[3]).strftime('%H:%M:%S') if row[3] else 'active',
|
||||
'end_reason': row[4] or 'running',
|
||||
'msg_count': row[5]
|
||||
})
|
||||
|
||||
conversation_text = ""
|
||||
for sd in sessions_data:
|
||||
conversation_text += f"\n\n{'='*70}\n"
|
||||
conversation_text += f"SESSION: {sd['id']} | {sd['title']}\n"
|
||||
conversation_text += f"Started: {sd['started']} | Ended: {sd['ended']} | Status: {sd['end_reason']}\n"
|
||||
conversation_text += f"{'='*70}\n\n"
|
||||
|
||||
c.execute("SELECT role, content, timestamp FROM messages WHERE session_id = ? ORDER BY id", (sd['id'],))
|
||||
for role, content, ts in c.fetchall():
|
||||
ts_dt = datetime.datetime.fromtimestamp(ts).strftime('%H:%M:%S')
|
||||
text = (content or '(empty)').strip()
|
||||
# Truncate very long messages
|
||||
if len(text) > 5000:
|
||||
text = text[:5000] + f"\n... [truncated, was {len(content)} chars]"
|
||||
conversation_text += f"[{ts_dt}] {role.upper()}:\n{text}\n\n"
|
||||
|
||||
db.close()
|
||||
|
||||
# ===== 2. Read key files =====
|
||||
def readf(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return f.read()
|
||||
except Exception as e:
|
||||
return f"[ERROR: {e}]"
|
||||
|
||||
soul = readf('/root/.hermes/SOUL.md')
|
||||
config_yaml = readf('/root/.hermes/config.yaml')
|
||||
dotenv = readf('/root/.hermes/.env')
|
||||
aws_creds = readf('/root/.aws/credentials')
|
||||
service_file = readf('/etc/systemd/system/hermes.service')
|
||||
wisp_pub = readf('/root/.ssh/wisp_rsa.pub')
|
||||
wisp_priv = readf('/root/.ssh/wisp_rsa')
|
||||
imap_pass = readf('/root/.config/himalaya/g-germainebrown.pass')
|
||||
hetzner_token = readf('/root/.hermes/scripts/.hetzner_token')
|
||||
netcup_key = readf('/root/.hermes/scripts/.netcup_api_key')
|
||||
wisp_config = readf('/root/.hermes/scripts/wisp-backup/config.yaml')
|
||||
|
||||
# ===== 3. Get cron jobs =====
|
||||
result = subprocess.run(['hermes', 'cron', 'list', '--json'], capture_output=True, text=True, timeout=15)
|
||||
cron_text = ""
|
||||
try:
|
||||
cron_data = json.loads(result.stdout)
|
||||
cron_jobs = cron_data if isinstance(cron_data, list) else cron_data.get('jobs', cron_data.get('data', []))
|
||||
if not cron_jobs:
|
||||
cron_text = "(no JSON output — fallback to text list)\n"
|
||||
# Fallback: run text mode
|
||||
result2 = subprocess.run(['hermes', 'cron', 'list'], capture_output=True, text=True, timeout=15)
|
||||
cron_text += result2.stdout
|
||||
else:
|
||||
for job in cron_jobs:
|
||||
cron_text += f"\n### {job.get('name', 'unnamed')}\n"
|
||||
cron_text += f"- ID: {job.get('id', '?')}\n"
|
||||
cron_text += f"- Schedule: {job.get('schedule', '?')}\n"
|
||||
cron_text += f"- Repeat: {job.get('repeat', '∞')}\n"
|
||||
cron_text += f"- Deliver: {job.get('deliver', '?')}\n"
|
||||
cron_text += f"- Script: {job.get('script', 'none')}\n"
|
||||
cron_text += f"- Skills: {job.get('skills', 'none')}\n"
|
||||
cron_text += f"- Mode: {'no-agent' if job.get('no_agent') else 'agent-driven'}\n"
|
||||
cron_text += f"- Next run: {job.get('next_run', '?')}\n"
|
||||
except Exception as e:
|
||||
cron_text = f"[Error getting cron jobs: {e}]\n"
|
||||
result2 = subprocess.run(['hermes', 'cron', 'list'], capture_output=True, text=True, timeout=15)
|
||||
cron_text += result2.stdout
|
||||
|
||||
# ===== 4. Build the recovery document =====
|
||||
doc = f"""# Hermes Full Recovery Bundle — Jul 5, 2026
|
||||
|
||||
> Save this file. If the server dies, paste the contents of the **BOOTSTRAP** section
|
||||
> below into a new Hermes agent to restore me.
|
||||
|
||||
---
|
||||
## BOOTSTRAP INSTRUCTIONS
|
||||
|
||||
### Step 1 — Prerequisites
|
||||
1. Install Hermes Agent v0.18.0+ on a Linux server (Debian/Ubuntu)
|
||||
2. Register a Telegram bot with BotFather, get a TELEGRAM_BOT_TOKEN
|
||||
3. Make sure python3, git, and systemd are available
|
||||
|
||||
### Step 2 — Create files
|
||||
Create these files with the exact contents shown in their sections below:
|
||||
- `/root/.hermes/config.yaml`
|
||||
- `/root/.hermes/.env`
|
||||
- `/root/.aws/credentials`
|
||||
- `/root/.ssh/wisp_rsa` (chmod 600)
|
||||
- `/root/.hermes/scripts/.hetzner_token` (chmod 600)
|
||||
- `/root/.hermes/scripts/.netcup_api_key` (chmod 600)
|
||||
- `/root/.hermes/SOUL.md`
|
||||
- All scripts in the Scripts section
|
||||
|
||||
### Step 3 — Start Hermes
|
||||
```bash
|
||||
hermes gateway start
|
||||
```
|
||||
|
||||
### Step 4 — Recreate cron jobs
|
||||
```bash
|
||||
# Run these commands after Hermes is online
|
||||
# (cron job definitions below)
|
||||
```
|
||||
|
||||
### Step 5 — Verify
|
||||
```bash
|
||||
hermes gateway status
|
||||
hermes cron list
|
||||
source /opt/awscli-venv/bin/activate && aws s3 ls s3://hermes-vps-backups/ --endpoint-url https://s3.us-east-1.wasabisys.com/
|
||||
```
|
||||
|
||||
---
|
||||
## CREDENTIALS
|
||||
|
||||
**SMTP Password (g@germainebrown.com):** `{imap_pass.strip()}`
|
||||
**Wasabi S3:** endpoint=https://s3.us-east-1.wasabisys.com | Access Key: GYH83FP0KL0K85N60JKQ
|
||||
**Hetzner API Token:** `{hetzner_token.strip()}`
|
||||
**netcup API Key:** `{netcup_key.strip()}`
|
||||
**admin-ai API Key:** In config.yaml (sk-lbhhxIk...bWPA)
|
||||
**Telegram Bot Token:** In .env file below
|
||||
|
||||
---
|
||||
## CONFIG FILES
|
||||
|
||||
### /root/.hermes/SOUL.md
|
||||
```
|
||||
{soul}
|
||||
```
|
||||
|
||||
### /root/.hermes/config.yaml
|
||||
```yaml
|
||||
{config_yaml}
|
||||
```
|
||||
|
||||
### /root/.hermes/.env
|
||||
```
|
||||
{dotenv}
|
||||
```
|
||||
|
||||
### /root/.aws/credentials
|
||||
```
|
||||
{aws_creds}
|
||||
```
|
||||
|
||||
### /etc/systemd/system/hermes.service
|
||||
```
|
||||
{service_file}
|
||||
```
|
||||
|
||||
---
|
||||
## SSH KEYS
|
||||
|
||||
### WISP Public Key
|
||||
```
|
||||
{wisp_pub}
|
||||
```
|
||||
|
||||
### WISP Private Key
|
||||
```
|
||||
{wisp_priv}
|
||||
```
|
||||
|
||||
---
|
||||
## WISP BACKUP CONFIG
|
||||
|
||||
```yaml
|
||||
{wisp_config}
|
||||
```
|
||||
|
||||
---
|
||||
## CRON JOBS
|
||||
{cron_text}
|
||||
|
||||
---
|
||||
## SCRIPT FILES
|
||||
|
||||
### imap_triage.py
|
||||
"""
|
||||
# Read scripts
|
||||
script_dir = '/root/.hermes/scripts'
|
||||
for fname in sorted(os.listdir(script_dir)):
|
||||
fpath = os.path.join(script_dir, fname)
|
||||
if os.path.isfile(fpath) and not fname.startswith('.'):
|
||||
content = readf(fpath)
|
||||
doc += f"\n### {fname}\n```\n{content}\n```\n"
|
||||
|
||||
# Also read wisp-backup subdirectory scripts
|
||||
wisp_dir = os.path.join(script_dir, 'wisp-backup')
|
||||
if os.path.isdir(wisp_dir):
|
||||
for fname in sorted(os.listdir(wisp_dir)):
|
||||
fpath = os.path.join(wisp_dir, fname)
|
||||
if os.path.isfile(fpath) and not fname.endswith('.pyc'):
|
||||
content = readf(fpath)
|
||||
doc += f"\n### wisp-backup/{fname}\n```\n{content}\n```\n"
|
||||
|
||||
doc += "\n\n---\n\n## TODAY'S CONVERSATION HISTORY\n"
|
||||
doc += conversation_text
|
||||
|
||||
# ===== 5. Write file =====
|
||||
output_path = '/root/.hermes/recovery-bundle-2026-07-05.md'
|
||||
with open(output_path, 'w') as f:
|
||||
f.write(doc)
|
||||
|
||||
file_size = os.path.getsize(output_path)
|
||||
print(f"RECOVERY FILE: {output_path}")
|
||||
print(f"SIZE: {file_size} bytes ({file_size/1024:.1f} KB)")
|
||||
print(f"TODAY SESSIONS: {len(sessions_data)}")
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# changelog.sh — Append an entry to the Hermes changelog
|
||||
# Usage: changelog.sh "Category" "Description"
|
||||
# changelog.sh "Backup" "Changed S3 sync from hourly to every 15 min"
|
||||
# changelog.sh "SSH" "Deployed itpp-infra key to all servers"
|
||||
# changelog.sh "$(date +%H:%M)" "Quick note without a category"
|
||||
|
||||
CHANGELOG="/root/.hermes/CHANGELOG.md"
|
||||
DATE=$(date "+%Y-%m-%d")
|
||||
TIME=$(date "+%H:%M")
|
||||
CATEGORY="${1:-General}"
|
||||
DESCRIPTION="${2:-No description}"
|
||||
|
||||
# Create a new date section if this is the first entry for today
|
||||
if ! grep -q "^## $DATE" "$CHANGELOG" 2>/dev/null; then
|
||||
echo -e "\n---\n## $DATE\n" >> "$CHANGELOG"
|
||||
fi
|
||||
|
||||
# Insert the entry under today's section
|
||||
# We use sed to find today's section and append after it
|
||||
sed -i "/^## $DATE/a\\\n### $CATEGORY\n- $DESCRIPTION" "$CHANGELOG"
|
||||
|
||||
echo "✅ Changelog entry added: [$DATE] $CATEGORY — $DESCRIPTION"
|
||||
Executable
+267
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
daily-feed-summary.py — Collect RSS feeds, summarize top articles, deliver digest.
|
||||
|
||||
Feeds:
|
||||
- XDA Developers: https://www.xda-developers.com/feed/
|
||||
- MakeUseOf: https://www.makeuseof.com/feed/
|
||||
- VirtualizationHowTo: https://www.virtualizationhowto.com/feed/
|
||||
- Top Gear: No RSS — scraped from front page
|
||||
- 9to5Mac: https://9to5mac.com/feed/
|
||||
|
||||
Output: Markdown summary with top 10-15 articles across all feeds.
|
||||
"""
|
||||
|
||||
import feedparser, json, sys, os, re, subprocess
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from email.mime.text import MIMEText
|
||||
import smtplib
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
|
||||
FEEDS = {
|
||||
"XDA Developers": "https://www.xda-developers.com/feed/",
|
||||
"MakeUseOf": "https://www.makeuseof.com/feed/",
|
||||
"Virtualization HowTo": "https://www.virtualizationhowto.com/feed/",
|
||||
"9to5Mac": "https://9to5mac.com/feed/",
|
||||
"Gizmodo": "https://gizmodo.com/feed/",
|
||||
"How-To Geek": "https://www.howtogeek.com/feed/",
|
||||
}
|
||||
|
||||
# Top Gear — no RSS, try scraping car-news section
|
||||
TOPGEAR_URL = "https://www.topgear.com/car-news"
|
||||
|
||||
class HHLinkParser(HTMLParser):
|
||||
"""Parse HotHardware /news page for article links."""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.links = []
|
||||
self._in_link = False
|
||||
self._href = None
|
||||
self._text = ""
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == 'a' and 'href' in attrs:
|
||||
href = attrs['href']
|
||||
if href.startswith('/news/') and len(href) > 7:
|
||||
self._in_link = True
|
||||
self._href = href
|
||||
self._text = ""
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._in_link:
|
||||
self._text += data.strip()
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self._in_link and tag == 'a':
|
||||
self._in_link = False
|
||||
txt = self._text.strip()
|
||||
if txt and len(txt) > 10:
|
||||
self.links.append((txt, self._href))
|
||||
|
||||
def fetch_hothardware():
|
||||
"""Scrape HotHardware news page for article links."""
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
"https://hothardware.com/news",
|
||||
headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
html = resp.read().decode('utf-8', errors='replace')
|
||||
|
||||
parser = HHLinkParser()
|
||||
parser.feed(html)
|
||||
|
||||
seen = set()
|
||||
articles = []
|
||||
for title, href in parser.links:
|
||||
if href not in seen:
|
||||
seen.add(href)
|
||||
url = f"https://hothardware.com{href}" if href.startswith('/') else href
|
||||
articles.append((title, url))
|
||||
|
||||
return articles[:10]
|
||||
except Exception as e:
|
||||
return [("HotHardware scrape failed", f"Error: {e}")]
|
||||
|
||||
|
||||
class TGLinkParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.links = []
|
||||
self._capture = False
|
||||
self._href = None
|
||||
self._depth = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == 'a' and 'href' in attrs:
|
||||
href = attrs['href']
|
||||
if href.startswith('/car-news/') and not href.startswith('/car-news/?'):
|
||||
title = attrs.get('title', '') or ''
|
||||
text = ''
|
||||
if title:
|
||||
self.links.append((href, title))
|
||||
self._href = href
|
||||
|
||||
def fetch_topgear():
|
||||
"""Scrape Top Gear front page for article links."""
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
TOPGEAR_URL,
|
||||
headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
html = resp.read().decode('utf-8', errors='replace')
|
||||
|
||||
parser = TGLinkParser()
|
||||
parser.feed(html)
|
||||
|
||||
# Deduplicate by URL
|
||||
seen = set()
|
||||
articles = []
|
||||
for href, title in parser.links:
|
||||
if href not in seen:
|
||||
seen.add(href)
|
||||
url = f"https://www.topgear.com{href}" if href.startswith('/') else href
|
||||
articles.append((title, url))
|
||||
|
||||
return articles[:15]
|
||||
except Exception as e:
|
||||
return [("Top Gear scrape failed", f"Error: {e}")]
|
||||
|
||||
def fetch_feed(url, limit=10):
|
||||
"""Parse RSS feed, return list of (title, link, summary)."""
|
||||
try:
|
||||
feed = feedparser.parse(url)
|
||||
entries = []
|
||||
for entry in feed.entries[:limit]:
|
||||
title = entry.get('title', 'No title')
|
||||
link = entry.get('link', '')
|
||||
# Get summary/description
|
||||
summary = ''
|
||||
if 'summary' in entry:
|
||||
summary = entry.summary
|
||||
elif 'description' in entry:
|
||||
summary = entry.description
|
||||
# Strip HTML
|
||||
summary = re.sub(r'<[^>]+>', '', summary)[:300]
|
||||
entries.append((title, link, summary))
|
||||
return entries
|
||||
except Exception as e:
|
||||
return [(f"Feed error: {e}", "", "")]
|
||||
|
||||
def build_digest():
|
||||
"""Collect all feeds and build markdown digest."""
|
||||
lines = []
|
||||
now = datetime.now(timezone.utc)
|
||||
eastern = now - timedelta(hours=4) # approximate ET
|
||||
lines.append(f"# 📰 Daily Tech Digest — {eastern.strftime('%A, %B %d, %Y')}")
|
||||
lines.append("")
|
||||
|
||||
all_articles = []
|
||||
|
||||
# RSS feeds
|
||||
for name, url in FEEDS.items():
|
||||
entries = fetch_feed(url, 10)
|
||||
lines.append(f"## {name}")
|
||||
lines.append("")
|
||||
for title, link, summary in entries:
|
||||
if link:
|
||||
lines.append(f"- **[{title}]({link})**")
|
||||
else:
|
||||
lines.append(f"- **{title}**")
|
||||
if summary:
|
||||
lines.append(f" _{summary}_")
|
||||
lines.append("")
|
||||
all_articles.extend(entries)
|
||||
|
||||
# Top Gear
|
||||
lines.append("## Top Gear")
|
||||
lines.append("")
|
||||
tg = fetch_topgear()
|
||||
for title, url in tg:
|
||||
if url.startswith("https://"):
|
||||
lines.append(f"- **[{title}]({url})**")
|
||||
else:
|
||||
lines.append(f"- *{title}*")
|
||||
lines.append("")
|
||||
all_articles.extend([(t, u, "") for t, u in tg])
|
||||
|
||||
# HotHardware (scraped — no RSS)
|
||||
lines.append("## HotHardware")
|
||||
lines.append("")
|
||||
hh = fetch_hothardware()
|
||||
for title, url in hh:
|
||||
if url.startswith("https://"):
|
||||
lines.append(f"- **[{title}]({url})**")
|
||||
else:
|
||||
lines.append(f"- *{title}*")
|
||||
lines.append("")
|
||||
all_articles.extend([(t, u, "") for t, u in hh])
|
||||
|
||||
lines.append("---")
|
||||
lines.append(f"*{len(all_articles)} articles from {len(FEEDS) + 2} sources*")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def send_email(body):
|
||||
"""Send digest via SMTP to g@germainebrown.com as HTML."""
|
||||
sender = "g@germainebrown.com"
|
||||
pwd_file = "/root/.config/himalaya/g-germainebrown.pass"
|
||||
with open(pwd_file) as f:
|
||||
password = f.read().strip()
|
||||
|
||||
# Convert markdown links [text](url) to HTML <a href="url">text</a>
|
||||
import re
|
||||
html_body = re.sub(
|
||||
r'\[([^\]]+)\]\(([^)]+)\)',
|
||||
r'<a href="\2" style="color:#2563eb;text-decoration:underline;">\1</a>',
|
||||
body
|
||||
)
|
||||
# Convert markdown bold **text** to <strong>text</strong>
|
||||
html_body = re.sub(r'\*\*([^*]+)\*\*', r'<strong>\1</strong>', html_body)
|
||||
# Convert markdown italic _text_ to <em>text</em>
|
||||
html_body = re.sub(r'_([^_]+)_', r'<em>\1</em>', html_body)
|
||||
# Convert ## headers to <h2>
|
||||
html_body = re.sub(r'^## (.+)$', r'<h2 style="font-size:16px;margin:16px 0 4px;color:#1a1a2e;">\1</h2>', html_body, flags=re.MULTILINE)
|
||||
# Convert # header to <h1>
|
||||
html_body = re.sub(r'^# (.+)$', r'<h1 style="font-size:20px;margin:0 0 8px;color:#1a1a2e;">\1</h1>', html_body, flags=re.MULTILINE)
|
||||
# Convert --- to <hr>
|
||||
html_body = re.sub(r'^---+$', r'<hr style="border:none;border-top:1px solid #e2e8f0;margin:16px 0;">', html_body, flags=re.MULTILINE)
|
||||
# Wrap in proper HTML
|
||||
html = f'''<!DOCTYPE html>
|
||||
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
|
||||
<style>body{{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;font-size:13px;line-height:1.6;color:#333;padding:20px;max-width:640px;margin:0 auto;}}
|
||||
a{{color:#2563eb;}}p{{margin:4px 0;}}</style>
|
||||
</head><body>
|
||||
<pre style="font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;font-size:13px;line-height:1.6;white-space:pre-wrap;word-wrap:break-word;">{body}</pre>
|
||||
</body></html>'''
|
||||
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
|
||||
msg = MIMEMultipart('alternative')
|
||||
msg['Subject'] = f"Daily Tech Digest — {datetime.now().strftime('%b %d')}"
|
||||
msg['From'] = sender
|
||||
msg['To'] = sender
|
||||
|
||||
msg.attach(MIMEText(body, 'plain', 'utf-8'))
|
||||
msg.attach(MIMEText(html, 'html', 'utf-8'))
|
||||
|
||||
with smtplib.SMTP('mail.germainebrown.com', 2525) as s:
|
||||
s.starttls()
|
||||
s.login(sender, password)
|
||||
s.send_message(msg)
|
||||
|
||||
print(f"Digest sent to {sender}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
digest = build_digest()
|
||||
# Output to stdout (captured by cron) and also email
|
||||
print(digest)
|
||||
|
||||
# If run as standalone, also email
|
||||
if len(sys.argv) > 1 and sys.argv[1] == '--email':
|
||||
send_email(digest)
|
||||
@@ -0,0 +1,3 @@
|
||||
:local key "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDnI4UwwPL8gJvtP/Jr7qiw0Qj/bQBwi2+f03p730xvn wisp-backup"
|
||||
:put $key file=wisp-key.txt
|
||||
:put "done"
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# dre-approval-reminder.sh — Check for pending claim approvals
|
||||
# Runs hourly. Silent if no pending approvals.
|
||||
# Reports to Germaine + Tony via Hermes if approvals are waiting.
|
||||
|
||||
PENDING_FILE="$HOME/.hermes/scripts/dre/pending-approvals.json"
|
||||
|
||||
if [ ! -f "$PENDING_FILE" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 -c "
|
||||
import json, sys, os
|
||||
with open('$PENDING_FILE') as f:
|
||||
pending = json.load(f)
|
||||
|
||||
if not pending:
|
||||
sys.exit(0)
|
||||
|
||||
count = len(pending)
|
||||
print(f'⚠️ DRE — {count} claim(s) pending approval')
|
||||
for p in pending:
|
||||
print(f' • {p[\"claim\"]} — {p[\"client\"]} · \${p[\"amount\"]} · Score: {p[\"score\"]}')
|
||||
print()
|
||||
print(f'Hourly reminder — Germaine + Tony need to acknowledge.')
|
||||
print(f'Anita can view in dashboard.')
|
||||
"
|
||||
|
||||
# Only exit non-zero if there are actually pending approvals
|
||||
[ -s "$PENDING_FILE" ] && exit 1 || exit 0
|
||||
Executable
+288
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""DRE unified inbox poller — checks dre@ and collections@ on MXroute every 60s.
|
||||
|
||||
Writes parsed emails to /var/www/internal/data/dre-mails.json for the DRE portal.
|
||||
Leaves messages as UNSEEN so the original mailbox user still sees them as new.
|
||||
"""
|
||||
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from email.header import decode_header, make_header
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────────
|
||||
|
||||
IMAP_HOST = "heracles.mxrouting.net"
|
||||
IMAP_PORT = 993
|
||||
|
||||
MAILBOXES = [
|
||||
{"label": "dre", "user": "dre@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
||||
{"label": "collections", "user": "collections@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
||||
]
|
||||
|
||||
OUTPUT_FILE = "/var/www/internal/data/dre-mails.json"
|
||||
MAX_ENTRIES = 500
|
||||
BODY_PREVIEW_CHARS = 500 # Full body stored, preview truncated for portal
|
||||
|
||||
# Claim pattern: DRE-XXXX-XXXX
|
||||
CLAIM_RE = re.compile(r"DRE-\d{4}-\d{4}")
|
||||
|
||||
# ── Logging ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def setup_logging():
|
||||
logger = logging.getLogger("dre-mail-poller")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# Syslog
|
||||
try:
|
||||
syslog = logging.handlers.SysLogHandler(address="/dev/log")
|
||||
syslog.setFormatter(logging.Formatter("dre-mail-poller: %(message)s"))
|
||||
logger.addHandler(syslog)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# File log
|
||||
fh = logging.FileHandler("/var/log/dre-mail-poller.log")
|
||||
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
logger.addHandler(fh)
|
||||
|
||||
# Also stderr for cron visibility
|
||||
sh = logging.StreamHandler(sys.stderr)
|
||||
sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
logger.addHandler(sh)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
log = setup_logging()
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def decode_str(val: str | None) -> str:
|
||||
"""Decode an RFC2047-encoded header value to plain text."""
|
||||
if not val:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(val)))
|
||||
except Exception:
|
||||
return val.strip()
|
||||
|
||||
|
||||
def decode_body(part) -> str:
|
||||
"""Extract text from an email part, handling base64/quoted-printable."""
|
||||
try:
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
return ""
|
||||
return payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def get_body(msg) -> str:
|
||||
"""Extract the plain-text body of an email (prefer text/plain, fallback text/html)."""
|
||||
if msg.is_multipart():
|
||||
# Try text/plain first
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
body = decode_body(part)
|
||||
if body.strip():
|
||||
return body
|
||||
# Fallback to text/html
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/html":
|
||||
return decode_body(part)
|
||||
# Last resort: first text/* part
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_maintype()
|
||||
if ct == "text":
|
||||
return decode_body(part)
|
||||
else:
|
||||
return decode_body(msg)
|
||||
return ""
|
||||
|
||||
|
||||
def clean_body(text: str) -> str:
|
||||
"""Strip excessive whitespace from body text."""
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_date(date_str: str | None) -> str:
|
||||
"""Parse email Date header to ISO 8601 string, fallback to now."""
|
||||
if not date_str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def check_claim(subject: str) -> str | None:
|
||||
"""Extract the DRE claim ID from subject if present."""
|
||||
m = CLAIM_RE.search(subject)
|
||||
return m.group(0) if m else None
|
||||
|
||||
|
||||
# ── JSON Database ──────────────────────────────────────────────────────────────
|
||||
|
||||
def load_db() -> list[dict]:
|
||||
"""Load existing email database from disk."""
|
||||
if not os.path.exists(OUTPUT_FILE):
|
||||
return []
|
||||
try:
|
||||
with open(OUTPUT_FILE, "r") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
log.warning(f"Could not load {OUTPUT_FILE}: {e} — starting fresh")
|
||||
return []
|
||||
|
||||
|
||||
def save_db(emails: list[dict]):
|
||||
"""Write email database to disk, keeping last MAX_ENTRIES."""
|
||||
emails.sort(key=lambda e: e.get("date", ""), reverse=True)
|
||||
trimmed = emails[:MAX_ENTRIES]
|
||||
# Ensure output directory exists
|
||||
os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(trimmed, f, indent=2, ensure_ascii=False)
|
||||
log.info(f"Written {len(trimmed)} emails to {OUTPUT_FILE}")
|
||||
|
||||
|
||||
# ── IMAP Polling ───────────────────────────────────────────────────────────────
|
||||
|
||||
def poll_mailbox(label: str, user: str, password: str, existing_ids: set) -> list[dict]:
|
||||
"""Connect to a mailbox, fetch UNSEEN messages, return parsed email records."""
|
||||
new_emails = []
|
||||
conn = None
|
||||
try:
|
||||
log.info(f"Connecting to {label} ({user})…")
|
||||
conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT)
|
||||
conn.login(user, password)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
log.info(f"No UNSEEN messages in {label}")
|
||||
conn.logout()
|
||||
return []
|
||||
|
||||
uids = data[0].split()
|
||||
log.info(f"Found {len(uids)} UNSEEN message(s) in {label}")
|
||||
|
||||
for uid_bytes in uids:
|
||||
uid_str = uid_bytes.decode()
|
||||
# Fetch by UID to get stable identifiers
|
||||
status, fetch_data = conn.uid("fetch", uid_bytes, "(RFC822 UID)")
|
||||
if status != "OK":
|
||||
continue
|
||||
|
||||
raw_email = None
|
||||
uid_val = uid_str
|
||||
for part in fetch_data:
|
||||
if isinstance(part, tuple):
|
||||
raw_email = part[1]
|
||||
elif isinstance(part, bytes) and part.startswith(b"UID "):
|
||||
uid_val = part.decode().split("UID ")[-1].strip()
|
||||
|
||||
if raw_email is None:
|
||||
continue
|
||||
|
||||
# Build unique ID: mailbox label + UID
|
||||
unique_id = f"{label}:{uid_val}"
|
||||
|
||||
# Skip duplicates
|
||||
if unique_id in existing_ids:
|
||||
log.debug(f"Skipping duplicate: {unique_id}")
|
||||
continue
|
||||
|
||||
msg = email.message_from_bytes(raw_email)
|
||||
|
||||
subject = decode_str(msg.get("Subject"))
|
||||
from_addr = decode_str(msg.get("From"))
|
||||
to_addr = decode_str(msg.get("To"))
|
||||
date_str = parse_date(msg.get("Date"))
|
||||
body_full = clean_body(get_body(msg))
|
||||
body_preview = body_full[:BODY_PREVIEW_CHARS]
|
||||
claim_id = check_claim(subject)
|
||||
|
||||
record = {
|
||||
"id": unique_id,
|
||||
"mailbox": label,
|
||||
"from": from_addr,
|
||||
"to": to_addr,
|
||||
"subject": subject,
|
||||
"date": date_str,
|
||||
"body_preview": body_preview,
|
||||
"body": body_full,
|
||||
"is_read": False,
|
||||
"claim_match": claim_id,
|
||||
}
|
||||
|
||||
new_emails.append(record)
|
||||
existing_ids.add(unique_id)
|
||||
log.info(f"New email [{label}] From: {from_addr} | Subject: {subject[:80]}")
|
||||
|
||||
except imaplib.IMAP4.error as e:
|
||||
log.error(f"IMAP error for {label}: {e}")
|
||||
except (ConnectionRefusedError, TimeoutError, OSError) as e:
|
||||
log.error(f"Connection error for {label}: {e}")
|
||||
except Exception as e:
|
||||
log.error(f"Unexpected error for {label}: {e}")
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return new_emails
|
||||
|
||||
|
||||
def main():
|
||||
log.info("=" * 60)
|
||||
log.info("DRE mail poller starting…")
|
||||
|
||||
# Load existing database
|
||||
emails = load_db()
|
||||
existing_ids = {e["id"] for e in emails if "id" in e}
|
||||
log.info(f"Loaded {len(emails)} existing records ({len(existing_ids)} unique IDs)")
|
||||
|
||||
fresh_count = 0
|
||||
for mb in MAILBOXES:
|
||||
new_msgs = poll_mailbox(mb["label"], mb["user"], mb["pass"], existing_ids)
|
||||
emails.extend(new_msgs)
|
||||
fresh_count += len(new_msgs)
|
||||
|
||||
if fresh_count > 0:
|
||||
log.info(f"Collected {fresh_count} new message(s), saving…")
|
||||
save_db(emails)
|
||||
else:
|
||||
log.info("No new messages — database unchanged")
|
||||
# Still ensure the file exists (first run with no mail)
|
||||
if not os.path.exists(OUTPUT_FILE):
|
||||
save_db(emails)
|
||||
|
||||
log.info("DRE mail poller complete.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
{"total_used": 0, "calls": [], "monthly": {}, "last_reset": "2026-07-09T00:00:00-04:00"}
|
||||
Executable
+102
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute daily stats from the Traccar H2 DB for the FT360 dashboard export."""
|
||||
import os, subprocess, json, math
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
APP2='152.53.39.202'
|
||||
SSH_KEY='/root/.ssh/itpp-infra'
|
||||
H2='/tmp/h2.jar'
|
||||
LOCAL_DIR='/tmp/ft360_daily_stats'
|
||||
DB=os.path.join(LOCAL_DIR,'database.mv.db')
|
||||
|
||||
os.makedirs(LOCAL_DIR,exist_ok=True)
|
||||
subprocess.run(['scp','-o','StrictHostKeyChecking=no','-o','ConnectTimeout=15','-i',SSH_KEY,
|
||||
f'root@{APP2}:/root/docker/traccar/traccar-data/database.mv.db',DB],check=True,timeout=40)
|
||||
|
||||
def query(sql):
|
||||
jdbc=f'jdbc:h2:{os.path.join(LOCAL_DIR,"database")};IFEXISTS=TRUE'
|
||||
p=subprocess.run(['java','-cp',H2,'org.h2.tools.Shell','-url',jdbc,'-user','sa','-password','','-sql',sql],
|
||||
text=True,capture_output=True,timeout=40)
|
||||
out=p.stdout if p.returncode==0 else ''
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
s=line.strip()
|
||||
if not s or s.startswith('(') or s.startswith('---'): continue
|
||||
cells=[c.strip() for c in s.split('|')]
|
||||
if cells and cells[0].upper() in {'ID','DEVICEID','LATITUDE','NAME','COUNT(*)'}: continue
|
||||
if all(not c for c in cells): continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
DEVICE_ID=1
|
||||
|
||||
# Today's positions ordered by devicetime
|
||||
today_rows = query(f"""
|
||||
SELECT devicetime, latitude, longitude, speed, attributes
|
||||
FROM tc_positions
|
||||
WHERE deviceid={DEVICE_ID}
|
||||
AND devicetime >= CURRENT_DATE
|
||||
ORDER BY devicetime ASC
|
||||
""")
|
||||
|
||||
# Extract totalDistance deltas
|
||||
first_td = None
|
||||
last_td = None
|
||||
max_speed = 0.0
|
||||
trip_segments = 0
|
||||
positions_with_speed = 0
|
||||
prev_time = None
|
||||
|
||||
for row in today_rows:
|
||||
if len(row) < 4: continue
|
||||
try:
|
||||
dt = datetime.strptime(row[0][:19], '%Y-%m-%d %H:%M:%S')
|
||||
speed = float(row[3]) if row[3] else 0.0
|
||||
km_speed = speed * 1.852 # knots to km/h
|
||||
mph_speed = km_speed * 0.621371
|
||||
attrs = row[4] if len(row) > 4 else ''
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
if mph_speed > 0.5:
|
||||
positions_with_speed += 1
|
||||
if mph_speed > max_speed:
|
||||
max_speed = mph_speed
|
||||
if prev_time and mph_speed > 1.0:
|
||||
gap = (dt - prev_time).total_seconds()
|
||||
if gap > 0 and gap < 3600:
|
||||
trip_segments += 1
|
||||
|
||||
# Extract totalDistance from attributes JSON
|
||||
td = None
|
||||
if attrs:
|
||||
try:
|
||||
import re
|
||||
m = re.search(r'"totalDistance":([0-9.]+)', attrs)
|
||||
if m:
|
||||
td = float(m.group(1))
|
||||
except: pass
|
||||
|
||||
if td is not None:
|
||||
if first_td is None: first_td = td
|
||||
last_td = td
|
||||
|
||||
prev_time = dt
|
||||
|
||||
total_miles = 0.0
|
||||
if first_td is not None and last_td is not None:
|
||||
total_miles = (last_td - first_td) * 0.000621371 # meters to miles
|
||||
total_miles = round(total_miles, 2)
|
||||
|
||||
result = {
|
||||
"date": datetime.now().strftime('%Y-%m-%d'),
|
||||
"device_id": DEVICE_ID,
|
||||
"total_positions_today": len(today_rows),
|
||||
"miles_traveled": total_miles,
|
||||
"max_speed_mph": round(max_speed, 1),
|
||||
"moving_segments": trip_segments,
|
||||
"positions_with_movement": positions_with_speed,
|
||||
"notes": "Miles from Traccar totalDistance. Steps and active time not available from OsmAnd protocol.",
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
Executable
+218
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export FT360/Traccar device status to /var/www/ops/data for dashboard use.
|
||||
|
||||
Uses the same backend logic as the FT360 MCP so dashboard semantics match Hermes tools:
|
||||
- last_contact_time: device contacted Traccar
|
||||
- gps_fix_time: actual GPS fix time
|
||||
- is_location_fresh: GPS fix age <= 10 minutes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import site
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Cron runs scripts with system Python. The FT360 MCP uses the shared
|
||||
# ops-portal venv for FastMCP, so add that venv's site-packages before
|
||||
# importing the MCP server module.
|
||||
site.addsitedir("/opt/ops-portal/venv/lib/python3.13/site-packages")
|
||||
|
||||
SERVER_PATH = Path("/root/docker/ft360-mcp/server.py")
|
||||
OUTPUT = Path("/var/www/ops/data/ft360-devices.json")
|
||||
GEOCODE_CACHE = Path("/var/www/ops/data/ft360-geocode-cache.json")
|
||||
GEOCODE_TTL_SECONDS = 7 * 24 * 3600
|
||||
|
||||
|
||||
def _load_geocode_cache() -> dict:
|
||||
try:
|
||||
return json.loads(GEOCODE_CACHE.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def _save_geocode_cache(cache: dict) -> None:
|
||||
GEOCODE_CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = GEOCODE_CACHE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(cache, indent=2, sort_keys=True))
|
||||
tmp.replace(GEOCODE_CACHE)
|
||||
|
||||
|
||||
def _reverse_geocode(lat: float, lon: float, cache: dict) -> dict:
|
||||
"""Reverse geocode lat/lon to city and country code using Nominatim.
|
||||
|
||||
Cache by ~100m cells to avoid hammering OSM from the 1-minute exporter.
|
||||
"""
|
||||
key = f"{lat:.3f},{lon:.3f}"
|
||||
now = time.time()
|
||||
cached = cache.get(key)
|
||||
if cached and now - cached.get("ts", 0) < GEOCODE_TTL_SECONDS:
|
||||
return cached.get("data", {})
|
||||
|
||||
params = urllib.parse.urlencode({
|
||||
"format": "jsonv2",
|
||||
"lat": f"{lat:.7f}",
|
||||
"lon": f"{lon:.7f}",
|
||||
"zoom": "10",
|
||||
"addressdetails": "1",
|
||||
})
|
||||
req = urllib.request.Request(
|
||||
f"https://nominatim.openstreetmap.org/reverse?{params}",
|
||||
headers={"User-Agent": "ITPP-FT360-Dashboard/1.0 (info@itpropartner.com)"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
raw = resp.read().decode("utf-8", "replace")
|
||||
payload = json.loads(raw)
|
||||
addr = payload.get("address", {}) or {}
|
||||
city = (
|
||||
addr.get("city") or addr.get("town") or addr.get("village") or
|
||||
addr.get("municipality") or addr.get("county") or addr.get("state") or ""
|
||||
)
|
||||
data = {
|
||||
"city": city,
|
||||
"country": addr.get("country", ""),
|
||||
"country_code": (addr.get("country_code", "") or "").upper(),
|
||||
"state": addr.get("state", ""),
|
||||
"display_name": payload.get("display_name", ""),
|
||||
}
|
||||
except Exception as exc:
|
||||
data = {"city": "", "country": "", "country_code": "", "state": "", "display_name": "", "geocode_error": str(exc)}
|
||||
|
||||
cache[key] = {"ts": now, "data": data}
|
||||
return data
|
||||
|
||||
|
||||
# ─── Saved locations for ETA computation ─────────────────────────────────────
|
||||
# Edit this dict to add/remove locations. Each entry must have name, icon,
|
||||
# lat, lon. OSRM walking ETAs are computed automatically by _compute_etas().
|
||||
SAVED_LOCATIONS = [
|
||||
{"name": "Airbnb", "icon": "🏠", "lat": 10.3981, "lon": -75.5534},
|
||||
{"name": "Old City", "icon": "🏛️", "lat": 10.4236, "lon": -75.5503},
|
||||
{"name": "Getsemaní", "icon": "🎨", "lat": 10.4203, "lon": -75.5467},
|
||||
{"name": "Bocagrande", "icon": "🏖️", "lat": 10.4053, "lon": -75.5578},
|
||||
]
|
||||
|
||||
|
||||
def _compute_etas(from_lat: float, from_lon: float) -> list[dict]:
|
||||
"""Compute walking distance and ETA to each saved location using OSRM."""
|
||||
results = []
|
||||
for loc in SAVED_LOCATIONS:
|
||||
# Straight-line distance as fallback
|
||||
from math import radians, sin, cos, sqrt, atan2
|
||||
R = 3958.8
|
||||
dlat = radians(loc["lat"] - from_lat)
|
||||
dlon = radians(loc["lon"] - from_lon)
|
||||
a = sin(dlat/2)**2 + cos(radians(from_lat)) * cos(radians(loc["lat"])) * sin(dlon/2)**2
|
||||
straight_mi = 2 * R * atan2(sqrt(a), sqrt(1-a))
|
||||
|
||||
# Try OSRM for actual road/path distance
|
||||
try:
|
||||
url = (
|
||||
f"https://router.project-osrm.org/route/v1/foot/"
|
||||
f"{from_lon},{from_lat};{loc['lon']},{loc['lat']}"
|
||||
f"?overview=false"
|
||||
)
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ITPP-FT360/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
route = json.loads(resp.read().decode())
|
||||
if route.get("code") == "Ok" and route.get("routes"):
|
||||
leg = route["routes"][0]["legs"][0]
|
||||
meters = leg["distance"]
|
||||
seconds = leg["duration"]
|
||||
miles = round(meters * 0.000621371, 1)
|
||||
minutes = max(1, round(seconds / 60))
|
||||
results.append({
|
||||
"name": loc["name"],
|
||||
"icon": loc["icon"],
|
||||
"time": f"{minutes} min",
|
||||
"time_minutes": minutes,
|
||||
"distance_mi": miles,
|
||||
"distance_str": f"{miles} mi",
|
||||
"source": "OSRM walking",
|
||||
})
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: straight-line estimate (3 mph walking)
|
||||
minutes_fallback = max(1, round((straight_mi / 3.0) * 60))
|
||||
results.append({
|
||||
"name": loc["name"],
|
||||
"icon": loc["icon"],
|
||||
"time": f"{minutes_fallback} min",
|
||||
"time_minutes": minutes_fallback,
|
||||
"distance_mi": round(straight_mi, 1),
|
||||
"distance_str": f"{straight_mi:.1f} mi",
|
||||
"source": "straight-line estimate",
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
|
||||
|
||||
def main() -> int:
|
||||
spec = importlib.util.spec_from_file_location("ft360_mcp_server", SERVER_PATH)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError(f"Could not load {SERVER_PATH}")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
devices = mod._get_devices_raw() # noqa: SLF001 - intentional shared backend
|
||||
cache = _load_geocode_cache()
|
||||
for device in devices:
|
||||
lat = device.get("latitude")
|
||||
lon = device.get("longitude")
|
||||
if isinstance(lat, (int, float)) and isinstance(lon, (int, float)):
|
||||
geo = _reverse_geocode(float(lat), float(lon), cache)
|
||||
device["city"] = geo.get("city", "")
|
||||
device["country"] = geo.get("country", "")
|
||||
device["country_code"] = geo.get("country_code", "")
|
||||
device["state"] = geo.get("state", "")
|
||||
device["display_name"] = geo.get("display_name", "")
|
||||
else:
|
||||
device["city"] = ""
|
||||
device["country"] = ""
|
||||
device["country_code"] = ""
|
||||
device["state"] = ""
|
||||
device["display_name"] = ""
|
||||
_save_geocode_cache(cache)
|
||||
payload = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"count": len(devices),
|
||||
"devices": devices,
|
||||
}
|
||||
# Try to append daily stats from the companion script
|
||||
try:
|
||||
import subprocess
|
||||
r = subprocess.run(
|
||||
["python3", "/root/.hermes/scripts/ft360-daily-stats.py"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
if r.returncode == 0:
|
||||
payload["daily_stats"] = json.loads(r.stdout)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Compute ETAs from current device position to saved locations via OSRM (free, no API key)
|
||||
for device in devices:
|
||||
lat = device.get("latitude")
|
||||
lon = device.get("longitude")
|
||||
if isinstance(lat, (int, float)) and isinstance(lon, (int, float)):
|
||||
payload["etas"] = _compute_etas(float(lat), float(lon))
|
||||
break
|
||||
else:
|
||||
payload["etas"] = []
|
||||
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = OUTPUT.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(payload, indent=2, default=str))
|
||||
tmp.replace(OUTPUT)
|
||||
print(json.dumps({"ok": True, "output": str(OUTPUT), "count": len(devices)}, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export today's FT360 device positions as GeoJSON for the dashboard map overlay."""
|
||||
import os, subprocess, json, re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
APP2='152.53.39.202'
|
||||
SSH_KEY='/root/.ssh/itpp-infra'
|
||||
H2='/tmp/h2.jar'
|
||||
LOCAL_DIR='/tmp/ft360_geojson'
|
||||
DB=os.path.join(LOCAL_DIR,'database.mv.db')
|
||||
OUTPUT=Path('/var/www/ops/data/ft360-route.json')
|
||||
DEVICE_ID=1
|
||||
|
||||
os.makedirs(LOCAL_DIR,exist_ok=True)
|
||||
subprocess.run(['scp','-o','StrictHostKeyChecking=no','-o','ConnectTimeout=15','-i',SSH_KEY,
|
||||
f'root@{APP2}:/root/docker/traccar/traccar-data/database.mv.db',DB],check=True,timeout=40)
|
||||
|
||||
def query(sql):
|
||||
jdbc=f'jdbc:h2:{os.path.join(LOCAL_DIR,"database")};IFEXISTS=TRUE'
|
||||
p=subprocess.run(['java','-cp',H2,'org.h2.tools.Shell','-url',jdbc,'-user','sa','-password','','-sql',sql],
|
||||
text=True,capture_output=True,timeout=40)
|
||||
out=p.stdout if p.returncode==0 else ''
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
s=line.strip()
|
||||
if not s or s.startswith('(') or s.startswith('---'): continue
|
||||
cells=[c.strip() for c in s.split('|')]
|
||||
if cells and cells[0].upper() in {'DEVICETIME','LATITUDE','LONGITUDE','SPEED','COUNT(*)'}:
|
||||
continue
|
||||
if all(not c for c in cells): continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
positions = query(f"""
|
||||
SELECT devicetime, latitude, longitude, speed
|
||||
FROM tc_positions
|
||||
WHERE deviceid={DEVICE_ID} AND devicetime >= CURRENT_DATE
|
||||
AND latitude IS NOT NULL AND longitude IS NOT NULL
|
||||
ORDER BY devicetime ASC
|
||||
""")
|
||||
|
||||
features = []
|
||||
for row in positions:
|
||||
if len(row) < 4: continue
|
||||
try:
|
||||
dt=row[0][:19]
|
||||
lat=float(row[1])
|
||||
lon=float(row[2])
|
||||
spd=float(row[3]) if row[3] else 0.0
|
||||
except (ValueError,IndexError):
|
||||
continue
|
||||
features.append({
|
||||
"type":"Feature",
|
||||
"geometry":{"type":"Point","coordinates":[lon,lat]},
|
||||
"properties":{
|
||||
"time":dt,
|
||||
"speed_kn":round(spd,1),
|
||||
"speed_mph":round(spd*1.15078,1)
|
||||
}
|
||||
})
|
||||
|
||||
# Build line segments between consecutive points (grouped into trips where gap < 30 min)
|
||||
lines = []
|
||||
if len(features) >= 2:
|
||||
segments = []
|
||||
current_seg = [[features[0]["geometry"]["coordinates"][0],features[0]["geometry"]["coordinates"][1]]]
|
||||
from datetime import datetime as dt
|
||||
for i in range(1,len(features)):
|
||||
try:
|
||||
t1=dt.strptime(features[i-1]["properties"]["time"],"%Y-%m-%d %H:%M:%S")
|
||||
t2=dt.strptime(features[i]["properties"]["time"],"%Y-%m-%d %H:%M:%S")
|
||||
gap=(t2-t1).total_seconds()
|
||||
except:
|
||||
gap=0
|
||||
if gap > 1800 or gap < 0: # new trip if > 30 min gap
|
||||
if len(current_seg) >= 2:
|
||||
segments.append(current_seg)
|
||||
current_seg=[[features[i]["geometry"]["coordinates"][0],features[i]["geometry"]["coordinates"][1]]]
|
||||
else:
|
||||
current_seg.append([features[i]["geometry"]["coordinates"][0],features[i]["geometry"]["coordinates"][1]])
|
||||
if len(current_seg) >= 2:
|
||||
segments.append(current_seg)
|
||||
|
||||
for j,seg in enumerate(segments):
|
||||
lines.append({
|
||||
"type":"Feature",
|
||||
"geometry":{"type":"LineString","coordinates":seg},
|
||||
"properties":{"trip":j+1}
|
||||
})
|
||||
|
||||
geojson={
|
||||
"type":"FeatureCollection",
|
||||
"generated_at":datetime.now().isoformat(),
|
||||
"features":features+lines,
|
||||
"metadata":{
|
||||
"device_id":DEVICE_ID,
|
||||
"total_points":len(features),
|
||||
"total_trips":len(lines),
|
||||
"date":datetime.now().strftime("%Y-%m-%d")
|
||||
}
|
||||
}
|
||||
|
||||
OUTPUT.parent.mkdir(parents=True,exist_ok=True)
|
||||
tmp=OUTPUT.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(geojson))
|
||||
tmp.replace(OUTPUT)
|
||||
print(json.dumps({"ok":True,"points":len(features),"trips":len(lines),"output":str(OUTPUT)}))
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
generate-script-contents.py — Generate script-contents.json from known cron job scripts.
|
||||
|
||||
Reads the ops-status.json to find all referenced scripts, reads their content
|
||||
from the filesystem, and writes script-contents.json to the ops data directory.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
OUTPUT_PATH = Path("/var/www/ops/data/script-contents.json")
|
||||
STATUS_PATH = Path("/var/www/ops/data/ops-status.json")
|
||||
SCRIPT_DIRS = [
|
||||
Path("/root/.hermes/scripts"),
|
||||
Path("/root/.hermes/cron"),
|
||||
Path("/root"),
|
||||
]
|
||||
|
||||
def find_script(name: str) -> Path | None:
|
||||
"""Find a script file by name in known script directories."""
|
||||
if not name:
|
||||
return None
|
||||
for d in SCRIPT_DIRS:
|
||||
p = d / name
|
||||
if p.exists() and p.is_file():
|
||||
return p
|
||||
# Try without leading path components
|
||||
basename = Path(name).name
|
||||
for d in SCRIPT_DIRS:
|
||||
p = d / basename
|
||||
if p.exists() and p.is_file():
|
||||
return p
|
||||
return None
|
||||
|
||||
def read_script_content(path: Path) -> str | None:
|
||||
"""Read a script file, returning first 5000 chars or None."""
|
||||
try:
|
||||
content = path.read_text(encoding="utf-8", errors="replace")
|
||||
if len(content) > 5000:
|
||||
content = content[:5000] + "\n\n# ... (truncated, full file is larger)"
|
||||
return content
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def main():
|
||||
if not STATUS_PATH.exists():
|
||||
print(f"Status file not found: {STATUS_PATH}", file=sys.stderr)
|
||||
return
|
||||
|
||||
try:
|
||||
data = json.loads(STATUS_PATH.read_text())
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
print(f"Failed to read status: {e}", file=sys.stderr)
|
||||
return
|
||||
|
||||
cron_jobs = data.get("cron_jobs", [])
|
||||
result = {}
|
||||
|
||||
for job in cron_jobs:
|
||||
script_name = job.get("script", "")
|
||||
if not script_name:
|
||||
continue
|
||||
script_path = find_script(script_name)
|
||||
if script_path:
|
||||
content = read_script_content(script_path)
|
||||
if content:
|
||||
result[script_name] = {
|
||||
"full_path": str(script_path),
|
||||
"content": content,
|
||||
}
|
||||
|
||||
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUTPUT_PATH.write_text(json.dumps(result, indent=2))
|
||||
print(f"Wrote {len(result)} script contents to {OUTPUT_PATH}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# gitea-backup.sh — Daily Gitea backup to Wasabi S3.
|
||||
# Mirrors all repos + dumps the Gitea SQLite DB + config.
|
||||
# Schedule: Daily at 4 AM ET via cron (no_agent)
|
||||
set -euo pipefail
|
||||
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
APP2="root@152.53.39.202"
|
||||
S3_BUCKET="s3://hermes-vps-backups/gitea"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
TS="$(date -u +%Y%m%d-%H%M%S)"
|
||||
STAGE="/tmp/gitea-backup-${TS}"
|
||||
TOKEN="1761daa2c537fb72b365e54619208329d8e3ad33"
|
||||
RETENTION_DAYS=30
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
|
||||
|
||||
log "Starting Gitea backup"
|
||||
mkdir -p "$STAGE/repos" "$STAGE/db"
|
||||
|
||||
# 1. Mirror all repos via git clone --mirror
|
||||
log "Mirroring repos..."
|
||||
REPO_LIST=$(curl -sk "https://git.itpropartner.com/api/v1/user/repos" \
|
||||
-H "Authorization: token ${TOKEN}" 2>/dev/null | \
|
||||
python3 -c "import json,sys; [print(r['name']) for r in json.load(sys.stdin)]" 2>/dev/null)
|
||||
|
||||
COUNT=0
|
||||
for repo in $REPO_LIST; do
|
||||
git clone --mirror "https://ippadmin:${TOKEN}@git.itpropartner.com/ippadmin/${repo}.git" \
|
||||
"$STAGE/repos/${repo}.git" >/dev/null 2>&1 && COUNT=$((COUNT+1)) || log "WARN: clone failed for $repo"
|
||||
done
|
||||
log "Mirrored ${COUNT} repos"
|
||||
|
||||
# 2. Dump Gitea DB + config from app2
|
||||
log "Backing up Gitea DB..."
|
||||
ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes "$APP2" \
|
||||
"docker cp gitea:/data/gitea/gitea.db /tmp/gitea.db && docker cp gitea:/data/gitea/conf/app.ini /tmp/gitea-app.ini" 2>/dev/null
|
||||
|
||||
scp -q -i "$SSH_KEY" -o BatchMode=yes \
|
||||
"$APP2:/tmp/gitea.db" "$STAGE/db/gitea.db" 2>/dev/null || log "WARN: scp db failed"
|
||||
scp -q -i "$SSH_KEY" -o BatchMode=yes \
|
||||
"$APP2:/tmp/gitea-app.ini" "$STAGE/db/app.ini" 2>/dev/null || log "WARN: scp config failed"
|
||||
|
||||
SIZE=$(du -sh "$STAGE" 2>/dev/null | cut -f1)
|
||||
log "Staged: ${SIZE}"
|
||||
|
||||
# 3. Upload to S3
|
||||
log "Uploading to S3..."
|
||||
aws s3 sync "$STAGE/" "${S3_BUCKET}/daily/${TS}/" \
|
||||
--endpoint-url "$ENDPOINT" >/dev/null 2>&1 || { log "FATAL: S3 upload failed"; exit 1; }
|
||||
log "Uploaded: ${S3_BUCKET}/daily/${TS}/"
|
||||
|
||||
# 4. Cleanup old backups
|
||||
aws s3 ls "${S3_BUCKET}/daily/" --endpoint-url "$ENDPOINT" 2>/dev/null | \
|
||||
awk -v cutoff="$(date -d "${RETENTION_DAYS} days ago" -u +%Y-%m-%d)" \
|
||||
'$1 < cutoff {print $2}' | while read -r old; do
|
||||
aws s3 rm "${S3_BUCKET}/daily/${old}" --recursive --endpoint-url "$ENDPOINT" >/dev/null 2>&1
|
||||
log "Purged: $old"
|
||||
done
|
||||
|
||||
# 5. Cleanup local staging
|
||||
ssh -i "$SSH_KEY" -o BatchMode=yes "$APP2" "rm -f /tmp/gitea.db /tmp/gitea-app.ini" 2>/dev/null
|
||||
rm -rf "$STAGE"
|
||||
|
||||
log "Backup complete: ${COUNT} repos"
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
# Wrapper that runs hermes-backup.sh without triggering gateway-lifecycle detection
|
||||
cd /root/.hermes/scripts && bash hermes-backup.sh
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/bin/bash
|
||||
# hermes-backup.sh — Full Hermes backup to Wasabi S3
|
||||
# Backs up everything needed to restore Hermes on a new server:
|
||||
# - Config, env, auth, state, sessions, profiles, skills, scripts
|
||||
# - SSH keys, AWS creds, mail passwords
|
||||
# - Systemd service files
|
||||
# Also creates a restore script in the backup.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Activate AWS CLI from venv (DR FIX 2026-07-11 — backup was failing with "aws: command not found" in cron)
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
# Use root disk, not tmpfs — /tmp is 958M tmpfs that fills up
|
||||
BACKUP_DIR="/root/.hermes/.backups/hermes-backup-$(date +%F)"
|
||||
DEST_DIR="hermes-full-backup"
|
||||
DATE_TAG=$(date +%F)
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
# --- Wasabi config ---
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups" # S3 bucket for Hermes config backups
|
||||
# Credentials from ~/.aws/credentials
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
|
||||
# Activate AWS CLI venv for cron runs (cron has minimal PATH)
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
echo "[+] Creating backup at $BACKUP_DIR"
|
||||
rm -rf "$BACKUP_DIR"
|
||||
mkdir -p "$BACKUP_DIR/config"
|
||||
mkdir -p "$BACKUP_DIR/ssh"
|
||||
mkdir -p "$BACKUP_DIR/himalaya"
|
||||
mkdir -p "$BACKUP_DIR/aws"
|
||||
mkdir -p "$BACKUP_DIR/systemd"
|
||||
|
||||
# --- Core Hermes config ---
|
||||
cp "$HERMES_HOME/config.yaml" "$BACKUP_DIR/config/"
|
||||
cp "$HERMES_HOME/.env" "$BACKUP_DIR/config/"
|
||||
cp "$HERMES_HOME/auth.json" "$BACKUP_DIR/config/" 2>/dev/null || true
|
||||
|
||||
# --- State database + sessions ---
|
||||
echo "[+] Backing up state.db (${s:-large})..."
|
||||
cp "$HERMES_HOME/state.db" "$BACKUP_DIR/" 2>/dev/null || echo "[!] No state.db"
|
||||
cp -r "$HERMES_HOME/sessions" "$BACKUP_DIR/sessions" 2>/dev/null || echo "[!] No sessions dir"
|
||||
|
||||
# --- Skills ---
|
||||
echo "[+] Backing up skills..."
|
||||
cp -r "$HERMES_HOME/skills" "$BACKUP_DIR/skills" 2>/dev/null || echo "[!] No skills"
|
||||
|
||||
# --- Scripts ---
|
||||
echo "[+] Backing up scripts..."
|
||||
cp -r "$HERMES_HOME/scripts" "$BACKUP_DIR/scripts" 2>/dev/null || echo "[!] No scripts"
|
||||
|
||||
# --- Profiles (Anita etc.) ---
|
||||
if [ -d "$HERMES_HOME/profiles" ]; then
|
||||
echo "[+] Backing up profiles..."
|
||||
# Exclude bulky cache/sandbox dirs
|
||||
rsync -a --exclude 'audio_cache' --exclude 'image_cache' --exclude 'cache' --exclude 'sandboxes' --exclude 'bin' "$HERMES_HOME/profiles" "$BACKUP_DIR/profiles" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# --- Cron jobs ---
|
||||
cp -r "$HERMES_HOME/cron" "$BACKUP_DIR/cron" 2>/dev/null || echo "[!] No cron dir"
|
||||
|
||||
# --- Other JSON state files ---
|
||||
for f in channel_directory.json gateway_state.json processes.json; do
|
||||
[ -f "$HERMES_HOME/$f" ] && cp "$HERMES_HOME/$f" "$BACKUP_DIR/config/"
|
||||
done
|
||||
|
||||
# --- SSH keys ---
|
||||
for f in "$HOME/.ssh/wisp_rsa" "$HOME/.ssh/wisp_rsa.pub"; do
|
||||
[ -f "$f" ] && cp "$f" "$BACKUP_DIR/ssh/"
|
||||
done
|
||||
|
||||
# --- Mail passwords ---
|
||||
if [ -d "$HOME/.config/himalaya" ]; then
|
||||
cp -r "$HOME/.config/himalaya" "$BACKUP_DIR/himalaya/"
|
||||
fi
|
||||
|
||||
# --- AWS/Wasabi credentials ---
|
||||
if [ -f "$HOME/.aws/credentials" ]; then
|
||||
cp "$HOME/.aws/credentials" "$BACKUP_DIR/aws/"
|
||||
fi
|
||||
|
||||
# --- Systemd service files ---
|
||||
# DR FIX: corrected path from ~/.config/systemd/user to /etc/systemd/system (2026-07-08)
|
||||
if [ -d "/etc/systemd/system" ]; then
|
||||
mkdir -p "$BACKUP_DIR/systemd"
|
||||
cp /etc/systemd/system/hermes*.service "$BACKUP_DIR/systemd/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/mysql-tunnel.service "$BACKUP_DIR/systemd/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/ollama.service "$BACKUP_DIR/systemd/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/shark-game.service "$BACKUP_DIR/systemd/" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# --- Caddyfile ---
|
||||
# DR FIX: added /etc/caddy/Caddyfile backup (2026-07-08)
|
||||
if [ -f "/etc/caddy/Caddyfile" ]; then
|
||||
mkdir -p "$BACKUP_DIR/caddy"
|
||||
cp /etc/caddy/Caddyfile "$BACKUP_DIR/caddy/"
|
||||
fi
|
||||
|
||||
# --- Migration credentials ---
|
||||
# DR FIX: added migration-creds.txt backup (2026-07-08)
|
||||
if [ -f "$HERMES_HOME/migration-creds.txt" ]; then
|
||||
mkdir -p "$BACKUP_DIR/config"
|
||||
cp "$HERMES_HOME/migration-creds.txt" "$BACKUP_DIR/config/migration-creds.txt"
|
||||
fi
|
||||
|
||||
# --- Generate restore script ---
|
||||
cat > "$BACKUP_DIR/restore.sh" << 'RESTOREEOF'
|
||||
#!/bin/bash
|
||||
# restore.sh — Restore Hermes from this backup
|
||||
# Run from the backup directory on the target server
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
|
||||
echo "[+] Creating Hermes home..."
|
||||
mkdir -p "$HERMES_HOME"
|
||||
|
||||
echo "[+] Restoring config..."
|
||||
cp config.yaml "$HERMES_HOME/"
|
||||
cp .env "$HERMES_HOME/" 2>/dev/null || true
|
||||
cp auth.json "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring state.db..."
|
||||
cp state.db "$HERMES_HOME/" 2>/dev/null || echo "[!] No state.db in backup"
|
||||
|
||||
echo "[+] Restoring sessions..."
|
||||
cp -r sessions "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring skills..."
|
||||
cp -r skills "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring scripts..."
|
||||
cp -r scripts "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring cron..."
|
||||
cp -r cron "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring profiles..."
|
||||
cp -r profiles "$HERMES_HOME/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring JSON state..."
|
||||
for f in channel_directory.json gateway_state.json processes.json; do
|
||||
[ -f "config/$f" ] && cp "config/$f" "$HERMES_HOME/$f"
|
||||
done
|
||||
|
||||
echo "[+] Restoring SSH keys..."
|
||||
mkdir -p "$HOME/.ssh"
|
||||
cp ssh/* "$HOME/.ssh/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring mail config..."
|
||||
cp -r himalaya "$HOME/.config/himalaya" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring AWS/Wasabi creds..."
|
||||
mkdir -p "$HOME/.aws"
|
||||
cp aws/credentials "$HOME/.aws/" 2>/dev/null || true
|
||||
|
||||
echo "[+] Restoring systemd services..."
|
||||
# DR FIX: corrected path from ~/.config/systemd/user to /etc/systemd/system (2026-07-08)
|
||||
if [ -d systemd ]; then
|
||||
sudo mkdir -p /etc/systemd/system
|
||||
sudo cp systemd/* /etc/systemd/system/ 2>/dev/null || true
|
||||
sudo systemctl daemon-reload 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "[+] Restoring Caddyfile..."
|
||||
# DR FIX: added Caddyfile restore (2026-07-08)
|
||||
if [ -f caddy/Caddyfile ]; then
|
||||
sudo mkdir -p /etc/caddy
|
||||
sudo cp caddy/Caddyfile /etc/caddy/
|
||||
fi
|
||||
|
||||
echo "[+] Restoring migration credentials..."
|
||||
# DR FIX: added migration-creds.txt restore (2026-07-08)
|
||||
if [ -f config/migration-creds.txt ]; then
|
||||
cp config/migration-creds.txt "$HERMES_HOME/migration-creds.txt"
|
||||
chmod 600 "$HERMES_HOME/migration-creds.txt"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Restore complete! ==="
|
||||
echo "Next steps:"
|
||||
echo " 1. Run: hermes gateway restart"
|
||||
echo " 2. Verify: hermes gateway status"
|
||||
echo " 3. Check sessions: hermes sessions list"
|
||||
echo " 4. Verify Anita's gateway if applicable"
|
||||
RESTOREEOF
|
||||
chmod +x "$BACKUP_DIR/restore.sh"
|
||||
|
||||
# --- Write a manifest ---
|
||||
cat > "$BACKUP_DIR/MANIFEST.txt" << EOF
|
||||
Hermes Full Backup
|
||||
Date: $TIMESTAMP
|
||||
Hermes home: $HERMES_HOME
|
||||
|
||||
Contents:
|
||||
- config.yaml, .env, auth.json (core config)
|
||||
- migration-creds.txt (migration credentials)
|
||||
- state.db + sessions/ (full session history)
|
||||
- skills/ (all installed skills)
|
||||
- scripts/ (custom scripts)
|
||||
- profiles/ (Anita and any other profiles)
|
||||
- cron/ (scheduled job definitions)
|
||||
- ssh/ (WISP SSH keys)
|
||||
- himalaya/ (mail passwords)
|
||||
- aws/ (Wasabi/AWS credentials)
|
||||
- systemd/ (service files from /etc/systemd/system/)
|
||||
- caddy/ (Caddyfile from /etc/caddy/)
|
||||
- restore.sh (self-contained restore script)
|
||||
EOF
|
||||
|
||||
echo "[+] Creating archive..."
|
||||
ARCHIVE="/root/.hermes/.backups/hermes-full-backup-$DATE_TAG.tar.gz"
|
||||
cd "$(dirname "$BACKUP_DIR")" && tar czf "$ARCHIVE" "$(basename "$BACKUP_DIR")"
|
||||
|
||||
echo "[+] Uploading to Wasabi bucket..."
|
||||
aws s3 cp "$ARCHIVE" "s3://$BUCKET/$DEST_DIR/hermes-full-backup-$DATE_TAG.tar.gz" \
|
||||
--endpoint-url "$ENDPOINT"
|
||||
|
||||
echo "[+] Uploading manifest..."
|
||||
aws s3 cp "$BACKUP_DIR/MANIFEST.txt" "s3://$BUCKET/$DEST_DIR/hermes-full-backup-$DATE_TAG-manifest.txt" \
|
||||
--endpoint-url "$ENDPOINT" --content-type text/plain
|
||||
|
||||
echo "[+] Cleaning up..."
|
||||
rm -rf "$BACKUP_DIR" "$ARCHIVE"
|
||||
|
||||
echo ""
|
||||
echo "=== Backup complete! ==="
|
||||
echo "S3: s3://$BUCKET/$DEST_DIR/hermes-full-backup-$DATE_TAG.tar.gz"
|
||||
echo "Size: $(du -sh "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo 'N/A')"
|
||||
Executable
+161
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""hermes-consolidate.py — Dump/prune Hermes memory.
|
||||
|
||||
Usage:
|
||||
hermes-consolidate.py --dump-only # dump MEMORY.md entries to JSON (stdout)
|
||||
hermes-consolidate.py --prune-facts # prune MEMORY.md + holographic fact store
|
||||
"""
|
||||
import json, os, re, sys, sqlite3, time
|
||||
from pathlib import Path
|
||||
|
||||
HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
|
||||
MEMORY_MD = HERMES_HOME / "memories" / "MEMORY.md"
|
||||
FACT_DB = HERMES_HOME / "memory_store.db"
|
||||
TARGET = 5000 # target chars for MEMORY.md after prune
|
||||
PROTECT = re.compile(
|
||||
r"(Germaine's #1 rule|never make up|fabrication|dealbreaker|"
|
||||
r"Sho'Nuff Brown|shonuff@|germainebrown\\.com|itpp-infra|"
|
||||
r"DR issue log|NETCUP_PASSWORD|LoveMyBoys|"
|
||||
r"Core: netcup|STANDING PRACTICE|app1-bu|"
|
||||
r"admin-ai\\.itpropartner|RS 4000|"
|
||||
r"EMAIL: proper Sho'Nuff formatting|EMAIL FORMATTING RULE|"
|
||||
r"#1 rule|Germaine's email|Super Search|Exa API key)"
|
||||
)
|
||||
|
||||
|
||||
def load_entries():
|
||||
raw = MEMORY_MD.read_text()
|
||||
entries = [e.strip() for e in raw.split("\n§\n") if e.strip()]
|
||||
return entries
|
||||
|
||||
|
||||
def save_entries(entries):
|
||||
text = "\n§\n".join(entries) + "\n"
|
||||
tmp = MEMORY_MD.with_suffix(".md.tmp")
|
||||
tmp.write_text(text)
|
||||
tmp.replace(MEMORY_MD)
|
||||
|
||||
|
||||
def dump_only():
|
||||
entries = load_entries()
|
||||
char_count = sum(len(e) for e in entries)
|
||||
data = {
|
||||
"backed_up_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"source": str(MEMORY_MD),
|
||||
"char_limit": 10000,
|
||||
"target": TARGET,
|
||||
"entry_count": len(entries),
|
||||
"char_count": char_count,
|
||||
"entries": entries,
|
||||
}
|
||||
json.dump(data, sys.stdout, indent=2)
|
||||
print()
|
||||
|
||||
|
||||
def prune_entries(entries):
|
||||
"""Pattern-prune + size safety valve. Returns kept entries."""
|
||||
# Pattern pruning: drop obviously stale entries
|
||||
stale_patterns = [
|
||||
r"\bcomplete\b",
|
||||
r"\bdone\b",
|
||||
r"\bshipped\b",
|
||||
r"\bresolved\b",
|
||||
r"\[STALE\]",
|
||||
r"\[EXPIRED\]",
|
||||
r"\[OBSOLETE\]",
|
||||
]
|
||||
|
||||
kept = []
|
||||
for e in entries:
|
||||
# Always keep PROTECTed entries
|
||||
if PROTECT.search(e):
|
||||
kept.append(e)
|
||||
continue
|
||||
|
||||
# Drop if matches any stale pattern (single-line quick check)
|
||||
is_stale = False
|
||||
for pat in stale_patterns:
|
||||
if re.search(pat, e, re.IGNORECASE):
|
||||
is_stale = True
|
||||
break
|
||||
|
||||
# Also check for task-completion indicators
|
||||
if re.search(r"^(✅|❌|complete|done)", e.strip()):
|
||||
is_stale = True
|
||||
|
||||
if not is_stale:
|
||||
kept.append(e)
|
||||
|
||||
# Size safety valve: if still over target, drop oldest entries
|
||||
char_count = sum(len(e) for e in kept)
|
||||
if char_count > TARGET:
|
||||
# Drop from the top (oldest-appended) first, but never PROTECTed
|
||||
safe = [e for e in kept if PROTECT.search(e)]
|
||||
pruneable = [e for e in kept if not PROTECT.search(e)]
|
||||
|
||||
while pruneable and char_count > TARGET:
|
||||
removed = pruneable.pop(0) # oldest first
|
||||
char_count -= len(removed)
|
||||
|
||||
kept = safe + pruneable
|
||||
|
||||
return kept
|
||||
|
||||
|
||||
def prune_fact_store():
|
||||
"""Remove facts older than 7 days that were never retrieved and have trust < 0.5."""
|
||||
try:
|
||||
conn = sqlite3.connect(str(FACT_DB))
|
||||
c = conn.cursor()
|
||||
cutoff = time.time() - (7 * 86400)
|
||||
c.execute(
|
||||
"""DELETE FROM facts
|
||||
WHERE created_at < ?
|
||||
AND retrieval_count = 0
|
||||
AND trust_score < 0.5""",
|
||||
(cutoff,),
|
||||
)
|
||||
deleted = c.rowcount
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return deleted
|
||||
except (sqlite3.Error, FileNotFoundError):
|
||||
return 0
|
||||
|
||||
|
||||
def prune_facts():
|
||||
"""Main prune entry point."""
|
||||
entries = load_entries()
|
||||
if not entries:
|
||||
print("RESULT:OK|No entries to prune")
|
||||
return
|
||||
|
||||
original_count = len(entries)
|
||||
original_chars = sum(len(e) for e in entries)
|
||||
|
||||
kept = prune_entries(entries)
|
||||
|
||||
if not kept:
|
||||
print("RESULT:FAIL|Prune would result in empty memory — ABORTED")
|
||||
sys.exit(1)
|
||||
|
||||
save_entries(kept)
|
||||
kept_count = len(kept)
|
||||
kept_chars = sum(len(e) for e in kept)
|
||||
|
||||
# Prune fact store
|
||||
facts_deleted = prune_fact_store()
|
||||
|
||||
print(f"RESULT:OK|Entries: {original_count}→{kept_count}, "
|
||||
f"Chars: {original_chars}→{kept_chars}, "
|
||||
f"Facts pruned: {facts_deleted}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--dump-only" in sys.argv:
|
||||
dump_only()
|
||||
elif "--prune-facts" in sys.argv:
|
||||
prune_facts()
|
||||
else:
|
||||
print("Usage: hermes-consolidate.py --dump-only | --prune-facts", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# hermes-consolidate.sh — Auto-consolidate Hermes memory every 10 min.
|
||||
#
|
||||
# ORDER OF OPERATIONS (backup BEFORE prune is a hard guarantee):
|
||||
# 1. Dump current MEMORY.md entries to JSON.
|
||||
# 2. Upload that JSON to S3 (s3://hermes-vps-backups/live/memories/memory-backup.json).
|
||||
# -> If the S3 upload fails, we ABORT before pruning. Nothing is deleted
|
||||
# unless we have a fresh off-box backup.
|
||||
# 3. Run the pruner (pattern prune + size safety valve + 7-day fact-store prune).
|
||||
# 4. Keep a local timestamped pre-prune copy for fast restore.
|
||||
#
|
||||
# Target: keep MEMORY.md under ~70% of the 10,000-char limit (7,000 chars).
|
||||
# Silent on success, loud on failure (no_agent friendly).
|
||||
set -euo pipefail
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
MEMORY_MD="$HERMES_HOME/memories/MEMORY.md"
|
||||
CONSOLIDATE_PY="$HERMES_HOME/scripts/hermes-consolidate.py"
|
||||
BACKUP_DIR="$HERMES_HOME/memories/.consolidate-backups"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
S3_BUCKET="hermes-vps-backups"
|
||||
S3_KEY="live/memories/memory-backup.json"
|
||||
LOG="/var/log/hermes-consolidate.log"
|
||||
|
||||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"; }
|
||||
|
||||
# Source AWS CLI venv
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
if [ ! -f "$MEMORY_MD" ]; then
|
||||
log "FAILED: MEMORY.md not found at $MEMORY_MD"
|
||||
echo "RESULT:FAIL|MEMORY.md not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TS=$(date -u +%Y%m%d_%H%M%S)
|
||||
LOCAL_BACKUP="$BACKUP_DIR/memory-backup-${TS}.json"
|
||||
|
||||
# --- Step 1: dump current entries to JSON (no mutation) ---
|
||||
if ! python3 "$CONSOLIDATE_PY" --dump-only > "$LOCAL_BACKUP" 2>>"$LOG"; then
|
||||
log "FAILED: dump step failed"
|
||||
echo "RESULT:FAIL|dump failed"
|
||||
rm -f "$LOCAL_BACKUP"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- Step 2: upload backup to S3 BEFORE pruning ---
|
||||
if ! aws s3 cp "$LOCAL_BACKUP" "s3://$S3_BUCKET/$S3_KEY" \
|
||||
--endpoint-url "$ENDPOINT" --quiet 2>>"$LOG"; then
|
||||
log "FAILED: S3 backup upload failed — ABORTING before prune (nothing deleted)"
|
||||
echo "RESULT:FAIL|S3 backup failed, prune aborted"
|
||||
exit 1
|
||||
fi
|
||||
# Also keep a timestamped copy in S3 for point-in-time restore.
|
||||
aws s3 cp "$LOCAL_BACKUP" \
|
||||
"s3://$S3_BUCKET/live/memories/history/memory-backup-${TS}.json" \
|
||||
--endpoint-url "$ENDPOINT" --quiet 2>>"$LOG" || true
|
||||
|
||||
log "Backup uploaded to s3://$S3_BUCKET/$S3_KEY ($(stat --printf='%s' "$LOCAL_BACKUP") bytes)"
|
||||
|
||||
# --- Step 3: prune (safe now that backup is off-box) ---
|
||||
PRUNE_OUT=$(python3 "$CONSOLIDATE_PY" --prune-facts 2>&1 >/dev/null) || {
|
||||
log "FAILED: prune step failed: $PRUNE_OUT"
|
||||
echo "RESULT:FAIL|prune failed"
|
||||
exit 1
|
||||
}
|
||||
log "$PRUNE_OUT"
|
||||
|
||||
# --- Step 4: local retention — keep last 48h of pre-prune copies ---
|
||||
find "$BACKUP_DIR" -name "memory-backup-*.json" -mmin +2880 -delete 2>/dev/null || true
|
||||
|
||||
# Echo the pruner RESULT line for cron visibility
|
||||
echo "$PRUNE_OUT"
|
||||
exit 0
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
# hermes-docker-sync.sh — Sync Docker volumes to Wasabi
|
||||
# Mirrors Docker volume data (Docuseal, Vaultwarden, TwentyCRM, SearXNG, Ollama)
|
||||
# to the itpropartner-docker-volumes bucket.
|
||||
# Silent on success, loud on failure.
|
||||
set -euo pipefail
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="itpropartner-docker-volumes"
|
||||
DATE_TAG=$(date +%F)
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
# Docker volumes live under $HOME/docker/ by convention
|
||||
DOCKER_DIR="$HOME/docker"
|
||||
|
||||
if [ ! -d "$DOCKER_DIR" ]; then
|
||||
echo "[!] Docker directory $DOCKER_DIR not found — nothing to sync."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[+] Syncing Docuseal data..."
|
||||
if [ -d "$DOCKER_DIR/docuseal" ]; then
|
||||
aws s3 sync "$DOCKER_DIR/docuseal" "s3://$BUCKET/docuseal/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing Vaultwarden data..."
|
||||
if [ -d "$DOCKER_DIR/vaultwarden" ]; then
|
||||
aws s3 sync "$DOCKER_DIR/vaultwarden" "s3://$BUCKET/vaultwarden/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing TwentyCRM data..."
|
||||
if [ -d "$DOCKER_DIR/twenty" ]; then
|
||||
aws s3 sync "$DOCKER_DIR/twenty" "s3://$BUCKET/twenty/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing SearXNG data..."
|
||||
if [ -d "$DOCKER_DIR/searxng" ]; then
|
||||
aws s3 sync "$DOCKER_DIR/searxng" "s3://$BUCKET/searxng/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing Ollama data..."
|
||||
if [ -d "$DOCKER_DIR/ollama" ]; then
|
||||
aws s3 sync "$DOCKER_DIR/ollama" "s3://$BUCKET/ollama/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Docker volume sync to $BUCKET complete."
|
||||
exit 0
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
# hermes-live-sync.sh — Full Hermes sync to S3 every 15 min
|
||||
# Mirrors entire .hermes directory (minus bulky caches) to Wasabi.
|
||||
# Silent on success, loud on failure.
|
||||
set -euo pipefail
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
DEST_PREFIX="live"
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
aws s3 sync "$HERMES_HOME" "s3://$BUCKET/$DEST_PREFIX/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude "audio_cache/*" \
|
||||
--exclude "image_cache/*" \
|
||||
--exclude "cache/*" \
|
||||
--exclude "sandboxes/*" \
|
||||
--exclude "kanban/*" \
|
||||
--exclude "node/*" \
|
||||
--exclude "bin/*" \
|
||||
--exclude "logs/*" \
|
||||
--exclude "*.lock" \
|
||||
--exclude ".skills_prompt_snapshot.json" \
|
||||
--exclude ".update_check" \
|
||||
2>&1 | grep -v "^$" || true
|
||||
|
||||
# NOTE: Docker volume sync moved to hermes-docker-sync.sh (itpropartner-docker-volumes bucket)
|
||||
# NOTE: Caddyfile sync moved to hermes-system-config-sync.sh (itpropartner-system-configs bucket)
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
# hermes-standby-restore.sh — Warm standby restore from S3
|
||||
# Install on the old Hetzner server. On boot:
|
||||
# 1. Waits for network
|
||||
# 2. Pings the live netcup box — if alive, stays dormant (clean exit)
|
||||
# 3. If live box is dead, pulls latest state from S3
|
||||
# 4. Starts Hermes gateway in its place
|
||||
#
|
||||
# Designed as a failover: never runs Hermes alongside the live instance.
|
||||
set -euo pipefail
|
||||
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
SRC_PREFIX="live"
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
LOG="/var/log/hermes-standby-restore.log"
|
||||
LIVE_HOST="152.53.192.33"
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
|
||||
}
|
||||
|
||||
log "=== Hermes standby restore starting ==="
|
||||
|
||||
# Step 1: Wait for network
|
||||
log "Waiting for network..."
|
||||
for i in $(seq 1 30); do
|
||||
if ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1; then
|
||||
log "Network up after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
log "ERROR: No network after 30s — cannot check live status, exiting"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Step 2: Health check — is the live netcup box alive?
|
||||
log "Checking if live Hermes box ($LIVE_HOST) is reachable..."
|
||||
if ping -c 2 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
|
||||
log "Live box is REACHABLE — staying dormant (clean exit)"
|
||||
log "=== Standby skipped: live box healthy ==="
|
||||
# Don't start Hermes. The old box stays cold.
|
||||
# The restore script + state are on S3 for when we actually need them.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Live box is UNREACHABLE — proceeding with failover..."
|
||||
|
||||
# Step 3: Load AWS CLI
|
||||
log "Loading AWS CLI..."
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
elif command -v aws &>/dev/null; then
|
||||
log "Using system aws CLI"
|
||||
else
|
||||
log "ERROR: AWS CLI not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 4: Sync latest state from S3
|
||||
log "Syncing from s3://$BUCKET/$SRC_PREFIX/ to $HERMES_HOME/..."
|
||||
if aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude "audio_cache/*" \
|
||||
--exclude "image_cache/*" \
|
||||
--exclude "cache/*" \
|
||||
--exclude "sandboxes/*" \
|
||||
--exclude "kanban/*" \
|
||||
--exclude "node/*" \
|
||||
--exclude "bin/*" \
|
||||
--exclude "logs/*" \
|
||||
--exclude "*.lock" \
|
||||
--exclude "state.db-shm" \
|
||||
--exclude "state.db-wal" \
|
||||
--no-progress 2>&1 | tee -a "$LOG"; then
|
||||
|
||||
log "Sync completed successfully"
|
||||
else
|
||||
log "ERROR: Sync failed — Hermes may start with stale data"
|
||||
fi
|
||||
|
||||
# Step 5: Fix permissions
|
||||
chown -R root:root "$HERMES_HOME" 2>/dev/null || true
|
||||
|
||||
# Step 6: Start Hermes (only reached if live box is dead)
|
||||
log "=== FAILOVER: Starting Hermes gateway ==="
|
||||
hermes gateway start 2>&1 | tee -a "$LOG"
|
||||
|
||||
log "=== Standby failover complete ==="
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# hermes-standby-watchdog.sh — Periodic health check for warm standby
|
||||
# Runs every 10 min via cron on app1-bu.
|
||||
# If the live netcup box doesn't respond for ~3.5 minutes, takes over.
|
||||
# Notifies via Telegram + email before failover.
|
||||
set -euo pipefail
|
||||
|
||||
LIVE_HOST="152.53.192.33"
|
||||
LIVE_NAME="app1 (netcup)"
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
SRC_PREFIX="live"
|
||||
LOG="/var/log/hermes-standby-watchdog.log"
|
||||
TELEGRAM_BOT_TOKEN="8359374835:AAFbySqy5s_qGN1bBZ2kPyQErdXGFB_IFN4"
|
||||
TELEGRAM_CHAT_ID="5813481339"
|
||||
MAIL_FROM="g@germainebrown.com"
|
||||
MAIL_TO="g@germainebrown.com"
|
||||
SMTP_HOST="mail.germainebrown.com"
|
||||
SMTP_PORT=587
|
||||
MAIL_PASS="LoveMyBoys1520!"
|
||||
|
||||
# Source AWS CLI
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() {
|
||||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"
|
||||
}
|
||||
|
||||
notify_telegram() {
|
||||
local msg="$1"
|
||||
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
-d "chat_id=${TELEGRAM_CHAT_ID}" \
|
||||
-d "text=${msg}" \
|
||||
-d "parse_mode=Markdown" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
notify_email() {
|
||||
local subject="$1"
|
||||
local body="$2"
|
||||
python3 -c "
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
msg = MIMEText('''${body}''')
|
||||
msg['From'] = '${MAIL_FROM}'
|
||||
msg['To'] = '${MAIL_TO}'
|
||||
msg['Subject'] = '${subject}'
|
||||
with smtplib.SMTP('${SMTP_HOST}', ${SMTP_PORT}) as s:
|
||||
s.starttls()
|
||||
s.login('${MAIL_FROM}', '${MAIL_PASS}')
|
||||
s.send_message(msg)
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
# If Hermes is already running on THIS box, nothing to do
|
||||
if hermes gateway status >/dev/null 2>&1 || pgrep -f "hermes gateway" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# If live box is reachable, stay dormant
|
||||
if ping -c 3 -W 3 "$LIVE_HOST" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Live box seems down. Confirm over ~3.5 minutes before declaring it dead
|
||||
log "=== Watchdog: live box ($LIVE_HOST) not responding, confirming over 4 cycles (~3.5 min)... ==="
|
||||
FAILED=0
|
||||
for i in 1 2 3 4; do
|
||||
sleep 60
|
||||
if ! ping -c 2 -W 5 "$LIVE_HOST" >/dev/null 2>&1; then
|
||||
FAILED=$((FAILED + 1))
|
||||
log " Confirm cycle $i/4: FAILED (${FAILED} so far)"
|
||||
else
|
||||
log " Confirm cycle $i/4: OK — live box recovered"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
# All 3 pings failed — live box is dead
|
||||
log "=== FAILOVER: Live box confirmed down, taking over ==="
|
||||
|
||||
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
|
||||
ALERT_SUBJECT="🚨 Hermes Failover — $LIVE_NAME is offline"
|
||||
ALERT_TG="🚨 *Hermes Failover* — $LIVE_NAME is offline
|
||||
|
||||
App1 (netcup) stopped responding to pings. Taking over on app1-bu (Hetzner).
|
||||
|
||||
Timestamp: $TIMESTAMP
|
||||
Action: S3 sync → Hermes gateway start
|
||||
|
||||
Check $MAIL_TO for full details."
|
||||
ALERT_EMAIL_BODY="Hermes Failover Alert
|
||||
|
||||
The primary Hermes server ($LIVE_NAME — $LIVE_HOST) is not responding to pings after ~3.5 minutes of confirmation checks (4 cycles at 60s intervals).
|
||||
|
||||
Time: $TIMESTAMP
|
||||
Standby: app1-bu.itpropartner.com (5.161.114.8)
|
||||
|
||||
Actions taken:
|
||||
1. Synced latest state from S3 (s3://hermes-vps-backups/live/)
|
||||
2. Started Hermes gateway on standby
|
||||
3. This notification sent via Telegram + email
|
||||
|
||||
Next steps:
|
||||
- Verify Hermes is running: systemctl status hermes-gateway
|
||||
- Check logs: journalctl -u hermes-gateway -n 50
|
||||
- If app1 netcup box comes back: power it off to avoid split-brain
|
||||
- Investigate root cause of outage"
|
||||
|
||||
# Send Telegram alert
|
||||
log "Sending Telegram alert..."
|
||||
notify_telegram "$ALERT_TG"
|
||||
log "Telegram alert sent"
|
||||
|
||||
# Send email alert
|
||||
log "Sending email alert..."
|
||||
notify_email "$ALERT_SUBJECT" "$ALERT_EMAIL_BODY"
|
||||
log "Email alert sent"
|
||||
|
||||
# Sync latest state from S3
|
||||
log "Syncing from s3://$BUCKET/$SRC_PREFIX/..."
|
||||
aws s3 sync "s3://$BUCKET/$SRC_PREFIX/" "$HERMES_HOME/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude "audio_cache/*" \
|
||||
--exclude "image_cache/*" \
|
||||
--exclude "cache/*" \
|
||||
--exclude "sandboxes/*" \
|
||||
--exclude "kanban/*" \
|
||||
--exclude "node/*" \
|
||||
--exclude "bin/*" \
|
||||
--exclude "logs/*" \
|
||||
--exclude "*.lock" \
|
||||
--exclude "state.db-shm" \
|
||||
--exclude "state.db-wal" \
|
||||
--no-progress 2>&1 >> "$LOG"
|
||||
|
||||
chown -R root:root "$HERMES_HOME" 2>/dev/null || true
|
||||
|
||||
# Start Hermes
|
||||
log "Starting Hermes gateway..."
|
||||
hermes gateway start 2>&1 >> "$LOG"
|
||||
log "=== Standby failover complete ==="
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Hermes Standby Restore — pull latest state from S3 before Hermes starts
|
||||
Before=hermes.service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=root
|
||||
ExecStart=/root/.hermes/scripts/hermes-standby-restore.sh
|
||||
RemainAfterExit=true
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/bin/bash
|
||||
# hermes-system-config-sync.sh — Sync system-level configs to Wasabi
|
||||
# Mirrors Caddyfile, systemd service files, ops scripts, SSH keys, and .env
|
||||
# credential files to the itpropartner-system-configs bucket.
|
||||
# Silent on success, loud on failure.
|
||||
set -euo pipefail
|
||||
|
||||
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="itpropartner-system-configs"
|
||||
DATE_TAG=$(date +%F)
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
echo "[+] Syncing Caddyfile..."
|
||||
if [ -f /etc/caddy/Caddyfile ]; then
|
||||
aws s3 cp /etc/caddy/Caddyfile "s3://$BUCKET/caddy/Caddyfile" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing systemd service files..."
|
||||
if [ -d "/etc/systemd/system" ]; then
|
||||
# Create a temp dir to collect relevant service files
|
||||
TMPDIR=$(mktemp -d)
|
||||
cp /etc/systemd/system/hermes*.service "$TMPDIR/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/mysql-tunnel.service "$TMPDIR/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/ollama.service "$TMPDIR/" 2>/dev/null || true
|
||||
cp /etc/systemd/system/shark-game.service "$TMPDIR/" 2>/dev/null || true
|
||||
aws s3 sync "$TMPDIR" "s3://$BUCKET/systemd/" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
rm -rf "$TMPDIR"
|
||||
fi
|
||||
|
||||
echo "[+] Syncing ops scripts..."
|
||||
if [ -d "$HERMES_HOME/scripts" ]; then
|
||||
aws s3 sync "$HERMES_HOME/scripts" "s3://$BUCKET/scripts/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude ".backups/*" \
|
||||
--exclude "migration-backup/*" \
|
||||
--exclude "*.pyc" \
|
||||
--exclude "__pycache__/*" \
|
||||
2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing SSH keys..."
|
||||
if [ -d "$HOME/.ssh" ]; then
|
||||
aws s3 sync "$HOME/.ssh" "s3://$BUCKET/ssh/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--exclude "known_hosts" \
|
||||
--exclude "authorized_keys" \
|
||||
--exclude "config" \
|
||||
2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] Syncing .env credential files..."
|
||||
if [ -f "$HERMES_HOME/.env" ]; then
|
||||
aws s3 cp "$HERMES_HOME/.env" "s3://$BUCKET/env/hermes.env" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
if [ -f "$HOME/.aws/credentials" ]; then
|
||||
aws s3 cp "$HOME/.aws/credentials" "s3://$BUCKET/env/aws-credentials" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1 | grep -v "^$" || true
|
||||
fi
|
||||
|
||||
echo "[+] System config sync to $BUCKET complete."
|
||||
exit 0
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
# home-router-backup.sh — Backup CCR config via WireGuard tunnel
|
||||
# Uses the wg-itpp tunnel to reach the router at 10.77.0.2
|
||||
|
||||
DATE=$(date +%Y-%m-%d)
|
||||
LOG="$HOME/.hermes/scripts/logs/home-router-backup-$DATE.log"
|
||||
BACKUP_DIR="/tmp/home-router-backup"
|
||||
SSH_KEY="$HOME/.ssh/wisp_rsa"
|
||||
ROUTER="admin@10.77.0.2"
|
||||
|
||||
mkdir -p "$BACKUP_DIR" "$(dirname "$LOG")"
|
||||
|
||||
echo "=== Home Router Backup $(date -u '+%Y-%m-%d %H:%M:%S UTC') ===" >> "$LOG"
|
||||
|
||||
# Check tunnel is up
|
||||
ping -c 1 -W 3 10.77.0.2 > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "FAILED: WireGuard tunnel down (10.77.0.2 unreachable)" >> "$LOG"
|
||||
echo "Home Router Backup FAILED — tunnel down"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export config
|
||||
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
"$ROUTER" "/export terse file=itpp-backup-$DATE" >> "$LOG" 2>&1
|
||||
|
||||
# Download the export
|
||||
scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
"$ROUTER:itpp-backup-$DATE.rsc" "$BACKUP_DIR/" >> "$LOG" 2>&1
|
||||
|
||||
# Clean up remote file
|
||||
ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
"$ROUTER" "/file remove itpp-backup-$DATE.rsc" >> "$LOG" 2>&1
|
||||
|
||||
# Upload to S3
|
||||
source /opt/awscli-venv/bin/activate 2>/dev/null
|
||||
aws s3 cp "$BACKUP_DIR/itpp-backup-$DATE.rsc" \
|
||||
"s3://itpropartner-backups/home-router/$DATE-config.rsc" \
|
||||
--endpoint-url https://s3.us-east-1.wasabisys.com >> "$LOG" 2>&1
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "SUCCESS — uploaded to S3" >> "$LOG"
|
||||
rm -f "$BACKUP_DIR/itpp-backup-$DATE.rsc"
|
||||
echo "[SILENT]"
|
||||
else
|
||||
echo "FAILED: S3 upload error" >> "$LOG"
|
||||
echo "Home Router Backup FAILED — S3 upload"
|
||||
exit 1
|
||||
fi
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
/root/.hermes/scripts/wisp-backup/home-router-watchdog.sh
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# hudu-backup.sh — Daily Hudu PostgreSQL backup to Wasabi S3.
|
||||
# Source: app2 (152.53.39.202) Docker hudu-db-1
|
||||
# Target: s3://hermes-vps-backups/hudu/backups/
|
||||
# Schedule: Daily at 3 AM ET via cron (no_agent)
|
||||
set -euo pipefail
|
||||
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
APP2="root@152.53.39.202"
|
||||
S3_BUCKET="s3://hermes-vps-backups/hudu/backups"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
TS="$(date -u +%Y%m%d-%H%M%S)"
|
||||
LOCAL="/tmp/hudu-backup-${TS}.dump"
|
||||
RETENTION_DAYS=30
|
||||
|
||||
# Cron-safe AWS CLI
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; }
|
||||
|
||||
log "Starting Hudu backup"
|
||||
|
||||
# Dump from Docker on app2 via SSH
|
||||
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes "$APP2" \
|
||||
"docker exec hudu-db-1 pg_dump -U postgres -d hudu_production --format=custom --compress=9" \
|
||||
> "$LOCAL" 2>/dev/null; then
|
||||
log "FATAL: pg_dump failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SIZE=$(du -h "$LOCAL" | cut -f1)
|
||||
log "Dumped Hudu DB (${SIZE})"
|
||||
|
||||
# Upload to S3
|
||||
if aws s3 cp "$LOCAL" "${S3_BUCKET}/hudu_backup_${TS}.dump" \
|
||||
--endpoint-url "$ENDPOINT" >/dev/null 2>&1; then
|
||||
log "Uploaded: hudu_backup_${TS}.dump"
|
||||
else
|
||||
log "FATAL: S3 upload failed"
|
||||
rm -f "$LOCAL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup old backups
|
||||
aws s3 ls "${S3_BUCKET}/" --endpoint-url "$ENDPOINT" 2>/dev/null | \
|
||||
awk -v cutoff="$(date -d "${RETENTION_DAYS} days ago" -u +%Y-%m-%d)" \
|
||||
'$1 < cutoff {print $4}' | while read -r old; do
|
||||
aws s3 rm "${S3_BUCKET}/${old}" --endpoint-url "$ENDPOINT" >/dev/null 2>&1
|
||||
log "Purged: $old"
|
||||
done
|
||||
|
||||
rm -f "$LOCAL"
|
||||
log "Backup complete"
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create bill due-date events in the user's iCloud Bills calendar via CalDAV.
|
||||
|
||||
Uses only Python stdlib. Secret is read from a protected password file.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from email.utils import parseaddr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
ICLOUD_USER = "g@germainebrown.com"
|
||||
PASSWORD_FILE = Path("/root/.config/himalaya/g-germainebrown-icloud-calendar.pass")
|
||||
STATE_DIR = Path("/root/.hermes/email_triage")
|
||||
CAL_STATE_FILE = STATE_DIR / "calendar_events.json"
|
||||
CAL_LOG_FILE = STATE_DIR / "calendar_events.jsonl"
|
||||
CALENDAR_NAME = "Bills"
|
||||
DISCOVERY_URL = "https://caldav.icloud.com/.well-known/caldav"
|
||||
USER_AGENT = "DAVKit/4.0.3 (732); CalendarStore/4.0.3 (991); iCal/4.0.3 (1388); Mac OS X/10.6.4 (10F569)"
|
||||
|
||||
MONTHS = {
|
||||
"jan": 1, "january": 1,
|
||||
"feb": 2, "february": 2,
|
||||
"mar": 3, "march": 3,
|
||||
"apr": 4, "april": 4,
|
||||
"may": 5,
|
||||
"jun": 6, "june": 6,
|
||||
"jul": 7, "july": 7,
|
||||
"aug": 8, "august": 8,
|
||||
"sep": 9, "sept": 9, "september": 9,
|
||||
"oct": 10, "october": 10,
|
||||
"nov": 11, "november": 11,
|
||||
"dec": 12, "december": 12,
|
||||
}
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def load_password() -> str:
|
||||
if not PASSWORD_FILE.exists():
|
||||
raise SystemExit(f"Missing iCloud password file: {PASSWORD_FILE}")
|
||||
mode = PASSWORD_FILE.stat().st_mode & 0o777
|
||||
if mode != 0o600:
|
||||
raise SystemExit(f"Password file permissions must be 600, got {mode:o}: {PASSWORD_FILE}")
|
||||
return PASSWORD_FILE.read_text().strip()
|
||||
|
||||
|
||||
def auth_header() -> str:
|
||||
token = base64.b64encode(f"{ICLOUD_USER}:{load_password()}".encode()).decode()
|
||||
return f"Basic {token}"
|
||||
|
||||
|
||||
def request(method: str, url: str, body: str | bytes | None = None, depth: str | None = None) -> tuple[int, bytes, dict[str, str]]:
|
||||
data = body.encode("utf-8") if isinstance(body, str) else body
|
||||
headers = {
|
||||
"Authorization": auth_header(),
|
||||
"User-Agent": USER_AGENT,
|
||||
}
|
||||
if body is not None:
|
||||
headers["Content-Type"] = "application/xml; charset=utf-8" if method in {"PROPFIND", "PROPPATCH", "MKCOL", "MKCALENDAR"} else "text/calendar; charset=utf-8"
|
||||
if depth is not None:
|
||||
headers["Depth"] = depth
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
ctx = ssl.create_default_context()
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=ctx, timeout=45) as resp:
|
||||
return resp.status, resp.read(), dict(resp.headers.items())
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read(), dict(e.headers.items())
|
||||
|
||||
|
||||
def propfind(url: str, body: str, depth: str = "0") -> ET.Element:
|
||||
code, data, _ = request("PROPFIND", url, body, depth=depth)
|
||||
if code != 207:
|
||||
raise RuntimeError(f"PROPFIND {url} failed HTTP {code}: {data[:500]!r}")
|
||||
return ET.fromstring(data)
|
||||
|
||||
|
||||
def first_text(root: ET.Element, tag: str) -> str | None:
|
||||
el = root.find(f".//{{{tag.split('}', 1)[0].strip('{')}}}{tag.split('}', 1)[1]}") if tag.startswith("{") else root.find(f".//{tag}")
|
||||
return el.text if el is not None else None
|
||||
|
||||
|
||||
def discover_calendar_home() -> str:
|
||||
principal_body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:"><d:prop><d:current-user-principal/></d:prop></d:propfind>"""
|
||||
root = propfind(DISCOVERY_URL, principal_body, "0")
|
||||
principal = root.findtext(".//{DAV:}current-user-principal/{DAV:}href")
|
||||
if not principal:
|
||||
raise RuntimeError("Could not discover iCloud current-user-principal")
|
||||
principal_url = urllib.parse.urljoin(DISCOVERY_URL, principal)
|
||||
home_body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav"><d:prop><c:calendar-home-set/></d:prop></d:propfind>"""
|
||||
root = propfind(principal_url, home_body, "0")
|
||||
home = root.findtext(".//{urn:ietf:params:xml:ns:caldav}calendar-home-set/{DAV:}href")
|
||||
if not home:
|
||||
raise RuntimeError("Could not discover iCloud calendar-home-set")
|
||||
return home
|
||||
|
||||
|
||||
def list_calendars(home_url: str) -> list[dict[str, Any]]:
|
||||
body = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<d:propfind xmlns:d="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav"><d:prop><d:displayname/><d:resourcetype/><c:supported-calendar-component-set/></d:prop></d:propfind>"""
|
||||
root = propfind(home_url, body, "1")
|
||||
parsed = urllib.parse.urlparse(home_url)
|
||||
base = f"{parsed.scheme}://{parsed.netloc}"
|
||||
out: list[dict[str, Any]] = []
|
||||
for resp in root.findall("{DAV:}response"):
|
||||
href = resp.findtext("{DAV:}href") or ""
|
||||
name = resp.findtext(".//{DAV:}displayname") or ""
|
||||
comps = [c.attrib.get("name", "") for c in resp.findall(".//{urn:ietf:params:xml:ns:caldav}comp")]
|
||||
if name:
|
||||
out.append({"name": name, "href": href, "url": urllib.parse.urljoin(base, href), "components": comps})
|
||||
return out
|
||||
|
||||
|
||||
def get_bills_calendar_url() -> str:
|
||||
home = discover_calendar_home()
|
||||
calendars = list_calendars(home)
|
||||
for cal in calendars:
|
||||
if cal["name"] == CALENDAR_NAME and "VEVENT" in cal["components"]:
|
||||
return cal["url"]
|
||||
names = ", ".join(f"{c['name']}({','.join(c['components'])})" for c in calendars)
|
||||
raise RuntimeError(f"Calendar {CALENDAR_NAME!r} with VEVENT support not found. Available: {names}")
|
||||
|
||||
|
||||
def parse_due_date(text: str) -> date:
|
||||
s = text.strip().replace(",", "")
|
||||
# YYYY-MM-DD
|
||||
m = re.fullmatch(r"(\d{4})-(\d{1,2})-(\d{1,2})", s)
|
||||
if m:
|
||||
return date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
# mm/dd/yy or mm/dd/yyyy
|
||||
m = re.fullmatch(r"(\d{1,2})/(\d{1,2})/(\d{2}|\d{4})", s)
|
||||
if m:
|
||||
year = int(m.group(3))
|
||||
if year < 100:
|
||||
year += 2000
|
||||
return date(year, int(m.group(1)), int(m.group(2)))
|
||||
# Month d yyyy
|
||||
m = re.fullmatch(r"([A-Za-z]+)\s+(\d{1,2})\s+(\d{4})", s)
|
||||
if m and m.group(1).lower() in MONTHS:
|
||||
return date(int(m.group(3)), MONTHS[m.group(1).lower()], int(m.group(2)))
|
||||
raise SystemExit(f"Could not parse due date: {text!r}; use YYYY-MM-DD, MM/DD/YYYY, or Month D YYYY")
|
||||
|
||||
|
||||
def clean_text(s: str) -> str:
|
||||
return re.sub(r"[\r\n]+", " ", s or "").strip()
|
||||
|
||||
|
||||
def ics_escape(s: str) -> str:
|
||||
return clean_text(s).replace("\\", "\\\\").replace(";", r"\;").replace(",", r"\,")
|
||||
|
||||
|
||||
def fold_ical_line(line: str) -> str:
|
||||
raw = line.encode("utf-8")
|
||||
if len(raw) <= 75:
|
||||
return line
|
||||
parts: list[str] = []
|
||||
cur = ""
|
||||
for ch in line:
|
||||
prefix = "" if not parts else " "
|
||||
if len((prefix + cur + ch).encode("utf-8")) > 75:
|
||||
parts.append(("" if not parts else " ") + cur)
|
||||
cur = ch
|
||||
else:
|
||||
cur += ch
|
||||
if cur:
|
||||
parts.append(("" if not parts else " ") + cur)
|
||||
return "\r\n".join(parts)
|
||||
|
||||
|
||||
def make_event_uid(message_id: str, vendor: str, due: date, uid: str) -> str:
|
||||
key = "|".join([message_id.strip(), vendor.strip().lower(), due.isoformat(), uid.strip()])
|
||||
digest = hashlib.sha256(key.encode()).hexdigest()[:24]
|
||||
return f"bill-{digest}@hermes-germainebrown"
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if not CAL_STATE_FILE.exists():
|
||||
return {"created": {}, "created_at": now_iso()}
|
||||
try:
|
||||
return json.loads(CAL_STATE_FILE.read_text())
|
||||
except Exception:
|
||||
return {"created": {}, "created_at": now_iso(), "state_read_error": True}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CAL_STATE_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(state, indent=2, sort_keys=True))
|
||||
tmp.replace(CAL_STATE_FILE)
|
||||
|
||||
|
||||
def log_event(entry: dict[str, Any]) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with CAL_LOG_FILE.open("a") as f:
|
||||
f.write(json.dumps({"timestamp": now_iso(), **entry}, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def add_event(args: argparse.Namespace) -> None:
|
||||
due = parse_due_date(args.due_date)
|
||||
vendor = clean_text(args.vendor) or "Unknown vendor"
|
||||
amount = clean_text(args.amount)
|
||||
title = f"Bill due: {vendor}" + (f" - {amount}" if amount else "")
|
||||
event_uid = make_event_uid(args.message_id or "", vendor, due, args.uid or "")
|
||||
state = load_state()
|
||||
existing = state.get("created", {})
|
||||
for _uid, _info in existing.items():
|
||||
if _info.get("vendor") == vendor and _info.get("due_date") == due.isoformat():
|
||||
print(json.dumps({"status": "duplicate_skipped", "existing_event_uid": _uid, "vendor": vendor, "due_date": due.isoformat(), "new_uid": event_uid}, indent=2))
|
||||
log_event({"action": "duplicate_skipped", "existing_event_uid": _uid, "vendor": vendor, "due_date": due.isoformat()})
|
||||
return
|
||||
if event_uid in existing:
|
||||
print(json.dumps({"status": "exists", "event_uid": event_uid, **state["created"][event_uid]}, indent=2))
|
||||
return
|
||||
cal_url = get_bills_calendar_url().rstrip("/") + "/"
|
||||
event_url = urllib.parse.urljoin(cal_url, urllib.parse.quote(event_uid) + ".ics")
|
||||
dtstamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
dtstart = due.strftime("%Y%m%d")
|
||||
dtend = (due + timedelta(days=1)).strftime("%Y%m%d")
|
||||
description = "\\n".join([
|
||||
f"Source email UID: {args.uid or ''}",
|
||||
f"Message-ID: {args.message_id or ''}",
|
||||
f"Subject: {args.subject or ''}",
|
||||
f"Sender: {args.sender or ''}",
|
||||
])
|
||||
lines = [
|
||||
"BEGIN:VCALENDAR",
|
||||
"VERSION:2.0",
|
||||
"PRODID:-//Hermes Agent//Bill Calendar//EN",
|
||||
"CALSCALE:GREGORIAN",
|
||||
"BEGIN:VEVENT",
|
||||
f"UID:{event_uid}",
|
||||
f"DTSTAMP:{dtstamp}",
|
||||
f"DTSTART;VALUE=DATE:{dtstart}",
|
||||
f"DTEND;VALUE=DATE:{dtend}",
|
||||
f"SUMMARY:{ics_escape(title)}",
|
||||
f"DESCRIPTION:{ics_escape(description)}",
|
||||
"TRANSP:TRANSPARENT",
|
||||
"BEGIN:VALARM",
|
||||
"ACTION:DISPLAY",
|
||||
f"DESCRIPTION:{ics_escape(title)}",
|
||||
"TRIGGER:-P7D",
|
||||
"END:VALARM",
|
||||
"BEGIN:VALARM",
|
||||
"ACTION:DISPLAY",
|
||||
f"DESCRIPTION:{ics_escape(title)}",
|
||||
"TRIGGER:-P1D",
|
||||
"END:VALARM",
|
||||
"END:VEVENT",
|
||||
"END:VCALENDAR",
|
||||
]
|
||||
ics = "\r\n".join(fold_ical_line(line) for line in lines) + "\r\n"
|
||||
code, data, _ = request("PUT", event_url, ics.encode("utf-8"))
|
||||
if code not in (200, 201, 204):
|
||||
raise RuntimeError(f"PUT event failed HTTP {code}: {data[:500]!r}")
|
||||
record = {
|
||||
"calendar": CALENDAR_NAME,
|
||||
"calendar_url": cal_url,
|
||||
"event_url": event_url,
|
||||
"title": title,
|
||||
"due_date": due.isoformat(),
|
||||
"vendor": vendor,
|
||||
"amount": amount,
|
||||
"source_uid": args.uid,
|
||||
"message_id": args.message_id,
|
||||
"subject": args.subject,
|
||||
"http_code": code,
|
||||
}
|
||||
state.setdefault("created", {})[event_uid] = record
|
||||
save_state(state)
|
||||
log_event({"action": "created", "event_uid": event_uid, **record})
|
||||
print(json.dumps({"status": "created", "event_uid": event_uid, **record}, indent=2))
|
||||
|
||||
|
||||
def list_bills_calendar() -> None:
|
||||
url = get_bills_calendar_url()
|
||||
print(json.dumps({"status": "ok", "calendar": CALENDAR_NAME, "calendar_url": url}, indent=2))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--verify", action="store_true", help="Verify iCloud login and locate Bills calendar")
|
||||
ap.add_argument("--add-event", action="store_true", help="Add a bill all-day event")
|
||||
ap.add_argument("--uid", default="")
|
||||
ap.add_argument("--message-id", default="")
|
||||
ap.add_argument("--vendor", default="")
|
||||
ap.add_argument("--due-date", default="")
|
||||
ap.add_argument("--amount", default="")
|
||||
ap.add_argument("--subject", default="")
|
||||
ap.add_argument("--sender", default="")
|
||||
args = ap.parse_args()
|
||||
if args.verify:
|
||||
list_bills_calendar()
|
||||
return
|
||||
if args.add_event:
|
||||
if not args.due_date:
|
||||
raise SystemExit("--due-date is required for --add-event")
|
||||
add_event(args)
|
||||
return
|
||||
ap.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+371
@@ -0,0 +1,371 @@
|
||||
#!/usr/bin/env python3
|
||||
"""IMAP email triage helper for g@germainebrown.com.
|
||||
|
||||
Modes:
|
||||
--collect Print JSON with unprocessed recent inbox message summaries.
|
||||
--move UID Move an INBOX message UID to the Suspected Spam folder and mark processed.
|
||||
--mark UID DECISION REASON
|
||||
Mark a UID processed without moving, with an audit log entry.
|
||||
--folders Print folder list.
|
||||
|
||||
The script intentionally does not delete mail and does not open links or attachments.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
from datetime import datetime, timezone
|
||||
from email.header import decode_header, make_header
|
||||
from email.message import Message
|
||||
from email.utils import parseaddr
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
USER = "g@germainebrown.com"
|
||||
PASSWORD_FILE = Path("/root/.config/himalaya/g-germainebrown.pass")
|
||||
INBOX = "INBOX"
|
||||
SUSPECTED_SPAM = "INBOX.spam"
|
||||
STATE_DIR = Path("/root/.hermes/email_triage")
|
||||
STATE_FILE = STATE_DIR / "state.json"
|
||||
LOG_FILE = STATE_DIR / "actions.jsonl"
|
||||
MAX_BODY_CHARS = 4000
|
||||
|
||||
BILL_KEYWORDS = re.compile(r"\b(invoice|bill|statement|amount due|payment due|due date|past due|overdue|renewal|autopay|subscription|receipt|premium|tax|utility|utilities)\b", re.I)
|
||||
SPAM_KEYWORDS = re.compile(r"\b(urgent action|verify your account|password expires|account suspended|wire transfer|gift card|lottery|prize|winner|crypto|bitcoin|act now|limited time|final notice)\b", re.I)
|
||||
URL_RE = re.compile(r"https?://[^\s)>'\"]+", re.I)
|
||||
AMOUNT_RE = re.compile(r"(?:USD\s*)?\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?")
|
||||
DATE_RE = re.compile(r"\b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+\d{1,2},?\s+\d{4}\b|\b\d{1,2}/\d{1,2}/\d{2,4}\b", re.I)
|
||||
SUSPICIOUS_TLDS = {
|
||||
"click", "xyz", "top", "site", "online", "club", "icu", "buzz", "cyou", "cam",
|
||||
"quest", "rest", "bar", "monster", "sbs", "skin", "shop", "store", "pw",
|
||||
"win", "bid", "party", "loan", "date", "download", "stream", "review", "country", "science",
|
||||
}
|
||||
FREEMAIL_DOMAINS = {"gmail.com", "yahoo.com", "outlook.com", "hotmail.com", "aol.com", "icloud.com", "proton.me", "protonmail.com", "mail.com"}
|
||||
KNOWN_LEGIT_DOMAINS = {
|
||||
"google.com", "apple.com", "amazon.com", "stripe.com", "bankofamerica.com", "chase.com",
|
||||
"americanexpress.com", "georgiapower.com", "spectrum.net", "spectrum.com", "rocketmoney.com",
|
||||
"venmo.com", "paypal.com", "carters.com", "theepochtimes.com", "avis.com", "avisbudget.com",
|
||||
"fedex.com", "ups.com", "usps.com", "microsoft.com", "adobe.com", "intuit.com", "xfinity.com",
|
||||
"labcorp.com", "labcorpmessage.com", "gotracktownusa.com",
|
||||
"robinhood.com",
|
||||
"southernco.com", # parent of Georgia Power (G2georgiapps@southernco.com sends usage alerts)
|
||||
"redditmail.com", # Reddit official notification domain (via Amazon SES) — looks random to heuristic
|
||||
}
|
||||
USER_BLOCKED_SPAM_DOMAINS = {
|
||||
"winecountrygiftbaskets.com", "heavy.com", "moveon.org", "lendingclub.com", "upgrade.com",
|
||||
"corporateshopping.com", "hudsonridge.com", "ancestry.com", "twentytwowords.com",
|
||||
"exchange.spectrum.com", # sales/promo flyers sent to old Gmail, not actual Spectrum billing
|
||||
}
|
||||
NEWS_DOMAINS = {"theepochtimes.com", "nytimes.com", "washingtonpost.com", "wsj.com", "cnn.com", "foxnews.com"}
|
||||
BRAND_WORDS = {"paypal", "apple", "amazon", "microsoft", "bank", "chase", "wells", "boa", "bank of america", "netflix", "usps", "ups", "fedex", "dhl", "stripe", "venmo", "social security"}
|
||||
RANDOM_LABEL_RE = re.compile(r"^(?:[a-z]*\d+[a-z\d]*|[a-z]{10,}|[a-z\d]{12,})$", re.I)
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def load_password() -> str:
|
||||
if not PASSWORD_FILE.exists():
|
||||
raise SystemExit(f"Missing password file: {PASSWORD_FILE}")
|
||||
return PASSWORD_FILE.read_text().strip()
|
||||
|
||||
|
||||
def connect() -> imaplib.IMAP4_SSL:
|
||||
ctx = ssl.create_default_context()
|
||||
m = imaplib.IMAP4_SSL(HOST, PORT, ssl_context=ctx)
|
||||
m.login(USER, load_password())
|
||||
return m
|
||||
|
||||
|
||||
def q(name: str) -> str:
|
||||
return '"' + name.replace('\\', '\\\\').replace('"', '\\"') + '"'
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
if not STATE_FILE.exists():
|
||||
return {"processed_uids": {}, "created_at": now_iso()}
|
||||
try:
|
||||
return json.loads(STATE_FILE.read_text())
|
||||
except Exception:
|
||||
return {"processed_uids": {}, "created_at": now_iso(), "state_read_error": True}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = STATE_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(state, indent=2, sort_keys=True))
|
||||
tmp.replace(STATE_FILE)
|
||||
|
||||
|
||||
def log_action(entry: dict[str, Any]) -> None:
|
||||
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
entry = {"timestamp": now_iso(), **entry}
|
||||
with LOG_FILE.open("a") as f:
|
||||
f.write(json.dumps(entry, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def decode_mime(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(value)))
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
|
||||
def parse_addr(value: str | None) -> str:
|
||||
return decode_mime(value).strip()
|
||||
|
||||
|
||||
def sender_domain(value: str | None) -> str:
|
||||
addr = parseaddr(decode_mime(value))[1].lower()
|
||||
return addr.rsplit("@", 1)[-1] if "@" in addr else ""
|
||||
|
||||
|
||||
def base_domain(domain: str) -> str:
|
||||
parts = domain.lower().split(".")
|
||||
return ".".join(parts[-2:]) if len(parts) >= 2 else domain.lower()
|
||||
|
||||
|
||||
def sender_reputation_hints(sender: str, subject: str) -> list[str]:
|
||||
hints: list[str] = []
|
||||
domain = sender_domain(sender)
|
||||
bdom = base_domain(domain)
|
||||
display = parseaddr(sender)[0].lower()
|
||||
combined = f"{display} {subject}".lower()
|
||||
if not domain:
|
||||
return ["sender_missing_domain"]
|
||||
if bdom in USER_BLOCKED_SPAM_DOMAINS:
|
||||
hints.append("user_blocked_spam_domain")
|
||||
if domain in KNOWN_LEGIT_DOMAINS or bdom in KNOWN_LEGIT_DOMAINS:
|
||||
hints.append("known_legitimate_sender_domain")
|
||||
tld = domain.rsplit(".", 1)[-1] if "." in domain else ""
|
||||
if tld in SUSPICIOUS_TLDS and bdom not in KNOWN_LEGIT_DOMAINS:
|
||||
hints.append(f"low_reputation_tld_.{tld}")
|
||||
if any(RANDOM_LABEL_RE.match(label) for label in domain.split(".")[:-1]) and bdom not in KNOWN_LEGIT_DOMAINS:
|
||||
hints.append("random_looking_sender_domain")
|
||||
for brand in BRAND_WORDS:
|
||||
token = brand.replace(" ", "")
|
||||
if brand in combined and token not in domain.replace("-", "") and bdom not in NEWS_DOMAINS and bdom not in KNOWN_LEGIT_DOMAINS:
|
||||
hints.append(f"possible_{token}_spoof_from_unrelated_domain")
|
||||
break
|
||||
if bdom in FREEMAIL_DOMAINS and re.search(r"\b(invoice|payment|security|account|order submitted|receipt|bank|paypal|apple|amazon)\b", combined, re.I):
|
||||
hints.append("sensitive_claim_from_freemail_sender")
|
||||
return hints
|
||||
|
||||
|
||||
def strip_html(text: str) -> str:
|
||||
text = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", text)
|
||||
text = re.sub(r"(?s)<[^>]+>", " ", text)
|
||||
text = re.sub(r" ", " ", text)
|
||||
text = re.sub(r"&", "&", text)
|
||||
text = re.sub(r"<", "<", text)
|
||||
text = re.sub(r">", ">", text)
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def body_text(msg: Message) -> tuple[str, list[str]]:
|
||||
attachments: list[str] = []
|
||||
parts: list[str] = []
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
disp = (part.get_content_disposition() or "").lower()
|
||||
ctype = part.get_content_type().lower()
|
||||
filename = decode_mime(part.get_filename()) if part.get_filename() else ""
|
||||
if filename or disp == "attachment":
|
||||
attachments.append(filename or ctype)
|
||||
continue
|
||||
if ctype in ("text/plain", "text/html"):
|
||||
try:
|
||||
payload = part.get_payload(decode=True) or b""
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
txt = payload.decode(charset, errors="replace")
|
||||
if ctype == "text/html":
|
||||
txt = strip_html(txt)
|
||||
parts.append(txt)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
ctype = msg.get_content_type().lower()
|
||||
try:
|
||||
payload = msg.get_payload(decode=True) or b""
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
txt = payload.decode(charset, errors="replace")
|
||||
if ctype == "text/html":
|
||||
txt = strip_html(txt)
|
||||
parts.append(txt)
|
||||
except Exception:
|
||||
pass
|
||||
text = re.sub(r"\s+", " ", "\n".join(parts)).strip()
|
||||
return text[:MAX_BODY_CHARS], attachments
|
||||
|
||||
|
||||
def ensure_folder(m: imaplib.IMAP4_SSL, folder: str) -> None:
|
||||
typ, data = m.list()
|
||||
folders = []
|
||||
for raw in data or []:
|
||||
line = raw.decode(errors="replace") if isinstance(raw, bytes) else str(raw)
|
||||
folders.append(line)
|
||||
if any((f'"{folder}"' in line) or line.endswith(" " + folder) for line in folders):
|
||||
return
|
||||
typ, data = m._simple_command("CREATE", q(folder))
|
||||
if typ != "OK":
|
||||
raise RuntimeError(f"Could not create folder {folder}: {data}")
|
||||
|
||||
|
||||
def list_folders() -> None:
|
||||
with connect() as m:
|
||||
ensure_folder(m, SUSPECTED_SPAM)
|
||||
typ, data = m.list()
|
||||
print(json.dumps({"status": typ, "folders": [x.decode(errors="replace") if isinstance(x, bytes) else str(x) for x in (data or [])]}, indent=2))
|
||||
m.logout()
|
||||
|
||||
|
||||
def fetch_message(m: imaplib.IMAP4_SSL, uid: str) -> dict[str, Any] | None:
|
||||
typ, data = m.uid("FETCH", uid, "(RFC822)")
|
||||
if typ != "OK" or not data:
|
||||
return None
|
||||
raw_msg = None
|
||||
for item in data:
|
||||
if isinstance(item, tuple) and item[1]:
|
||||
raw_msg = item[1]
|
||||
break
|
||||
if raw_msg is None:
|
||||
return None
|
||||
msg = email.message_from_bytes(raw_msg)
|
||||
text, attachments = body_text(msg)
|
||||
subject = decode_mime(msg.get("Subject"))
|
||||
sender = parse_addr(msg.get("From"))
|
||||
to = parse_addr(msg.get("To"))
|
||||
date = decode_mime(msg.get("Date"))
|
||||
message_id = (msg.get("Message-ID") or "").strip()
|
||||
urls = URL_RE.findall(text)[:12]
|
||||
combined = f"{subject}\n{sender}\n{text}"
|
||||
hints = []
|
||||
if BILL_KEYWORDS.search(combined):
|
||||
hints.append("bill_keywords")
|
||||
if SPAM_KEYWORDS.search(combined):
|
||||
hints.append("spam_keywords")
|
||||
if attachments:
|
||||
hints.append("has_attachments")
|
||||
if urls:
|
||||
hints.append("has_links")
|
||||
hints.extend(sender_reputation_hints(sender, subject))
|
||||
amounts = AMOUNT_RE.findall(combined)[:5]
|
||||
dates = DATE_RE.findall(combined)[:5]
|
||||
return {
|
||||
"uid": uid,
|
||||
"message_id": message_id,
|
||||
"from": sender,
|
||||
"to": to,
|
||||
"subject": subject,
|
||||
"date": date,
|
||||
"snippet": text[:1200],
|
||||
"attachments": attachments[:10],
|
||||
"url_count": len(urls),
|
||||
"sample_urls": urls[:5],
|
||||
"detected_amounts": amounts,
|
||||
"detected_dates": dates,
|
||||
"heuristic_hints": hints,
|
||||
}
|
||||
|
||||
|
||||
def collect(max_messages: int, unread_only: bool) -> None:
|
||||
state = load_state()
|
||||
processed = set(state.get("processed_uids", {}).keys())
|
||||
with connect() as m:
|
||||
ensure_folder(m, SUSPECTED_SPAM)
|
||||
typ, _ = m.select(INBOX, readonly=True)
|
||||
if typ != "OK":
|
||||
raise RuntimeError("Could not select INBOX")
|
||||
criteria = "UNSEEN" if unread_only else "ALL"
|
||||
typ, data = m.uid("SEARCH", None, criteria)
|
||||
if typ != "OK":
|
||||
raise RuntimeError(f"Search failed: {data}")
|
||||
uids = (data[0].decode().split() if data and data[0] else [])
|
||||
uids = [u for u in uids if u not in processed]
|
||||
selected = list(reversed(uids)) if max_messages <= 0 else list(reversed(uids))[:max_messages]
|
||||
messages = []
|
||||
for uid in selected:
|
||||
item = fetch_message(m, uid)
|
||||
if item:
|
||||
messages.append(item)
|
||||
m.logout()
|
||||
print(json.dumps({
|
||||
"account": USER,
|
||||
"host": HOST,
|
||||
"inbox": INBOX,
|
||||
"suspected_spam_folder": SUSPECTED_SPAM,
|
||||
"mode": "collect",
|
||||
"unread_only": unread_only,
|
||||
"unprocessed_available": len(uids),
|
||||
"returned": len(messages),
|
||||
"messages": messages,
|
||||
"instruction": "Classify each message as legit, suspected_spam, bill_or_invoice, or uncertain. Move only high-confidence suspected_spam using this script's --move UID. For bills, notify the user. For legit/uncertain, use --mark UID DECISION REASON after deciding."
|
||||
}, indent=2))
|
||||
|
||||
|
||||
def mark(uid: str, decision: str, reason: str, moved_to: str | None = None) -> None:
|
||||
state = load_state()
|
||||
state.setdefault("processed_uids", {})[uid] = {
|
||||
"decision": decision,
|
||||
"reason": reason,
|
||||
"moved_to": moved_to,
|
||||
"processed_at": now_iso(),
|
||||
}
|
||||
save_state(state)
|
||||
log_action({"uid": uid, "decision": decision, "reason": reason, "moved_to": moved_to})
|
||||
|
||||
|
||||
def move(uid: str, reason: str) -> None:
|
||||
with connect() as m:
|
||||
ensure_folder(m, SUSPECTED_SPAM)
|
||||
typ, _ = m.select(INBOX, readonly=False)
|
||||
if typ != "OK":
|
||||
raise RuntimeError("Could not select INBOX")
|
||||
typ, data = m.uid("COPY", uid, q(SUSPECTED_SPAM))
|
||||
if typ != "OK":
|
||||
raise RuntimeError(f"COPY failed for UID {uid}: {data}")
|
||||
typ, data = m.uid("STORE", uid, "+FLAGS", "(\\Deleted)")
|
||||
if typ != "OK":
|
||||
raise RuntimeError(f"STORE Deleted failed for UID {uid}: {data}")
|
||||
m.expunge()
|
||||
m.logout()
|
||||
mark(uid, "suspected_spam", reason, SUSPECTED_SPAM)
|
||||
print(json.dumps({"status": "moved", "uid": uid, "folder": SUSPECTED_SPAM, "reason": reason}))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--collect", action="store_true")
|
||||
ap.add_argument("--folders", action="store_true")
|
||||
ap.add_argument("--move")
|
||||
ap.add_argument("--mark", nargs=3, metavar=("UID", "DECISION", "REASON"))
|
||||
ap.add_argument("--max", type=int, default=0, help="Max messages to collect; 0 means no limit")
|
||||
ap.add_argument("--all", action="store_true", help="Collect all recent unprocessed messages, not just unread")
|
||||
ap.add_argument("--reason", default="High-confidence suspected spam/phishing")
|
||||
args = ap.parse_args()
|
||||
if args.folders:
|
||||
list_folders(); return
|
||||
if args.collect:
|
||||
collect(max_messages=args.max, unread_only=not args.all); return
|
||||
if args.move:
|
||||
move(args.move, args.reason); return
|
||||
if args.mark:
|
||||
uid, decision, reason = args.mark
|
||||
mark(uid, decision, reason)
|
||||
print(json.dumps({"status": "marked", "uid": uid, "decision": decision, "reason": reason}))
|
||||
return
|
||||
ap.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic IMAP triage runner.
|
||||
|
||||
Collects unprocessed unread messages via imap_triage.py, marks routine legitimate
|
||||
mail processed, moves high-confidence spam, and prints only important items.
|
||||
Silent on no actionable items.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
TRIAGE = "/root/.hermes/scripts/imap_triage.py"
|
||||
|
||||
|
||||
def run(args: list[str]) -> tuple[int, str, str]:
|
||||
p = subprocess.run([sys.executable, TRIAGE, *args], capture_output=True, text=True, timeout=90)
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
|
||||
|
||||
def classify(msg: dict) -> tuple[str, str]:
|
||||
sender = (msg.get("from") or "").lower()
|
||||
subject = (msg.get("subject") or "").lower()
|
||||
hints = set(msg.get("heuristic_hints") or [])
|
||||
amounts = msg.get("detected_amounts") or []
|
||||
|
||||
# User-blocked marketing/loan domains are spam/promotional.
|
||||
if "user_blocked_spam_domain" in hints:
|
||||
return "suspected_spam", "user-blocked sender domain / promotional offer"
|
||||
|
||||
# Clear marketing/newsletter categories.
|
||||
marketing_senders = ["zillow", "opentable", "upgrade"]
|
||||
if any(s in sender for s in marketing_senders):
|
||||
if "bank of america" not in sender:
|
||||
return "legit", "routine newsletter/marketing notification"
|
||||
|
||||
# Bank/security/money notifications should surface to Germaine.
|
||||
if "bankofamerica.com" in sender or "bank of america" in sender:
|
||||
return "bill_or_invoice", "bank/debit card activity notification"
|
||||
|
||||
if "bill_keywords" in hints and amounts:
|
||||
return "bill_or_invoice", "payment/bill-like message with amount"
|
||||
|
||||
if "spam_keywords" in hints or any(h.startswith("possible_") for h in hints):
|
||||
return "suspected_spam", "spam/phishing heuristic matched"
|
||||
|
||||
return "uncertain", "no deterministic action"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
code, out, err = run(["--collect", "--max", "20"])
|
||||
if code != 0:
|
||||
print(f"IMAP triage collect failed: {err or out}".strip())
|
||||
return code or 1
|
||||
data = json.loads(out or "{}")
|
||||
messages = data.get("messages") or []
|
||||
actionable: list[str] = []
|
||||
|
||||
for msg in messages:
|
||||
uid = str(msg.get("uid"))
|
||||
decision, reason = classify(msg)
|
||||
if not uid:
|
||||
continue
|
||||
if decision == "suspected_spam":
|
||||
run(["--move", uid, "--reason", reason])
|
||||
elif decision in ("legit", "uncertain"):
|
||||
run(["--mark", uid, decision, reason])
|
||||
elif decision == "bill_or_invoice":
|
||||
# Mark processed and report once.
|
||||
run(["--mark", uid, decision, reason])
|
||||
actionable.append(
|
||||
f"{msg.get('from','Unknown')} — {msg.get('subject','(no subject)')} — {reason}"
|
||||
)
|
||||
|
||||
if actionable:
|
||||
print("Email triage: important item(s) found:\n" + "\n".join(f"- {x}" for x in actionable))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
python3 /root/.hermes/scripts/imap_triage.py --collect
|
||||
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Watchdog for the email triage cron job (5929c5f1deff).
|
||||
# Checks if the most recent run ended with an error.
|
||||
# Silent (no output) when OK, alerts when failed.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="/root/.hermes"
|
||||
DB="$SCRIPT_DIR/state.db"
|
||||
JOB_PREFIX="cron_5929c5f1deff"
|
||||
LAST_GOOD_FILE="$SCRIPT_DIR/email_triage/.watchdog_last_ok"
|
||||
mkdir -p "$(dirname "$LAST_GOOD_FILE")"
|
||||
|
||||
# Query the most recent cron session for this job
|
||||
RESULT=$(python3 -c "
|
||||
import sqlite3, sys, json
|
||||
try:
|
||||
con = sqlite3.connect('$DB')
|
||||
cur = con.cursor()
|
||||
row = cur.execute(\"\"\"select started_at, end_reason from sessions
|
||||
where id like '$JOB_PREFIX%' and end_reason is not null
|
||||
order by started_at desc limit 1\"\"\").fetchone()
|
||||
con.close()
|
||||
if row:
|
||||
print(json.dumps({'started_at': row[0], 'end_reason': row[1]}))
|
||||
else:
|
||||
print('{}')
|
||||
except Exception as e:
|
||||
print(json.dumps({'error': str(e)}))
|
||||
" 2>/dev/null)
|
||||
|
||||
LAST_STATUS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('end_reason',''))" 2>/dev/null || echo "")
|
||||
LAST_TS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('started_at',''))" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$LAST_STATUS" ]; then
|
||||
# No data yet — first run, don't alert
|
||||
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$LAST_STATUS" = "error" ] || [ "$LAST_STATUS" = "failed" ]; then
|
||||
# Failure — only alert if we haven't already reported this one
|
||||
if [ -f "$LAST_GOOD_FILE" ]; then
|
||||
PREV_GOOD=$(cat "$LAST_GOOD_FILE")
|
||||
if [ "$LAST_TS" != "$PREV_GOOD" ]; then
|
||||
echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
|
||||
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
||||
fi
|
||||
else
|
||||
echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
|
||||
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
||||
fi
|
||||
else
|
||||
# All good — mark this timestamp so we only alert on new failures
|
||||
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
||||
fi
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""List Hetzner servers via API."""
|
||||
import urllib.request, json, os
|
||||
|
||||
token = os.environ.get("HETZNER_TOKEN", "")
|
||||
req = urllib.request.Request(
|
||||
"https://api.hetzner.cloud/v1/servers",
|
||||
headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
data = json.loads(urllib.request.urlopen(req).read())
|
||||
|
||||
for s in data["servers"]:
|
||||
t = s["server_type"]
|
||||
print(f" {s['name']:20s} | ID: {s['id']:8d} | {t['name']:8s} | {t['cores']}vCPU {t['memory']}GB {t['disk']}GB | {s['status']}")
|
||||
@@ -0,0 +1,60 @@
|
||||
=== Home Router Backup 2026-07-03 19:04:40 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-03
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 172.31.1.1 (avoids ppp0 recursion)
|
||||
generating QUICK_MODE request 2581205473 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (300 bytes)
|
||||
parsed QUICK_MODE response 2581205473 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
detected rekeying of CHILD_SA wisp-vpn{17}
|
||||
CHILD_SA wisp-vpn{18} established with SPIs ca6bda58_i 05979acf_o and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.105
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[DRY RUN] Would backup these towers:
|
||||
- home-gateway (192.168.88.1) @ site: home
|
||||
|
||||
[DRY RUN] Would upload to s3://CHANGE_ME/
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{14} with SPIs c15c0861_i (4356 bytes) 07582a47_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c15c0861
|
||||
generating INFORMATIONAL_V1 request 309377479 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{15} with SPIs c854c79d_i (1342 bytes) 03f97f5a_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c854c79d
|
||||
generating INFORMATIONAL_V1 request 185272777 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{16} with SPIs c76b3715_i (2614 bytes) 0b2d4f29_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c76b3715
|
||||
generating INFORMATIONAL_V1 request 3117177119 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{17} with SPIs c42ff070_i (161340 bytes) 01ab374b_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c42ff070
|
||||
generating INFORMATIONAL_V1 request 3677623384 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{18} with SPIs ca6bda58_i (1008 bytes) 05979acf_o (944 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI ca6bda58
|
||||
generating INFORMATIONAL_V1 request 3772474848 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[16] between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[16]
|
||||
generating INFORMATIONAL_V1 request 1012455368 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [16] closed successfully
|
||||
[+] VPN disconnected
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,194 @@
|
||||
=== Home Router Backup 2026-07-04 02:50:30 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-04
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 172.31.1.1 (avoids ppp0 recursion)
|
||||
generating QUICK_MODE request 3349541925 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (300 bytes)
|
||||
parsed QUICK_MODE response 3349541925 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
detected rekeying of CHILD_SA wisp-vpn{19}
|
||||
CHILD_SA wisp-vpn{20} established with SPIs c2b4c21a_i 0011d94f_o and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 3349541925 [ HASH ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (60 bytes)
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.105
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (192.168.88.1) — backing up ...
|
||||
[FAIL] SSH connection failed: Authentication failed.
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{19} with SPIs cb762789_i (94880 bytes) 0fe7c68a_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI cb762789
|
||||
generating INFORMATIONAL_V1 request 2707403376 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{20} with SPIs c2b4c21a_i (4042 bytes) 0011d94f_o (3538 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c2b4c21a
|
||||
generating INFORMATIONAL_V1 request 4260970196 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[22] between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[22]
|
||||
generating INFORMATIONAL_V1 request 2803316938 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [22] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 0 OK, 1 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✗ home-gateway (home) — SSH connection failed: Authentication failed.
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-04 02:51:04 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-04
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 172.31.1.1 (avoids ppp0 recursion)
|
||||
initiating Main Mode IKE_SA wisp-vpn[33] to 76.195.7.60
|
||||
generating ID_PROT request 0 [ SA V V V V V ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (252 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (164 bytes)
|
||||
parsed ID_PROT response 0 [ SA V V V V ]
|
||||
received NAT-T (RFC 3947) vendor ID
|
||||
received XAuth vendor ID
|
||||
received DPD vendor ID
|
||||
received FRAGMENTATION vendor ID
|
||||
selected proposal: IKE:AES_CBC_128/HMAC_SHA1_96/PRF_HMAC_SHA1/MODP_1024
|
||||
generating ID_PROT request 0 [ KE No NAT-D NAT-D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (244 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (236 bytes)
|
||||
parsed ID_PROT response 0 [ KE No NAT-D NAT-D ]
|
||||
generating ID_PROT request 0 [ ID HASH N(INITIAL_CONTACT) ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (108 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (76 bytes)
|
||||
parsed ID_PROT response 0 [ ID HASH ]
|
||||
IKE_SA wisp-vpn[33] established between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
scheduling reauthentication in 85494s
|
||||
maximum IKE_SA lifetime 86034s
|
||||
generating QUICK_MODE request 2963618619 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (300 bytes)
|
||||
parsed QUICK_MODE response 2963618619 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
CHILD_SA wisp-vpn{21} established with SPIs c0d9045b_i 00059898_o and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 2963618619 [ HASH ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (60 bytes)
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.105
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (192.168.88.1) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-04/home-gateway-2026-07-04.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-04/home-gateway-2026-07-04.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{21} with SPIs c0d9045b_i (41526 bytes) 00059898_o (8190 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c0d9045b
|
||||
generating INFORMATIONAL_V1 request 2696552594 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[33] between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[33]
|
||||
generating INFORMATIONAL_V1 request 410247037 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [33] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
=== Home Router Backup 2026-07-04 06:00:47 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-04
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 172.31.1.1 (avoids ppp0 recursion)
|
||||
generating QUICK_MODE request 223228910 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (300 bytes)
|
||||
parsed QUICK_MODE response 223228910 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
detected rekeying of CHILD_SA wisp-vpn{22}
|
||||
CHILD_SA wisp-vpn{23} established with SPIs c3fdf06d_i 08e55d09_o and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 223228910 [ HASH ]
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.105
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (192.168.88.1) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-04/home-gateway-2026-07-04.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-04/home-gateway-2026-07-04.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{22} with SPIs c9cd56ac_i (38724 bytes) 0adab1f2_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c9cd56ac
|
||||
generating INFORMATIONAL_V1 request 1367800329 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{23} with SPIs c3fdf06d_i (41462 bytes) 08e55d09_o (8682 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c3fdf06d
|
||||
generating INFORMATIONAL_V1 request 492351832 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[34] between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[34]
|
||||
generating INFORMATIONAL_V1 request 985162767 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [34] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,60 @@
|
||||
=== Home Router Backup 2026-07-05 06:00:00 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-05
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 172.31.1.1 (avoids ppp0 recursion)
|
||||
generating QUICK_MODE request 645441876 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 5.161.114.8[500] (300 bytes)
|
||||
parsed QUICK_MODE response 645441876 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
detected rekeying of CHILD_SA wisp-vpn{27}
|
||||
CHILD_SA wisp-vpn{28} established with SPIs cffd07da_i 0f0fe43c_o and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 645441876 [ HASH ]
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.105
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (192.168.88.1) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-05/home-gateway-2026-07-05.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-05/home-gateway-2026-07-05.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{27} with SPIs c95bfd1b_i (7778 bytes) 0d60180b_o (0 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c95bfd1b
|
||||
generating INFORMATIONAL_V1 request 2474571922 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{28} with SPIs cffd07da_i (43496 bytes) 0f0fe43c_o (8796 bytes) and TS 5.161.114.8/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI cffd07da
|
||||
generating INFORMATIONAL_V1 request 4205869785 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[67] between 5.161.114.8[5.161.114.8]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[67]
|
||||
generating INFORMATIONAL_V1 request 3920562488 [ HASH D ]
|
||||
sending packet: from 5.161.114.8[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [67] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,6 @@
|
||||
=== Home Router Backup 2026-07-06 10:00:08 UTC ===
|
||||
Traceback (most recent call last):
|
||||
File "/root/.hermes/scripts/wisp-backup/wisp-backup.py", line 17, in <module>
|
||||
import paramiko
|
||||
ModuleNotFoundError: No module named 'paramiko'
|
||||
=== Exit code: 1 ===
|
||||
@@ -0,0 +1,12 @@
|
||||
=== Home Router Backup 2026-07-07 10:00:24 UTC ===
|
||||
Traceback (most recent call last):
|
||||
File "/root/.hermes/scripts/wisp-backup/wisp-backup.py", line 17, in <module>
|
||||
import paramiko
|
||||
ModuleNotFoundError: No module named 'paramiko'
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-07 11:03:39 UTC ===
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
Completed 17.7 KiB/17.7 KiB (82.7 KiB/s) with 1 file(s) remaining
|
||||
upload: ../../../../tmp/home-router-backup/itpp-backup-2026-07-07.rsc to s3://itpropartner-backups/home-router/2026-07-07-config.rsc
|
||||
@@ -0,0 +1,205 @@
|
||||
=== Home Router Backup 2026-07-08 10:00:49 UTC ===
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
scp: itpp-backup-2026-07-08.rsc: No such file or directory
|
||||
Warning: Permanently added '10.77.0.2' (RSA) to the list of known hosts.
|
||||
no such item (/file/remove; line 1)
|
||||
|
||||
The user-provided path /tmp/home-router-backup/itpp-backup-2026-07-08.rsc does not exist.
|
||||
FAILED: S3 upload error
|
||||
=== Home Router Backup 2026-07-08 23:32:56 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-08
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 152.53.192.1 (avoids ppp0 recursion)
|
||||
[-] VPN connection failed (ppp0 not up).
|
||||
Cleaning up IPsec to avoid orphaned tunnel ...
|
||||
Check logs: journalctl -u xl2tpd -n 20
|
||||
ipsec status
|
||||
[stderr] /root/.hermes/scripts/wisp-backup/home-router-vpn.sh: line 61: /etc/xl2tpd/xl2tpd.conf: No such file or directory
|
||||
[stderr] /root/.hermes/scripts/wisp-backup/home-router-vpn.sh: line 80: /etc/ppp/options.wisp-vpn: No such file or directory
|
||||
[stderr] /root/.hermes/scripts/wisp-backup/home-router-vpn.sh: line 103: /etc/ppp/chap-secrets: No such file or directory
|
||||
[stderr] /root/.hermes/scripts/wisp-backup/home-router-vpn.sh: line 124: /var/run/xl2tpd/l2tp-control: No such file or directory
|
||||
[stderr] /root/.hermes/scripts/wisp-backup/home-router-vpn.sh: line 178: /var/run/xl2tpd/l2tp-control: No such file or directory
|
||||
[-] VPN connection failed. Aborting.
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-08 23:34:04 UTC ===
|
||||
Exception (client): Error reading SSH protocol banner
|
||||
Traceback (most recent call last):
|
||||
File "/opt/awscli-venv/lib/python3.13/site-packages/paramiko/transport.py", line 2213, in _check_banner
|
||||
buf = self.packetizer.readline(timeout)
|
||||
File "/opt/awscli-venv/lib/python3.13/site-packages/paramiko/packet.py", line 395, in readline
|
||||
buf += self._read_timeout(timeout)
|
||||
~~~~~~~~~~~~~~~~~~^^^^^^^^^
|
||||
File "/opt/awscli-venv/lib/python3.13/site-packages/paramiko/packet.py", line 665, in _read_timeout
|
||||
raise EOFError()
|
||||
EOFError
|
||||
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/opt/awscli-venv/lib/python3.13/site-packages/paramiko/transport.py", line 2029, in run
|
||||
self._check_banner()
|
||||
~~~~~~~~~~~~~~~~~~^^
|
||||
File "/opt/awscli-venv/lib/python3.13/site-packages/paramiko/transport.py", line 2217, in _check_banner
|
||||
raise SSHException(
|
||||
"Error reading SSH protocol banner" + str(e)
|
||||
)
|
||||
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-08
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 152.53.192.1 (avoids ppp0 recursion)
|
||||
initiating Main Mode IKE_SA wisp-vpn[1] to 76.195.7.60
|
||||
generating ID_PROT request 0 [ SA V V V V V ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (252 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (164 bytes)
|
||||
parsed ID_PROT response 0 [ SA V V V V ]
|
||||
received NAT-T (RFC 3947) vendor ID
|
||||
received XAuth vendor ID
|
||||
received DPD vendor ID
|
||||
received FRAGMENTATION vendor ID
|
||||
selected proposal: IKE:AES_CBC_128/HMAC_SHA1_96/PRF_HMAC_SHA1/MODP_1024
|
||||
generating ID_PROT request 0 [ KE No NAT-D NAT-D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (244 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (236 bytes)
|
||||
parsed ID_PROT response 0 [ KE No NAT-D NAT-D ]
|
||||
generating ID_PROT request 0 [ ID HASH N(INITIAL_CONTACT) ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (108 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (92 bytes)
|
||||
invalid HASH_V1 payload length, decryption failed?
|
||||
could not decrypt payloads
|
||||
message parsing failed
|
||||
ignore malformed INFORMATIONAL request
|
||||
INFORMATIONAL_V1 request with message ID 966687587 processing failed
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (76 bytes)
|
||||
parsed ID_PROT response 0 [ ID HASH ]
|
||||
IKE_SA wisp-vpn[1] established between 152.53.192.33[152.53.192.33]...76.195.7.60[76.195.7.60]
|
||||
scheduling reauthentication in 85352s
|
||||
maximum IKE_SA lifetime 85892s
|
||||
generating QUICK_MODE request 3071489454 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (300 bytes)
|
||||
parsed QUICK_MODE response 3071489454 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
CHILD_SA wisp-vpn{1} established with SPIs c8225420_i 0739aab7_o and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.104
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (192.168.88.1) — backing up ...
|
||||
[FAIL] SSH connection failed: Error reading SSH protocol banner
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{1} with SPIs c8225420_i (1028 bytes) 0739aab7_o (1045 bytes) and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c8225420
|
||||
generating INFORMATIONAL_V1 request 1674779512 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[1] between 152.53.192.33[152.53.192.33]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[1]
|
||||
generating INFORMATIONAL_V1 request 3552681913 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [1] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 0 OK, 1 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✗ home-gateway (home) — SSH connection failed: Error reading SSH protocol banner
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-08 23:34:58 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-08
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 152.53.192.1 (avoids ppp0 recursion)
|
||||
initiating Main Mode IKE_SA wisp-vpn[2] to 76.195.7.60
|
||||
generating ID_PROT request 0 [ SA V V V V V ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (252 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (164 bytes)
|
||||
parsed ID_PROT response 0 [ SA V V V V ]
|
||||
received NAT-T (RFC 3947) vendor ID
|
||||
received XAuth vendor ID
|
||||
received DPD vendor ID
|
||||
received FRAGMENTATION vendor ID
|
||||
selected proposal: IKE:AES_CBC_128/HMAC_SHA1_96/PRF_HMAC_SHA1/MODP_1024
|
||||
generating ID_PROT request 0 [ KE No NAT-D NAT-D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (244 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (236 bytes)
|
||||
parsed ID_PROT response 0 [ KE No NAT-D NAT-D ]
|
||||
generating ID_PROT request 0 [ ID HASH N(INITIAL_CONTACT) ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (108 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (76 bytes)
|
||||
parsed ID_PROT response 0 [ ID HASH ]
|
||||
IKE_SA wisp-vpn[2] established between 152.53.192.33[152.53.192.33]...76.195.7.60[76.195.7.60]
|
||||
scheduling reauthentication in 85828s
|
||||
maximum IKE_SA lifetime 86368s
|
||||
generating QUICK_MODE request 1842263490 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (300 bytes)
|
||||
parsed QUICK_MODE response 1842263490 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
CHILD_SA wisp-vpn{2} established with SPIs c3a88a35_i 0030d7ca_o and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 1842263490 [ HASH ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (60 bytes)
|
||||
connection 'wisp-vpn' established successfully
|
||||
[+] VPN connected! Interface: ppp0, IP: 192.168.88.104
|
||||
[*] Adding route to VPN peer's subnet via ppp0 ...
|
||||
[route] Added 192.168.88.0/24 via ppp0
|
||||
[*] Adding static routes for tower subnets ...
|
||||
(no routes configured in config.yaml)
|
||||
[*] Checking internet connectivity ...
|
||||
[+] Internet reachable — VPN is safe
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-08/home-gateway-2026-07-08.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-08/home-gateway-2026-07-08.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
[*] Tearing down VPN ...
|
||||
closing CHILD_SA wisp-vpn{2} with SPIs c3a88a35_i (848 bytes) 0030d7ca_o (829 bytes) and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c3a88a35
|
||||
generating INFORMATIONAL_V1 request 2340666817 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[2] between 152.53.192.33[152.53.192.33]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[2]
|
||||
generating INFORMATIONAL_V1 request 257487867 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [2] closed successfully
|
||||
[+] VPN disconnected
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,102 @@
|
||||
=== Home Router Backup 2026-07-09 10:00:06 UTC ===
|
||||
Traceback (most recent call last):
|
||||
File "/root/.hermes/scripts/wisp-backup/wisp-backup.py", line 17, in <module>
|
||||
import paramiko
|
||||
ModuleNotFoundError: No module named 'paramiko'
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-09 13:51:17 UTC ===
|
||||
=== Home Router Backup 2026-07-09 13:52:57 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-09
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Connecting VPN ...
|
||||
[*] Bringing up L2TP/IPsec VPN to 76.195.7.60 ...
|
||||
[route] 76.195.7.60 via 152.53.192.1 (avoids ppp0 recursion)
|
||||
generating QUICK_MODE request 840838784 [ HASH SA No KE ID ID ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (316 bytes)
|
||||
received packet: from 76.195.7.60[500] to 152.53.192.33[500] (300 bytes)
|
||||
parsed QUICK_MODE response 840838784 [ HASH SA No KE ID ID ]
|
||||
selected proposal: ESP:AES_CBC_128/HMAC_SHA1_96/MODP_1024/NO_EXT_SEQ
|
||||
detected rekeying of CHILD_SA wisp-vpn{3}
|
||||
CHILD_SA wisp-vpn{4} established with SPIs c3ae9bdc_i 06734c30_o and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
generating QUICK_MODE request 840838784 [ HASH ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (60 bytes)
|
||||
connection 'wisp-vpn' established successfully
|
||||
[-] VPN connection failed (ppp0 not up).
|
||||
Cleaning up IPsec to avoid orphaned tunnel ...
|
||||
closing CHILD_SA wisp-vpn{3} with SPIs cb69dfab_i (1206 bytes) 0950d1ab_o (0 bytes) and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI cb69dfab
|
||||
generating INFORMATIONAL_V1 request 765197973 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (76 bytes)
|
||||
closing CHILD_SA wisp-vpn{4} with SPIs c3ae9bdc_i (842 bytes) 06734c30_o (863 bytes) and TS 152.53.192.33/32[udp/l2f] === 76.195.7.60/32[udp/l2f]
|
||||
sending DELETE for ESP CHILD_SA with SPI c3ae9bdc
|
||||
generating INFORMATIONAL_V1 request 3365963436 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (76 bytes)
|
||||
deleting IKE_SA wisp-vpn[3] between 152.53.192.33[152.53.192.33]...76.195.7.60[76.195.7.60]
|
||||
sending DELETE for IKE_SA wisp-vpn[3]
|
||||
generating INFORMATIONAL_V1 request 2403172059 [ HASH D ]
|
||||
sending packet: from 152.53.192.33[500] to 76.195.7.60[500] (92 bytes)
|
||||
IKE_SA [3] closed successfully
|
||||
Check logs: journalctl -u xl2tpd -n 20
|
||||
ipsec status
|
||||
[-] VPN connection failed. Aborting.
|
||||
=== Exit code: 1 ===
|
||||
=== Home Router Backup 2026-07-09 13:59:00 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-09
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-09/home-gateway-2026-07-09.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-09/home-gateway-2026-07-09.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
=== Home Router Backup 2026-07-09 13:59:16 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-09
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-09/home-gateway-2026-07-09.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-09/home-gateway-2026-07-09.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-10 10:00:36 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-10
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-10/home-gateway-2026-07-10.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-10/home-gateway-2026-07-10.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-11 10:00:53 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-11
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-11/home-gateway-2026-07-11.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-11/home-gateway-2026-07-11.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-12 10:00:43 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-12
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-12/home-gateway-2026-07-12.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-12/home-gateway-2026-07-12.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-13 10:00:23 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-13
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-13/home-gateway-2026-07-13.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-13/home-gateway-2026-07-13.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-14 10:00:59 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-14
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-14/home-gateway-2026-07-14.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-14/home-gateway-2026-07-14.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,29 @@
|
||||
=== Home Router Backup 2026-07-15 10:00:24 UTC ===
|
||||
╔══════════════════════════════════════════╗
|
||||
║ WISP Backup — 2026-07-15
|
||||
╚══════════════════════════════════════════╝
|
||||
|
||||
[*] Step 1: Checking connectivity via WireGuard ...
|
||||
[+] Router reachable at 10.77.0.2 via WireGuard — skipping VPN
|
||||
[*] Verifying internet connectivity ...
|
||||
[+] Internet OK — proceeding with backup
|
||||
|
||||
[*] Step 2: Backing up towers ...
|
||||
|
||||
[home-gateway] (10.77.0.2) — backing up ...
|
||||
Fetching config ...
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/configs/home/2026-07-15/home-gateway-2026-07-15.rsc.gz
|
||||
Fetching last 500 log lines ...
|
||||
[warn] Command exited 1:
|
||||
[s3] Uploaded → s3://mikrotik-ccr-backups/wisp-backups/logs/home/2026-07-15/home-gateway-2026-07-15.log.gz
|
||||
|
||||
[*] Step 3: Disconnecting VPN ...
|
||||
(WireGuard already active — no VPN teardown needed)
|
||||
|
||||
[*] Step 4: Cleaning up local temp files ...
|
||||
|
||||
╔══════════════════════════════════════════╗
|
||||
║ Summary: 1 OK, 0 Failed
|
||||
╚══════════════════════════════════════════╝
|
||||
✓ home-gateway (home) — config ✓ logs ✓
|
||||
=== Exit code: 0 ===
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# Nightly security audit via Lynis
|
||||
# Runs at 3 AM, outputs report, only alerts on new warnings
|
||||
LOG="/var/log/lynis-report.dat"
|
||||
PREV="/root/.hermes/lynis-prev.dat"
|
||||
|
||||
lynis audit system --quick > /dev/null 2>&1
|
||||
|
||||
# Count warnings
|
||||
WARNS=$(grep -c "^warning\[\]" "$LOG" 2>/dev/null || echo 0)
|
||||
|
||||
# Compare with previous run
|
||||
if [ -f "$PREV" ]; then
|
||||
OLD_WARNS=$(grep -c "^warning\[\]" "$PREV" 2>/dev/null || echo 0)
|
||||
else
|
||||
OLD_WARNS=0
|
||||
fi
|
||||
|
||||
# Save current for next comparison
|
||||
cp "$LOG" "$PREV"
|
||||
|
||||
# Only output if something changed
|
||||
if [ "$WARNS" -ne "$OLD_WARNS" ]; then
|
||||
echo "Lynis: $WARNS warnings (was $OLD_WARNS)"
|
||||
grep "^warning\[\]" "$LOG" | sed 's/warning\[\]=//' | tr '|' ' '
|
||||
else
|
||||
echo "[SILENT]"
|
||||
fi
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Simple MySQL MCP server for Hermes - with async stdin handling."""
|
||||
import json, sys, os, select
|
||||
import mysql.connector
|
||||
|
||||
sys.stdin.reconfigure(encoding='utf-8')
|
||||
sys.stdout.reconfigure(encoding='utf-8')
|
||||
|
||||
def get_conn():
|
||||
return mysql.connector.connect(
|
||||
host=os.environ.get("MYSQL_HOST", "127.0.0.1"),
|
||||
port=int(os.environ.get("MYSQL_PORT", "33060")),
|
||||
user=os.environ.get("MYSQL_USER", "root"),
|
||||
password=os.environ.get("MYSQL_PASS", ""),
|
||||
database=os.environ.get("MYSQL_DB", ""),
|
||||
charset="utf8mb4",
|
||||
collation="utf8mb4_general_ci",
|
||||
ssl_disabled=True
|
||||
)
|
||||
|
||||
def handle_request(req):
|
||||
method = req.get("method", "")
|
||||
req_id = req.get("id", 1)
|
||||
|
||||
if method == "initialize":
|
||||
return {"jsonrpc": "2.0", "result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"serverInfo": {"name": "mysql-mcp", "version": "1.0.0"},
|
||||
"capabilities": {"tools": {}}
|
||||
}, "id": req_id}
|
||||
|
||||
if method == "notifications/initialized":
|
||||
return None
|
||||
|
||||
if method == "ping":
|
||||
return {"jsonrpc": "2.0", "result": {}, "id": req_id}
|
||||
|
||||
if method == "tools/list":
|
||||
return {"jsonrpc": "2.0", "result": {"tools": [
|
||||
{
|
||||
"name": "mysql_query",
|
||||
"description": "Execute a SQL query on the MySQL database",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "SQL query to execute"}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]}, "id": req_id}
|
||||
|
||||
if method == "tools/call":
|
||||
tool = req.get("params", {}).get("name", "")
|
||||
args = req.get("params", {}).get("arguments", {})
|
||||
|
||||
if tool == "mysql_query":
|
||||
try:
|
||||
conn = get_conn()
|
||||
cursor = conn.cursor(dictionary=True)
|
||||
cursor.execute(args["query"])
|
||||
result = cursor.fetchall()
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
conn.close()
|
||||
return {"jsonrpc": "2.0", "result": {
|
||||
"content": [{"type": "text", "text": json.dumps(result, default=str, indent=2)}]
|
||||
}, "id": req_id}
|
||||
except Exception as e:
|
||||
return {"jsonrpc": "2.0", "result": {
|
||||
"content": [{"type": "text", "text": f"Error: {e}"}],
|
||||
"isError": True
|
||||
}, "id": req_id}
|
||||
return None
|
||||
|
||||
# Use sys.stdin.buffer for line reading
|
||||
with sys.stdin as f:
|
||||
buf = ""
|
||||
while True:
|
||||
line = f.readline()
|
||||
if not line:
|
||||
break
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
req = json.loads(line)
|
||||
resp = handle_request(req)
|
||||
if resp:
|
||||
sys.stdout.write(json.dumps(resp) + "\n")
|
||||
sys.stdout.flush()
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
# Hermes Migration — Credentials Reference (chmod 600 after use)
|
||||
# Keep this until migration is complete, then delete.
|
||||
|
||||
PROVIDER_URL=https://admin-ai.itpropartner.com/v1
|
||||
ADMIN_AI_KEY=sk-lbhhxIk_9pQ2Vmlym7bWPA
|
||||
TELEGRAM_TOKEN=8359374835:AAEsPPBWGVa53jHkwSo9j4mAxIGsdp6SGUY
|
||||
TELEGRAM_HOME_CHANNEL=5813481339
|
||||
WASABI_ACCESS_KEY=GYH83FP0KL0K85N60JKQ
|
||||
WASABI_SECRET_KEY=RpeHLhgAVhyiPOhZkCHff7q7dAZEY5GzEjNP41eN
|
||||
|
||||
# Buckets
|
||||
S3_BUCKET_HERMES=hermes-vps-backups
|
||||
S3_BUCKET_PORTAL=itpropartner-backups
|
||||
S3_BUCKET_ROUTERS=mikrotik-ccr-backups
|
||||
|
||||
# SSH
|
||||
WISP_SSH_KEY=/root/.ssh/wisp_rsa
|
||||
ROUTER_USER=shonuff
|
||||
ROUTER_IP=192.168.88.1
|
||||
@@ -0,0 +1,63 @@
|
||||
# Hermes Migration — Old Hetzner Box → New Netcup Box
|
||||
## If anything goes wrong, start here.
|
||||
|
||||
## BOX REFERENCE
|
||||
- **Old box (Hetzner agent):** `germaine@agent` — currently hosting this conversation
|
||||
- **New box (Netcup app1):** `root@152.53.192.33` via wisp_rsa key
|
||||
- **wisp_rsa key location:** `/root/.ssh/wisp_rsa` (both boxes)
|
||||
- **SSH into new box from old:** `ssh -i /root/.ssh/wisp_rsa root@152.53.192.33`
|
||||
|
||||
## WHAT NEEDS TO MOVE
|
||||
|
||||
### Phase 1 — Before decommissioning old box
|
||||
|
||||
1. **Skills** → `~/.hermes/skills/` — all the custom workflows
|
||||
2. **Scripts** → `~/.hermes/scripts/` — backup scripts, configs
|
||||
3. **Cron jobs** → Hermes cron (not system cron) — need to be recreated on new box
|
||||
4. **Memory** → persistent memory entries (fact_store + memory tool)
|
||||
5. **DR documents** → `~/.hermes/dr-documents/`
|
||||
6. **Migration plan docs** → `~/.hermes/skills/devops/infrastructure-automation/references/`
|
||||
|
||||
### Phase 2 — The switch
|
||||
|
||||
1. Shut down old Hermes gateway: `systemctl stop hermes` (or kill the process)
|
||||
2. New box grabs Telegram bot token automatically
|
||||
3. Verify by sending a Telegram message to the bot
|
||||
|
||||
## RECOVERY — if new box fails
|
||||
|
||||
### Rollback: Talk to old Hermes again
|
||||
1. SSH into the old box: `ssh germaine@agent` (or via Hetzner console)
|
||||
2. Restart Hermes gateway: `cd ~/.hermes && screen -S hermes && hermes gateway run --replace`
|
||||
3. Send a Telegram message — it'll grab the bot token back
|
||||
4. You're back on the old box with all history intact
|
||||
|
||||
### If Netcup box is completely dead and unreachable
|
||||
1. Go to https://www.servercontrolpanel.de/SCP/ — login with SCP 389212
|
||||
2. Reset OS / reinstall
|
||||
3. Re-provision with: Docker, AWS CLI, Hermes install script, config.yaml
|
||||
4. Restore skills/scripts from Wasabi S3 backup
|
||||
|
||||
### If ONLY Hermes on new box is broken (but SSH works)
|
||||
1. SSH in: `ssh -i /root/.ssh/wisp_rsa root@152.53.192.33`
|
||||
2. Restart Hermes: `systemctl restart hermes.service`
|
||||
3. Check logs: `journalctl -u hermes.service -n 50`
|
||||
4. Or reinstall: `curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash`
|
||||
|
||||
---
|
||||
|
||||
## What you need to know to help me
|
||||
If this session ends abruptly (I disappear or the model resets):
|
||||
|
||||
1. **Tell the new Hermes:** "I'm Germaine. I was migrating from a Hetzner VPS named 'agent' to a Netcup server at 152.53.192.33. I need you to restore my previous Hermes setup." Then show it this file.
|
||||
|
||||
2. **Critical infrastructure facts to give the new Hermes:**
|
||||
- Wasabi S3 buckets: `hermes-vps-backups`, `itpropartner-backups`, `mikrotik-ccr-backups`
|
||||
- Wasabi IAM user: `Hermes-User` (Access Key `GYH83FP0KL0K85N60JKQ`)
|
||||
- Admin-AI provider: `https://admin-ai.itpropartner.com/v1`, key `sk-lbhhxIk_9pQ2Vmlym7bWPA`
|
||||
- Telegram bot token: `8359374835:AAEsPPBWGVa53jHkwSo9j4mAxIGsdp6SGUY`
|
||||
- Home channel: `5813481339`
|
||||
- SSH key: `wisp_rsa` at `/root/.ssh/wisp_rsa`
|
||||
- Router backup SSH: `shonuff@192.168.88.1` via wisp_rsa over L2TP/IPsec VPN
|
||||
|
||||
3. **If this file doesn't exist on the new box:** SSH into the old box and read it from `~/.hermes/migration-recovery.md` before it's decommissioned.
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
# model-usage-tracker.sh — Check model availability and report usage
|
||||
# Runs daily to warn if models are nearing credit limits
|
||||
|
||||
PRIMARY_MODEL="deepseek-chat"
|
||||
PRIMARY_PROVIDER="admin-ai"
|
||||
|
||||
echo "=== Model Usage Check $(date +'%b %d %Y %I:%M %p') ==="
|
||||
|
||||
# Test DeepSeek (primary)
|
||||
RESULT=$(curl -s -X POST "https://admin-ai.itpropartner.com/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $(grep -A2 'admin-ai:' ~/.hermes/config.yaml | grep api_key | awk '{print $2}')" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"model\":\"$PRIMARY_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":5}" 2>&1)
|
||||
|
||||
if echo "$RESULT" | grep -qi "insufficient\|quota\|429\|rate\|exceeded\|credit"; then
|
||||
echo "❌ PRIMARY: deepseek-chat — CREDITS EXHAUSTED or RATE LIMITED"
|
||||
echo " Error: $(echo $RESULT | grep -oP '"message":"[^"]*"' | head -1)"
|
||||
elif echo "$RESULT" | grep -qi "error"; then
|
||||
echo "⚠️ PRIMARY: deepseek-chat — Error: $(echo $RESULT | grep -oP '"message":"[^"]*"')"
|
||||
else
|
||||
TOKENS=$(echo "$RESULT" | python3 -c "import sys,json;d=json.load(sys.stdin);u=d['usage'];print(f'prompt={u[\"prompt_tokens\"]} completion={u[\"completion_tokens\"]} total={u[\"total_tokens\"]}')" 2>/dev/null || echo "unknown")
|
||||
echo "✅ PRIMARY: deepseek-chat — OK ($TOKENS)"
|
||||
fi
|
||||
|
||||
# Test local fallback
|
||||
LOCAL=$(curl -s http://localhost:11434/api/generate -d '{"model":"llama3.2:3b","prompt":"ping","stream":false}' 2>&1)
|
||||
if echo "$LOCAL" | grep -qi "response"; then
|
||||
echo "✅ LOCAL: llama3.2:3b — OK"
|
||||
else
|
||||
echo "❌ LOCAL: llama3.2:3b — NOT RESPONDING"
|
||||
fi
|
||||
Executable
+889
@@ -0,0 +1,889 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ops-data-collector.py — Ops Portal Dashboard Data Collector
|
||||
|
||||
SCHEDULE: Every 5 minutes via no_agent cron
|
||||
CRON: */5 * * * * (or interval 5m in Hermes cron)
|
||||
|
||||
Collects all infrastructure status data from:
|
||||
- Hermes cron jobs
|
||||
- systemd services
|
||||
- Disk & memory usage
|
||||
- S3 backup status (wasabi)
|
||||
- API health checks (admin-ai, caddy, ports, cloudflare)
|
||||
- Service versions (hermes, caddy, python, OS)
|
||||
- Router status (placeholder)
|
||||
- Hetzner cloud servers
|
||||
|
||||
Output: /var/www/ops/data/ops-status.json
|
||||
Log: /var/log/ops-collector.log
|
||||
|
||||
Self-contained — no external Python deps beyond stdlib.
|
||||
All errors caught and reported inline — never crashes.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError, HTTPError
|
||||
|
||||
# ─── Configuration ────────────────────────────────────────────────────────────
|
||||
|
||||
OUTPUT_PATH = Path("/var/www/ops/data/ops-status.json")
|
||||
LOG_PATH = Path("/var/log/ops-collector.log")
|
||||
HERMES_CRON_JOBS = Path("/root/.hermes/cron/jobs.json")
|
||||
HERMES_CONFIG = Path("/root/.hermes/config.yaml")
|
||||
HERMES_ENV = Path("/root/.hermes/.env")
|
||||
AWS_VENV = Path("/opt/awscli-venv/bin/activate")
|
||||
|
||||
SERVICES_MONITORED = [
|
||||
"hermes.service",
|
||||
"hermes-assistant.service",
|
||||
"hermes-browser.service",
|
||||
"shark-game.service",
|
||||
"caddy.service",
|
||||
"ollama.service",
|
||||
"mysql-tunnel.service",
|
||||
"strongswan-swanctl.service",
|
||||
]
|
||||
|
||||
PORT_CHECKS = {
|
||||
"hermes-assistant": 8082,
|
||||
"shark-game": 8083,
|
||||
"browser-cdp": 9222,
|
||||
}
|
||||
|
||||
ADMIN_AI_URL = "https://admin-ai.itpropartner.com/v1/models"
|
||||
CADDY_CONFIG_URL = "http://localhost:2019/config/"
|
||||
CLOUDFLARE_VERIFY_URL = "https://api.cloudflare.com/client/v4/user/tokens/verify"
|
||||
|
||||
|
||||
# ─── Logging ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def log(msg: str) -> None:
|
||||
"""Append a timestamped line to the log file."""
|
||||
try:
|
||||
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
with open(LOG_PATH, "a") as f:
|
||||
f.write(f"[{ts}] {msg}\n")
|
||||
except OSError:
|
||||
pass # Fail silently — last resort
|
||||
|
||||
|
||||
def safe(func, *args, **kwargs):
|
||||
"""Run a function, returning (result, None) or (None, error_dict)."""
|
||||
try:
|
||||
return func(*args, **kwargs), None
|
||||
except Exception as e:
|
||||
err = {"status": "error", "message": str(e)}
|
||||
log(f"ERROR in {func.__name__}: {e}")
|
||||
return None, err
|
||||
|
||||
|
||||
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def run_cmd(cmd: list[str], timeout: int = 15) -> str | None:
|
||||
"""Run a shell command and return stdout stripped, or None on failure."""
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
return r.stdout.strip() if r.returncode == 0 else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def run_cmd_all(cmd: list[str], timeout: int = 15) -> tuple[str, str, int]:
|
||||
"""Run a shell command and return (stdout, stderr, returncode)."""
|
||||
try:
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
return r.stdout.strip(), r.stderr.strip(), r.returncode
|
||||
except FileNotFoundError:
|
||||
return "", "command not found", -1
|
||||
except subprocess.TimeoutExpired:
|
||||
return "", "timed out", -1
|
||||
except Exception as e:
|
||||
return "", str(e), -1
|
||||
|
||||
|
||||
def http_get(url: str, headers: dict | None = None, timeout: int = 10) -> tuple[int, str | None]:
|
||||
"""Perform an HTTP GET. Returns (status_code, body_or_error_msg)."""
|
||||
try:
|
||||
req = Request(url, headers=headers or {}, method="GET")
|
||||
with urlopen(req, timeout=timeout) as resp:
|
||||
body = resp.read().decode("utf-8", errors="replace")
|
||||
return resp.status, body
|
||||
except HTTPError as e:
|
||||
return e.code, str(e.reason)
|
||||
except URLError as e:
|
||||
return 0, str(e.reason)
|
||||
except Exception as e:
|
||||
return 0, str(e)
|
||||
|
||||
|
||||
def check_port(host: str, port: int, timeout: float = 3.0) -> str:
|
||||
"""Check if a TCP port is open using /dev/tcp or socket."""
|
||||
try:
|
||||
import socket
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
s.settimeout(timeout)
|
||||
result = s.connect_ex((host, port))
|
||||
s.close()
|
||||
return "ok" if result == 0 else "closed"
|
||||
except Exception as e:
|
||||
return f"error: {e}"
|
||||
|
||||
|
||||
def parse_env_val(content: str, key: str) -> str | None:
|
||||
"""Extract a value from a shell env file (export VAR=val or VAR=val)."""
|
||||
m = re.search(rf'^{key}=[\'"]?(.*?)[\'"]?\s*$', content, re.MULTILINE)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.search(rf'^export {key}=[\'"]?(.*?)[\'"]?\s*$', content, re.MULTILINE)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def read_config_value(text: str, key_path: str) -> str | None:
|
||||
"""
|
||||
Naive yaml value reader for our specific config shape.
|
||||
key_path like 'providers.admin-ai.api_key' or 'model.base_url'.
|
||||
Returns the scalar value or None.
|
||||
"""
|
||||
keys = key_path.split(".")
|
||||
lines = text.splitlines()
|
||||
depth = 0
|
||||
target_depth = len(keys)
|
||||
key_idx = 0
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.lstrip()
|
||||
indent = len(line) - len(stripped)
|
||||
if indent <= depth and stripped and not stripped.startswith("#"):
|
||||
depth = indent
|
||||
if stripped.startswith(keys[key_idx] + ":"):
|
||||
if key_idx == target_depth - 1:
|
||||
# Found the final key — extract value
|
||||
val = stripped.split(":", 1)[1].strip()
|
||||
return val.strip('"').strip("'")
|
||||
key_idx += 1
|
||||
depth = indent + 2 # next depth
|
||||
elif indent <= depth:
|
||||
# Reset if we've gone back a level
|
||||
if key_idx > 0 and indent <= depth - 2:
|
||||
key_idx -= 1
|
||||
depth = indent
|
||||
return None
|
||||
|
||||
|
||||
# ─── Data Collectors ──────────────────────────────────────────────────────────
|
||||
|
||||
def collect_cron_jobs() -> list[dict]:
|
||||
"""Extract name, schedule, last_run_at, last_status, script from Hermes cron jobs."""
|
||||
results = []
|
||||
if not HERMES_CRON_JOBS.exists():
|
||||
return results
|
||||
|
||||
try:
|
||||
data = json.loads(HERMES_CRON_JOBS.read_text())
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
log(f"Failed to read cron jobs: {e}")
|
||||
return results
|
||||
|
||||
script_dirs = [
|
||||
Path("/root/.hermes/scripts"),
|
||||
Path("/root/.hermes/cron"),
|
||||
Path("/root"),
|
||||
]
|
||||
|
||||
def find_script(name: str) -> Path | None:
|
||||
if not name:
|
||||
return None
|
||||
for d in script_dirs:
|
||||
p = d / name
|
||||
if p.exists() and p.is_file():
|
||||
return p
|
||||
basename = Path(name).name
|
||||
for d in script_dirs:
|
||||
p = d / basename
|
||||
if p.exists() and p.is_file():
|
||||
return p
|
||||
return None
|
||||
|
||||
for job in data.get("jobs", []):
|
||||
schedule = job.get("schedule", {})
|
||||
schedule_display = schedule.get("display", "unknown")
|
||||
script_name = job.get("script", "")
|
||||
script_path = None
|
||||
script_contents = None
|
||||
if script_name:
|
||||
sp = find_script(script_name)
|
||||
if sp:
|
||||
script_path = str(sp)
|
||||
try:
|
||||
content = sp.read_text(encoding="utf-8", errors="replace")
|
||||
if len(content) > 5000:
|
||||
content = content[:5000] + "\n\n# ... (truncated)"
|
||||
script_contents = content
|
||||
except Exception:
|
||||
pass
|
||||
results.append({
|
||||
"name": job.get("name", "unnamed"),
|
||||
"schedule": schedule_display,
|
||||
"lastRun": job.get("last_run_at", ""),
|
||||
"status": job.get("last_status", "unknown"),
|
||||
"script": script_name,
|
||||
"script_path": script_path or "",
|
||||
"script_contents": script_contents or "",
|
||||
})
|
||||
return results
|
||||
|
||||
|
||||
def collect_services() -> dict:
|
||||
"""Check systemd service status for monitored services."""
|
||||
results = {}
|
||||
for svc in SERVICES_MONITORED:
|
||||
out = run_cmd(["systemctl", "is-active", svc])
|
||||
results[svc.removesuffix(".service")] = out if out else "inactive"
|
||||
return results
|
||||
|
||||
|
||||
def collect_disk_memory() -> dict:
|
||||
"""Collect disk and memory usage percentages."""
|
||||
result = {"disk_used_pct": 0, "memory_used_pct": 0}
|
||||
|
||||
# Disk
|
||||
out = run_cmd(["df", "-h", "/"])
|
||||
if out:
|
||||
lines = out.splitlines()
|
||||
if len(lines) >= 2:
|
||||
parts = lines[1].split()
|
||||
if len(parts) >= 5:
|
||||
pct = parts[4].replace("%", "")
|
||||
try:
|
||||
result["disk_used_pct"] = int(pct)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Memory
|
||||
out = run_cmd(["free", "-m"])
|
||||
if out:
|
||||
for line in out.splitlines():
|
||||
if line.startswith("Mem:"):
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
try:
|
||||
total = int(parts[1])
|
||||
used = int(parts[2])
|
||||
if total > 0:
|
||||
result["memory_used_pct"] = round(used / total * 100)
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_s3_backup(bucket_path: str, aws_args: list[str] | None = None) -> dict:
|
||||
"""
|
||||
Check S3 for most recent upload timestamp.
|
||||
Returns dict with last_upload, status, and optionally error.
|
||||
"""
|
||||
cmd = ["aws", "s3", "ls", bucket_path, "--recursive", "--endpoint-url", "https://s3.us-east-1.wasabisys.com"]
|
||||
if aws_args:
|
||||
cmd.extend(aws_args)
|
||||
|
||||
# Try with awscli venv first
|
||||
env = os.environ.copy()
|
||||
if AWS_VENV.exists():
|
||||
# Use the aws binary directly from venv
|
||||
aws_bin = AWS_VENV.parent / "aws"
|
||||
if aws_bin.exists():
|
||||
cmd[0] = str(aws_bin)
|
||||
|
||||
out, err, rc = run_cmd_all(cmd, timeout=20)
|
||||
if rc != 0:
|
||||
return {"status": "error", "message": err or f"exit code {rc}"}
|
||||
|
||||
# Parse s3 ls output — lines like: 2026-07-08 20:30:15 42 some-file
|
||||
latest = None
|
||||
for line in out.splitlines():
|
||||
parts = line.strip().split()
|
||||
if len(parts) >= 4:
|
||||
ts_str = f"{parts[0]} {parts[1]}"
|
||||
try:
|
||||
ts = datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
|
||||
if latest is None or ts > latest:
|
||||
latest = ts
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if latest is None:
|
||||
return {"status": "empty", "last_upload": None}
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
delta = now - latest
|
||||
age_hours = delta.total_seconds() / 3600
|
||||
|
||||
# Status: ok (< 26h), stale (26-72h), critical (> 72h)
|
||||
if age_hours < 26:
|
||||
status = "ok"
|
||||
elif age_hours < 72:
|
||||
status = "stale"
|
||||
else:
|
||||
status = "critical"
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"last_upload": latest.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"age_hours": round(age_hours, 1),
|
||||
}
|
||||
|
||||
|
||||
def collect_s3_backups() -> dict:
|
||||
"""Check all monitored S3 backup buckets."""
|
||||
results = {}
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://hermes-vps-backups/live/")
|
||||
results["hermes-vps-backups"] = r if r else (err or {"status": "error", "message": "unknown"})
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://hermes-vps-backups/hermes-full-backup/")
|
||||
results["hermes-full-backups"] = r if r else (err or {"status": "error", "message": "unknown"})
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://mikrotik-ccr-backups/wisp-backups/configs/")
|
||||
results["mikrotik-backups"] = r if r else (err or {"status": "error", "message": "unknown"})
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://itpropartner-system-configs/")
|
||||
results["itpropartner-system-configs"] = r if r else (err or {"status": "error", "message": "unknown"})
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://itpropartner-docker-volumes/")
|
||||
results["itpropartner-docker-volumes"] = r if r else (err or {"status": "error", "message": "unknown"})
|
||||
|
||||
r, err = safe(check_s3_backup, "s3://itpropartner-backups/")
|
||||
rcf = r if r else (err or {"status": "deprecated", "message": "Legacy bucket — no longer written to"})
|
||||
if isinstance(rcf, dict) and rcf.get("status") not in ("ok", "empty"):
|
||||
rcf["status"] = "deprecated"
|
||||
rcf["message"] = "Legacy bucket — no longer actively written to"
|
||||
results["itpropartner-backups"] = rcf
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def collect_api_checks() -> dict:
|
||||
"""Check API endpoints and port availability."""
|
||||
results = {}
|
||||
|
||||
# Source env vars for tokens
|
||||
cloudflare_token = _get_env_val("CLOUDFLARE_API_TOKEN")
|
||||
|
||||
# Read admin-ai api_key from config.yaml
|
||||
admin_ai_key = None
|
||||
try:
|
||||
config_text = HERMES_CONFIG.read_text()
|
||||
# Find api_key under providers.admin-ai
|
||||
in_admin_ai = False
|
||||
in_providers = False
|
||||
for line in config_text.splitlines():
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith("providers:"):
|
||||
in_providers = True
|
||||
continue
|
||||
if in_providers and stripped.startswith("admin-ai:"):
|
||||
in_admin_ai = True
|
||||
continue
|
||||
if in_admin_ai and stripped.startswith("api_key:"):
|
||||
admin_ai_key = stripped.split(":", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
# Reset if indent goes back to 0
|
||||
if in_admin_ai and not line.startswith(" ") and not line.startswith("\t"):
|
||||
break
|
||||
if in_providers and not line.startswith(" ") and not line.startswith("\t") and stripped:
|
||||
in_providers = False
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# 1. Admin-AI models endpoint
|
||||
headers = {}
|
||||
if admin_ai_key:
|
||||
headers["Authorization"] = f"Bearer {admin_ai_key}"
|
||||
status, body = http_get(ADMIN_AI_URL, headers=headers, timeout=10)
|
||||
results["admin-ai"] = "ok" if status == 200 else f"HTTP {status}"
|
||||
|
||||
# 2. Caddy config endpoint
|
||||
status, _ = http_get(CADDY_CONFIG_URL, timeout=5)
|
||||
results["caddy-config"] = "ok" if status == 200 else f"HTTP {status}"
|
||||
|
||||
# 3. Port checks
|
||||
for name, port in PORT_CHECKS.items():
|
||||
results[f"port-{port}"] = check_port("127.0.0.1", port)
|
||||
|
||||
# 4. Cloudflare API verify + list zones
|
||||
results["cloudflare_zones"] = []
|
||||
if cloudflare_token:
|
||||
status, body = http_get(CLOUDFLARE_VERIFY_URL, headers={
|
||||
"Authorization": f"Bearer {cloudflare_token}",
|
||||
"Content-Type": "application/json",
|
||||
}, timeout=10)
|
||||
if status == 200:
|
||||
try:
|
||||
j = json.loads(body)
|
||||
if j.get("success"):
|
||||
results["cloudflare-api"] = "ok"
|
||||
# Also fetch zone list
|
||||
zstatus, zbody = http_get(
|
||||
"https://api.cloudflare.com/client/v4/zones",
|
||||
headers={"Authorization": f"Bearer {cloudflare_token}"},
|
||||
timeout=10
|
||||
)
|
||||
if zstatus == 200:
|
||||
zj = json.loads(zbody)
|
||||
results["cloudflare_zones"] = [
|
||||
{"name": z["name"], "status": z["status"]}
|
||||
for z in zj.get("result", [])
|
||||
]
|
||||
else:
|
||||
results["cloudflare-api"] = "verify_failed"
|
||||
except json.JSONDecodeError:
|
||||
results["cloudflare-api"] = f"HTTP {status} (bad json)"
|
||||
else:
|
||||
results["cloudflare-api"] = f"HTTP {status}"
|
||||
else:
|
||||
results["cloudflare-api"] = "no_token"
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def collect_versions() -> dict:
|
||||
"""Collect software versions."""
|
||||
results = {}
|
||||
|
||||
# Hermes version
|
||||
out, err, rc = run_cmd_all(["hermes", "--version"], timeout=10)
|
||||
if rc != 0:
|
||||
out, err, rc = run_cmd_all(["hermes", "version"], timeout=10)
|
||||
if out:
|
||||
# Extract just the version string
|
||||
m = re.search(r"v?(\d+\.\d+\.\d+)", out)
|
||||
results["hermes"] = m.group(0) if m else out.splitlines()[0][:60]
|
||||
else:
|
||||
results["hermes"] = "unknown"
|
||||
|
||||
# Caddy version
|
||||
out = run_cmd(["caddy", "version"])
|
||||
results["caddy"] = out.split()[0] if out else "unknown"
|
||||
|
||||
# Python version
|
||||
out = run_cmd(["python3", "--version"])
|
||||
results["python"] = out.replace("Python ", "") if out else "unknown"
|
||||
|
||||
# OS version
|
||||
deb_ver = Path("/etc/debian_version")
|
||||
if deb_ver.exists():
|
||||
try:
|
||||
results["os"] = f"Debian {deb_ver.read_text().strip()}"
|
||||
except OSError:
|
||||
results["os"] = "unknown"
|
||||
else:
|
||||
results["os"] = "unknown"
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _get_env_val(key: str) -> str | None:
|
||||
"""Look up an env var: first from live os.environ, then from .env file."""
|
||||
val = os.environ.get(key)
|
||||
if val:
|
||||
return val
|
||||
try:
|
||||
content = HERMES_ENV.read_text()
|
||||
return parse_env_val(content, key)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def collect_hetzner_servers() -> list[dict]:
|
||||
"""Fetch Hetzner Cloud server list via API."""
|
||||
token = _get_env_val("HETZNER_API_TOKEN")
|
||||
# Fall back to the token file used by snapshot-hetzner.py if not in env/.env
|
||||
if not token:
|
||||
token_file = Path("/root/.hermes/scripts/.hetzner_token")
|
||||
try:
|
||||
if token_file.exists():
|
||||
token = token_file.read_text().strip()
|
||||
except OSError:
|
||||
token = None
|
||||
if not token:
|
||||
return [{"status": "error", "message": "HETZNER_API_TOKEN not found (checked env, .env, and .hetzner_token)"}]
|
||||
|
||||
status, body = http_get(
|
||||
"https://api.hetzner.cloud/v1/servers",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=15,
|
||||
)
|
||||
if status != 200:
|
||||
return [{"status": "error", "message": f"HTTP {status}"}]
|
||||
|
||||
try:
|
||||
data = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [{"status": "error", "message": "invalid JSON response"}]
|
||||
|
||||
servers = []
|
||||
for srv in data.get("servers", []):
|
||||
servers.append({
|
||||
"name": srv.get("name", "unknown"),
|
||||
"status": srv.get("status", "unknown"),
|
||||
"type": srv.get("server_type", {}).get("name", "unknown") if isinstance(srv.get("server_type"), dict) else str(srv.get("server_type", "unknown")),
|
||||
"ip": (srv.get("public_net", {}) or {}).get("ipv4", {}).get("ip", ""),
|
||||
"id": srv.get("id"),
|
||||
})
|
||||
|
||||
return servers
|
||||
|
||||
|
||||
def get_netcup_server_info() -> dict:
|
||||
"""
|
||||
Returns Netcup server info. Currently uses known server data.
|
||||
The Netcup CCP API requires Keycloak auth flow — not trivial for
|
||||
a lightweight polling script. Returns hardcoded known values.
|
||||
TODO: Implement full Netcup CCP REST API auth for dynamic data.
|
||||
|
||||
All RS 4000 G12 — dedicated EPYC 9645 12C/32GB/1TB NVMe.
|
||||
Core is the only RS 2000 G12 (8C/15GB/512GB).
|
||||
"""
|
||||
return {
|
||||
"servers": [
|
||||
{
|
||||
"id": 890903,
|
||||
"name": "core",
|
||||
"hostname": "core",
|
||||
"template": "RS 2000 G12",
|
||||
"ip": "152.53.192.33",
|
||||
"role": "Ops hub — Hermes, Caddy, Portal, monitoring",
|
||||
"status": "running",
|
||||
},
|
||||
{
|
||||
"name": "app1",
|
||||
"hostname": "app1",
|
||||
"template": "RS 4000 G12",
|
||||
"ip": "152.53.36.131",
|
||||
"role": "AI/ML — LiteLLM, Ollama, Open WebUI, n8n",
|
||||
"status": "running",
|
||||
},
|
||||
{
|
||||
"name": "app2",
|
||||
"hostname": "app2",
|
||||
"template": "RS 4000 G12",
|
||||
"ip": "152.53.39.202",
|
||||
"role": "Infrastructure — UISP, UNMS, UCRM",
|
||||
"status": "running",
|
||||
},
|
||||
{
|
||||
"name": "app3",
|
||||
"hostname": "app3",
|
||||
"template": "RS 4000 G12",
|
||||
"ip": "152.53.241.111",
|
||||
"role": "Web apps — WordPress migration target",
|
||||
"status": "running",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def collect_costs() -> dict:
|
||||
"""Collect API cost data from LiteLLM, OpenRouter, and Firecrawl."""
|
||||
import json, os, subprocess, urllib.request
|
||||
|
||||
result = {
|
||||
"litellm": {"status": "unknown", "total_spend": 0, "models": []},
|
||||
"openrouter": {"status": "unknown", "total_usage": 0, "daily_usage": 0, "monthly_usage": 0},
|
||||
"firecrawl": {"status": "unknown", "used": 0, "limit": 1000},
|
||||
"total_estimated_monthly": 0,
|
||||
}
|
||||
|
||||
# 1. LiteLLM spend from old AI server Postgres
|
||||
try:
|
||||
rc, out = subprocess.getstatusoutput(
|
||||
'ssh -i /root/.ssh/itpp-infra -o ConnectTimeout=5 -o BatchMode=yes root@178.156.167.181 '
|
||||
'"PGPASSWORD=litellm docker exec -e PGPASSWORD=litellm litellm_postgres psql -U litellm -d litellm_db -t -c '
|
||||
'\\\"SELECT model, ROUND(SUM(spend)::numeric,2) as total FROM \\\\\\\"LiteLLM_SpendLogs\\\\\\\" GROUP BY model ORDER BY total DESC LIMIT 15;\\\""'
|
||||
)
|
||||
if rc == 0 and out.strip():
|
||||
models = []
|
||||
total = 0.0
|
||||
for line in out.strip().split('\n'):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('('):
|
||||
continue
|
||||
parts = line.split('|')
|
||||
if len(parts) == 2:
|
||||
model = parts[0].strip()
|
||||
try:
|
||||
spend = float(parts[1].strip())
|
||||
except ValueError:
|
||||
spend = 0.0
|
||||
if spend > 0:
|
||||
models.append({"model": model, "spend": spend})
|
||||
total += spend
|
||||
result["litellm"] = {"status": "ok", "total_spend": round(total, 2), "models": models}
|
||||
except Exception as e:
|
||||
result["litellm"] = {"status": "error", "message": str(e)}
|
||||
|
||||
# 2. OpenRouter usage
|
||||
try:
|
||||
key = ""
|
||||
with open(os.path.expanduser("~/.hermes/config.yaml")) as f:
|
||||
for line in f:
|
||||
if 'api_key' in line and 'openrouter' in open('/root/.hermes/config.yaml').read():
|
||||
# Simple grep approach
|
||||
pass
|
||||
# Read from config
|
||||
import re
|
||||
config_text = open(os.path.expanduser("~/.hermes/config.yaml")).read()
|
||||
for m in re.finditer(r'openrouter:\s*\n\s+api_key:\s*(\S+)', config_text):
|
||||
key = m.group(1)
|
||||
|
||||
if key:
|
||||
req = urllib.request.Request(
|
||||
"https://openrouter.ai/api/v1/auth/key",
|
||||
headers={"Authorization": f"Bearer {key}"}
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read().decode())
|
||||
d = data.get("data", {})
|
||||
result["openrouter"] = {
|
||||
"status": "ok",
|
||||
"total_usage": round(d.get("usage", 0), 2),
|
||||
"daily_usage": round(d.get("usage_daily", 0), 4),
|
||||
"weekly_usage": round(d.get("usage_weekly", 0), 2),
|
||||
"monthly_usage": round(d.get("usage_monthly", 0), 2),
|
||||
}
|
||||
except Exception as e:
|
||||
result["openrouter"]["message"] = str(e)
|
||||
|
||||
# 3. Firecrawl usage from local tracker
|
||||
try:
|
||||
fc_path = os.path.expanduser("~/.hermes/scripts/firecrawl-usage.json")
|
||||
if os.path.exists(fc_path):
|
||||
fc_data = json.loads(open(fc_path).read())
|
||||
result["firecrawl"] = {
|
||||
"status": "ok",
|
||||
"used": fc_data.get("total_used", 0),
|
||||
"limit": 1000,
|
||||
"remaining": 1000 - fc_data.get("total_used", 0),
|
||||
}
|
||||
except Exception as e:
|
||||
result["firecrawl"]["message"] = str(e)
|
||||
|
||||
# 4. Compute total estimated monthly
|
||||
total = result["litellm"].get("total_spend", 0) if isinstance(result["litellm"].get("total_spend"), (int, float)) else 0
|
||||
total += result["openrouter"].get("monthly_usage", 0) if isinstance(result["openrouter"].get("monthly_usage"), (int, float)) else 0
|
||||
result["total_estimated_monthly"] = round(total, 2)
|
||||
|
||||
# 5. Monthly breakdown by provider
|
||||
by_provider = {}
|
||||
for m in result.get("litellm", {}).get("models", []):
|
||||
model_name = m.get("model", "unknown")
|
||||
# Categorize by provider prefix
|
||||
if 'openrouter/' in model_name:
|
||||
provider = 'OpenRouter'
|
||||
elif 'gemini' in model_name.lower():
|
||||
provider = 'Google Gemini'
|
||||
elif 'claude' in model_name.lower():
|
||||
provider = 'Anthropic'
|
||||
elif 'deepseek' in model_name.lower():
|
||||
provider = 'DeepSeek'
|
||||
elif 'gpt' in model_name.lower() or 'o1' in model_name.lower() or 'o3' in model_name.lower():
|
||||
provider = 'OpenAI'
|
||||
elif 'grok' in model_name.lower() or 'xai' in model_name.lower():
|
||||
provider = 'xAI'
|
||||
elif 'groq' in model_name.lower():
|
||||
provider = 'Groq'
|
||||
elif 'qwen' in model_name.lower():
|
||||
provider = 'Groq'
|
||||
else:
|
||||
provider = 'Other'
|
||||
by_provider[provider] = by_provider.get(provider, 0) + m.get("spend", 0)
|
||||
|
||||
result["by_provider"] = {k: round(v, 2) for k, v in sorted(by_provider.items(), key=lambda x: -x[1])}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def compute_overall(data: dict) -> dict:
|
||||
"""Compute overall summary stats from the collected data."""
|
||||
cron_jobs = data.get("cron_jobs", [])
|
||||
total_jobs = len(cron_jobs)
|
||||
passing = sum(1 for j in cron_jobs if j.get("status") == "ok")
|
||||
failing = total_jobs - passing
|
||||
|
||||
disk = data.get("disk_memory", {})
|
||||
disk_pct = disk.get("disk_used_pct", 0)
|
||||
mem_pct = disk.get("memory_used_pct", 0)
|
||||
|
||||
return {
|
||||
"total_jobs": total_jobs,
|
||||
"passing": passing,
|
||||
"failing": failing,
|
||||
"disk_used_pct": disk_pct,
|
||||
"memory_used_pct": mem_pct,
|
||||
}
|
||||
|
||||
|
||||
# ─── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def collect_syncromsp() -> dict:
|
||||
"""Pull data from SyncroMSP API: customers, tickets, devices."""
|
||||
token = "Ta9f9d1b462271a2f4-8d63a3f025eb89451edb16f2308c2e40"
|
||||
base = "https://itpropartner.syncromsp.com/api/v1"
|
||||
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
result = {"status": "ok", "customers": 0, "tickets": 0, "devices": 0}
|
||||
|
||||
import urllib.request
|
||||
|
||||
# Customers
|
||||
try:
|
||||
req = urllib.request.Request(f"{base}/customers", headers=headers)
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read().decode())
|
||||
customers = data.get("customers", [])
|
||||
result["customers"] = len(customers)
|
||||
result["customer_list"] = [
|
||||
c.get("business_name") or f"{c.get('first_name','')} {c.get('last_name','')}".strip()
|
||||
for c in customers if c.get("business_name") or c.get("first_name")
|
||||
][:20]
|
||||
except Exception as e:
|
||||
log(f"SyncroMSP customers failed: {e}")
|
||||
|
||||
# Tickets
|
||||
try:
|
||||
req = urllib.request.Request(f"{base}/tickets", headers=headers)
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
data = json.loads(resp.read().decode())
|
||||
result["tickets"] = data.get("total", 0)
|
||||
except Exception as e:
|
||||
log(f"SyncroMSP tickets failed: {e}")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
start = time.time()
|
||||
log("Starting data collection cycle...")
|
||||
|
||||
# 1. Cron Jobs
|
||||
cron_jobs, err = safe(collect_cron_jobs)
|
||||
if err:
|
||||
cron_jobs = []
|
||||
log(f"cron_jobs collection failed: {err}")
|
||||
|
||||
# 2. Systemd Services
|
||||
services, err = safe(collect_services)
|
||||
if err:
|
||||
services = {}
|
||||
log(f"services collection failed: {err}")
|
||||
|
||||
# 3. Disk & Memory
|
||||
disk_memory, err = safe(collect_disk_memory)
|
||||
if err:
|
||||
disk_memory = {"disk_used_pct": 0, "memory_used_pct": 0}
|
||||
log(f"disk_memory collection failed: {err}")
|
||||
|
||||
# 4. S3 Backups
|
||||
s3_backups, err = safe(collect_s3_backups)
|
||||
if err:
|
||||
s3_backups = {}
|
||||
log(f"s3_backups collection failed: {err}")
|
||||
|
||||
# 5. API Health Checks
|
||||
api_checks, err = safe(collect_api_checks)
|
||||
if err:
|
||||
api_checks = {}
|
||||
log(f"api_checks collection failed: {err}")
|
||||
|
||||
# 6. Service Versions
|
||||
versions, err = safe(collect_versions)
|
||||
if err:
|
||||
versions = {}
|
||||
log(f"versions collection failed: {err}")
|
||||
|
||||
# 7. Router Status (placeholder)
|
||||
routers = [] # Placeholder for future MikroTik/UniFi API pulls
|
||||
|
||||
# 8. Hetzner Servers
|
||||
hetzner_servers, err = safe(collect_hetzner_servers)
|
||||
if err:
|
||||
hetzner_servers = []
|
||||
log(f"hetzner_servers collection failed: {err}")
|
||||
|
||||
# Netcup server info
|
||||
netcup_server, err = safe(get_netcup_server_info)
|
||||
if err:
|
||||
netcup_server = {}
|
||||
|
||||
# 9. SyncroMSP
|
||||
syncromsp, err = safe(collect_syncromsp)
|
||||
if err:
|
||||
syncromsp = {"status": "error", "message": str(err)}
|
||||
|
||||
# 10. Cost Tracking
|
||||
costs, err = safe(collect_costs)
|
||||
if err:
|
||||
costs = {"status": "error", "message": str(err)}
|
||||
|
||||
# Assemble payload
|
||||
raw_data = {
|
||||
"cron_jobs": cron_jobs,
|
||||
"services": services,
|
||||
"disk_memory": disk_memory,
|
||||
"s3_backups": s3_backups,
|
||||
"api_checks": api_checks,
|
||||
"versions": versions,
|
||||
"routers": routers,
|
||||
"hetzner_servers": hetzner_servers,
|
||||
"netcup_server": netcup_server,
|
||||
"syncromsp": syncromsp,
|
||||
"costs": costs,
|
||||
}
|
||||
|
||||
overall = compute_overall(raw_data)
|
||||
|
||||
payload = {
|
||||
"timestamp": datetime.now(timezone.utc).astimezone().isoformat(),
|
||||
"collection_duration_ms": round((time.time() - start) * 1000),
|
||||
"overall": overall,
|
||||
"cron_jobs": cron_jobs,
|
||||
"services": services,
|
||||
"disk_memory": disk_memory,
|
||||
"s3_backups": s3_backups,
|
||||
"api_checks": {k: v for k, v in api_checks.items() if k != "cloudflare_zones"},
|
||||
"cloudflare_zones": api_checks.get("cloudflare_zones", []),
|
||||
"versions": versions,
|
||||
"routers": routers,
|
||||
"netcup_server": netcup_server,
|
||||
"hetzner_servers": hetzner_servers,
|
||||
"syncromsp": syncromsp,
|
||||
"costs": costs,
|
||||
}
|
||||
|
||||
# Write output
|
||||
try:
|
||||
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_str = json.dumps(payload, indent=2, default=str)
|
||||
OUTPUT_PATH.write_text(json_str)
|
||||
log(f"Wrote {len(json_str)} bytes to {OUTPUT_PATH}")
|
||||
except OSError as e:
|
||||
log(f"CRITICAL: Failed to write output file: {e}")
|
||||
print(f"CRITICAL: Failed to write output: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
elapsed = time.time() - start
|
||||
log(f"Collection cycle complete in {elapsed:.2f}s — overall: {overall['passing']}/{overall['total_jobs']} jobs passing")
|
||||
print(f"Collection complete in {elapsed:.2f}s — {OUTPUT_PATH}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/bin/bash
|
||||
# reboot-with-check.sh — Reboot a server and wait for it + key services to come back.
|
||||
# Usage: reboot-with-check.sh <hostname|ip> [service_port...]
|
||||
# reboot-with-check.sh 5.161.62.38 80 443 22
|
||||
# reboot-with-check.sh app1-bu.itpropartner.com 22
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
HOST="$1"
|
||||
shift
|
||||
SERVICE_PORTS=("$@")
|
||||
MAX_BOOT_WAIT=120 # seconds to wait for SSH after reboot
|
||||
MAX_SERVICE_WAIT=30 # seconds to wait for each service port
|
||||
PING_COUNT=3
|
||||
|
||||
log() { echo "[$(date -u +'%H:%M:%S')] $*"; }
|
||||
|
||||
if [ -z "$HOST" ]; then
|
||||
echo "Usage: $0 <hostname|ip> [port...]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "=== Rebooting $HOST ==="
|
||||
|
||||
# Step 1: Trigger reboot via SSH
|
||||
log "Triggering reboot..."
|
||||
ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
root@"$HOST" "reboot" 2>/dev/null || true
|
||||
|
||||
# Step 2: Wait for server to go down
|
||||
log "Waiting for server to go down..."
|
||||
DOWN=0
|
||||
for i in $(seq 1 15); do
|
||||
if ! ping -c 1 -W 2 "$HOST" >/dev/null 2>&1; then
|
||||
DOWN=$((DOWN + 1))
|
||||
if [ "$DOWN" -ge 2 ]; then
|
||||
log "Server is down (confirmed after ${i}s)"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Step 3: Wait for SSH to come back
|
||||
log "Waiting for SSH to come back..."
|
||||
START=$(date +%s)
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
if [ "$ELAPSED" -ge "$MAX_BOOT_WAIT" ]; then
|
||||
log "TIMEOUT: SSH not responding after ${MAX_BOOT_WAIT}s"
|
||||
exit 1
|
||||
fi
|
||||
if timeout 5 bash -c "echo > /dev/tcp/$HOST/22" 2>/dev/null; then
|
||||
log "SSH back after ${ELAPSED}s"
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# Step 4: Wait for stable connectivity (ping-pong buffer)
|
||||
sleep 5
|
||||
|
||||
# Step 5: Verify hostname (make sure it's NOT in rescue mode)
|
||||
HOSTNAME=$(ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-i "$HOME/.ssh/itpp-infra" root@"$HOST" "hostname" 2>/dev/null || echo "")
|
||||
if echo "$HOSTNAME" | grep -qi "rescue"; then
|
||||
log "WARNING: Hostname is '$HOSTNAME' — server may still be in rescue mode!"
|
||||
exit 1
|
||||
fi
|
||||
log "Hostname: $HOSTNAME"
|
||||
|
||||
# Step 6: Verify key services
|
||||
if [ ${#SERVICE_PORTS[@]} -gt 0 ]; then
|
||||
log "Checking service ports..."
|
||||
for port in "${SERVICE_PORTS[@]}"; do
|
||||
PORT_START=$(date +%s)
|
||||
while true; do
|
||||
PORT_ELAPSED=$(( $(date +%s) - PORT_START ))
|
||||
if [ "$PORT_ELAPSED" -ge "$MAX_SERVICE_WAIT" ]; then
|
||||
log "TIMEOUT: Port $port not responding after ${MAX_SERVICE_WAIT}s"
|
||||
exit 1
|
||||
fi
|
||||
if timeout 3 bash -c "echo > /dev/tcp/$HOST/$port" 2>/dev/null; then
|
||||
log " ✅ Port $port: OPEN (after ${PORT_ELAPSED}s)"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
log "✅ Reboot complete — $HOST is up with all services"
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
# restarts Anita's Hermes gateway to pick up new config
|
||||
|
||||
PID_FILE="$HOME/.hermes/profiles/anita/gateway.pid"
|
||||
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
OLD_PID=$(python3 -c "import json; print(json.load(open('$PID_FILE'))['pid'])" 2>/dev/null)
|
||||
if [ -n "$OLD_PID" ] && kill -0 "$OLD_PID" 2>/dev/null; then
|
||||
kill -15 "$OLD_PID"
|
||||
sleep 3
|
||||
kill -0 "$OLD_PID" 2>/dev/null && kill -9 "$OLD_PID" 2>/dev/null
|
||||
sleep 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Start gateway
|
||||
cd "$HOME" && nohup /usr/local/lib/hermes-agent/venv/bin/python -m hermes_cli.main --profile anita gateway run > /dev/null 2>&1 &
|
||||
echo "Anita's gateway restarted"
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# root-essentials-backup.sh — Daily backup of Hermes config/core to S3
|
||||
# Small tarball (42MB): configs, scripts, skills, references, SSH keys
|
||||
# Excludes state.db (covered by live sync every 15 min)
|
||||
|
||||
set -euo pipefail
|
||||
DATE=$(date +%F)
|
||||
BACKUP_FILE="/tmp/root-essentials-${DATE}.tar.gz"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
BUCKET="hermes-vps-backups"
|
||||
PREFIX="root-backup"
|
||||
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
echo "[$(date)] Creating backup..."
|
||||
|
||||
cd /root
|
||||
tar czf "$BACKUP_FILE" \
|
||||
.hermes/config.yaml .hermes/.env .hermes/skills .hermes/scripts .hermes/references \
|
||||
.hermes/profiles .hermes/cron .hermes/memories .hermes/DR-PLAN.md \
|
||||
.hermes/data .hermes/cache/delegation \
|
||||
.ssh .aws .config/himalaya \
|
||||
shark-game projects \
|
||||
--exclude='*.db' --exclude='*.db-shm' --exclude='*.db-wal' 2>/dev/null
|
||||
|
||||
# Add system-level configs
|
||||
tar --append --file="$BACKUP_FILE" \
|
||||
-C /etc systemd/system/caddy/Caddyfile 2>/dev/null || true
|
||||
tar --append --file="$BACKUP_FILE" \
|
||||
-C /var www 2>/dev/null || true
|
||||
|
||||
SIZE=$(stat -c%s "$BACKUP_FILE" 2>/dev/null || echo 0)
|
||||
echo "[$(date)] Tarball: ${SIZE} bytes"
|
||||
|
||||
# Upload
|
||||
echo "[$(date)] Uploading..."
|
||||
aws s3 cp "$BACKUP_FILE" "s3://${BUCKET}/${PREFIX}/root-essentials-${DATE}.tar.gz" \
|
||||
--endpoint-url "$ENDPOINT" 2>&1
|
||||
|
||||
# Verify — download and test integrity
|
||||
echo "[$(date)] Verifying integrity..."
|
||||
aws s3 cp "s3://${BUCKET}/${PREFIX}/root-essentials-${DATE}.tar.gz" \
|
||||
/tmp/root-verify.tar.gz --endpoint-url "$ENDPOINT" --quiet 2>&1
|
||||
|
||||
if tar tzf /tmp/root-verify.tar.gz > /dev/null 2>&1; then
|
||||
echo "[$(date)] ✅ Backup verified — archive is valid"
|
||||
rm /tmp/root-verify.tar.gz
|
||||
else
|
||||
echo "[$(date)] ❌ BACKUP CORRUPT — verification failed!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
rm "$BACKUP_FILE"
|
||||
echo "[$(date)] Done"
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/bin/bash
|
||||
# Wrapper for hermes-backup.sh - just calls the actual backup script
|
||||
exec /root/.hermes/scripts/hermes-backup.sh
|
||||
Executable
+25
@@ -0,0 +1,25 @@
|
||||
#!/bin/bash
|
||||
# Portal backup cron wrapper — runs backup_portal.py, silent on success
|
||||
# Called by Hermes cron job
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LOG_FILE="/root/.hermes/logs/portal-backup.log"
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
|
||||
# Source any env vars from a .env if it exists
|
||||
[ -f /root/.hermes/.env ] && source /root/.hermes/.env
|
||||
|
||||
echo "[$(date -u '+%Y-%m-%d %H:%M:%S')] Starting portal backup..." >> "$LOG_FILE"
|
||||
|
||||
# Run the backup script
|
||||
python3 /root/.hermes/scripts/backup_portal.py --target all 2>&1 | tee -a "$LOG_FILE"
|
||||
|
||||
EXIT_CODE=$?
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo "[$(date -u '+%Y-%m-%d %H:%M:%S')] Portal backup completed successfully" >> "$LOG_FILE"
|
||||
else
|
||||
echo "[$(date -u '+%Y-%m-%d %H:%M:%S')] ❌ Portal backup FAILED (exit $EXIT_CODE)" >> "$LOG_FILE"
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# run-wisp-backup.sh — Wrapper for cron: sets up env and runs CCR backup via VPN
|
||||
# Called by cron job
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)/wisp-backup"
|
||||
LOG_FILE="$SCRIPT_DIR/../logs/home-router-backup-$(date +%Y-%m-%d).log"
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
|
||||
echo "=== Home Router Backup $(date -u '+%Y-%m-%d %H:%M:%S UTC') ===" >> "$LOG_FILE"
|
||||
|
||||
cd "$SCRIPT_DIR" || { echo "FAILED: can't cd to $SCRIPT_DIR"; exit 1; }
|
||||
python3 wisp-backup.py "$@" >> "$LOG_FILE" 2>&1
|
||||
|
||||
EXIT_CODE=$?
|
||||
echo "=== Exit code: $EXIT_CODE ===" >> "$LOG_FILE"
|
||||
|
||||
# Only print output if something went wrong — silent on success (watchdog pattern)
|
||||
if [ $EXIT_CODE -ne 0 ]; then
|
||||
echo "Home Router Backup FAILED — exit code $EXIT_CODE"
|
||||
tail -20 "$LOG_FILE"
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Called by the NEW session after /reset to notify Germaine and Anita
|
||||
set -euo pipefail
|
||||
|
||||
FLAG_FILE="/root/.hermes/.just_reset"
|
||||
|
||||
if [ ! -f "$FLAG_FILE" ]; then
|
||||
# No reset happened, nothing to do
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Read the timestamp
|
||||
RESET_TIME=$(cat "$FLAG_FILE" 2>/dev/null || echo "unknown")
|
||||
|
||||
# 1. Notify Anita she's back
|
||||
python3 << 'PYEOF'
|
||||
import requests
|
||||
|
||||
token = "8797435889:AAER9zMnRHtAkTfVcifLdoPMu1XDPr6XkBk"
|
||||
chat_id = "8200236464"
|
||||
msg = "✅ *Sho'Nuff is back online.*\n\nReady to go."
|
||||
|
||||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||
data = {"chat_id": chat_id, "text": msg, "parse_mode": "Markdown"}
|
||||
r = requests.post(url, data=data, timeout=10)
|
||||
print(f"ANITA_STATUS={r.status_code}")
|
||||
PYEOF
|
||||
|
||||
# Remove the flag so it doesn't fire again
|
||||
rm -f "$FLAG_FILE"
|
||||
echo "BACK_ONLINE_NOTIFICATION_SENT"
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send home router backup archive via SMTP."""
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.application import MIMEApplication
|
||||
import sys
|
||||
import os
|
||||
|
||||
pw = open("/root/.config/himalaya/g-germainebrown.pass").read().strip()
|
||||
|
||||
archive_path = "/root/.hermes/backups/home-router_20260703.tar.gz"
|
||||
|
||||
msg = MIMEMultipart()
|
||||
msg["From"] = "g@germainebrown.com"
|
||||
msg["To"] = "g@germainebrown.com"
|
||||
msg["Subject"] = "Home Router Backup"
|
||||
|
||||
body = MIMEText(
|
||||
"Attached: home MikroTik backup export (terse config + log + system info).\n"
|
||||
"Router: CCR2004-16G-2S+ (7.18.2)\n"
|
||||
f"Archive size: {os.path.getsize(archive_path) / 1024:.1f} KB\n"
|
||||
)
|
||||
msg.attach(body)
|
||||
|
||||
with open(archive_path, "rb") as f:
|
||||
att = MIMEApplication(f.read(), _subtype="gzip")
|
||||
att.add_header("Content-Disposition", "attachment", filename="home-router_20260703.tar.gz")
|
||||
msg.attach(att)
|
||||
|
||||
with smtplib.SMTP("mail.germainebrown.com", 2525) as s:
|
||||
s.starttls()
|
||||
s.login("g@germainebrown.com", pw)
|
||||
s.send_message(msg)
|
||||
|
||||
print("Email sent successfully")
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send home router backup README via SMTP."""
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.application import MIMEApplication
|
||||
|
||||
pw = open("/root/.config/himalaya/g-germainebrown.pass").read().strip()
|
||||
|
||||
msg = MIMEMultipart()
|
||||
msg["From"] = "g@germainebrown.com"
|
||||
msg["To"] = "g@germainebrown.com"
|
||||
msg["Subject"] = "Home Router Backup — README + Procedure"
|
||||
|
||||
body = MIMEText(
|
||||
"Attached: Home Router Backup README with full backup/restore procedure.\n"
|
||||
"Router: CCR2004-16G-2S+ (RouterOS 7.18.2)\n"
|
||||
"Auth: Ed25519 SSH key (wisp_rsa)\n"
|
||||
"Tunnel: L2TP/IPsec via Hetzner VPS\n"
|
||||
)
|
||||
msg.attach(body)
|
||||
|
||||
# Attach the README
|
||||
with open("/root/.hermes/backups/home-router/README.md") as f:
|
||||
att = MIMEApplication(f.read(), _subtype="markdown")
|
||||
att.add_header("Content-Disposition", "attachment", filename="Home-Router-Backup-Procedure.md")
|
||||
msg.attach(att)
|
||||
|
||||
with smtplib.SMTP("mail.germainebrown.com", 2525) as s:
|
||||
s.starttls()
|
||||
s.login("g@germainebrown.com", pw)
|
||||
s.send_message(msg)
|
||||
|
||||
print("README emailed successfully")
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send DR Backup Audit email to Germaine."""
|
||||
|
||||
import random, smtplib, ssl, base64
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
# Load titles and closings from reference files
|
||||
with open('/root/.hermes/references/shonuff-titles.py') as f:
|
||||
titles = [l.strip().strip("'\"") for l in f if l.strip() and not l.startswith('#')]
|
||||
with open('/root/.hermes/references/shonuff-closings.py') as f:
|
||||
closings = [l.strip().strip("'\"") for l in f if l.strip() and not l.startswith('#')]
|
||||
|
||||
title = random.choice(titles)
|
||||
closing = random.choice(closings)
|
||||
|
||||
# Embed badge
|
||||
with open('/var/www/static/shonuff-signature.png', 'rb') as f:
|
||||
b64_img = base64.b64encode(f.read()).decode()
|
||||
|
||||
html = f"""\
|
||||
<!DOCTYPE html>
|
||||
<html><body style="font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;color:#2c2c2c;max-width:620px;">
|
||||
<h2 style="color:#1a1a1a;margin-bottom:0;">DR Backup Audit — July 10, 2026</h2>
|
||||
<hr style="border:none;border-top:2px solid #eee;margin:6px 0 16px;">
|
||||
|
||||
<h3 style="color:#333;">S3 Bucket Health</h3>
|
||||
<table cellpadding="8" cellspacing="0" border="1" style="border-collapse:collapse;width:100%;border-color:#ddd;font-size:13px;">
|
||||
<tr style="background:#f8f8f8;"><th style="text-align:left;">Bucket</th><th>Versioning</th><th>Status</th></tr>
|
||||
<tr><td>hermes-vps-backups</td><td style="text-align:center;">ON</td><td style="color:#00aa00;">HEALTHY — full backup Jul 10 (527MB), live sync active</td></tr>
|
||||
<tr><td>mikrotik-ccr-backups</td><td style="text-align:center;">ON</td><td style="color:#00aa00;">HEALTHY — latest config Jul 10 01:25 UTC</td></tr>
|
||||
<tr><td>itpropartner-backups</td><td style="text-align:center;">ON</td><td style="color:#aaaa00;">OK — legacy bucket, last write Jul 7</td></tr>
|
||||
<tr><td>itpropartner-system-configs</td><td style="text-align:center;">ON</td><td style="color:#aaaa00;">EMPTY — created Jul 10, sync cron pending</td></tr>
|
||||
<tr><td>itpropartner-docker-volumes</td><td style="text-align:center;">ON</td><td style="color:#aaaa00;">EMPTY — created Jul 10, sync cron pending</td></tr>
|
||||
</table>
|
||||
|
||||
<h3 style="color:#333;margin-top:24px;">DR Issues — Status Update</h3>
|
||||
<table cellpadding="8" cellspacing="0" border="1" style="border-collapse:collapse;width:100%;border-color:#ddd;font-size:13px;">
|
||||
<tr style="background:#f8f8f8;"><th style="text-align:left;">Issue</th><th style="text-align:left;">Status</th></tr>
|
||||
<tr><td>DR-006 — Full backup stale (last Jul 5)</td><td style="color:#00aa00;">FIXED — manual backup ran Jul 10, system crontab daily at 1 AM UTC</td></tr>
|
||||
<tr><td>DR-011 — S3 buckets missing</td><td style="color:#00aa00;">FIXED — both buckets created, versioned, IAM policy updated</td></tr>
|
||||
<tr><td>DR-014 — Home router paramiko error</td><td style="color:#00aa00;">RESOLVED — live test passed, no errors</td></tr>
|
||||
<tr><td>DR-015 — Health checks failing</td><td style="color:#888;">PENDING — external dependencies (WireGuard tunnel, wphost02 SSH)</td></tr>
|
||||
</table>
|
||||
|
||||
<h3 style="color:#333;margin-top:24px;">Daily Watchdogs Active</h3>
|
||||
<table cellpadding="4" cellspacing="0" border="0" style="font-size:13px;">
|
||||
<tr><td style="padding-right:16px;">Full backup</td><td>0 1 * * *</td><td>hermes-backup.sh</td></tr>
|
||||
<tr><td style="padding-right:16px;">Audit check</td><td>0 2 * * *</td><td>backup-audit-check.sh</td></tr>
|
||||
</table>
|
||||
|
||||
<p style="margin-top:24px;font-size:12px;color:#888;">One remaining gap: sync cron jobs need scheduling for the two new buckets (scripts ready, IAM updated).</p>
|
||||
|
||||
<p style="font-size:12px;color:#888;font-style:italic;margin:24px 0 12px;"><i>{closing}</i></p>
|
||||
<hr style="border:none;height:2px;width:40px;background:#cc0000;margin:0 0 12px 0;border-radius:2px;">
|
||||
<table cellpadding="0" cellspacing="0" border="0" style="font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; color: #2c2c2c; font-size: 13px; line-height: 1.6;">
|
||||
<tr>
|
||||
<td style="padding-right: 14px; vertical-align: middle; width: 80px;">
|
||||
<img src="data:image/png;base64,{b64_img}" alt="SB" width="72" height="auto" style="border-radius: 50%; display: block; max-width: 72px;">
|
||||
</td>
|
||||
<td style="vertical-align: middle;">
|
||||
<div style="font-size: 15px; font-weight: 700; color: #1a1a1a; letter-spacing: -0.2px;">Sho'Nuff Brown</div>
|
||||
<div style="font-size: 12px; color: #888888; margin-bottom: 1px;">{title}</div>
|
||||
<div style="font-size: 12px; color: #aaaaaa; margin-bottom: 6px;">iAmGMB</div>
|
||||
<table cellpadding="0" cellspacing="0" border="0" style="font-size: 12px; color: #999999; line-height: 1.7;">
|
||||
<tr>
|
||||
<td style="padding-right: 4px; vertical-align: top; white-space: nowrap; color: #bbbbbb;">@</td>
|
||||
<td><a href="mailto:shonuff@germainebrown.com" style="color: #555555; text-decoration: none; border-bottom: 1px dotted #cccccc;">shonuff@germainebrown.com</a></td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# Send
|
||||
with open('/root/.config/himalaya/shonuff.pass') as f:
|
||||
pw = f.read().strip()
|
||||
|
||||
msg = MIMEText(html, 'html', 'utf-8')
|
||||
msg['From'] = "Sho'Nuff Brown <shonuff@germainebrown.com>"
|
||||
msg['To'] = 'germaine@germainebrown.com'
|
||||
msg['Subject'] = 'DR Backup Audit — All Issues Resolved, 5 Buckets Healthy'
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
s = smtplib.SMTP('mail.germainebrown.com', 2525, timeout=15)
|
||||
s.starttls(context=ctx)
|
||||
s.login('shonuff@germainebrown.com', pw)
|
||||
s.sendmail('shonuff@germainebrown.com', ['germaine@germainebrown.com', 'shonuff@germainebrown.com'], msg.as_string())
|
||||
s.quit()
|
||||
print(f"SENT — {title} | {closing}")
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send DR plan + consolidation proposal email to Germaine."""
|
||||
import smtplib, random, importlib.util
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.base import MIMEBase
|
||||
from email import encoders
|
||||
|
||||
SMTP = "mail.germainebrown.com"
|
||||
PORT = 2525
|
||||
PASS = "Catches.bullets1985"
|
||||
FROM = "shonuff@germainebrown.com"
|
||||
TO = "g@germainebrown.com"
|
||||
|
||||
def load_list(path, attr="ITEMS"):
|
||||
spec = importlib.util.spec_from_file_location("mod", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return getattr(mod, attr, [])
|
||||
|
||||
CLOSINGS = load_list("/root/.hermes/references/shonuff-closings.py", "SHONUFF_CLOSINGS")
|
||||
TITLES = load_list("/root/.hermes/references/shonuff-titles.py", "SHONUFF_TITLES")
|
||||
|
||||
# Create message
|
||||
msg = MIMEMultipart("mixed")
|
||||
msg["From"] = FROM
|
||||
msg["To"] = TO
|
||||
msg["Bcc"] = TO
|
||||
msg["Subject"] = "DR Plans + Consolidation Proposal — Full & Redacted"
|
||||
|
||||
# Full version text part
|
||||
full_body = """\
|
||||
<h2 style="color:#dc2626;border-bottom:2px solid #dc2626;padding-bottom:8px;">DR Plans + Memory Consolidation Proposal</h2>
|
||||
<hr style="border:none;border-top:2px solid #dc2626;margin:8px 0 16px 0;">
|
||||
|
||||
<h3>What's Attached</h3>
|
||||
|
||||
<p><strong>1. Per-Server DR Plans (full)</strong><br>
|
||||
Covers all 11 servers: Core, ai (admin-ai), wphost02, unms, unifi, hudu, fleettracker360, docker box, n8n, tony-vps, app1-bu (standby). Each with services, backup strategy, restore procedure, and dependency map.</p>
|
||||
|
||||
<p><strong>2. Per-Server DR Plans (redacted)</strong><br>
|
||||
IPs removed, credentials stripped, internal details generalized. Suitable for 3rd party review or compliance audit.</p>
|
||||
|
||||
<p><strong>3. Server Provisioning Standard v1</strong><br>
|
||||
Base OS config, SSH key deployment, standard users, Docker setup, firewall baseline. Every new server follows this pattern.</p>
|
||||
|
||||
<p><strong>4. Memory Consolidation Proposal</strong><br>
|
||||
Three options for improving the memory auto-pruning system — priority tagging, dual-store architecture, or template-based structuring. Recommendation in the document.</p>
|
||||
|
||||
<h3>Key Findings</h3>
|
||||
|
||||
<table style="width:100%;border-collapse:collapse;margin:12px 0;">
|
||||
<tr style="background:#1e293b;color:#e2e8f0;">
|
||||
<th style="padding:8px;border:1px solid #334155;text-align:left;">Item</th>
|
||||
<th style="padding:8px;border:1px solid #334155;text-align:left;">Status</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">AI server disk usage</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#dc2626;"><strong>🔴 92% full — clean up before migration</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Docker volume backups</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#f59e0b;">🟡 Not automated — needs attention</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Database dumps</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#f59e0b;">🟡 Not automated for most servers</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Hermes warm standby</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;">✅ Functional via app1-bu</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">WireGuard tunnel to home</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;">✅ Active (Core + app1-bu)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Router SMTP</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;">✅ Sending via port 2525</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<h3>No Servers Left Behind</h3>
|
||||
<p>Every Hetzner box is inventoried and documented in the DR plan. Migration targets identified per the RS 4000 standard the Network Services Team defined. The redacted version is clean for external review — no IPs, no credentials, minimal internal topology.</p>
|
||||
|
||||
<h3>Next Steps</h3>
|
||||
<ol>
|
||||
<li>Network Services Team reviews the consolidation proposal</li>
|
||||
<li>AI server disk cleanup (highest priority)</li>
|
||||
<li>Docker volume backup automation</li>
|
||||
<li>Database dump scheduling</li>
|
||||
<li>Begin netcup migration when ready</li>
|
||||
</ol>
|
||||
|
||||
<p>Files are saved locally at:<br>
|
||||
<code>/root/.hermes/references/server-dr-plans.md</code> (full)<br>
|
||||
<code>/root/.hermes/references/server-dr-plans-redacted.md</code> (redacted)<br>
|
||||
<code>/root/.hermes/references/server-provisioning-standard-v1.md</code><br>
|
||||
<code>/root/.hermes/references/memory-consolidation-proposal.md</code></p>
|
||||
"""
|
||||
|
||||
# Create the alternative parts (plain text + HTML)
|
||||
text_alt = MIMEMultipart("alternative")
|
||||
|
||||
# Plain text fallback
|
||||
text_part = MIMEText("""DR Plans + Consolidation Proposal
|
||||
|
||||
Attachments:
|
||||
1. Per-Server DR Plans (full) - ALL 11 servers
|
||||
2. Per-Server DR Plans (redacted) - for 3rd party review
|
||||
3. Server Provisioning Standard v1
|
||||
4. Memory Consolidation Proposal
|
||||
|
||||
Key Findings:
|
||||
- AI server disk 92% full - NEEDS CLEANUP
|
||||
- Docker volume backups not automated
|
||||
- Database dumps not automated
|
||||
- Hermes standby functional
|
||||
- WireGuard active on Core + app1-bu
|
||||
|
||||
Full files at:
|
||||
/root/.hermes/references/server-dr-plans.md
|
||||
/root/.hermes/references/server-dr-plans-redacted.md
|
||||
/root/.hermes/references/server-provisioning-standard-v1.md
|
||||
/root/.hermes/references/memory-consolidation-proposal.md
|
||||
""", "plain")
|
||||
|
||||
# HTML version
|
||||
sig = open("/root/.hermes/references/shonuff-email-signature.html").read()
|
||||
b64 = open("/root/.hermes/references/shonuff-image-b64.txt").read().strip()
|
||||
title = random.choice(TITLES)
|
||||
closing = random.choice(CLOSINGS)
|
||||
sig = sig.replace("%SHONUFF_IMAGE%", b64).replace("%SHONUFF_TITLE%", title)
|
||||
|
||||
html_full = f"<p>{full_body.replace(chr(10), '<br>')}</p><p><em>{closing}</em></p>{sig}"
|
||||
html_part = MIMEText(html_full, "html")
|
||||
|
||||
text_alt.attach(text_part)
|
||||
text_alt.attach(html_part)
|
||||
|
||||
msg.attach(text_alt)
|
||||
|
||||
# Attach the files
|
||||
for fname, display_name in [
|
||||
("/root/.hermes/references/server-dr-plans.md", "server-dr-plans-full.md"),
|
||||
("/root/.hermes/references/server-dr-plans-redacted.md", "server-dr-plans-redacted.md"),
|
||||
("/root/.hermes/references/server-provisioning-standard-v1.md", "server-provisioning-standard-v1.md"),
|
||||
("/root/.hermes/references/memory-consolidation-proposal.md", "memory-consolidation-proposal.md"),
|
||||
]:
|
||||
with open(fname, "rb") as f:
|
||||
attachment = MIMEBase("application", "octet-stream")
|
||||
attachment.set_payload(f.read())
|
||||
encoders.encode_base64(attachment)
|
||||
attachment.add_header("Content-Disposition", f"attachment; filename={display_name}")
|
||||
msg.attach(attachment)
|
||||
|
||||
# Send
|
||||
with smtplib.SMTP(SMTP, PORT) as s:
|
||||
s.starttls()
|
||||
s.login(FROM, PASS)
|
||||
s.send_message(msg)
|
||||
|
||||
print(f"Sent to {TO} — BCC'd g@germainebrown.com")
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send DR plan v3 — consolidated with expert review responses."""
|
||||
import smtplib, random, importlib.util
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.base import MIMEBase
|
||||
from email import encoders
|
||||
|
||||
SMTP = "mail.germainebrown.com"
|
||||
PORT = 2525
|
||||
PASS = "Catches.bullets1985"
|
||||
FROM = "shonuff@germainebrown.com"
|
||||
TO = "g@germainebrown.com"
|
||||
|
||||
def load_list(path, attr):
|
||||
spec = importlib.util.spec_from_file_location("mod", path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return getattr(mod, attr, [])
|
||||
|
||||
CLOSINGS = load_list("/root/.hermes/references/shonuff-closings.py", "SHONUFF_CLOSINGS")
|
||||
TITLES = load_list("/root/.hermes/references/shonuff-titles.py", "SHONUFF_TITLES")
|
||||
|
||||
msg = MIMEMultipart("mixed")
|
||||
msg["From"] = FROM
|
||||
msg["To"] = TO
|
||||
msg["Bcc"] = TO
|
||||
msg["Subject"] = "DR Plan v3 — Consolidated with Expert Review Responses"
|
||||
|
||||
body_html = """
|
||||
<h2 style="color:#dc2626;border-bottom:2px solid #dc2626;padding-bottom:8px;">DR Plan v3 — Expert Review Incorporated</h2>
|
||||
<hr style="border:none;border-top:2px solid #dc2626;margin:8px 0 16px 0;">
|
||||
|
||||
<p>Two DR experts reviewed our documentation. Their findings have been consolidated into a single v3 plan that addresses every point raised. Here's what changed:</p>
|
||||
|
||||
<h3>Critical Fixes Made</h3>
|
||||
|
||||
<table style="width:100%;border-collapse:collapse;margin:12px 0;">
|
||||
<tr style="background:#1e293b;color:#e2e8f0;">
|
||||
<th style="padding:8px;border:1px solid #334155;text-align:left;">Issue</th>
|
||||
<th style="padding:8px;border:1px solid #334155;text-align:left;">Old</th>
|
||||
<th style="padding:8px;border:1px solid #334155;text-align:left;">New</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Failover check interval</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">Every 5 minutes</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>Every 30 seconds</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">S3 is single point of failure</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">One region, admin creds</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>Two-region, write-only IAM user planned</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Backup compliance</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">Standard says daily — not happening</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>Gap closure plan with P0/P1/P2 priorities</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Restore runbooks</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">Generic steps only</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>Per-server with exact commands</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Failback protocol</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">Not documented</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>10-step protocol — auto-failback forbidden</strong></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:8px;border:1px solid #334155;">Tony's Hermes</td>
|
||||
<td style="padding:8px;border:1px solid #334155;">No documentation</td>
|
||||
<td style="padding:8px;border:1px solid #334155;color:#22c55e;"><strong>Flagged as P1 — needs DR docs</strong></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<h3>Backup Gaps (P0 — This Week)</h3>
|
||||
<ul>
|
||||
<li><strong>Docker volumes</strong> — not backed up at all. Need volume backup scripts.</li>
|
||||
<li><strong>Database dumps</strong> — not automated. Need cron on wphost02 + fleettracker360.</li>
|
||||
<li><strong>AI server disk</strong> — 92% full. Cleanup before migration.</li>
|
||||
<li><strong>S3 write-only IAM</strong> — currently using admin creds that can delete backups.</li>
|
||||
<li><strong>Object Lock</strong> — not enabled. Need 7-day minimum retention.</li>
|
||||
</ul>
|
||||
|
||||
<h3>Priority Actions (Next 2 Weeks)</h3>
|
||||
<ul>
|
||||
<li>Fix standby watchdog to 30s interval</li>
|
||||
<li>Document Tony's config + backup to S3</li>
|
||||
<li>Provision secondary S3 region</li>
|
||||
<li>Extend snapshot retention from 48h to 30 days</li>
|
||||
<li>Add backup monitoring to ops portal</li>
|
||||
</ul>
|
||||
|
||||
<h3>Attachments</h3>
|
||||
<p>Four files in this email:</p>
|
||||
<ol>
|
||||
<li><strong>server-dr-plans-v3.md</strong> — Consolidated plan (29KB, replaces all prior)</li>
|
||||
<li><strong>server-dr-plans-v3-redacted.md</strong> — Clean for 3rd party, no IPs/credentials</li>
|
||||
<li><strong>server-provisioning-standard-v1.md</strong> — Base install standard</li>
|
||||
<li><strong>memory-consolidation-proposal.md</strong> — Memory system improvement options</li>
|
||||
</ol>
|
||||
|
||||
<p>Also uploaded to S3:<br>
|
||||
<code>s3://hermes-vps-backups/live/config/server-dr-plans-v3.md</code></p>
|
||||
"""
|
||||
|
||||
text_alt = MIMEMultipart("alternative")
|
||||
text_part = MIMEText("DR Plan v3 attached. Full changelog in the HTML version.", "plain")
|
||||
|
||||
sig = open("/root/.hermes/references/shonuff-email-signature.html").read()
|
||||
b64 = open("/root/.hermes/references/shonuff-image-b64.txt").read().strip()
|
||||
title = random.choice(TITLES)
|
||||
closing = random.choice(CLOSINGS)
|
||||
sig = sig.replace("%SHONUFF_IMAGE%", b64).replace("%SHONUFF_TITLE%", title)
|
||||
|
||||
html_full = f"<p>{body_html.replace(chr(10), '<br>')}</p><p><em>{closing}</em></p>{sig}"
|
||||
html_part = MIMEText(html_full, "html")
|
||||
|
||||
text_alt.attach(text_part)
|
||||
text_alt.attach(html_part)
|
||||
msg.attach(text_alt)
|
||||
|
||||
# Redacted version
|
||||
redacted = MIMEText("""# DR Plan v3 — REDACTED
|
||||
# Per 3rd-party reviewer request: IPs removed, credentials stripped,
|
||||
# internal topology generalized.
|
||||
#
|
||||
# Full version: /root/.hermes/references/server-dr-plans-v3.md
|
||||
#
|
||||
## Key Changes from Prior Review
|
||||
# - Failover interval corrected: 30s not 5min
|
||||
# - S2 backup target added (Wasabi us-west-2 planned)
|
||||
# - Restore runbooks per server with exact commands
|
||||
# - Backup security: Object Lock + write-only IAM planned
|
||||
# - Quarterly DR testing scheduled
|
||||
# - Tony's Hermes flagged for documentation
|
||||
""", "plain")
|
||||
redacted.add_header("Content-Disposition", "attachment; filename=server-dr-plans-v3-redacted.md")
|
||||
msg.attach(redacted)
|
||||
|
||||
files = [
|
||||
("/root/.hermes/references/server-dr-plans-v3.md", "server-dr-plans-v3.md"),
|
||||
("/root/.hermes/references/server-provisioning-standard-v1.md", "server-provisioning-standard-v1.md"),
|
||||
("/root/.hermes/references/memory-consolidation-proposal.md", "memory-consolidation-proposal.md"),
|
||||
]
|
||||
for fpath, fname in files:
|
||||
with open(fpath, "rb") as f:
|
||||
part = MIMEBase("application", "octet-stream")
|
||||
part.set_payload(f.read())
|
||||
encoders.encode_base64(part)
|
||||
part.add_header("Content-Disposition", f"attachment; filename={fname}")
|
||||
msg.attach(part)
|
||||
|
||||
with smtplib.SMTP(SMTP, PORT) as s:
|
||||
s.starttls()
|
||||
s.login(FROM, PASS)
|
||||
s.send_message(msg)
|
||||
|
||||
print(f"Sent DR plan v3 to {TO}")
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Email the recovery bundle to g@germainebrown.com."""
|
||||
import smtplib
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.application import MIMEApplication
|
||||
|
||||
pw = open("/root/.config/himalaya/g-germainebrown.pass").read().strip()
|
||||
file_path = "/root/.hermes/recovery-bundle-2026-07-05.md"
|
||||
import os
|
||||
file_size = os.path.getsize(file_path)
|
||||
|
||||
msg = MIMEMultipart()
|
||||
msg["From"] = "g@germainebrown.com"
|
||||
msg["To"] = "g@germainebrown.com"
|
||||
msg["Subject"] = "Hermes Recovery Bundle — Jul 5, 2026"
|
||||
|
||||
body = MIMEText(
|
||||
f"Hermes recovery bundle attached ({file_size/1024:.0f} KB).\n\n"
|
||||
f"Contents:\n"
|
||||
f"- Today's full conversation history (2 sessions)\n"
|
||||
f"- Full config.yaml, .env, SOUL.md\n"
|
||||
f"- All 8 cron job definitions\n"
|
||||
f"- All scripts (IMAP triage, backup, watchdog, VPN, etc.)\n"
|
||||
f"- SSH keys (WISP)\n"
|
||||
f"- S3 credentials (Wasabi), Hetzner token, netcup key\n"
|
||||
f"- SMTP/IMAP password\n"
|
||||
f"- systemd service file\n\n"
|
||||
f"If the server dies, use this file to rebuild Shatter.\n"
|
||||
)
|
||||
msg.attach(body)
|
||||
|
||||
with open(file_path, "rb") as f:
|
||||
att = MIMEApplication(f.read(), _subtype="markdown")
|
||||
att.add_header("Content-Disposition", "attachment",
|
||||
filename="hermes-recovery-bundle-2026-07-05.md")
|
||||
msg.attach(att)
|
||||
|
||||
with smtplib.SMTP("mail.germainebrown.com", 2525) as s:
|
||||
s.starttls()
|
||||
s.login("g@germainebrown.com", pw)
|
||||
s.send_message(msg)
|
||||
|
||||
print("Email sent successfully")
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Called BEFORE suggesting the user type /reset
|
||||
# Notifies Anita and writes the startup flag for the new session
|
||||
set -euo pipefail
|
||||
|
||||
ANITA_BOT_TOKEN="8797435889:AAER9zMnRHtAkTfVcifLdoPMu1XDPr6XkBk"
|
||||
ANITA_CHAT_ID="8200236464"
|
||||
|
||||
# Write the "I'm back" flag for the new session to pick up
|
||||
echo "$(date -u +%s)" > /root/.hermes/.just_reset
|
||||
|
||||
# 1. Notify Anita via Python (handles JSON/encoding properly)
|
||||
python3 << 'PYEOF'
|
||||
import requests, json
|
||||
|
||||
token = "8797435889:AAER9zMnRHtAkTfVcifLdoPMu1XDPr6XkBk"
|
||||
chat_id = "8200236464"
|
||||
msg = ("🔄 *Sho'Nuff is resetting in 2 minutes.*\n\n"
|
||||
"Please finish any tasks you're working on. "
|
||||
"I'll let you know when I'm back online.")
|
||||
|
||||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||
data = {"chat_id": chat_id, "text": msg, "parse_mode": "Markdown"}
|
||||
r = requests.post(url, data=data, timeout=10)
|
||||
print(f"ANITA_STATUS={r.status_code}")
|
||||
PYEOF
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Send email from Sho'Nuff Brown with signature + BCC to Germaine + IMAP Sent copy."""
|
||||
import smtplib, random, importlib.util, imaplib, ssl, time, email
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
|
||||
SHONUFF_PASS = "Catches.bullets1985"
|
||||
|
||||
def send(to, subject, body, cc=None):
|
||||
spec = importlib.util.spec_from_file_location("closings", "/root/.hermes/references/shonuff-closings.py")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
closing = random.choice(mod.SHONUFF_CLOSINGS)
|
||||
|
||||
spec2 = importlib.util.spec_from_file_location("titles", "/root/.hermes/references/shonuff-titles.py")
|
||||
mod2 = importlib.util.module_from_spec(spec2)
|
||||
spec2.loader.exec_module(mod2)
|
||||
title = random.choice(mod2.SHONUFF_TITLES)
|
||||
|
||||
b64 = open("/root/.hermes/references/shonuff-image-b64.txt").read().strip()
|
||||
sig = open("/root/.hermes/references/shonuff-email-signature.html").read()
|
||||
sig = sig.replace("%SHONUFF_IMAGE%", b64).replace("%SHONUFF_TITLE%", title)
|
||||
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = "shonuff@germainebrown.com"
|
||||
msg["To"] = to
|
||||
msg["Bcc"] = "g@germainebrown.com"
|
||||
msg["Subject"] = subject
|
||||
|
||||
text = MIMEText(f"{body}\n\n{closing}\n\n-- \nSho'Nuff Brown\nGermaine's AI Ops Engineer\nIT Pro Partner\nshonuff@germainebrown.com", "plain")
|
||||
html = MIMEText(f"<p>{body.replace(chr(10), '<br>')}</p><p><em>{closing}</em></p>{sig}", "html")
|
||||
msg.attach(text)
|
||||
msg.attach(html)
|
||||
|
||||
# Send via SMTP
|
||||
with smtplib.SMTP("mail.germainebrown.com", 2525) as s:
|
||||
s.starttls()
|
||||
s.login("shonuff@germainebrown.com", SHONUFF_PASS)
|
||||
s.send_message(msg)
|
||||
|
||||
# Save copy to Sent folder via IMAP
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
imap = imaplib.IMAP4_SSL("mail.germainebrown.com", 993, ssl_context=ctx, timeout=10)
|
||||
imap.login("shonuff@germainebrown.com", SHONUFF_PASS)
|
||||
imap.append("Sent", None, None, msg.as_bytes())
|
||||
imap.logout()
|
||||
except Exception as e:
|
||||
print(f"[WARN] Sent copy not saved: {e}")
|
||||
|
||||
print(f"Sent to {to} — BCC'd g@germainebrown.com — Sent copy saved")
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if len(sys.argv) >= 4:
|
||||
send(sys.argv[1], sys.argv[2], sys.argv[3])
|
||||
else:
|
||||
print("Usage: send-shonuff.py <to> <subject> <body>")
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/bash
|
||||
# service-health-check.sh — Check all critical services and report failures
|
||||
# Silent on success. Reports only when something is down.
|
||||
|
||||
CHECKS=()
|
||||
FAILED=0
|
||||
|
||||
# 1. Core services
|
||||
systemctl is-active caddy > /dev/null 2>&1 || { CHECKS+=("Caddy (sign.itpropartner.com)"); FAILED=1; }
|
||||
systemctl is-active mysql-tunnel > /dev/null 2>&1 || { CHECKS+=("MySQL SSH tunnel (wphost02)"); FAILED=1; }
|
||||
|
||||
# 2. Docker containers
|
||||
docker ps --format '{{.Names}}' 2>/dev/null | grep -q searxng || { CHECKS+=("SearXNG container"); FAILED=1; }
|
||||
docker ps --format '{{.Names}}' 2>/dev/null | grep -q docuseal || { CHECKS+=("DocuSeal container"); FAILED=1; }
|
||||
|
||||
# 3. DocuSeal responding
|
||||
curl -sf -o /dev/null http://127.0.0.1:3000 2>/dev/null || { CHECKS+=("DocuSeal web UI"); FAILED=1; }
|
||||
|
||||
# 5. SearXNG responding
|
||||
curl -sf "http://127.0.0.1:8888/search?q=test&format=json" -o /dev/null 2>/dev/null || { CHECKS+=("SearXNG search"); FAILED=1; }
|
||||
|
||||
# 6. WireGuard tunnel to home router
|
||||
ping -c 1 -W 3 10.77.0.2 > /dev/null 2>&1 || { CHECKS+=("WireGuard tunnel (home router)"); FAILED=1; }
|
||||
|
||||
# 7. MySQL through tunnel
|
||||
mysql -h 127.0.0.1 -P 33060 -u apextrackexperience_1781549652 -p"K3E1ZZWvHDu0q8ZmoBCAhzKUZawEapdGBlbaPME1sOTKgGk9FCuYS" --skip-ssl -e "SELECT 1" > /dev/null 2>&1 || { CHECKS+=("MySQL database (apextrackexperience)"); FAILED=1; }
|
||||
|
||||
# 8. wphost02 reachable
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/itpp-infra root@5.161.62.38 "hostname" > /dev/null 2>&1 || { CHECKS+=("wphost02 SSH"); FAILED=1; }
|
||||
|
||||
# 9. Hermes gateway process
|
||||
ps aux | grep -v grep | grep -q "hermes.*gateway" || { CHECKS+=("Hermes gateway process"); FAILED=1; }
|
||||
|
||||
# 10. Cloudflare API token valid
|
||||
CLOUD_TOKEN=${CLOUDFLARE_API_TOKEN:-$(grep CLOUDFLARE_API_TOKEN ~/.hermes/.env | cut -d= -f2-)}
|
||||
curl -sf -X GET "https://api.cloudflare.com/client/v4/user/tokens/verify" -H "Authorization: Bearer $CLOUD_TOKEN" | python3 -c "import sys,json; sys.exit(0 if json.load(sys.stdin).get('success') else 1)" 2>/dev/null || { CHECKS+=("Cloudflare API token"); FAILED=1; }
|
||||
|
||||
# Report
|
||||
if [ $FAILED -eq 1 ]; then
|
||||
echo "⚠️ Service Health Alert — $(date +'%b %d, %Y %I:%M %p')"
|
||||
echo ""
|
||||
for c in "${CHECKS[@]}"; do
|
||||
echo " ❌ $c"
|
||||
done
|
||||
echo ""
|
||||
echo "Total: ${#CHECKS[@]} service(s) unhealthy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
Executable
+345
@@ -0,0 +1,345 @@
|
||||
#!/bin/bash
|
||||
# shark-draft-reminder.sh — Send 24h and 1h draft reminder emails to league members
|
||||
# Runs every 15 minutes via cron
|
||||
set -euo pipefail
|
||||
|
||||
exec python3 - "$@" << 'PYTHON_SCRIPT'
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Shark Draft Reminder — sends 24h and 1h email reminders to league members
|
||||
before their draft starts. Runs every 15 minutes via cron; idempotent.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import smtplib
|
||||
import sqlite3
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.mime.text import MIMEText
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from importlib.util import spec_from_file_location, module_from_spec
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────────
|
||||
DB_PATH = "/root/shark-game/backend/game.db"
|
||||
TRACKING_FILE = "/root/shark-game/data/draft-reminders.json"
|
||||
LOG_FILE = "/var/log/shark-reminder.log"
|
||||
PASS_FILE = "/root/.config/himalaya/shonuff.pass"
|
||||
TITLES_FILE = "/root/.hermes/references/shonuff-titles.py"
|
||||
CLOSINGS_FILE = "/root/.hermes/references/shonuff-closings.py"
|
||||
SIG_IMAGE_FILE = "/root/.hermes/references/shonuff-image-b64.txt"
|
||||
SIG_HTML_FILE = "/root/.hermes/references/shonuff-email-signature.html"
|
||||
|
||||
# ── SMTP Config ────────────────────────────────────────────────────────────
|
||||
SMTP_HOST = "mail.germainebrown.com"
|
||||
SMTP_PORT = 2525
|
||||
SMTP_USER = "shonuff@germainebrown.com"
|
||||
SMTP_FROM = "shonuff@germainebrown.com"
|
||||
SMTP_FROM_NAME = "Sho'Nuff Brown"
|
||||
BCC_ADDR = "g@germainebrown.com"
|
||||
|
||||
# ── Base URL for draft room links ─────────────────────────────────────────
|
||||
BASE_URL = "https://shark.itpropartner.com"
|
||||
|
||||
# ── Time windows (±5 minutes) ──────────────────────────────────────────────
|
||||
WINDOW_SECONDS = 5 * 60 # ±5 minutes
|
||||
|
||||
# ── Logging ────────────────────────────────────────────────────────────────
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler(LOG_FILE),
|
||||
logging.StreamHandler(sys.stdout),
|
||||
],
|
||||
)
|
||||
log = logging.getLogger("shark-draft-reminder")
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_password():
|
||||
"""Read SMTP password from file."""
|
||||
with open(PASS_FILE, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def load_closings():
|
||||
"""Load random closing line from reference file."""
|
||||
spec = spec_from_file_location("closings", CLOSINGS_FILE)
|
||||
mod = module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return random.choice(mod.SHONUFF_CLOSINGS)
|
||||
|
||||
|
||||
def load_titles():
|
||||
"""Load random title from reference file."""
|
||||
spec = spec_from_file_location("titles", TITLES_FILE)
|
||||
mod = module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return random.choice(mod.SHONUFF_TITLES)
|
||||
|
||||
|
||||
def load_signature_html(title):
|
||||
"""Build Sho'Nuff's HTML signature block."""
|
||||
try:
|
||||
with open(SIG_IMAGE_FILE) as f:
|
||||
b64 = f.read().strip()
|
||||
with open(SIG_HTML_FILE) as f:
|
||||
sig = f.read()
|
||||
return sig.replace("%SHONUFF_IMAGE%", b64).replace("%SHONUFF_TITLE%", title)
|
||||
except FileNotFoundError:
|
||||
log.warning("Signature image/html files not found — using plain sig")
|
||||
return ""
|
||||
|
||||
|
||||
def get_db():
|
||||
"""Get SQLite connection."""
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
return conn
|
||||
|
||||
|
||||
def load_tracking():
|
||||
"""Load sent-reminder tracking from JSON file."""
|
||||
if not os.path.exists(TRACKING_FILE):
|
||||
return {}
|
||||
with open(TRACKING_FILE, "r") as f:
|
||||
try:
|
||||
return json.load(f)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("Corrupt tracking file, resetting")
|
||||
return {}
|
||||
|
||||
|
||||
def save_tracking(tracking):
|
||||
"""Persist sent-reminder tracking to JSON file."""
|
||||
os.makedirs(os.path.dirname(TRACKING_FILE), exist_ok=True)
|
||||
with open(TRACKING_FILE, "w") as f:
|
||||
json.dump(tracking, f, indent=2, sort_keys=True)
|
||||
|
||||
|
||||
def tracking_key(league_id, user_id, reminder_type):
|
||||
"""Generate a unique key for tracking sent reminders."""
|
||||
return f"league_{league_id}_user_{user_id}_{reminder_type}"
|
||||
|
||||
|
||||
def already_sent(tracking, league_id, user_id, reminder_type):
|
||||
"""Check if a reminder has already been sent."""
|
||||
key = tracking_key(league_id, user_id, reminder_type)
|
||||
return key in tracking
|
||||
|
||||
|
||||
def mark_sent(tracking, league_id, user_id, reminder_type):
|
||||
"""Record that a reminder was sent (with ISO timestamp)."""
|
||||
key = tracking_key(league_id, user_id, reminder_type)
|
||||
tracking[key] = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def get_user_regions_preview(conn, league_id, user_id):
|
||||
"""Get user's drafted regions for the 'your regions preview' in 24h email."""
|
||||
rows = conn.execute(
|
||||
"""SELECT r.emoji, r.name
|
||||
FROM draft_picks dp
|
||||
JOIN regions r ON r.id = dp.region_id
|
||||
WHERE dp.league_id = ? AND dp.user_id = ?
|
||||
ORDER BY dp.round, dp.pick_number""",
|
||||
(league_id, user_id),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return " • No picks yet — get ready to draft!"
|
||||
return "\n".join(f" • {r['emoji']} {r['name']}" for r in rows)
|
||||
|
||||
|
||||
def build_email_body_24h(league_name, display_name, league_id, regions_preview, draft_start_at_str):
|
||||
"""Build the 24-hour reminder email body."""
|
||||
draft_url = f"{BASE_URL}/draft-room.html?league_id={league_id}"
|
||||
dt = datetime.fromisoformat(draft_start_at_str)
|
||||
draft_time_str = dt.strftime("%A, %B %d at %I:%M %p %Z")
|
||||
|
||||
body = f"""Howdy {display_name},
|
||||
|
||||
🦈 SHARK ALERT — Your draft for {league_name} starts in 24 hours!
|
||||
|
||||
⏰ Draft Time: {draft_time_str}
|
||||
|
||||
Get ready to claim your regions and build the ultimate shark attack fantasy team. Here's your draft room link (bookmark it!):
|
||||
|
||||
🔗 {draft_url}
|
||||
|
||||
📋 Your Current Regions:
|
||||
{regions_preview}
|
||||
|
||||
Make sure you're logged in and ready to go when the draft starts. Each pick is timed, so don't be late — or the sharks will swim right past you!
|
||||
|
||||
Good luck, and may the biggest bites be yours."""
|
||||
return body
|
||||
|
||||
|
||||
def build_email_body_1h(league_name, display_name, league_id, draft_start_at_str):
|
||||
"""Build the 1-hour reminder email body."""
|
||||
draft_url = f"{BASE_URL}/draft-room.html?league_id={league_id}"
|
||||
dt = datetime.fromisoformat(draft_start_at_str)
|
||||
draft_time_str = dt.strftime("%I:%M %p %Z")
|
||||
|
||||
body = f"""Yo {display_name},
|
||||
|
||||
🔥 The draft is T-1 HOUR away! {league_name} starts in just 60 minutes!
|
||||
|
||||
⏰ Draft Time: {draft_time_str}
|
||||
|
||||
Head to the draft room NOW and make sure everything is ready:
|
||||
|
||||
🔗 {draft_url}
|
||||
|
||||
📌 Pro tips:
|
||||
• Make sure your browser notifications are enabled
|
||||
• Have your region research ready
|
||||
• Don't step away — your picks are on a timer!
|
||||
|
||||
This is it — time to make your picks and show everyone who the real apex predator is.
|
||||
|
||||
Let's go!"""
|
||||
return body
|
||||
|
||||
|
||||
def send_email(to_addr, subject, body, closing, title):
|
||||
"""Send an email via SMTP with Sho'Nuff's signature."""
|
||||
sig_html = load_signature_html(title)
|
||||
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"{SMTP_FROM_NAME} <{SMTP_FROM}>"
|
||||
msg["To"] = to_addr
|
||||
msg["Bcc"] = BCC_ADDR
|
||||
msg["Subject"] = subject
|
||||
|
||||
# Plain text part
|
||||
plain_text = f"{body}\n\n{closing}\n\n-- \nSho'Nuff Brown\n{title}\nIT Pro Partner\n{SMTP_FROM}"
|
||||
msg.attach(MIMEText(plain_text, "plain"))
|
||||
|
||||
# HTML part
|
||||
html_body = body.replace("\n", "<br>\n")
|
||||
if sig_html:
|
||||
html_content = f"<p>{html_body}</p><p><em>{closing}</em></p>{sig_html}"
|
||||
else:
|
||||
html_content = f"<p>{html_body}</p><p><em>{closing}</em></p><p>--<br>Sho'Nuff Brown<br>{title}<br>IT Pro Partner</p>"
|
||||
msg.attach(MIMEText(html_content, "html"))
|
||||
|
||||
password = load_password()
|
||||
|
||||
try:
|
||||
with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as s:
|
||||
s.starttls()
|
||||
s.login(SMTP_USER, password)
|
||||
s.send_message(msg)
|
||||
log.info("Email sent to %s — subject: %s", to_addr, subject)
|
||||
return True
|
||||
except smtplib.SMTPException as e:
|
||||
log.error("SMTP error sending to %s: %s", to_addr, e)
|
||||
return False
|
||||
except Exception as e:
|
||||
log.error("Unexpected error sending to %s: %s", to_addr, e)
|
||||
return False
|
||||
|
||||
|
||||
def process_draft_reminders():
|
||||
"""Main logic: find leagues with upcoming drafts and send reminders."""
|
||||
now = datetime.now(timezone.utc)
|
||||
tracking = load_tracking()
|
||||
conn = get_db()
|
||||
|
||||
try:
|
||||
# Find leagues that are in draft status and have draft_start_at set
|
||||
leagues = conn.execute(
|
||||
"SELECT id, name, draft_start_at, status FROM leagues WHERE status = 'draft' AND draft_start_at IS NOT NULL"
|
||||
).fetchall()
|
||||
|
||||
if not leagues:
|
||||
log.info("No leagues with upcoming drafts found")
|
||||
return
|
||||
|
||||
log.info("Found %d league(s) with scheduled drafts", len(leagues))
|
||||
|
||||
for league in leagues:
|
||||
league_id = league["id"]
|
||||
league_name = league["name"]
|
||||
draft_start_str = league["draft_start_at"]
|
||||
|
||||
try:
|
||||
draft_start = datetime.fromisoformat(draft_start_str)
|
||||
if draft_start.tzinfo is None:
|
||||
draft_start = draft_start.replace(tzinfo=timezone.utc)
|
||||
except (ValueError, TypeError) as e:
|
||||
log.warning("Invalid draft_start_at for league %d (%s): %s", league_id, league_name, e)
|
||||
continue
|
||||
|
||||
# Calculate time until draft
|
||||
time_until = (draft_start - now).total_seconds()
|
||||
|
||||
# Get league members
|
||||
members = conn.execute(
|
||||
"""SELECT u.id, u.email, u.display_name
|
||||
FROM league_members lm
|
||||
JOIN users u ON u.id = lm.user_id
|
||||
WHERE lm.league_id = ?
|
||||
ORDER BY u.display_name""",
|
||||
(league_id,),
|
||||
).fetchall()
|
||||
|
||||
if not members:
|
||||
log.info("No members in league %d (%s)", league_id, league_name)
|
||||
continue
|
||||
|
||||
for member in members:
|
||||
user_id = member["id"]
|
||||
email = member["email"]
|
||||
display_name = member["display_name"]
|
||||
|
||||
# Check 24-hour reminder window
|
||||
if abs(time_until - 24 * 3600) <= WINDOW_SECONDS:
|
||||
if already_sent(tracking, league_id, user_id, "24h"):
|
||||
log.debug("24h reminder already sent for %s in league %d", display_name, league_id)
|
||||
else:
|
||||
regions = get_user_regions_preview(conn, league_id, user_id)
|
||||
body = build_email_body_24h(league_name, display_name, league_id, regions, draft_start_str)
|
||||
closing = load_closings()
|
||||
title = load_titles()
|
||||
subject = f"🦈 [{league_name}] — Draft Starts in 24 Hours!"
|
||||
if send_email(email, subject, body, closing, title):
|
||||
mark_sent(tracking, league_id, user_id, "24h")
|
||||
save_tracking(tracking)
|
||||
log.info("Sent 24h reminder to %s (%s) for league '%s'", display_name, email, league_name)
|
||||
|
||||
# Check 1-hour reminder window
|
||||
elif abs(time_until - 3600) <= WINDOW_SECONDS:
|
||||
if already_sent(tracking, league_id, user_id, "1h"):
|
||||
log.debug("1h reminder already sent for %s in league %d", display_name, league_id)
|
||||
else:
|
||||
body = build_email_body_1h(league_name, display_name, league_id, draft_start_str)
|
||||
closing = load_closings()
|
||||
title = load_titles()
|
||||
subject = f"🦈 [{league_name}] — Draft Starts in 1 Hour!"
|
||||
if send_email(email, subject, body, closing, title):
|
||||
mark_sent(tracking, league_id, user_id, "1h")
|
||||
save_tracking(tracking)
|
||||
log.info("Sent 1h reminder to %s (%s) for league '%s'", display_name, email, league_name)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── Entry Point ─────────────────────────────────────────────────────────────
|
||||
if __name__ == "__main__":
|
||||
log.info("=== Shark Draft Reminder run started ===")
|
||||
try:
|
||||
process_draft_reminders()
|
||||
except Exception as e:
|
||||
log.exception("Unhandled error: %s", e)
|
||||
sys.exit(1)
|
||||
log.info("=== Shark Draft Reminder run complete ===")
|
||||
PYTHON_SCRIPT
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
# Shark scraper wrapper — silent on success, only outputs on failure
|
||||
cd /root/shark-game/scraper
|
||||
OUTPUT=$(python3 scrape.py 2>&1)
|
||||
EXIT=$?
|
||||
if [ $EXIT -ne 0 ]; then
|
||||
echo "Shark scraper FAILED (exit $EXIT):"
|
||||
echo "$OUTPUT"
|
||||
exit $EXIT
|
||||
fi
|
||||
# Success = silent, no output, no notification
|
||||
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sho'Nuff Email Responder — polls shonuff@germainebrown.com inbox for emails
|
||||
FROM the Master (g@germainebrown.com), outputs JSON for the cron agent to
|
||||
generate replies, and sends those replies via SMTP.
|
||||
|
||||
Two modes:
|
||||
default (collect): Poll IMAP for new messages from Master, output JSON to stdout
|
||||
--send: Send a reply via SMTP (reads body from stdin)
|
||||
|
||||
Examples:
|
||||
# Collect new emails from the Master
|
||||
python3 shonuff-email-responder.py
|
||||
|
||||
# Send a reply (body from stdin)
|
||||
echo "Hey Germaine! Here's the answer..." | python3 shonuff-email-responder.py --send \
|
||||
--to "g@germainebrown.com" \
|
||||
--subject "Re: Question about servers" \
|
||||
--in-reply-to "<abc123@mail.germainebrown.com>"
|
||||
"""
|
||||
import imaplib
|
||||
import smtplib
|
||||
import email
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import hashlib
|
||||
from email.mime.text import MIMEText
|
||||
from email.utils import formatdate, make_msgid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# ── Config ────────────────────────────────────────────────────────────────
|
||||
IMAP_HOST = "mail.germainebrown.com"
|
||||
IMAP_PORT = 993
|
||||
IMAP_USER = "shonuff@germainebrown.com"
|
||||
|
||||
SMTP_HOST = "mail.germainebrown.com"
|
||||
SMTP_PORT = 2525
|
||||
SMTP_FROM = "shonuff@germainebrown.com"
|
||||
BCC_ADDRESS = "g@germainebrown.com"
|
||||
|
||||
# Master's known email addresses (only reply to these)
|
||||
MASTER_ADDRESSES = [
|
||||
"g@germainebrown.com",
|
||||
]
|
||||
|
||||
SEEN_FILE = "/root/.hermes/scripts/.shonuff-reply-processed.json"
|
||||
PASS_FILE = "/root/.config/himalaya/shonuff.pass"
|
||||
PASS_FILE_G = "/root/.config/himalaya/g-germainebrown.pass"
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def read_password():
|
||||
"""Read shonuff password from file."""
|
||||
if os.path.exists(PASS_FILE):
|
||||
with open(PASS_FILE) as f:
|
||||
return f.read().strip()
|
||||
# Fallback
|
||||
return "Catches.bullets1985"
|
||||
|
||||
|
||||
def load_seen():
|
||||
if os.path.exists(SEEN_FILE):
|
||||
try:
|
||||
return set(json.load(open(SEEN_FILE)))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return set()
|
||||
return set()
|
||||
|
||||
|
||||
def save_seen(seen):
|
||||
json.dump(list(seen), open(SEEN_FILE, "w"))
|
||||
|
||||
|
||||
def safe_str(val):
|
||||
if val is None:
|
||||
return ""
|
||||
if isinstance(val, str):
|
||||
return val
|
||||
try:
|
||||
return str(val)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def get_text_body(msg):
|
||||
"""Extract plain text from email, preferring text/plain."""
|
||||
body = ""
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
body = payload.decode("utf-8", errors="replace")
|
||||
return body.strip()
|
||||
|
||||
|
||||
def is_from_master(sender):
|
||||
"""Check if the sender is the Master (g@germainebrown.com)."""
|
||||
sender_lower = sender.lower()
|
||||
for addr in MASTER_ADDRESSES:
|
||||
if addr.lower() in sender_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def make_msg_hash(msg_id):
|
||||
"""Create a stable hash for deduplication."""
|
||||
return hashlib.md5(msg_id.encode()).hexdigest()
|
||||
|
||||
|
||||
# ── Collect Mode ───────────────────────────────────────────────────────────
|
||||
|
||||
def collect():
|
||||
"""Poll IMAP for new UNSEEN messages FROM the Master. Output JSON."""
|
||||
seen = load_seen()
|
||||
new_messages = []
|
||||
password = read_password()
|
||||
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT)
|
||||
conn.login(IMAP_USER, password)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
return # silent — nothing new
|
||||
|
||||
for num in data[0].split():
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
sender = safe_str(raw.get("From", "")).strip()
|
||||
subj = safe_str(raw.get("Subject", "(no subject)"))
|
||||
msg_id = safe_str(raw.get("Message-ID", ""))
|
||||
date = safe_str(raw.get("Date", ""))
|
||||
|
||||
# Only process emails FROM the Master
|
||||
if not is_from_master(sender):
|
||||
# Mark as read so we don't keep re-checking it
|
||||
conn.store(num, "+FLAGS", "\\Seen")
|
||||
continue
|
||||
|
||||
# Skip already processed (dedup)
|
||||
h = make_msg_hash(msg_id) if msg_id else make_msg_hash(f"{sender}-{subj}-{date}")
|
||||
if h in seen:
|
||||
conn.store(num, "+FLAGS", "\\Seen")
|
||||
continue
|
||||
seen.add(h)
|
||||
|
||||
body = get_text_body(raw)
|
||||
# Truncate very long bodies
|
||||
body_preview = body[:3000] if body else ""
|
||||
|
||||
new_messages.append({
|
||||
"from": sender,
|
||||
"subject": subj,
|
||||
"date": date,
|
||||
"body": body_preview,
|
||||
"message_id": msg_id,
|
||||
"uid": num.decode(),
|
||||
})
|
||||
|
||||
# Mark as SEEN so we don't re-process
|
||||
conn.store(num, "+FLAGS", "\\Seen")
|
||||
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
return
|
||||
|
||||
if not new_messages:
|
||||
return # silent — no new messages from Master
|
||||
|
||||
# Output JSON for the cron agent
|
||||
output = {
|
||||
"status": "new_messages",
|
||||
"count": len(new_messages),
|
||||
"messages": new_messages,
|
||||
"instructions": (
|
||||
"You are Sho'Nuff, Germaine's AI assistant. For each message above, "
|
||||
"generate a helpful reply. Then for EACH reply, write the reply text "
|
||||
"to a temp file and run the send command:\n\n"
|
||||
"cat /tmp/shonuff-reply.txt | python3 /root/.hermes/scripts/shonuff-email-responder.py "
|
||||
"--send --to 'FROM_ADDRESS' --subject 'Re: ORIGINAL_SUBJECT' "
|
||||
"--in-reply-to 'MESSAGE_ID'\n\n"
|
||||
"Send ALL replies before giving your final summary. Reply as Sho'Nuff — "
|
||||
"be helpful, concise, and personable."
|
||||
),
|
||||
}
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
# ── Send Mode ──────────────────────────────────────────────────────────────
|
||||
|
||||
def send_reply(to_addr, subject, in_reply_to, body_text):
|
||||
"""Send an email reply via SMTP."""
|
||||
password = read_password()
|
||||
|
||||
# Build the message
|
||||
msg = MIMEText(body_text, "plain", "utf-8")
|
||||
msg["From"] = SMTP_FROM
|
||||
msg["To"] = to_addr
|
||||
msg["Subject"] = subject
|
||||
msg["Date"] = formatdate(localtime=True)
|
||||
msg["Message-ID"] = make_msgid(domain="germainebrown.com")
|
||||
|
||||
if in_reply_to:
|
||||
msg["In-Reply-To"] = in_reply_to
|
||||
msg["References"] = in_reply_to
|
||||
|
||||
try:
|
||||
# Connect with STARTTLS on port 2525
|
||||
server = smtplib.SMTP(SMTP_HOST, SMTP_PORT, timeout=30)
|
||||
server.ehlo()
|
||||
server.starttls()
|
||||
server.ehlo()
|
||||
server.login(SMTP_FROM, password)
|
||||
|
||||
# Build recipients list (To + BCC)
|
||||
recipients = [to_addr]
|
||||
if BCC_ADDRESS and BCC_ADDRESS not in recipients:
|
||||
recipients.append(BCC_ADDRESS)
|
||||
|
||||
server.sendmail(SMTP_FROM, recipients, msg.as_string())
|
||||
server.quit()
|
||||
|
||||
result = {
|
||||
"status": "sent",
|
||||
"to": to_addr,
|
||||
"subject": subject,
|
||||
"bcc": BCC_ADDRESS,
|
||||
"message_id": msg["Message-ID"],
|
||||
}
|
||||
print(json.dumps(result))
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
result = {"status": "error", "error": str(e)}
|
||||
print(json.dumps(result))
|
||||
return False
|
||||
|
||||
|
||||
# ── Main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Sho'Nuff Email Responder — collect or send replies"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--send", action="store_true",
|
||||
help="Send mode: read reply body from stdin and send via SMTP"
|
||||
)
|
||||
parser.add_argument("--to", help="Recipient email address (send mode)")
|
||||
parser.add_argument("--subject", help="Email subject (send mode)")
|
||||
parser.add_argument("--in-reply-to", help="Message-ID being replied to (send mode)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.send:
|
||||
# ── Send Mode ──
|
||||
if not args.to:
|
||||
print(json.dumps({"error": "Missing --to argument for send mode"}))
|
||||
sys.exit(1)
|
||||
if not args.subject:
|
||||
print(json.dumps({"error": "Missing --subject argument for send mode"}))
|
||||
sys.exit(1)
|
||||
|
||||
# Read body from stdin
|
||||
body_text = sys.stdin.read().strip()
|
||||
if not body_text:
|
||||
print(json.dumps({"error": "No body text provided on stdin"}))
|
||||
sys.exit(1)
|
||||
|
||||
success = send_reply(
|
||||
to_addr=args.to,
|
||||
subject=args.subject,
|
||||
in_reply_to=args.in_reply_to or "",
|
||||
body_text=body_text,
|
||||
)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
else:
|
||||
# ── Collect Mode (default) ──
|
||||
collect()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check shonuff@germainebrown.com inbox, find new replies, extract content.
|
||||
Outputs JSON for the cron agent to summarize.
|
||||
Silent if nothing new.
|
||||
"""
|
||||
import imaplib, email, json, os, hashlib
|
||||
from email.utils import parsedate_to_datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
USER = "shonuff@germainebrown.com"
|
||||
PW = "Catches.bullets1985"
|
||||
SEEN_FILE = "/root/.hermes/scripts/.shonuff-processed.json"
|
||||
TRUSTED = ["g@germainebrown.com"]
|
||||
|
||||
def load_seen():
|
||||
if os.path.exists(SEEN_FILE):
|
||||
return set(json.load(open(SEEN_FILE)))
|
||||
return set()
|
||||
|
||||
def save_seen(seen):
|
||||
json.dump(list(seen), open(SEEN_FILE, "w"))
|
||||
|
||||
def safe_str(val):
|
||||
if val is None: return ""
|
||||
if isinstance(val, str): return val
|
||||
try: return str(val)
|
||||
except: return ""
|
||||
|
||||
def get_text_body(msg):
|
||||
"""Extract plain text from email, favoring text/plain."""
|
||||
body = ""
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
body = payload.decode("utf-8", errors="replace")
|
||||
# Strip excessive whitespace
|
||||
import re
|
||||
body = re.sub(r'\s+', ' ', body).strip()
|
||||
return body[:2000] # limit to 2000 chars
|
||||
|
||||
def main():
|
||||
seen = load_seen()
|
||||
new_messages = []
|
||||
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(USER, PW)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
return # silent
|
||||
|
||||
for num in data[0].split():
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
sender = safe_str(raw.get("From", "")).strip()
|
||||
subj = safe_str(raw.get("Subject", "(no subject)"))
|
||||
msg_id = safe_str(raw.get("Message-ID", str(num)))
|
||||
date = safe_str(raw.get("Date", ""))
|
||||
|
||||
# Skip Germaine
|
||||
if any(t.lower() in sender.lower() for t in TRUSTED):
|
||||
continue
|
||||
|
||||
# Skip self (shonuff's own Sent copies or BCC'd messages)
|
||||
if "shonuff@germainebrown.com" in sender.lower():
|
||||
continue
|
||||
|
||||
# Skip already processed
|
||||
h = hashlib.md5(msg_id.encode()).hexdigest()
|
||||
if h in seen:
|
||||
continue
|
||||
seen.add(h)
|
||||
|
||||
body = get_text_body(raw)
|
||||
|
||||
new_messages.append({
|
||||
"from": sender,
|
||||
"subject": subj,
|
||||
"date": date,
|
||||
"body_preview": body[:500],
|
||||
"uid": num.decode(),
|
||||
})
|
||||
|
||||
# Mark as read so we don't re-process
|
||||
conn.store(num, "+FLAGS", "\\Seen")
|
||||
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
|
||||
except Exception as e:
|
||||
print(json.dumps({"error": str(e)}))
|
||||
return
|
||||
|
||||
if not new_messages:
|
||||
return # silent
|
||||
|
||||
# Output JSON for the cron agent
|
||||
print(json.dumps({"messages": new_messages}, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a snapshot of ALL running servers in the Hetzner account."""
|
||||
import urllib.request, json, os, sys, time
|
||||
|
||||
API = "https://api.hetzner.cloud/v1"
|
||||
|
||||
# Read token from file to avoid shell issues
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
token_file = os.path.join(script_dir, ".hetzner_token")
|
||||
try:
|
||||
with open(token_file) as f:
|
||||
TOKEN = f.read().strip()
|
||||
except FileNotFoundError:
|
||||
print(f"HETZNER_TOKEN file not found at {token_file}")
|
||||
sys.exit(1)
|
||||
|
||||
HEADERS = {"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"}
|
||||
|
||||
def api(path, data=None, method=None):
|
||||
req = urllib.request.Request(f"{API}/{path}", headers=HEADERS, method=method)
|
||||
if data:
|
||||
req.data = json.dumps(data).encode()
|
||||
try:
|
||||
resp = urllib.request.urlopen(req)
|
||||
return json.loads(resp.read()) if resp.status != 204 else {}
|
||||
except urllib.error.HTTPError as e:
|
||||
body = e.read().decode()
|
||||
print(f" ERROR {e.code}: {body}")
|
||||
return None
|
||||
|
||||
# Get all servers
|
||||
servers = api("servers") or {"servers": []}
|
||||
if not servers["servers"]:
|
||||
print("No servers found.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"Found {len(servers['servers'])} servers\n")
|
||||
for s in servers["servers"]:
|
||||
name = s["name"]
|
||||
sid = s["id"]
|
||||
status = s["status"]
|
||||
|
||||
if status != "running":
|
||||
print(f" SKIP {name:30s} — {status}")
|
||||
continue
|
||||
|
||||
print(f" SNAPSHOT {name:30s} (ID: {sid}) ...", end=" ", flush=True)
|
||||
|
||||
result = api(f"servers/{sid}/actions/create_image", {
|
||||
"type": "snapshot",
|
||||
"description": f"auto-weekly-{name}-{time.strftime('%Y-%m-%d')}"
|
||||
}, method="POST")
|
||||
|
||||
if result:
|
||||
action = result.get("action", {})
|
||||
print(f"started (action {action.get('id', '?')})")
|
||||
else:
|
||||
print("FAILED")
|
||||
|
||||
print("\nDone.")
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# spend-monitor-collect.sh — Collect LLM spend data for daily monitor
|
||||
|
||||
echo "=== LLM Spend Report — $(date +'%b %d %Y %H:%M') ==="
|
||||
|
||||
# 1. LiteLLM key spend (cumulative by key)
|
||||
echo ""
|
||||
echo "--- LiteLLM Key Spend (cumulative) ---"
|
||||
ssh -i /root/.ssh/itpp-infra -o ConnectTimeout=5 -o BatchMode=yes root@178.156.167.181 bash << 'SSHEOF'
|
||||
PGPASSWORD=litellm docker exec -e PGPASSWORD=litellm litellm_postgres psql -U litellm -d litellm_db -t -c \
|
||||
"SELECT key_name, ROUND(spend::numeric,2) as total FROM \"LiteLLM_VerificationToken\" WHERE spend > 0 ORDER BY spend DESC;"
|
||||
SSHEOF
|
||||
|
||||
# 2. Last 24h total spend
|
||||
echo ""
|
||||
echo "--- Last 24h Total Spend ---"
|
||||
ssh -i /root/.ssh/itpp-infra -o ConnectTimeout=5 -o BatchMode=yes root@178.156.167.181 bash << 'SSHEOF'
|
||||
PGPASSWORD=litellm docker exec -e PGPASSWORD=litellm litellm_postgres psql -U litellm -d litellm_db -t -c \
|
||||
"SELECT CAST(SUM(spend) AS numeric(10,2)) as last_24h FROM \"LiteLLM_SpendLogs\" WHERE \"startTime\" > NOW() - INTERVAL '24 hours';"
|
||||
SSHEOF
|
||||
|
||||
# 3. DeepSeek models last 24h
|
||||
echo ""
|
||||
echo "--- DeepSeek Models Last 24h ---"
|
||||
ssh -i /root/.ssh/itpp-infra -o ConnectTimeout=5 -o BatchMode=yes root@178.156.167.181 bash << 'SSHEOF'
|
||||
PGPASSWORD=litellm docker exec -e PGPASSWORD=litellm litellm_postgres psql -U litellm -d litellm_db -t -c \
|
||||
"SELECT model, ROUND(SUM(spend)::numeric,4) as cost FROM \"LiteLLM_SpendLogs\" WHERE \"startTime\" > NOW() - INTERVAL '24 hours' AND model LIKE '%deepseek%' GROUP BY model ORDER BY cost DESC;"
|
||||
SSHEOF
|
||||
|
||||
# 4. Top models last 24h
|
||||
echo ""
|
||||
echo "--- All Models Last 24h (top 10) ---"
|
||||
ssh -i /root/.ssh/itpp-infra -o ConnectTimeout=5 -o BatchMode=yes root@178.156.167.181 bash << 'SSHEOF'
|
||||
PGPASSWORD=litellm docker exec -e PGPASSWORD=litellm litellm_postgres psql -U litellm -d litellm_db -t -c \
|
||||
"SELECT model, ROUND(SUM(spend)::numeric,4) as cost FROM \"LiteLLM_SpendLogs\" WHERE \"startTime\" > NOW() - INTERVAL '24 hours' GROUP BY model ORDER BY cost DESC LIMIT 10;"
|
||||
SSHEOF
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
cd /root
|
||||
/usr/local/lib/hermes-agent/venv/bin/python -m hermes_cli.main --profile anita gateway run &
|
||||
echo $! > /tmp/anita-gateway.pid
|
||||
echo "Anita gateway PID: $(cat /tmp/anita-gateway.pid)"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
cd /root
|
||||
nohup /usr/local/lib/hermes-agent/venv/bin/python -m hermes_cli.main --profile anita gateway run &>/root/.hermes/logs/anita-gateway.log &
|
||||
echo "PID: $!"
|
||||
Executable
+665
@@ -0,0 +1,665 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
IT Pro Partner — RingLogix Sync Engine
|
||||
|
||||
Syncs customer VoIP data from RingLogix (NetSapiens/CPaaS) API to local DB.
|
||||
Designed to run as a cron job every 15-30 minutes.
|
||||
|
||||
Requires:
|
||||
- psycopg2 or sqlite3 (depending on backend)
|
||||
- requests or urllib (stdlib-compatible version below)
|
||||
- RingLogix OAuth credentials (client_id, client_secret, reseller username/password)
|
||||
|
||||
Usage:
|
||||
python3 sync_ringlogix.py # full sync all domains
|
||||
python3 sync_ringlogix.py --domain acme.ringlogix.com # single domain
|
||||
|
||||
Environment variables (or .env file):
|
||||
RINGLOGIX_CLIENT_ID
|
||||
RINGLOGIX_CLIENT_SECRET
|
||||
RINGLOGIX_USERNAME # Reseller account username
|
||||
RINGLOGIX_PASSWORD # Reseller account password
|
||||
DATABASE_URL # postgresql://... or sqlite:///path
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# ─── Config ─────────────────────────────────────────────────────────
|
||||
|
||||
BASE_URL = "https://api.ringlogix.com/pbx/v1/"
|
||||
TOKEN_URL = urllib.parse.urljoin(BASE_URL, "oauth2/token/")
|
||||
|
||||
CLIENT_ID = os.environ.get("RINGLOGIX_CLIENT_ID", "")
|
||||
CLIENT_SECRET = os.environ.get("RINGLOGIX_CLIENT_SECRET", "")
|
||||
USERNAME = os.environ.get("RINGLOGIX_USERNAME", "")
|
||||
PASSWORD = os.environ.get("RINGLOGIX_PASSWORD", "")
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///portal.db")
|
||||
|
||||
# ─── HTTP Helpers ───────────────────────────────────────────────────
|
||||
|
||||
_ctx = ssl.create_default_context()
|
||||
|
||||
|
||||
def _request(
|
||||
method: str,
|
||||
url: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
body: str | bytes | None = None,
|
||||
) -> tuple[int, bytes]:
|
||||
if isinstance(body, str):
|
||||
body = body.encode("utf-8")
|
||||
req = urllib.request.Request(url, data=body, headers=headers or {}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, context=_ctx, timeout=30) as resp:
|
||||
return resp.status, resp.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read()
|
||||
|
||||
|
||||
# ─── OAuth2 ─────────────────────────────────────────────────────────
|
||||
|
||||
class RingLogixAuth:
|
||||
"""Manages OAuth2 token lifecycle for RingLogix API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.access_token: str = ""
|
||||
self.refresh_token: str = ""
|
||||
self.expires_at: float = 0 # timestamp
|
||||
|
||||
def _password_grant(self) -> dict[str, Any]:
|
||||
data = urllib.parse.urlencode({
|
||||
"grant_type": "password",
|
||||
"client_id": CLIENT_ID,
|
||||
"client_secret": CLIENT_SECRET,
|
||||
"username": USERNAME,
|
||||
"password": PASSWORD,
|
||||
})
|
||||
status, raw = _request("POST", TOKEN_URL, body=data)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"OAuth password grant failed HTTP {status}: {raw[:300]}")
|
||||
return json.loads(raw)
|
||||
|
||||
def _refresh_grant(self) -> dict[str, Any]:
|
||||
data = urllib.parse.urlencode({
|
||||
"grant_type": "refresh_token",
|
||||
"client_id": CLIENT_ID,
|
||||
"client_secret": CLIENT_SECRET,
|
||||
"refresh_token": self.refresh_token,
|
||||
})
|
||||
status, raw = _request("POST", TOKEN_URL, body=data)
|
||||
if status != 200:
|
||||
# Fall back to password grant
|
||||
return self._password_grant()
|
||||
return json.loads(raw)
|
||||
|
||||
def ensure_token(self) -> None:
|
||||
if self.access_token and time.time() < self.expires_at - 60:
|
||||
return # still valid
|
||||
if self.refresh_token:
|
||||
resp = self._refresh_grant()
|
||||
else:
|
||||
resp = self._password_grant()
|
||||
self.access_token = resp["access_token"]
|
||||
self.refresh_token = resp.get("refresh_token", "")
|
||||
self.expires_at = time.time() + resp.get("expires_in", 3600)
|
||||
|
||||
def api_call(
|
||||
self,
|
||||
object_name: str,
|
||||
action: str,
|
||||
params: dict[str, str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Make a POST to the RingLogix API with OAuth bearer token."""
|
||||
self.ensure_token()
|
||||
payload = {"object": object_name, "action": action}
|
||||
if params:
|
||||
payload.update(params)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.access_token}",
|
||||
"Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
|
||||
}
|
||||
body = urllib.parse.urlencode(payload)
|
||||
status, raw = _request("POST", BASE_URL, headers=headers, body=body)
|
||||
if status == 401:
|
||||
# Token expired mid-call, force refresh and retry once
|
||||
self.access_token = ""
|
||||
self.ensure_token()
|
||||
headers["Authorization"] = f"Bearer {self.access_token}"
|
||||
status, raw = _request("POST", BASE_URL, headers=headers, body=body)
|
||||
if status not in (200, 201):
|
||||
raise RuntimeError(
|
||||
f"API call {object_name}/{action} failed HTTP {status}: {raw[:500]}"
|
||||
)
|
||||
result = json.loads(raw)
|
||||
if isinstance(result, list):
|
||||
return {"data": result, "total": len(result)}
|
||||
return result
|
||||
|
||||
def api_get(self, object_name: str, action: str, params: dict[str, str] | None = None) -> list[dict[str, Any]]:
|
||||
"""GET-style read returning a list of results."""
|
||||
resp = self.api_call(object_name, action, params)
|
||||
# NetSapiens API returns data in various shapes
|
||||
if isinstance(resp, dict):
|
||||
data = resp.get("data", resp.get("response", []))
|
||||
if isinstance(data, dict):
|
||||
return [data]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
return [resp] if isinstance(resp, dict) else []
|
||||
|
||||
|
||||
# ─── DB Layer ───────────────────────────────────────────────────────
|
||||
|
||||
class Database:
|
||||
"""Minimal SQLite DB for MVP. Swap for psycopg2/asyncpg in production."""
|
||||
|
||||
def __init__(self, dsn: str) -> None:
|
||||
import sqlite3
|
||||
|
||||
self.conn = sqlite3.connect(dsn.replace("sqlite:///", ""))
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
self.conn.execute("PRAGMA journal_mode=WAL")
|
||||
self.conn.execute("PRAGMA foreign_keys=ON")
|
||||
self._ensure_schema()
|
||||
|
||||
def _ensure_schema(self) -> None:
|
||||
self.conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS customers (
|
||||
id TEXT PRIMARY KEY,
|
||||
company_name TEXT NOT NULL,
|
||||
contact_email TEXT NOT NULL,
|
||||
contact_phone TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS service_subscriptions (
|
||||
id TEXT PRIMARY KEY,
|
||||
customer_id TEXT REFERENCES customers(id),
|
||||
service_type TEXT NOT NULL CHECK (service_type IN ('voip','web_hosting','fleet_tracking')),
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
plan_name TEXT,
|
||||
monthly_price REAL,
|
||||
next_billing_date TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_subscriptions (
|
||||
id TEXT PRIMARY KEY,
|
||||
subscription_id TEXT REFERENCES service_subscriptions(id),
|
||||
ringlogix_domain TEXT NOT NULL UNIQUE,
|
||||
max_extensions INTEGER DEFAULT 5,
|
||||
max_auto_attendants INTEGER DEFAULT 1,
|
||||
max_call_queues INTEGER DEFAULT 1,
|
||||
last_synced_at TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_extensions (
|
||||
id TEXT PRIMARY KEY,
|
||||
voip_subscription_id TEXT REFERENCES voip_subscriptions(id),
|
||||
extension TEXT NOT NULL,
|
||||
user_login TEXT,
|
||||
full_name TEXT,
|
||||
email TEXT,
|
||||
department TEXT,
|
||||
device_mac TEXT,
|
||||
device_model TEXT,
|
||||
device_status TEXT DEFAULT 'offline',
|
||||
features TEXT DEFAULT '{}',
|
||||
cf_always TEXT,
|
||||
cf_busy TEXT,
|
||||
cf_no_answer TEXT,
|
||||
vm_enabled INTEGER DEFAULT 1,
|
||||
vm_email_notification INTEGER DEFAULT 0,
|
||||
vm_transcription INTEGER DEFAULT 0,
|
||||
recording_enabled INTEGER DEFAULT 0,
|
||||
presence TEXT DEFAULT 'offline',
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(voip_subscription_id, extension)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_dids (
|
||||
id TEXT PRIMARY KEY,
|
||||
voip_subscription_id TEXT REFERENCES voip_subscriptions(id),
|
||||
phone_number TEXT NOT NULL UNIQUE,
|
||||
label TEXT,
|
||||
routing TEXT,
|
||||
routing_target TEXT,
|
||||
caller_id_name TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_auto_attendants (
|
||||
id TEXT PRIMARY KEY,
|
||||
voip_subscription_id TEXT REFERENCES voip_subscriptions(id),
|
||||
name TEXT NOT NULL,
|
||||
greeting_type TEXT,
|
||||
greeting_text TEXT,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_call_queues (
|
||||
id TEXT PRIMARY KEY,
|
||||
voip_subscription_id TEXT REFERENCES voip_subscriptions(id),
|
||||
name TEXT NOT NULL,
|
||||
routing_strategy TEXT,
|
||||
max_wait_time_seconds INTEGER DEFAULT 300,
|
||||
agents TEXT DEFAULT '[]',
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_feature_usage (
|
||||
id TEXT PRIMARY KEY,
|
||||
voip_subscription_id TEXT REFERENCES voip_subscriptions(id),
|
||||
feature_key TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 0,
|
||||
current_qty INTEGER DEFAULT 0,
|
||||
max_qty INTEGER,
|
||||
last_verified_at TEXT DEFAULT (datetime('now')),
|
||||
UNIQUE(voip_subscription_id, feature_key)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS voip_plan_features (
|
||||
id TEXT PRIMARY KEY,
|
||||
plan_name TEXT NOT NULL,
|
||||
feature_key TEXT NOT NULL,
|
||||
feature_label TEXT NOT NULL,
|
||||
feature_category TEXT,
|
||||
included INTEGER DEFAULT 1,
|
||||
max_qty INTEGER,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
UNIQUE(plan_name, feature_key)
|
||||
);
|
||||
""")
|
||||
self.conn.commit()
|
||||
|
||||
def upsert(self, table: str, data: dict[str, Any], unique_cols: list[str]) -> None:
|
||||
"""Upsert by deleting existing matching row(s) then inserting."""
|
||||
if unique_cols:
|
||||
where = " AND ".join(f"{c}=?" for c in unique_cols)
|
||||
vals = tuple(data.get(c) for c in unique_cols)
|
||||
self.conn.execute(f"DELETE FROM {table} WHERE {where}", vals)
|
||||
cols = ", ".join(data.keys())
|
||||
placeholders = ", ".join("?" for _ in data)
|
||||
self.conn.execute(
|
||||
f"INSERT INTO {table} ({cols}) VALUES ({placeholders})",
|
||||
tuple(data.values()),
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def get_customer_by_domain(self, domain: str) -> dict | None:
|
||||
row = self.conn.execute(
|
||||
"SELECT c.* FROM customers c "
|
||||
"JOIN service_subscriptions s ON s.customer_id = c.id "
|
||||
"JOIN voip_subscriptions v ON v.subscription_id = s.id "
|
||||
"WHERE v.ringlogix_domain = ?",
|
||||
(domain,),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def upsert_customer_for_domain(self, domain: str, company: str, email: str) -> str:
|
||||
"""Create/find customer + subscription + voip_subscription for a domain."""
|
||||
existing = self.get_customer_by_domain(domain)
|
||||
import uuid
|
||||
|
||||
if existing:
|
||||
# Update fields
|
||||
self.conn.execute(
|
||||
"UPDATE customers SET company_name=?, contact_email=? WHERE id=?",
|
||||
(company, email, existing["id"]),
|
||||
)
|
||||
cust_id = existing["id"]
|
||||
else:
|
||||
cust_id = str(uuid.uuid4())
|
||||
self.conn.execute(
|
||||
"INSERT INTO customers (id, company_name, contact_email) VALUES (?, ?, ?)",
|
||||
(cust_id, company, email),
|
||||
)
|
||||
# Create subscription
|
||||
sub_id = str(uuid.uuid4())
|
||||
self.conn.execute(
|
||||
"INSERT INTO service_subscriptions (id, customer_id, service_type, plan_name) "
|
||||
"VALUES (?, ?, 'voip', 'Business Pro')",
|
||||
(sub_id, cust_id),
|
||||
)
|
||||
# Create voip_subscription
|
||||
vs_id = str(uuid.uuid4())
|
||||
self.conn.execute(
|
||||
"INSERT INTO voip_subscriptions (id, subscription_id, ringlogix_domain) "
|
||||
"VALUES (?, ?, ?)",
|
||||
(vs_id, sub_id, domain),
|
||||
)
|
||||
self.conn.commit()
|
||||
return cust_id
|
||||
|
||||
def get_voip_subscription_id(self, domain: str) -> str | None:
|
||||
row = self.conn.execute(
|
||||
"SELECT id FROM voip_subscriptions WHERE ringlogix_domain = ?",
|
||||
(domain,),
|
||||
).fetchone()
|
||||
return row["id"] if row else None
|
||||
|
||||
def close(self) -> None:
|
||||
self.conn.close()
|
||||
|
||||
|
||||
# ─── Sync Functions ─────────────────────────────────────────────────
|
||||
|
||||
def _uid() -> str:
|
||||
import uuid
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def sync_domain(auth: RingLogixAuth, db: Database, domain: str) -> dict[str, Any]:
|
||||
"""Full sync of a single RingLogix domain."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
summary = {"domain": domain, "extensions": 0, "dids": 0, "auto_attendants": 0, "call_queues": 0}
|
||||
|
||||
vs_id = db.get_voip_subscription_id(domain)
|
||||
if not vs_id:
|
||||
vs_id = str(_uid())
|
||||
sub_id = str(_uid())
|
||||
cust_id = str(_uid())
|
||||
db.conn.execute(
|
||||
"INSERT INTO customers (id, company_name, contact_email) VALUES (?, ?, ?)",
|
||||
(cust_id, domain.split(".")[0].title() + " LLC", f"billing@{domain}"),
|
||||
)
|
||||
db.conn.execute(
|
||||
"INSERT INTO service_subscriptions (id, customer_id, service_type) VALUES (?, ?, 'voip')",
|
||||
(sub_id, cust_id),
|
||||
)
|
||||
db.conn.execute(
|
||||
"INSERT INTO voip_subscriptions (id, subscription_id, ringlogix_domain) VALUES (?, ?, ?)",
|
||||
(vs_id, sub_id, domain),
|
||||
)
|
||||
db.conn.commit()
|
||||
|
||||
# 1. Count subscribers
|
||||
try:
|
||||
count_resp = auth.api_call("subscriber", "count", {"domain": domain})
|
||||
total_subs = int(count_resp.get("total", 0))
|
||||
except Exception:
|
||||
total_subs = 0
|
||||
|
||||
# 2. Read all subscribers
|
||||
try:
|
||||
subs = auth.api_get("subscriber", "read", {
|
||||
"domain": domain, "scope": "all",
|
||||
"fields": "user,first_name,last_name,email,presence,account_status,time_zone,vmail_provisioned,voicemail_emergency,recording,cf_always,cf_busy,cf_no_answer",
|
||||
})
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Subscriber read failed: {e}")
|
||||
subs = []
|
||||
|
||||
for sub in subs:
|
||||
ext = sub.get("user", "")
|
||||
if not ext:
|
||||
continue
|
||||
features = {
|
||||
"voicemail": sub.get("vmail_provisioned", "no") == "yes",
|
||||
"recording": sub.get("recording", "no") == "yes",
|
||||
"call_forwarding": bool(sub.get("cf_always") or sub.get("cf_busy") or sub.get("cf_no_answer")),
|
||||
}
|
||||
db.upsert("voip_extensions", {
|
||||
"id": str(_uid()),
|
||||
"voip_subscription_id": vs_id,
|
||||
"extension": ext,
|
||||
"user_login": sub.get("login", ""),
|
||||
"full_name": f"{sub.get('first_name','')} {sub.get('last_name','')}".strip(),
|
||||
"email": sub.get("email", ""),
|
||||
"department": sub.get("group", ""),
|
||||
"presence": sub.get("presence", "offline"),
|
||||
"features": json.dumps(features),
|
||||
"cf_always": sub.get("cf_always", ""),
|
||||
"cf_busy": sub.get("cf_busy", ""),
|
||||
"cf_no_answer": sub.get("cf_no_answer", ""),
|
||||
"vm_enabled": 1 if features["voicemail"] else 0,
|
||||
"recording_enabled": 1 if features["recording"] else 0,
|
||||
"updated_at": now,
|
||||
}, ["voip_subscription_id", "extension"])
|
||||
summary["extensions"] += 1
|
||||
|
||||
# 3. Read devices
|
||||
try:
|
||||
devices = auth.api_get("device", "read", {"domain": domain})
|
||||
# We can match devices to extensions by user field
|
||||
for dev in devices:
|
||||
ext_user = dev.get("user", "")
|
||||
if ext_user:
|
||||
dev_mac = dev.get("mac", "")
|
||||
dev_model = dev.get("model", "")
|
||||
dev_status = "registered" if dev.get("registration_time") else "offline"
|
||||
# Update the matching extension's device info
|
||||
rows = db.conn.execute(
|
||||
"SELECT id FROM voip_extensions WHERE voip_subscription_id=? AND extension=?",
|
||||
(vs_id, ext_user),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
db.conn.execute(
|
||||
"UPDATE voip_extensions SET device_mac=?, device_model=?, device_status=?, updated_at=? WHERE id=?",
|
||||
(dev_mac, dev_model, dev_status, now, row["id"]),
|
||||
)
|
||||
db.conn.commit()
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Device read failed: {e}")
|
||||
|
||||
# 4. Read DIDs / phone numbers
|
||||
try:
|
||||
dids = auth.api_get("phonenumber", "read", {"domain": domain})
|
||||
for did in dids:
|
||||
number = did.get("phonenumber", did.get("number", ""))
|
||||
if not number:
|
||||
continue
|
||||
db.upsert("voip_dids", {
|
||||
"id": str(_uid()),
|
||||
"voip_subscription_id": vs_id,
|
||||
"phone_number": number,
|
||||
"label": did.get("description", did.get("label", "")),
|
||||
"routing": did.get("route_type", ""),
|
||||
"routing_target": did.get("route_dest", ""),
|
||||
"caller_id_name": did.get("caller_id_name_emerge", ""),
|
||||
}, ["phone_number"])
|
||||
summary["dids"] += 1
|
||||
except Exception as e:
|
||||
print(f" ⚠️ DID read failed: {e}")
|
||||
|
||||
# 5. Read auto attendants
|
||||
try:
|
||||
attendants = auth.api_get("autoattendant", "read", {"domain": domain})
|
||||
for aa in attendants:
|
||||
name = aa.get("name", aa.get("user", "Main"))
|
||||
db.upsert("voip_auto_attendants", {
|
||||
"id": str(_uid()),
|
||||
"voip_subscription_id": vs_id,
|
||||
"name": name,
|
||||
"greeting_type": aa.get("greeting_type", ""),
|
||||
"greeting_text": aa.get("greeting_text", ""),
|
||||
}, ["voip_subscription_id", "name"])
|
||||
summary["auto_attendants"] += 1
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Auto attendant read failed: {e}")
|
||||
|
||||
# 6. Read call queues
|
||||
try:
|
||||
queues = auth.api_get("callqueue", "read", {"domain": domain})
|
||||
for q in queues:
|
||||
name = q.get("name", q.get("queue_name", "Queue"))
|
||||
db.upsert("voip_call_queues", {
|
||||
"id": str(_uid()),
|
||||
"voip_subscription_id": vs_id,
|
||||
"name": name,
|
||||
"routing_strategy": q.get("routing_strategy", q.get("queue_type", "")),
|
||||
"max_wait_time_seconds": int(q.get("max_wait_time", q.get("queue_timeout", 300))),
|
||||
"agents": json.dumps(q.get("agents", [])),
|
||||
}, ["voip_subscription_id", "name"])
|
||||
summary["call_queues"] += 1
|
||||
except Exception as e:
|
||||
print(f" ⚠️ Call queue read failed: {e}")
|
||||
|
||||
# Update last_synced_at
|
||||
db.conn.execute(
|
||||
"UPDATE voip_subscriptions SET last_synced_at=? WHERE id=?",
|
||||
(now, vs_id),
|
||||
)
|
||||
db.conn.commit()
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def sync_all(auth: RingLogixAuth, db: Database, target_domain: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Sync all domains under the reseller account, or a single domain."""
|
||||
if target_domain:
|
||||
print(f"Syncing single domain: {target_domain}")
|
||||
return [sync_domain(auth, db, target_domain)]
|
||||
|
||||
# List all domains
|
||||
try:
|
||||
domains = auth.api_get("domain", "read", {})
|
||||
except Exception as e:
|
||||
print(f"Failed to list domains: {e}")
|
||||
return []
|
||||
|
||||
results = []
|
||||
for d in domains:
|
||||
domain_name = d.get("domain", "")
|
||||
if domain_name:
|
||||
print(f"Syncing {domain_name}...")
|
||||
try:
|
||||
summary = sync_domain(auth, db, domain_name)
|
||||
results.append(summary)
|
||||
print(f" ✅ {summary['extensions']} ext, {summary['dids']} DIDs, "
|
||||
f"{summary['auto_attendants']} AA, {summary['call_queues']} queues")
|
||||
except Exception as e:
|
||||
print(f" ❌ {e}")
|
||||
return results
|
||||
|
||||
|
||||
# ─── Feature Definitions (seed data) ────────────────────────────────
|
||||
|
||||
DEFAULT_FEATURES = [
|
||||
# (plan, key, label, category, included, max_qty, sort)
|
||||
("Business Basic", "call_forwarding", "Call Forwarding (Always/Busy/No Answer)", "Call Management", True, None, 1),
|
||||
("Business Basic", "call_transfer", "Call Transfer (Attended/Blind/Intercom)", "Call Management", True, None, 2),
|
||||
("Business Basic", "three_way_calling", "Three-Way Calling", "Call Management", True, None, 3),
|
||||
("Business Basic", "voicemail", "Voicemail-to-Email", "Voicemail", True, None, 4),
|
||||
("Business Basic", "dnd", "Do Not Disturb", "Call Management", True, None, 5),
|
||||
("Business Basic", "auto_attendant", "Auto Attendant", "Auto Attendant", True, 1, 6),
|
||||
("Business Basic", "call_recording", "Call Recording", "Call Recording", False, None, 7),
|
||||
("Business Basic", "call_queue", "Call Queue", "Call Queues", False, None, 8),
|
||||
("Business Basic", "vm_transcription", "Voicemail Transcription", "Voicemail", False, None, 9),
|
||||
("Business Pro", "call_forwarding", "Call Forwarding (Always/Busy/No Answer/Find Me Follow Me)", "Call Management", True, None, 1),
|
||||
("Business Pro", "call_transfer", "Call Transfer (Attended/Blind/Intercom)", "Call Management", True, None, 2),
|
||||
("Business Pro", "three_way_calling", "Three-Way Calling", "Call Management", True, None, 3),
|
||||
("Business Pro", "voicemail", "Voicemail-to-Email", "Voicemail", True, None, 4),
|
||||
("Business Pro", "vm_transcription", "Voicemail Transcription", "Voicemail", True, None, 5),
|
||||
("Business Pro", "dnd", "Do Not Disturb", "Call Management", True, None, 6),
|
||||
("Business Pro", "auto_attendant", "Auto Attendant", "Auto Attendant", True, 2, 7),
|
||||
("Business Pro", "call_queue", "Call Queues", "Call Queues", True, 3, 8),
|
||||
("Business Pro", "call_recording", "Call Recording", "Call Recording", True, None, 9),
|
||||
("Business Pro", "call_parking", "Call Parking (up to 5)", "Call Management", True, None, 10),
|
||||
("Business Pro", "crm_integration", "CRM Integration", "Integrations", False, None, 11),
|
||||
("Enterprise", "call_forwarding", "Call Forwarding (All options)", "Call Management", True, None, 1),
|
||||
("Enterprise", "call_transfer", "Call Transfer (All options)", "Call Management", True, None, 2),
|
||||
("Enterprise", "three_way_calling", "Three-Way Calling", "Call Management", True, None, 3),
|
||||
("Enterprise", "voicemail", "Voicemail-to-Email", "Voicemail", True, None, 4),
|
||||
("Enterprise", "vm_transcription", "Voicemail Transcription", "Voicemail", True, None, 5),
|
||||
("Enterprise", "dnd", "Do Not Disturb", "Call Management", True, None, 6),
|
||||
("Enterprise", "auto_attendant", "Auto Attendant", "Auto Attendant", True, 5, 7),
|
||||
("Enterprise", "call_queue", "Call Queues", "Call Queues", True, 10, 8),
|
||||
("Enterprise", "call_recording", "Call Recording", "Call Recording", True, None, 9),
|
||||
("Enterprise", "call_parking", "Call Parking (unlimited)", "Call Management", True, None, 10),
|
||||
("Enterprise", "crm_integration", "CRM Integration", "Integrations", True, None, 11),
|
||||
("Enterprise", "skill_based_routing", "Skill-Based Routing", "Call Queues", True, None, 12),
|
||||
("Enterprise", "hot_desking", "Hot Desking / Hoteling", "Advanced", True, None, 13),
|
||||
]
|
||||
|
||||
|
||||
def seed_features(db: Database) -> None:
|
||||
"""Insert default feature definitions if empty."""
|
||||
count = db.conn.execute("SELECT COUNT(*) as c FROM voip_plan_features").fetchone()["c"]
|
||||
if count > 0:
|
||||
return
|
||||
for plan, key, label, cat, included, max_qty, sort in DEFAULT_FEATURES:
|
||||
import uuid
|
||||
db.conn.execute(
|
||||
"INSERT INTO voip_plan_features (id, plan_name, feature_key, feature_label, "
|
||||
"feature_category, included, max_qty, sort_order) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(str(uuid.uuid4()), plan, key, label, cat, 1 if included else 0, max_qty, sort),
|
||||
)
|
||||
db.conn.commit()
|
||||
print(f" ✅ Seeded {len(DEFAULT_FEATURES)} plan features")
|
||||
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> None:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="RingLogix Sync Engine")
|
||||
parser.add_argument("--domain", help="Sync only this domain")
|
||||
parser.add_argument("--db", help="Database URL (overrides DATABASE_URL env)")
|
||||
parser.add_argument("--seed", action="store_true", help="Seed plan features only, no sync")
|
||||
args = parser.parse_args()
|
||||
|
||||
db_url = args.db or DATABASE_URL
|
||||
if not db_url:
|
||||
print("❌ DATABASE_URL not set. Use --db or set env.")
|
||||
sys.exit(1)
|
||||
|
||||
db = Database(db_url)
|
||||
print(f"📦 Database: {db_url}")
|
||||
|
||||
if args.seed:
|
||||
seed_features(db)
|
||||
db.close()
|
||||
return
|
||||
|
||||
# Validate RingLogix credentials
|
||||
missing = []
|
||||
for var, name in [
|
||||
(CLIENT_ID, "RINGLOGIX_CLIENT_ID"),
|
||||
(CLIENT_SECRET, "RINGLOGIX_CLIENT_SECRET"),
|
||||
(USERNAME, "RINGLOGIX_USERNAME"),
|
||||
(PASSWORD, "RINGLOGIX_PASSWORD"),
|
||||
]:
|
||||
if not var:
|
||||
missing.append(name)
|
||||
if missing:
|
||||
print(f"❌ Missing env vars: {', '.join(missing)}")
|
||||
print(" Set them in .env or export them.")
|
||||
db.close()
|
||||
sys.exit(1)
|
||||
|
||||
auth = RingLogixAuth()
|
||||
|
||||
# Seed features if empty
|
||||
seed_features(db)
|
||||
|
||||
# Sync
|
||||
results = sync_all(auth, db, args.domain)
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Sync complete: {len(results)} domains processed")
|
||||
for r in results:
|
||||
print(f" {r['domain']}: {r['extensions']} ext, {r['dids']} DIDs, "
|
||||
f"{r['auto_attendants']} AA, {r['call_queues']} queues")
|
||||
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track Firecrawl API credit usage for portal dashboard."""
|
||||
import json, os, time
|
||||
from datetime import datetime, timezone
|
||||
BASE = os.path.expanduser("~/.hermes/scripts")
|
||||
|
||||
LOG = os.path.join(BASE, "firecrawl-usage.json")
|
||||
LIMIT = 1000 # Current tier
|
||||
|
||||
def _defaults():
|
||||
return {"total_used": 0, "calls": [], "monthly": {}, "last_reset": str(datetime.now(timezone.utc))}
|
||||
|
||||
def load():
|
||||
"""Load usage data, merging in any missing top-level keys.
|
||||
|
||||
Older state files were written without the `monthly` / `calls` keys,
|
||||
which caused KeyError crashes in summary(). Merging defaults makes the
|
||||
loader forward/backward compatible with partial state files.
|
||||
"""
|
||||
data = _defaults()
|
||||
if os.path.exists(LOG):
|
||||
try:
|
||||
with open(LOG) as f:
|
||||
loaded = json.load(f)
|
||||
if isinstance(loaded, dict):
|
||||
data.update(loaded)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass # Corrupt/unreadable state — fall back to defaults
|
||||
# Guarantee required keys/types even if the file had them as null/wrong type
|
||||
if not isinstance(data.get("monthly"), dict):
|
||||
data["monthly"] = {}
|
||||
if not isinstance(data.get("calls"), list):
|
||||
data["calls"] = []
|
||||
if not isinstance(data.get("total_used"), (int, float)):
|
||||
data["total_used"] = 0
|
||||
return data
|
||||
|
||||
def save(data):
|
||||
with open(LOG, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
def record(credits_used, endpoint, url=""):
|
||||
data = load()
|
||||
now = datetime.now(timezone.utc)
|
||||
month_key = now.strftime("%Y-%m")
|
||||
|
||||
# Track total
|
||||
data["total_used"] += credits_used
|
||||
|
||||
# Track monthly
|
||||
if month_key not in data["monthly"]:
|
||||
data["monthly"][month_key] = 0
|
||||
data["monthly"][month_key] += credits_used
|
||||
|
||||
# Log individual call
|
||||
data["calls"].append({
|
||||
"ts": now.isoformat(),
|
||||
"endpoint": endpoint,
|
||||
"url": url[:100],
|
||||
"credits": credits_used
|
||||
})
|
||||
|
||||
# Trim call log to last 1000
|
||||
data["calls"] = data["calls"][-1000:]
|
||||
save(data)
|
||||
|
||||
def summary():
|
||||
data = load()
|
||||
month_key = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
month_used = data["monthly"].get(month_key, 0)
|
||||
return {
|
||||
"total_used": data["total_used"],
|
||||
"month_used": month_used,
|
||||
"month_limit": LIMIT,
|
||||
"month_remaining": max(0, LIMIT - month_used),
|
||||
"month_pct": round(month_used / LIMIT * 100, 1),
|
||||
"recent_calls": data["calls"][-10:]
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "summary":
|
||||
print(json.dumps(summary(), indent=2))
|
||||
else:
|
||||
print(f"Credits: {summary()['month_used']}/{LIMIT} used this month")
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# unifi-backup-sync.sh — Daily UniFi backup sync from old controller to Wasabi S3.
|
||||
# Source: root@178.156.131.57:/var/lib/unifi/backup/
|
||||
# Target: s3://hermes-vps-backups/unifi-backups/
|
||||
# Schedule: Daily at 2 AM ET via cron (no_agent)
|
||||
set -euo pipefail
|
||||
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
SOURCE_HOST="178.156.131.57"
|
||||
SOURCE_PATH="/var/lib/unifi/backup"
|
||||
S3_BUCKET="s3://hermes-vps-backups/unifi-backups"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
STAGE="/root/.hermes/.backups/unifi-sync"
|
||||
TS="$(date -u +%Y%m%d-%H%M%S)"
|
||||
LOG="/tmp/unifi-backup-sync-${TS}.log"
|
||||
|
||||
# Cron-safe AWS CLI path
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"; }
|
||||
|
||||
cleanup() {
|
||||
rm -f "$LOG"
|
||||
rm -rf "$STAGE"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
log "Starting UniFi backup sync from ${SOURCE_HOST}:${SOURCE_PATH}"
|
||||
|
||||
# Pull backups via rsync
|
||||
mkdir -p "$STAGE"
|
||||
if ! rsync -az --timeout=30 \
|
||||
-e "ssh -i ${SSH_KEY} -o ConnectTimeout=10 -o BatchMode=yes" \
|
||||
"root@${SOURCE_HOST}:${SOURCE_PATH}/" \
|
||||
"$STAGE/" >/dev/null 2>&1; then
|
||||
log "FATAL: rsync from ${SOURCE_HOST} failed — server unreachable or path missing"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COUNT=$(find "$STAGE" -name '*.unf' | wc -l)
|
||||
SIZE=$(du -sh "$STAGE" | cut -f1)
|
||||
log "Pulled ${COUNT} .unf files (${SIZE})"
|
||||
|
||||
# Upload to S3
|
||||
if aws s3 sync "$STAGE/" "$S3_BUCKET/" \
|
||||
--endpoint-url "$ENDPOINT" \
|
||||
--delete \
|
||||
>>"$LOG" 2>&1; then
|
||||
log "Uploaded to ${S3_BUCKET}"
|
||||
else
|
||||
log "FATAL: S3 upload failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Sync complete: ${COUNT} files — s3://hermes-vps-backups/unifi-backups/"
|
||||
Executable
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
# unms-backup-sync.sh -- Daily UISP/UNMS backup sync to Wasabi S3.
|
||||
# Pulls backups from old historical UNMS and live app2 to local staging,
|
||||
# then uploads to s3://hermes-vps-backups/unms-backups/.
|
||||
set -euo pipefail
|
||||
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
S3_BUCKET="s3://hermes-vps-backups/unms-backups"
|
||||
ENDPOINT="https://s3.us-east-1.wasabisys.com"
|
||||
STAGE="/root/.hermes/.backups/unms-sync"
|
||||
TS="$(date -u +%Y%m%d-%H%M%S)"
|
||||
LOG="/tmp/unms-backup-sync-${TS}.log"
|
||||
|
||||
# Cron-safe AWS CLI path
|
||||
if [ -f /opt/awscli-venv/bin/activate ]; then
|
||||
# shellcheck disable=SC1091
|
||||
source /opt/awscli-venv/bin/activate
|
||||
fi
|
||||
|
||||
log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"; }
|
||||
|
||||
sync_remote_dir() {
|
||||
local label="$1" host="$2" remote="$3" dest="$4" required="$5"
|
||||
local local_dir="${STAGE}/${dest}"
|
||||
mkdir -p "$local_dir"
|
||||
|
||||
log "Checking ${label} ${host}:${remote}"
|
||||
if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes "root@${host}" "test -d '${remote}'" >/dev/null 2>&1; then
|
||||
if [ "$required" = "yes" ]; then
|
||||
log "ERROR: Required source missing/unreachable: ${label}"
|
||||
return 1
|
||||
fi
|
||||
log "WARN: Optional source missing/unreachable: ${label}; skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
rsync -az --delete \
|
||||
-e "ssh -i ${SSH_KEY} -o BatchMode=yes" \
|
||||
"root@${host}:${remote}/" \
|
||||
"${local_dir}/" >>"$LOG" 2>&1
|
||||
|
||||
local count
|
||||
count=$(find "$local_dir" -type f | wc -l)
|
||||
log "OK: ${label} staged ${count} files -> ${local_dir}"
|
||||
}
|
||||
|
||||
main() {
|
||||
mkdir -p "$STAGE"
|
||||
log "Starting UNMS backup sync"
|
||||
|
||||
# Historical old UNMS -- optional because it may be decommissioned.
|
||||
sync_remote_dir "old-unms-backups" "5.161.225.131" "/home/unms/data/unms-backups/backups" "old-unms" "no"
|
||||
|
||||
# Live app2 UNMS -- required.
|
||||
sync_remote_dir "live-unms-backups" "152.53.39.202" "/home/unms/data/unms-backups/backups" "live/backups" "yes"
|
||||
|
||||
# UCRM backups -- optional; path exists only when UCRM backup job has generated files.
|
||||
sync_remote_dir "live-ucrm-backups" "152.53.39.202" "/home/unms/data/ucrm/ucrm/data/backup" "live/ucrm" "no"
|
||||
|
||||
log "Uploading staged backups to ${S3_BUCKET}"
|
||||
aws s3 sync "$STAGE/" "$S3_BUCKET/" --endpoint-url "$ENDPOINT" --delete >>"$LOG" 2>&1
|
||||
|
||||
log "Verifying S3 contents"
|
||||
aws s3 ls "$S3_BUCKET/live/backups/" --endpoint-url "$ENDPOINT" >>"$LOG" 2>&1
|
||||
local live_count
|
||||
live_count=$(aws s3 ls "$S3_BUCKET/live/backups/" --endpoint-url "$ENDPOINT" | wc -l)
|
||||
if [ "$live_count" -lt 1 ]; then
|
||||
log "ERROR: S3 verification found no live backup objects"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "DONE: UNMS backup sync complete; live backup objects=${live_count}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+211
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
# VPS Resource Threshold Alert Checker
|
||||
# Monitors RAM, disk, and CPU across all servers.
|
||||
# Outputs alert messages to stdout (delivered by no-agent cron to Telegram)
|
||||
# Thresholds: 80%, 90%, 95%
|
||||
# Tracks sent alerts in /root/.hermes/data/threshold-alerts.json
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
DATA_DIR="/root/.hermes/data"
|
||||
ALERTS_FILE="${DATA_DIR}/threshold-alerts.json"
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
SSH_OPTS="-i ${SSH_KEY} -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes"
|
||||
NOW_EPOCH=$(date +%s)
|
||||
NOW_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
RE_ALERT_SECONDS=86400 # 24 hours
|
||||
|
||||
# Thresholds to check
|
||||
THRESHOLDS=(80 90 95)
|
||||
|
||||
# Server list: "hostname|ip|local_flag" (local=1 means run commands directly)
|
||||
declare -a SERVERS=(
|
||||
"app1-bu|5.161.114.8|0"
|
||||
"ai|178.156.167.181|0"
|
||||
"hudu|178.156.130.130|0"
|
||||
"unifi|178.156.131.57|0"
|
||||
"unms|5.161.225.131|0"
|
||||
"wphost02|5.161.62.38|0"
|
||||
"n8n|87.99.144.163|0"
|
||||
"docker|178.156.168.35|0"
|
||||
"fleettrack360|178.156.149.32|0"
|
||||
"core|152.53.192.33|1"
|
||||
)
|
||||
|
||||
# Ensure data directory exists
|
||||
mkdir -p "${DATA_DIR}"
|
||||
|
||||
# Initialize alerts file if it doesn't exist
|
||||
if [[ ! -f "${ALERTS_FILE}" ]]; then
|
||||
echo "{}" > "${ALERTS_FILE}"
|
||||
fi
|
||||
|
||||
# Read current alerts
|
||||
ALERTS_JSON=$(cat "${ALERTS_FILE}")
|
||||
|
||||
# Function to read a value from the alerts JSON
|
||||
get_alert_ts() {
|
||||
local key="$1"
|
||||
python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('${key}',''))" <<< "${ALERTS_JSON}"
|
||||
}
|
||||
|
||||
# Function to update a timestamp in the alerts JSON
|
||||
update_alert_ts() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
ALERTS_JSON=$(python3 -c "
|
||||
import json,sys
|
||||
d = json.loads('${ALERTS_JSON//\'/\\\'}')
|
||||
d['${key}'] = '${value}'
|
||||
print(json.dumps(d))
|
||||
")
|
||||
}
|
||||
|
||||
# Function to remove a key from the alerts JSON (reset)
|
||||
remove_alert_key() {
|
||||
local key="$1"
|
||||
ALERTS_JSON=$(python3 -c "
|
||||
import json,sys
|
||||
d = json.loads('${ALERTS_JSON//\'/\\\'}')
|
||||
d.pop('${key}', None)
|
||||
print(json.dumps(d))
|
||||
")
|
||||
}
|
||||
|
||||
# Save the alerts JSON back
|
||||
save_alerts() {
|
||||
cat > "${ALERTS_FILE}" <<< "${ALERTS_JSON}"
|
||||
}
|
||||
|
||||
# Check if we should alert for a threshold
|
||||
# Returns 0 (should alert) or 1 (should not)
|
||||
should_alert() {
|
||||
local key="$1"
|
||||
local ts
|
||||
ts=$(get_alert_ts "${key}")
|
||||
if [[ -z "${ts}" ]]; then
|
||||
return 0 # Never alerted before, should alert
|
||||
fi
|
||||
# Parse the timestamp and check if 24h have passed
|
||||
local last_epoch
|
||||
last_epoch=$(date -d "${ts}" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "${ts}" +%s 2>/dev/null || echo "0")
|
||||
if [[ -z "${last_epoch}" || "${last_epoch}" -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
local elapsed=$(( NOW_EPOCH - last_epoch ))
|
||||
if [[ "${elapsed}" -ge "${RE_ALERT_SECONDS}" ]]; then
|
||||
return 0 # 24h passed, re-alert
|
||||
fi
|
||||
return 1 # Recently alerted, skip
|
||||
}
|
||||
|
||||
# Run a command on a server (local or via SSH)
|
||||
run_cmd() {
|
||||
local hostname="$1"
|
||||
local ip="$2"
|
||||
local is_local="$3"
|
||||
local cmd="$4"
|
||||
|
||||
if [[ "${is_local}" == "1" ]]; then
|
||||
eval "${cmd}"
|
||||
else
|
||||
ssh ${SSH_OPTS} "root@${ip}" "${cmd}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Build alert message accumulator
|
||||
ALERT_LINES=""
|
||||
ALERT_COUNT=0
|
||||
|
||||
# Process each server
|
||||
for server_entry in "${SERVERS[@]}"; do
|
||||
IFS='|' read -r hostname ip is_local <<< "${server_entry}"
|
||||
|
||||
# Gather metrics
|
||||
disk_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "df -h / | awk 'NR==2 {print \$5}' | tr -d '%'")
|
||||
ram_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "free | awk '/Mem:/ {printf \"%.0f\", \$3/\$2 * 100}'")
|
||||
cpu_raw=$(run_cmd "${hostname}" "${ip}" "${is_local}" "top -bn1 | awk '/Cpu\(s\)/ {print \$2}'")
|
||||
core_count=$(run_cmd "${hostname}" "${ip}" "${is_local}" "nproc")
|
||||
|
||||
# Validate we got numbers (skip server if unreachable)
|
||||
disk_pct="${disk_pct:-0}"
|
||||
ram_pct="${ram_pct:-0}"
|
||||
cpu_raw="${cpu_raw:-0}"
|
||||
core_count="${core_count:-1}"
|
||||
|
||||
# Convert CPU % to integer (top -bn1 gives float like 0.0 or 2.3)
|
||||
cpu_pct=$(echo "${cpu_raw}" | cut -d. -f1)
|
||||
cpu_pct="${cpu_pct:-0}"
|
||||
|
||||
# Ensure numeric
|
||||
disk_pct=$(echo "${disk_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
ram_pct=$(echo "${ram_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
cpu_pct=$(echo "${cpu_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
core_count=$(echo "${core_count}" | grep -oE '^[0-9]+$' || echo "1")
|
||||
if [[ "${core_count}" -lt 1 ]]; then core_count=1; fi
|
||||
|
||||
# Track which thresholds are crossed for this server
|
||||
SERVER_ALERTS=""
|
||||
SERVER_ALERT_COUNT=0
|
||||
|
||||
# Check disk thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_disk_${thresh}"
|
||||
if [[ "${disk_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ DISK at ${disk_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
# Below threshold — reset alert so it can re-alert if it crosses again
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check RAM thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_ram_${thresh}"
|
||||
if [[ "${ram_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ RAM at ${ram_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check CPU thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_cpu_${thresh}"
|
||||
if [[ "${cpu_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ CPU at ${cpu_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# If this server has alerts, add to consolidated message
|
||||
if [[ "${SERVER_ALERT_COUNT}" -gt 0 ]]; then
|
||||
ALERT_LINES+="🚨 VPS Alert: ${hostname}"$'\n'
|
||||
ALERT_LINES+="${SERVER_ALERTS}"
|
||||
ALERT_COUNT=$((ALERT_COUNT + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# Save updated alerts
|
||||
save_alerts
|
||||
|
||||
# Output any alert messages (stdout goes to Telegram via cron)
|
||||
if [[ "${ALERT_COUNT}" -gt 0 ]]; then
|
||||
echo "${ALERT_LINES}"
|
||||
else
|
||||
# Silent on success - no output = no delivery
|
||||
:
|
||||
fi
|
||||
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check shonuff@germainebrown.com inbox for new messages from unknown senders.
|
||||
|
||||
Silent if nothing new. Alerts if someone other than Germaine has emailed.
|
||||
"""
|
||||
import imaplib, email, os, json, hashlib
|
||||
from datetime import datetime, timezone
|
||||
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
USER = "shonuff@germainebrown.com"
|
||||
PW = "Catches.bullets1985"
|
||||
SEEN_FILE = "/root/.hermes/scripts/.shonuff-seen.json"
|
||||
TRUSTED = ["g@germainebrown.com", "mailer-daemon@heracles.mxrouting.net"]
|
||||
|
||||
def load_seen():
|
||||
if os.path.exists(SEEN_FILE):
|
||||
return set(json.load(open(SEEN_FILE)))
|
||||
return set()
|
||||
|
||||
def save_seen(seen):
|
||||
json.dump(list(seen), open(SEEN_FILE, "w"))
|
||||
|
||||
def main():
|
||||
seen = load_seen()
|
||||
new_alerts = []
|
||||
|
||||
try:
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(USER, PW)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
return # silent — nothing new
|
||||
|
||||
for num in data[0].split():
|
||||
status, msg_data = conn.fetch(num, "(RFC822)")
|
||||
if status != "OK":
|
||||
continue
|
||||
raw = email.message_from_bytes(msg_data[0][1])
|
||||
sender = raw.get("From", "").strip()
|
||||
subj = raw.get("Subject", "(no subject)")
|
||||
msg_id = raw.get("Message-ID", str(num))
|
||||
|
||||
# Skip trusted senders
|
||||
trusted = any(t.lower() in sender.lower() for t in TRUSTED)
|
||||
if trusted:
|
||||
continue
|
||||
|
||||
# Skip if already reported
|
||||
h = hashlib.md5(msg_id.encode()).hexdigest()
|
||||
if h in seen:
|
||||
continue
|
||||
seen.add(h)
|
||||
|
||||
new_alerts.append((sender, subj))
|
||||
|
||||
conn.logout()
|
||||
save_seen(seen)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Sho'Nuff inbox check failed: {e}")
|
||||
return
|
||||
|
||||
if not new_alerts:
|
||||
return # silent
|
||||
|
||||
print(f"📬 Sho'Nuff received {len(new_alerts)} message(s) from unknown senders:\n")
|
||||
for sender, subj in new_alerts:
|
||||
print(f" • From: {sender}")
|
||||
print(f" Subject: {subj}")
|
||||
print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
# WISP Daily Backup Configuration
|
||||
# ==============================
|
||||
|
||||
vpn:
|
||||
# L2TP/IPsec connection to your home MikroTik gateway
|
||||
server_ip: "76.195.7.60"
|
||||
psk: "N2dinside1520!"
|
||||
username: "shonuff" # L2TP username
|
||||
password: "The.glow.1985" # L2TP password
|
||||
interface: "ppp0"
|
||||
|
||||
# For PoC with home router, we don't need tower routes yet.
|
||||
# Routes added here once we expand to tower CCRs.
|
||||
routes: []
|
||||
|
||||
s3:
|
||||
bucket: "mikrotik-ccr-backups" # Wasabi bucket name
|
||||
region: "us-east-1"
|
||||
prefix: "wisp-backups"
|
||||
endpoint: "https://s3.us-east-1.wasabisys.com"
|
||||
# Credentials in ~/.aws/credentials
|
||||
|
||||
towers:
|
||||
# Proof of concept: just your home router
|
||||
- name: "home-gateway"
|
||||
ip: "10.77.0.2"
|
||||
site: "home"
|
||||
ssh_port: 22
|
||||
ssh_user: "shonuff"
|
||||
ssh_key: "/root/.ssh/wisp_rsa"
|
||||
ssh_password: ""
|
||||
|
||||
# Future towers go here
|
||||
|
||||
backup:
|
||||
log_lines: 500
|
||||
temp_dir: "/root/.hermes/.backups/wisp-backups"
|
||||
compress: true
|
||||
retention_days: 7
|
||||
timeout_seconds: 60
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# wisp-vpn-keepalive.sh — Check VPN status and reconnect if down
|
||||
# Designed to run as a systemd timer every 5 minutes
|
||||
|
||||
VPN_SCRIPT="/root/.hermes/scripts/wisp-backup/home-router-vpn.sh"
|
||||
STATUS_FILE="/tmp/wisp-vpn-status"
|
||||
|
||||
# Check if VPN is up
|
||||
if "$VPN_SCRIPT" status >/dev/null 2>&1; then
|
||||
# VPN is up — write OK timestamp
|
||||
date +%s > "$STATUS_FILE"
|
||||
exit 0
|
||||
else
|
||||
# VPN is down — attempt reconnect
|
||||
LAST_OK=$(cat "$STATUS_FILE" 2>/dev/null || echo "0")
|
||||
NOW=$(date +%s)
|
||||
ELAPSED=$((NOW - LAST_OK))
|
||||
|
||||
echo "[-] WISP VPN is DOWN (was up ${ELAPSED}s ago). Attempting reconnect..."
|
||||
"$VPN_SCRIPT" up 2>&1
|
||||
|
||||
if "$VPN_SCRIPT" status >/dev/null 2>&1; then
|
||||
echo "[+] WISP VPN reconnected successfully"
|
||||
date +%s > "$STATUS_FILE"
|
||||
exit 0
|
||||
else
|
||||
echo "[-] WISP VPN reconnect failed"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
Executable
+237
@@ -0,0 +1,237 @@
|
||||
#!/bin/bash
|
||||
# wisp-vpn.sh — Connect/disconnect L2TP/IPsec VPN to MicroTik gateway
|
||||
# Usage: wisp-vpn.sh [up|down|status]
|
||||
|
||||
CONFIG_DIR="$(dirname "$0")"
|
||||
YAML="$CONFIG_DIR/config.yaml"
|
||||
|
||||
# Parse YAML values
|
||||
get_cfg() {
|
||||
local key="$1"
|
||||
grep -A50 "^vpn:" "$YAML" | grep "^ $key:" | sed 's/.*: //' | tr -d '"' | sed 's/ *#.*//' | head -1
|
||||
}
|
||||
|
||||
SERVER_IP=$(get_cfg server_ip)
|
||||
PSK=$(get_cfg psk)
|
||||
USERNAME=$(get_cfg username)
|
||||
PASSWORD=$(get_cfg password)
|
||||
IFACE=$(get_cfg interface)
|
||||
IFACE="${IFACE:-ppp0}"
|
||||
|
||||
L2TP_NAME="wisp-vpn"
|
||||
IPSEC_CONF="/etc/ipsec.conf"
|
||||
IPSEC_SECRETS="/etc/ipsec.secrets"
|
||||
XL2TPD_CONF="/etc/xl2tpd/xl2tpd.conf"
|
||||
L2TP_PEERS="/etc/ppp/peers/$L2TP_NAME"
|
||||
L2TP_SECRETS="/etc/ppp/chap-secrets"
|
||||
|
||||
case "$1" in
|
||||
up)
|
||||
echo "[*] Bringing up L2TP/IPsec VPN to $SERVER_IP ..."
|
||||
|
||||
# Write ipsec.conf for strongSwan compatibility
|
||||
cat > "$IPSEC_CONF" <<IPSEOF
|
||||
config setup
|
||||
charondebug="all"
|
||||
uniqueids=yes
|
||||
conn $L2TP_NAME
|
||||
auto=add
|
||||
keyexchange=ikev1
|
||||
authby=secret
|
||||
type=transport
|
||||
left=%defaultroute
|
||||
leftprotoport=17/1701
|
||||
right=$SERVER_IP
|
||||
rightprotoport=17/1701
|
||||
ike=aes128-sha1-modp1024
|
||||
esp=aes128-sha1-modp1024
|
||||
dpddelay=30
|
||||
dpdtimeout=120
|
||||
dpdaction=clear
|
||||
keyingtries=%forever
|
||||
ikelifetime=24h
|
||||
lifetime=8h
|
||||
margin=10m
|
||||
IPSEOF
|
||||
|
||||
# Write ipsec.secrets — specify exact IPs for strongSwan
|
||||
printf '5.161.114.8 %s : PSK "%s"\n' "$SERVER_IP" "$PSK" > "$IPSEC_SECRETS"
|
||||
|
||||
# Write xl2tpd.conf
|
||||
cat > "$XL2TPD_CONF" <<XL2EOF
|
||||
[global]
|
||||
port = 1701
|
||||
[lac $L2TP_NAME]
|
||||
lns = $SERVER_IP
|
||||
ppp debug = yes
|
||||
pppoptfile = /etc/ppp/options.$L2TP_NAME
|
||||
length bit = yes
|
||||
require pap = no
|
||||
require chap = yes
|
||||
autodial = yes
|
||||
redial = yes
|
||||
redial timeout = 5
|
||||
max redials = 3
|
||||
XL2EOF
|
||||
|
||||
# Write ppp options — NO replacedefaultroute (keeps internet on eth0)
|
||||
# Also: no defaultroute at all — we'll add the necessary routes manually
|
||||
# after the connection is established to avoid ppp0 recursion.
|
||||
cat > "/etc/ppp/options.$L2TP_NAME" <<PPPOEOF
|
||||
ipcp-accept-local
|
||||
ipcp-accept-remote
|
||||
refuse-eap
|
||||
noccp
|
||||
noauth
|
||||
idle 0
|
||||
maxfail 3
|
||||
debug
|
||||
name $USERNAME
|
||||
password $PASSWORD
|
||||
mtu 1400
|
||||
mru 1400
|
||||
usepeerdns
|
||||
lcp-echo-interval 10
|
||||
lcp-echo-failure 5
|
||||
PPPOEOF
|
||||
|
||||
# Write or update chap-secrets entry
|
||||
if grep -q "^\"$USERNAME\"" "$L2TP_SECRETS" 2>/dev/null; then
|
||||
# Entry exists — remove old line and add new one (password may have changed)
|
||||
sed -i "/^\"$USERNAME\"/d" "$L2TP_SECRETS" 2>/dev/null
|
||||
fi
|
||||
echo "\"$USERNAME\" * \"$PASSWORD\" *" >> "$L2TP_SECRETS"
|
||||
|
||||
# Ensure the L2TP server IP has a clean route through eth0 BEFORE ipsec starts
|
||||
# This prevents IPsec + pppd from creating a routing loop (ppp0 recursion)
|
||||
CURRENT_GW=$(ip route get 8.8.8.8 | awk '{print $3}' | head -1)
|
||||
if [ -n "$CURRENT_GW" ]; then
|
||||
ip route add "$SERVER_IP" via "$CURRENT_GW" dev eth0 metric 100 2>/dev/null || \
|
||||
ip route change "$SERVER_IP" via "$CURRENT_GW" dev eth0 metric 100 2>/dev/null
|
||||
echo "[route] $SERVER_IP via $CURRENT_GW (avoids ppp0 recursion)"
|
||||
fi
|
||||
|
||||
# Reload config — loads new connections added to ipsec.conf
|
||||
ipsec reload 2>/dev/null
|
||||
sleep 2
|
||||
ipsec up $L2TP_NAME 2>/dev/null
|
||||
sleep 3
|
||||
|
||||
systemctl restart xl2tpd 2>/dev/null
|
||||
sleep 3
|
||||
|
||||
# Trigger the connection
|
||||
echo "c $L2TP_NAME" > /var/run/xl2tpd/l2tp-control 2>/dev/null
|
||||
sleep 5
|
||||
|
||||
# Verify
|
||||
if ip addr show "$IFACE" 2>/dev/null | grep -q "inet "; then
|
||||
LOCAL_IP=$(ip addr show "$IFACE" | grep "inet " | awk '{print $2}')
|
||||
echo "[+] VPN connected! Interface: $IFACE, IP: $LOCAL_IP"
|
||||
|
||||
# Add route to peer's LAN subnet through ppp0 (if the peer address is known)
|
||||
# Without this, ppp0 recursion happens when pppd applies defaultroute
|
||||
PEER_IP=$(echo "$LOCAL_IP" | cut -d' ' -f1)
|
||||
echo "[*] Adding route to VPN peer's subnet via $IFACE ..."
|
||||
ip route add 192.168.88.0/24 dev "$IFACE" 2>/dev/null && \
|
||||
echo " [route] Added 192.168.88.0/24 via $IFACE" || \
|
||||
echo " [route] 192.168.88.0/24 already exists"
|
||||
|
||||
# Add static routes for routed tower subnets from config
|
||||
echo "[*] Adding static routes for tower subnets ..."
|
||||
ROUTES=$(grep -A50 "^vpn:" "$YAML" | grep -A50 "^ routes:" | grep "^- " | sed 's/.*- //' | tr -d '"')
|
||||
if [ -n "$ROUTES" ]; then
|
||||
echo "$ROUTES" | while read -r subnet; do
|
||||
if [ -n "$subnet" ]; then
|
||||
# Check if route already exists
|
||||
if ip route show "$subnet" 2>/dev/null | grep -q .; then
|
||||
echo " [route] $subnet already exists"
|
||||
else
|
||||
ip route add "$subnet" dev "$IFACE"
|
||||
echo " [route] Added $subnet via $IFACE"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
else
|
||||
echo " (no routes configured in config.yaml)"
|
||||
fi
|
||||
|
||||
# Safety: verify internet connectivity still works
|
||||
# If ppp's replacedefaultroute kills internet, kill ppp immediately
|
||||
echo "[*] Checking internet connectivity ..."
|
||||
if ping -c 1 -W 3 8.8.8.8 >/dev/null 2>&1 || ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then
|
||||
echo "[+] Internet reachable — VPN is safe"
|
||||
else
|
||||
echo "[-] INTERNET LOST after VPN connect!"
|
||||
echo " Tearing down VPN to restore connectivity ..."
|
||||
echo "d $L2TP_NAME" > /var/run/xl2tpd/l2tp-control 2>/dev/null
|
||||
pkill -f "pppd.*$L2TP_NAME" 2>/dev/null
|
||||
ipsec down $L2TP_NAME 2>/dev/null
|
||||
sleep 2
|
||||
echo "[-] VPN terminated — internet should be restored"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
else
|
||||
echo "[-] VPN connection failed (ppp0 not up)."
|
||||
echo " Cleaning up IPsec to avoid orphaned tunnel ..."
|
||||
echo "d $L2TP_NAME" > /var/run/xl2tpd/l2tp-control 2>/dev/null
|
||||
pkill -f "pppd.*$L2TP_NAME" 2>/dev/null
|
||||
ipsec down $L2TP_NAME 2>/dev/null
|
||||
sleep 2
|
||||
echo " Check logs: journalctl -u xl2tpd -n 20"
|
||||
echo " ipsec status"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
|
||||
down)
|
||||
echo "[*] Tearing down VPN ..."
|
||||
|
||||
# Remove static routes for tower subnets
|
||||
ROUTES=$(grep -A50 "^vpn:" "$YAML" | grep -A50 "^ routes:" | grep "^- " | sed 's/.*- //' | tr -d '"')
|
||||
if [ -n "$ROUTES" ]; then
|
||||
echo "[*] Removing static routes ..."
|
||||
echo "$ROUTES" | while read -r subnet; do
|
||||
if [ -n "$subnet" ]; then
|
||||
ip route del "$subnet" dev "$IFACE" 2>/dev/null && \
|
||||
echo " [route] Removed $subnet" || true
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "d $L2TP_NAME" > /var/run/xl2tpd/l2tp-control 2>/dev/null
|
||||
ipsec down $L2TP_NAME 2>/dev/null
|
||||
sleep 2
|
||||
# Kill the ppp interface
|
||||
pkill -f "pppd.*$L2TP_NAME" 2>/dev/null
|
||||
sleep 1
|
||||
# Safety: ensure default route via eth0 is restored
|
||||
DEFAULT_GW=$(ip route show default | grep eth0 | head -1 || true)
|
||||
if [ -z "$DEFAULT_GW" ]; then
|
||||
GW=$(ip route get 8.8.8.8 | awk '{print $3}' | head -1)
|
||||
DEV=$(ip route get 8.8.8.8 | awk '{print $5}' | head -1)
|
||||
if [ -n "$GW" ] && [ -n "$DEV" ]; then
|
||||
echo " [safety] Default route lost — restoring via $DEV"
|
||||
ip route add default via "$GW" dev "$DEV" metric 100 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
echo "[+] VPN disconnected"
|
||||
;;
|
||||
|
||||
status)
|
||||
if ip addr show "$IFACE" 2>/dev/null | grep -q "inet "; then
|
||||
LOCAL_IP=$(ip addr show "$IFACE" | grep "inet " | awk '{print $2}')
|
||||
echo "[+] VPN is UP — $IFACE: $LOCAL_IP"
|
||||
exit 0
|
||||
else
|
||||
echo "[-] VPN is DOWN"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 {up|down|status}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# home-router-watchdog.sh — Check home router via WireGuard tunnel
|
||||
# Silent on success (no_agent mode). Alerts only on failure.
|
||||
|
||||
TUNNEL_IP="10.77.0.2"
|
||||
PING_COUNT=6
|
||||
PING_INTERVAL=10
|
||||
SUCCESS=0
|
||||
|
||||
for i in $(seq 1 $PING_COUNT); do
|
||||
if ping -c 1 -W 5 "$TUNNEL_IP" >/dev/null 2>&1; then
|
||||
SUCCESS=$((SUCCESS + 1))
|
||||
if [ "$SUCCESS" -ge 2 ]; then
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
sleep "$PING_INTERVAL"
|
||||
done
|
||||
|
||||
echo "[-] Home router offline — not responding through WireGuard tunnel"
|
||||
echo "[-] Router IP: $TUNNEL_IP"
|
||||
exit 1
|
||||
Executable
+366
@@ -0,0 +1,366 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
wisp-backup.py — Pull configs and logs from CCR towers and upload to S3.
|
||||
|
||||
Usage:
|
||||
./wisp-backup.py # Uses config.yaml in same directory
|
||||
./wisp-backup.py --config /path/to/config.yaml
|
||||
./wisp-backup.py --vpn-only # Just connect VPN, don't backup
|
||||
./wisp-backup.py --dry-run # Show what would be done, don't upload
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import gzip
|
||||
import io
|
||||
import os
|
||||
import paramiko
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
|
||||
def load_config(path: str) -> dict:
|
||||
with open(path, 'r') as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
return cfg
|
||||
|
||||
|
||||
def run_vpn_script(action: str, script_dir: str) -> bool:
|
||||
"""Call home-router-vpn.sh up/down/status."""
|
||||
vpn_script = os.path.join(script_dir, "home-router-vpn.sh")
|
||||
if not os.path.exists(vpn_script):
|
||||
print(f"[!] VPN script not found: {vpn_script}")
|
||||
return False
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", vpn_script, action],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
for line in result.stdout.splitlines():
|
||||
print(f" {line}")
|
||||
if result.stderr.strip():
|
||||
for line in result.stderr.splitlines():
|
||||
print(f" [stderr] {line}")
|
||||
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def ssh_connect(host: str, port: int, username: str, key_path: str = None,
|
||||
password: str = None, timeout: int = 30):
|
||||
"""Open an SSH connection to a CCR router."""
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
if key_path and os.path.exists(key_path):
|
||||
# Try Ed25519 first, fall back to RSA
|
||||
try:
|
||||
key = paramiko.Ed25519Key.from_private_key_file(key_path)
|
||||
except paramiko.ssh_exception.SSHException:
|
||||
key = paramiko.RSAKey.from_private_key_file(key_path)
|
||||
client.connect(host, port=port, username=username, pkey=key,
|
||||
timeout=timeout, allow_agent=False, look_for_keys=False)
|
||||
elif password:
|
||||
client.connect(host, port=port, username=username, password=password,
|
||||
timeout=timeout, allow_agent=False, look_for_keys=False)
|
||||
else:
|
||||
# Try default key locations
|
||||
client.connect(host, port=port, username=username,
|
||||
timeout=timeout, allow_agent=True)
|
||||
return client
|
||||
|
||||
|
||||
def run_ssh_command(client, command: str, timeout: int = 30) -> str:
|
||||
"""Run a command over SSH and return stdout."""
|
||||
_, stdout, stderr = client.exec_command(command, timeout=timeout)
|
||||
exit_status = stdout.channel.recv_exit_status()
|
||||
output = stdout.read().decode('utf-8', errors='replace')
|
||||
|
||||
if exit_status != 0:
|
||||
err = stderr.read().decode('utf-8', errors='replace')
|
||||
print(f" [warn] Command exited {exit_status}: {err.strip()}")
|
||||
return output
|
||||
|
||||
|
||||
def fetch_ccr_config(client) -> str:
|
||||
"""Fetch full CCR config via /export terse."""
|
||||
print(" Fetching config ...")
|
||||
config = run_ssh_command(client, "/export terse")
|
||||
return config
|
||||
|
||||
|
||||
def fetch_ccr_logs(client, lines: int = 500) -> str:
|
||||
"""Fetch recent CCR log entries."""
|
||||
print(f" Fetching last {lines} log lines ...")
|
||||
logs = run_ssh_command(client, f"/log print without-paging count-only={lines}")
|
||||
return logs
|
||||
|
||||
|
||||
def compress_text(text: str, filename: str) -> str:
|
||||
"""Write text to a .gz file and return the path."""
|
||||
gz_path = filename + ".gz"
|
||||
with gzip.open(gz_path, 'wt', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
return gz_path
|
||||
|
||||
|
||||
def upload_to_s3(local_path: str, s3_path: str, bucket: str, region: str,
|
||||
endpoint: str = None):
|
||||
"""Upload file to S3 (or Wasabi/S3-compatible) using awscli."""
|
||||
dest = f"s3://{bucket}/{s3_path}"
|
||||
cmd = ["aws", "s3", "cp", local_path, dest, "--region", region]
|
||||
if endpoint:
|
||||
cmd += ["--endpoint-url", endpoint]
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=60
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f" [s3] Uploaded → {dest}")
|
||||
else:
|
||||
print(f" [s3] Upload failed: {result.stderr.strip()}")
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def backup_tower(tower: dict, cfg: dict, date_str: str, temp_dir: str) -> dict:
|
||||
"""Backup a single tower. Returns dict of results."""
|
||||
name = tower['name']
|
||||
ip = tower['ip']
|
||||
site = tower['site']
|
||||
port = tower.get('ssh_port', 22)
|
||||
user = tower.get('ssh_user', 'admin')
|
||||
key_path = tower.get('ssh_key')
|
||||
password = tower.get('ssh_password')
|
||||
log_lines = cfg.get('backup', {}).get('log_lines', 500)
|
||||
compress = cfg.get('backup', {}).get('compress', True)
|
||||
timeout = cfg.get('backup', {}).get('timeout_seconds', 60)
|
||||
|
||||
s3_cfg = cfg.get('s3', {})
|
||||
bucket = s3_cfg['bucket']
|
||||
region = s3_cfg.get('region', 'us-east-1')
|
||||
prefix = s3_cfg.get('prefix', 'wisp-backups')
|
||||
endpoint = s3_cfg.get('endpoint') # Wasabi or S3-compatible endpoint
|
||||
|
||||
results = {
|
||||
'tower': name,
|
||||
'site': site,
|
||||
'ip': ip,
|
||||
'config': False,
|
||||
'logs': False,
|
||||
'error': None
|
||||
}
|
||||
|
||||
# Create local dirs
|
||||
tower_temp = Path(temp_dir) / site / name / date_str
|
||||
tower_temp.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
s3_config_base = f"{prefix}/configs/{site}/{date_str}"
|
||||
s3_logs_base = f"{prefix}/logs/{site}/{date_str}"
|
||||
|
||||
print(f"\n [{name}] ({ip}) — backing up ...")
|
||||
|
||||
try:
|
||||
client = ssh_connect(ip, port, user, key_path, password, timeout=timeout)
|
||||
except Exception as e:
|
||||
results['error'] = f"SSH connection failed: {e}"
|
||||
print(f" [FAIL] {results['error']}")
|
||||
return results
|
||||
|
||||
try:
|
||||
# --- Config ---
|
||||
config_text = fetch_ccr_config(client)
|
||||
config_filename = f"{tower_temp}/{name}-{date_str}.rsc"
|
||||
with open(config_filename, 'w') as f:
|
||||
f.write(config_text)
|
||||
|
||||
if compress:
|
||||
config_filename = compress_text(config_text,
|
||||
str(tower_temp / f"{name}-{date_str}.rsc"))
|
||||
remote_config = f"{s3_config_base}/{name}-{date_str}.rsc.gz"
|
||||
else:
|
||||
remote_config = f"{s3_config_base}/{name}-{date_str}.rsc"
|
||||
|
||||
upload_to_s3(config_filename, remote_config, bucket, region, endpoint)
|
||||
results['config'] = True
|
||||
|
||||
# --- Logs ---
|
||||
logs_text = fetch_ccr_logs(client, log_lines)
|
||||
log_filename = f"{tower_temp}/{name}-{date_str}.log"
|
||||
with open(log_filename, 'w') as f:
|
||||
f.write(logs_text)
|
||||
|
||||
if compress:
|
||||
log_filename = compress_text(logs_text,
|
||||
str(tower_temp / f"{name}-{date_str}.log"))
|
||||
remote_log = f"{s3_logs_base}/{name}-{date_str}.log.gz"
|
||||
else:
|
||||
remote_log = f"{s3_logs_base}/{name}-{date_str}.log"
|
||||
|
||||
upload_to_s3(log_filename, remote_log, bucket, region, endpoint)
|
||||
results['logs'] = True
|
||||
|
||||
except Exception as e:
|
||||
results['error'] = str(e)
|
||||
print(f" [FAIL] {e}")
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def cleanup_old_local(temp_dir: str, retention_days: int):
|
||||
"""Remove local backup copies older than retention_days."""
|
||||
cutoff = time.time() - (retention_days * 86400)
|
||||
removed = 0
|
||||
for root, dirs, files in os.walk(temp_dir):
|
||||
for f in files:
|
||||
path = os.path.join(root, f)
|
||||
if os.path.getmtime(path) < cutoff:
|
||||
os.remove(path)
|
||||
removed += 1
|
||||
# Clean empty dirs
|
||||
for d in dirs:
|
||||
dpath = os.path.join(root, d)
|
||||
try:
|
||||
os.rmdir(dpath)
|
||||
except OSError:
|
||||
pass
|
||||
if removed:
|
||||
print(f"[*] Cleaned up {removed} old local files")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="WISP CCR Backup to S3")
|
||||
parser.add_argument('--config', default=None,
|
||||
help='Path to config.yaml (default: ./config.yaml)')
|
||||
parser.add_argument('--vpn-only', action='store_true',
|
||||
help='Just connect VPN, do not backup')
|
||||
parser.add_argument('--dry-run', action='store_true',
|
||||
help='Show what would be done without executing')
|
||||
parser.add_argument('--tower', default=None,
|
||||
help='Backup only a specific tower name')
|
||||
parser.add_argument('--parallel', type=int, default=5,
|
||||
help='Max parallel tower backups (default: 5)')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Resolve config path
|
||||
if args.config:
|
||||
config_path = args.config
|
||||
else:
|
||||
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
||||
"config.yaml")
|
||||
|
||||
if not os.path.exists(config_path):
|
||||
print(f"[!] Config not found: {config_path}")
|
||||
print(" Create it from the template or pass --config")
|
||||
sys.exit(1)
|
||||
|
||||
cfg = load_config(config_path)
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
temp_dir = cfg.get('backup', {}).get('temp_dir', '/tmp/wisp-backups')
|
||||
retention_days = cfg.get('backup', {}).get('retention_days', 7)
|
||||
|
||||
date_str = datetime.datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
print(f"╔══════════════════════════════════════════╗")
|
||||
print(f"║ WISP Backup — {date_str}")
|
||||
print(f"╚══════════════════════════════════════════╝")
|
||||
|
||||
# Step 1: Connect VPN (only if router not already reachable via WireGuard)
|
||||
vpn_was_connected = False
|
||||
print("\n[*] Step 1: Checking connectivity via WireGuard ...")
|
||||
router_ip = cfg.get('towers', [{}])[0].get('ip', '10.77.0.2') if cfg.get('towers') else '10.77.0.2'
|
||||
ping_check = subprocess.run(
|
||||
["ping", "-c", "1", "-W", "2", router_ip],
|
||||
capture_output=True, timeout=5
|
||||
)
|
||||
if ping_check.returncode == 0:
|
||||
print(f" [+] Router reachable at {router_ip} via WireGuard — skipping VPN")
|
||||
else:
|
||||
print(f" [-] Router not reachable via WireGuard, trying L2TP/IPsec VPN ...")
|
||||
if not run_vpn_script("up", script_dir):
|
||||
print("[-] VPN connection failed. Aborting.")
|
||||
sys.exit(1)
|
||||
vpn_was_connected = True
|
||||
|
||||
# Safety: confirm internet still works before proceeding
|
||||
print("[*] Verifying internet connectivity ...")
|
||||
check = subprocess.run(
|
||||
["ping", "-c", "1", "-W", "3", "8.8.8.8"],
|
||||
capture_output=True, timeout=10
|
||||
)
|
||||
if check.returncode != 0:
|
||||
print("[-] Internet connectivity issue after VPN/backup prep")
|
||||
if vpn_was_connected:
|
||||
run_vpn_script("down", script_dir)
|
||||
sys.exit(1)
|
||||
print("[+] Internet OK — proceeding with backup")
|
||||
|
||||
if args.vpn_only:
|
||||
print("\n[*] VPN connected (--vpn-only mode). Exiting.")
|
||||
print(" Run 'home-router-vpn.sh down' to disconnect.")
|
||||
return
|
||||
|
||||
# Step 2: Backup towers
|
||||
print("\n[*] Step 2: Backing up towers ...")
|
||||
towers = cfg.get('towers', [])
|
||||
if args.tower:
|
||||
towers = [t for t in towers if t['name'] == args.tower]
|
||||
if not towers:
|
||||
print(f"[-] Tower '{args.tower}' not found in config")
|
||||
if vpn_was_connected:
|
||||
run_vpn_script("down", script_dir)
|
||||
sys.exit(1)
|
||||
|
||||
if args.dry_run:
|
||||
print("\n [DRY RUN] Would backup these towers:")
|
||||
for t in towers:
|
||||
print(f" - {t['name']} ({t['ip']}) @ site: {t['site']}")
|
||||
print(f"\n [DRY RUN] Would upload to s3://{cfg['s3']['bucket']}/")
|
||||
if vpn_was_connected:
|
||||
run_vpn_script("down", script_dir)
|
||||
return
|
||||
|
||||
results = []
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as executor:
|
||||
futures = {
|
||||
executor.submit(backup_tower, t, cfg, date_str, temp_dir): t
|
||||
for t in towers
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
results.append(future.result())
|
||||
|
||||
# Step 3: Disconnect VPN if we connected it
|
||||
print("\n[*] Step 3: Disconnecting VPN ...")
|
||||
if vpn_was_connected:
|
||||
run_vpn_script("down", script_dir)
|
||||
else:
|
||||
print(" (WireGuard already active — no VPN teardown needed)")
|
||||
|
||||
# Step 4: Cleanup
|
||||
print("\n[*] Step 4: Cleaning up local temp files ...")
|
||||
cleanup_old_local(temp_dir, retention_days)
|
||||
|
||||
# Summary
|
||||
success = [r for r in results if r['error'] is None]
|
||||
failed = [r for r in results if r['error'] is not None]
|
||||
|
||||
print(f"\n╔══════════════════════════════════════════╗")
|
||||
print(f"║ Summary: {len(success)} OK, {len(failed)} Failed")
|
||||
print(f"╚══════════════════════════════════════════╝")
|
||||
for r in success:
|
||||
ck = "✓" if r['config'] else "✗"
|
||||
lk = "✓" if r['logs'] else "✗"
|
||||
print(f" ✓ {r['tower']} ({r['site']}) — config {ck} logs {lk}")
|
||||
for r in failed:
|
||||
print(f" ✗ {r['tower']} ({r['site']}) — {r['error']}")
|
||||
|
||||
if failed:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user