hermes-recovery/scripts/vps-threshold-check.sh

#!/usr/bin/env bash
# VPS Resource Threshold Alert Checker
# Monitors RAM, disk, and CPU across all servers.
# Outputs alert messages to stdout (delivered by no-agent cron to Telegram)
# Thresholds: 80%, 90%, 95%
# Tracks sent alerts in /root/.hermes/data/threshold-alerts.json
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
DATA_DIR="/root/.hermes/data"
ALERTS_FILE="${DATA_DIR}/threshold-alerts.json"
SSH_KEY="/root/.ssh/itpp-infra"
SSH_OPTS="-i ${SSH_KEY} -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes"
NOW_EPOCH=$(date +%s)
NOW_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
RE_ALERT_SECONDS=86400  # 24 hours

# Thresholds to check
THRESHOLDS=(80 90 95)

# Server list: "hostname|ip|local_flag" (local=1 means run commands directly)
declare -a SERVERS=(
  "app1-bu|5.161.114.8|0"
  "ai|178.156.167.181|0"
  "hudu|178.156.130.130|0"
  "unifi|178.156.131.57|0"
  "unms|5.161.225.131|0"
  "wphost02|5.161.62.38|0"
  "n8n|87.99.144.163|0"
  "docker|178.156.168.35|0"
  "fleettrack360|178.156.149.32|0"
  "core|152.53.192.33|1"
)

# Ensure data directory exists
mkdir -p "${DATA_DIR}"

# Initialize alerts file if it doesn't exist
if [[ ! -f "${ALERTS_FILE}" ]]; then
  echo "{}" > "${ALERTS_FILE}"
fi

# Read current alerts
ALERTS_JSON=$(cat "${ALERTS_FILE}")

# Function to read a value from the alerts JSON
get_alert_ts() {
  local key="$1"
  python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('${key}',''))" <<< "${ALERTS_JSON}"
}

# Function to update a timestamp in the alerts JSON
update_alert_ts() {
  local key="$1"
  local value="$2"
  ALERTS_JSON=$(python3 -c "
import json,sys
d = json.loads('${ALERTS_JSON//\'/\\\'}')
d['${key}'] = '${value}'
print(json.dumps(d))
")
}

# Function to remove a key from the alerts JSON (reset)
remove_alert_key() {
  local key="$1"
  ALERTS_JSON=$(python3 -c "
import json,sys
d = json.loads('${ALERTS_JSON//\'/\\\'}')
d.pop('${key}', None)
print(json.dumps(d))
")
}

# Save the alerts JSON back
save_alerts() {
  cat > "${ALERTS_FILE}" <<< "${ALERTS_JSON}"
}

# Check if we should alert for a threshold
# Returns 0 (should alert) or 1 (should not)
should_alert() {
  local key="$1"
  local ts
  ts=$(get_alert_ts "${key}")
  if [[ -z "${ts}" ]]; then
    return 0  # Never alerted before, should alert
  fi
  # Parse the timestamp and check if 24h have passed
  local last_epoch
  last_epoch=$(date -d "${ts}" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "${ts}" +%s 2>/dev/null || echo "0")
  if [[ -z "${last_epoch}" || "${last_epoch}" -eq 0 ]]; then
    return 0
  fi
  local elapsed=$(( NOW_EPOCH - last_epoch ))
  if [[ "${elapsed}" -ge "${RE_ALERT_SECONDS}" ]]; then
    return 0  # 24h passed, re-alert
  fi
  return 1  # Recently alerted, skip
}

# Run a command on a server (local or via SSH)
run_cmd() {
  local hostname="$1"
  local ip="$2"
  local is_local="$3"
  local cmd="$4"

  if [[ "${is_local}" == "1" ]]; then
    eval "${cmd}"
  else
    ssh ${SSH_OPTS} "root@${ip}" "${cmd}" 2>/dev/null || true
  fi
}

# Build alert message accumulator
ALERT_LINES=""
ALERT_COUNT=0

# Process each server
for server_entry in "${SERVERS[@]}"; do
  IFS='|' read -r hostname ip is_local <<< "${server_entry}"

  # Gather metrics
  disk_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "df -h / | awk 'NR==2 {print \$5}' | tr -d '%'")
  ram_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "free | awk '/Mem:/ {printf \"%.0f\", \$3/\$2 * 100}'")
  cpu_raw=$(run_cmd "${hostname}" "${ip}" "${is_local}" "top -bn1 | awk '/Cpu\(s\)/ {print \$2}'")
  core_count=$(run_cmd "${hostname}" "${ip}" "${is_local}" "nproc")

  # Validate we got numbers (skip server if unreachable)
  disk_pct="${disk_pct:-0}"
  ram_pct="${ram_pct:-0}"
  cpu_raw="${cpu_raw:-0}"
  core_count="${core_count:-1}"

  # Convert CPU % to integer (top -bn1 gives float like 0.0 or 2.3)
  cpu_pct=$(echo "${cpu_raw}" | cut -d. -f1)
  cpu_pct="${cpu_pct:-0}"

  # Ensure numeric
  disk_pct=$(echo "${disk_pct}" | grep -oE '^[0-9]+$' || echo "0")
  ram_pct=$(echo "${ram_pct}" | grep -oE '^[0-9]+$' || echo "0")
  cpu_pct=$(echo "${cpu_pct}" | grep -oE '^[0-9]+$' || echo "0")
  core_count=$(echo "${core_count}" | grep -oE '^[0-9]+$' || echo "1")
  if [[ "${core_count}" -lt 1 ]]; then core_count=1; fi

  # Track which thresholds are crossed for this server
  SERVER_ALERTS=""
  SERVER_ALERT_COUNT=0

  # Check disk thresholds
  for thresh in "${THRESHOLDS[@]}"; do
    alert_key="${hostname}_disk_${thresh}"
    if [[ "${disk_pct}" -ge "${thresh}" ]]; then
      if should_alert "${alert_key}"; then
        SERVER_ALERTS+="  ⚠️ DISK at ${disk_pct}% (threshold: ${thresh}%)"$'\n'
        SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
        update_alert_ts "${alert_key}" "${NOW_ISO}"
      fi
    else
      # Below threshold — reset alert so it can re-alert if it crosses again
      remove_alert_key "${alert_key}"
    fi
  done

  # Check RAM thresholds
  for thresh in "${THRESHOLDS[@]}"; do
    alert_key="${hostname}_ram_${thresh}"
    if [[ "${ram_pct}" -ge "${thresh}" ]]; then
      if should_alert "${alert_key}"; then
        SERVER_ALERTS+="  ⚠️ RAM at ${ram_pct}% (threshold: ${thresh}%)"$'\n'
        SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
        update_alert_ts "${alert_key}" "${NOW_ISO}"
      fi
    else
      remove_alert_key "${alert_key}"
    fi
  done

  # Check CPU thresholds
  for thresh in "${THRESHOLDS[@]}"; do
    alert_key="${hostname}_cpu_${thresh}"
    if [[ "${cpu_pct}" -ge "${thresh}" ]]; then
      if should_alert "${alert_key}"; then
        SERVER_ALERTS+="  ⚠️ CPU at ${cpu_pct}% (threshold: ${thresh}%)"$'\n'
        SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
        update_alert_ts "${alert_key}" "${NOW_ISO}"
      fi
    else
      remove_alert_key "${alert_key}"
    fi
  done

  # If this server has alerts, add to consolidated message
  if [[ "${SERVER_ALERT_COUNT}" -gt 0 ]]; then
    ALERT_LINES+="🚨 VPS Alert: ${hostname}"$'\n'
    ALERT_LINES+="${SERVER_ALERTS}"
    ALERT_COUNT=$((ALERT_COUNT + 1))
  fi
done

# Save updated alerts
save_alerts

# Output any alert messages (stdout goes to Telegram via cron)
if [[ "${ALERT_COUNT}" -gt 0 ]]; then
  echo "${ALERT_LINES}"
else
  # Silent on success - no output = no delivery
  :
fi