Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README
This commit is contained in:
root
Executable
+211
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
# VPS Resource Threshold Alert Checker
|
||||
# Monitors RAM, disk, and CPU across all servers.
|
||||
# Outputs alert messages to stdout (delivered by no-agent cron to Telegram)
|
||||
# Thresholds: 80%, 90%, 95%
|
||||
# Tracks sent alerts in /root/.hermes/data/threshold-alerts.json
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
DATA_DIR="/root/.hermes/data"
|
||||
ALERTS_FILE="${DATA_DIR}/threshold-alerts.json"
|
||||
SSH_KEY="/root/.ssh/itpp-infra"
|
||||
SSH_OPTS="-i ${SSH_KEY} -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes"
|
||||
NOW_EPOCH=$(date +%s)
|
||||
NOW_ISO=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
RE_ALERT_SECONDS=86400 # 24 hours
|
||||
|
||||
# Thresholds to check
|
||||
THRESHOLDS=(80 90 95)
|
||||
|
||||
# Server list: "hostname|ip|local_flag" (local=1 means run commands directly)
|
||||
declare -a SERVERS=(
|
||||
"app1-bu|5.161.114.8|0"
|
||||
"ai|178.156.167.181|0"
|
||||
"hudu|178.156.130.130|0"
|
||||
"unifi|178.156.131.57|0"
|
||||
"unms|5.161.225.131|0"
|
||||
"wphost02|5.161.62.38|0"
|
||||
"n8n|87.99.144.163|0"
|
||||
"docker|178.156.168.35|0"
|
||||
"fleettrack360|178.156.149.32|0"
|
||||
"core|152.53.192.33|1"
|
||||
)
|
||||
|
||||
# Ensure data directory exists
|
||||
mkdir -p "${DATA_DIR}"
|
||||
|
||||
# Initialize alerts file if it doesn't exist
|
||||
if [[ ! -f "${ALERTS_FILE}" ]]; then
|
||||
echo "{}" > "${ALERTS_FILE}"
|
||||
fi
|
||||
|
||||
# Read current alerts
|
||||
ALERTS_JSON=$(cat "${ALERTS_FILE}")
|
||||
|
||||
# Function to read a value from the alerts JSON
|
||||
get_alert_ts() {
|
||||
local key="$1"
|
||||
python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('${key}',''))" <<< "${ALERTS_JSON}"
|
||||
}
|
||||
|
||||
# Function to update a timestamp in the alerts JSON
|
||||
update_alert_ts() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
ALERTS_JSON=$(python3 -c "
|
||||
import json,sys
|
||||
d = json.loads('${ALERTS_JSON//\'/\\\'}')
|
||||
d['${key}'] = '${value}'
|
||||
print(json.dumps(d))
|
||||
")
|
||||
}
|
||||
|
||||
# Function to remove a key from the alerts JSON (reset)
|
||||
remove_alert_key() {
|
||||
local key="$1"
|
||||
ALERTS_JSON=$(python3 -c "
|
||||
import json,sys
|
||||
d = json.loads('${ALERTS_JSON//\'/\\\'}')
|
||||
d.pop('${key}', None)
|
||||
print(json.dumps(d))
|
||||
")
|
||||
}
|
||||
|
||||
# Save the alerts JSON back
|
||||
save_alerts() {
|
||||
cat > "${ALERTS_FILE}" <<< "${ALERTS_JSON}"
|
||||
}
|
||||
|
||||
# Check if we should alert for a threshold
|
||||
# Returns 0 (should alert) or 1 (should not)
|
||||
should_alert() {
|
||||
local key="$1"
|
||||
local ts
|
||||
ts=$(get_alert_ts "${key}")
|
||||
if [[ -z "${ts}" ]]; then
|
||||
return 0 # Never alerted before, should alert
|
||||
fi
|
||||
# Parse the timestamp and check if 24h have passed
|
||||
local last_epoch
|
||||
last_epoch=$(date -d "${ts}" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "${ts}" +%s 2>/dev/null || echo "0")
|
||||
if [[ -z "${last_epoch}" || "${last_epoch}" -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
local elapsed=$(( NOW_EPOCH - last_epoch ))
|
||||
if [[ "${elapsed}" -ge "${RE_ALERT_SECONDS}" ]]; then
|
||||
return 0 # 24h passed, re-alert
|
||||
fi
|
||||
return 1 # Recently alerted, skip
|
||||
}
|
||||
|
||||
# Run a command on a server (local or via SSH)
|
||||
run_cmd() {
|
||||
local hostname="$1"
|
||||
local ip="$2"
|
||||
local is_local="$3"
|
||||
local cmd="$4"
|
||||
|
||||
if [[ "${is_local}" == "1" ]]; then
|
||||
eval "${cmd}"
|
||||
else
|
||||
ssh ${SSH_OPTS} "root@${ip}" "${cmd}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# Build alert message accumulator
|
||||
ALERT_LINES=""
|
||||
ALERT_COUNT=0
|
||||
|
||||
# Process each server
|
||||
for server_entry in "${SERVERS[@]}"; do
|
||||
IFS='|' read -r hostname ip is_local <<< "${server_entry}"
|
||||
|
||||
# Gather metrics
|
||||
disk_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "df -h / | awk 'NR==2 {print \$5}' | tr -d '%'")
|
||||
ram_pct=$(run_cmd "${hostname}" "${ip}" "${is_local}" "free | awk '/Mem:/ {printf \"%.0f\", \$3/\$2 * 100}'")
|
||||
cpu_raw=$(run_cmd "${hostname}" "${ip}" "${is_local}" "top -bn1 | awk '/Cpu\(s\)/ {print \$2}'")
|
||||
core_count=$(run_cmd "${hostname}" "${ip}" "${is_local}" "nproc")
|
||||
|
||||
# Validate we got numbers (skip server if unreachable)
|
||||
disk_pct="${disk_pct:-0}"
|
||||
ram_pct="${ram_pct:-0}"
|
||||
cpu_raw="${cpu_raw:-0}"
|
||||
core_count="${core_count:-1}"
|
||||
|
||||
# Convert CPU % to integer (top -bn1 gives float like 0.0 or 2.3)
|
||||
cpu_pct=$(echo "${cpu_raw}" | cut -d. -f1)
|
||||
cpu_pct="${cpu_pct:-0}"
|
||||
|
||||
# Ensure numeric
|
||||
disk_pct=$(echo "${disk_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
ram_pct=$(echo "${ram_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
cpu_pct=$(echo "${cpu_pct}" | grep -oE '^[0-9]+$' || echo "0")
|
||||
core_count=$(echo "${core_count}" | grep -oE '^[0-9]+$' || echo "1")
|
||||
if [[ "${core_count}" -lt 1 ]]; then core_count=1; fi
|
||||
|
||||
# Track which thresholds are crossed for this server
|
||||
SERVER_ALERTS=""
|
||||
SERVER_ALERT_COUNT=0
|
||||
|
||||
# Check disk thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_disk_${thresh}"
|
||||
if [[ "${disk_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ DISK at ${disk_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
# Below threshold — reset alert so it can re-alert if it crosses again
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check RAM thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_ram_${thresh}"
|
||||
if [[ "${ram_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ RAM at ${ram_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check CPU thresholds
|
||||
for thresh in "${THRESHOLDS[@]}"; do
|
||||
alert_key="${hostname}_cpu_${thresh}"
|
||||
if [[ "${cpu_pct}" -ge "${thresh}" ]]; then
|
||||
if should_alert "${alert_key}"; then
|
||||
SERVER_ALERTS+=" ⚠️ CPU at ${cpu_pct}% (threshold: ${thresh}%)"$'\n'
|
||||
SERVER_ALERT_COUNT=$((SERVER_ALERT_COUNT + 1))
|
||||
update_alert_ts "${alert_key}" "${NOW_ISO}"
|
||||
fi
|
||||
else
|
||||
remove_alert_key "${alert_key}"
|
||||
fi
|
||||
done
|
||||
|
||||
# If this server has alerts, add to consolidated message
|
||||
if [[ "${SERVER_ALERT_COUNT}" -gt 0 ]]; then
|
||||
ALERT_LINES+="🚨 VPS Alert: ${hostname}"$'\n'
|
||||
ALERT_LINES+="${SERVER_ALERTS}"
|
||||
ALERT_COUNT=$((ALERT_COUNT + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# Save updated alerts
|
||||
save_alerts
|
||||
|
||||
# Output any alert messages (stdout goes to Telegram via cron)
|
||||
if [[ "${ALERT_COUNT}" -gt 0 ]]; then
|
||||
echo "${ALERT_LINES}"
|
||||
else
|
||||
# Silent on success - no output = no delivery
|
||||
:
|
||||
fi
|
||||
Reference in New Issue
Block a user