56 lines
2.1 KiB
Bash
Executable File
56 lines
2.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Watchdog for the email triage cron job (5929c5f1deff).
|
|
# Checks if the most recent run ended with an error.
|
|
# Silent (no output) when OK, alerts when failed.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="/root/.hermes"
|
|
DB="$SCRIPT_DIR/state.db"
|
|
JOB_PREFIX="cron_5929c5f1deff"
|
|
LAST_GOOD_FILE="$SCRIPT_DIR/email_triage/.watchdog_last_ok"
|
|
mkdir -p "$(dirname "$LAST_GOOD_FILE")"
|
|
|
|
# Query the most recent cron session for this job
|
|
RESULT=$(python3 -c "
|
|
import sqlite3, sys, json
|
|
try:
|
|
con = sqlite3.connect('$DB')
|
|
cur = con.cursor()
|
|
row = cur.execute(\"\"\"select started_at, end_reason from sessions
|
|
where id like '$JOB_PREFIX%' and end_reason is not null
|
|
order by started_at desc limit 1\"\"\").fetchone()
|
|
con.close()
|
|
if row:
|
|
print(json.dumps({'started_at': row[0], 'end_reason': row[1]}))
|
|
else:
|
|
print('{}')
|
|
except Exception as e:
|
|
print(json.dumps({'error': str(e)}))
|
|
" 2>/dev/null)
|
|
|
|
LAST_STATUS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('end_reason',''))" 2>/dev/null || echo "")
|
|
LAST_TS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('started_at',''))" 2>/dev/null || echo "")
|
|
|
|
if [ -z "$LAST_STATUS" ]; then
|
|
# No data yet — first run, don't alert
|
|
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$LAST_STATUS" = "error" ] || [ "$LAST_STATUS" = "failed" ]; then
|
|
# Failure — only alert if we haven't already reported this one
|
|
if [ -f "$LAST_GOOD_FILE" ]; then
|
|
PREV_GOOD=$(cat "$LAST_GOOD_FILE")
|
|
if [ "$LAST_TS" != "$PREV_GOOD" ]; then
|
|
echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
|
|
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
|
fi
|
|
else
|
|
echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
|
|
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
|
fi
|
|
else
|
|
# All good — mark this timestamp so we only alert on new failures
|
|
echo "$LAST_TS" > "$LAST_GOOD_FILE"
|
|
fi
|