#!/usr/bin/env bash # Watchdog for the email triage cron job (5929c5f1deff). # Checks if the most recent run ended with an error. # Silent (no output) when OK, alerts when failed. set -euo pipefail SCRIPT_DIR="/root/.hermes" DB="$SCRIPT_DIR/state.db" JOB_PREFIX="cron_5929c5f1deff" LAST_GOOD_FILE="$SCRIPT_DIR/email_triage/.watchdog_last_ok" mkdir -p "$(dirname "$LAST_GOOD_FILE")" # Query the most recent cron session for this job RESULT=$(python3 -c " import sqlite3, sys, json try: con = sqlite3.connect('$DB') cur = con.cursor() row = cur.execute(\"\"\"select started_at, end_reason from sessions where id like '$JOB_PREFIX%' and end_reason is not null order by started_at desc limit 1\"\"\").fetchone() con.close() if row: print(json.dumps({'started_at': row[0], 'end_reason': row[1]})) else: print('{}') except Exception as e: print(json.dumps({'error': str(e)})) " 2>/dev/null) LAST_STATUS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('end_reason',''))" 2>/dev/null || echo "") LAST_TS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('started_at',''))" 2>/dev/null || echo "") if [ -z "$LAST_STATUS" ]; then # No data yet — first run, don't alert echo "$LAST_TS" > "$LAST_GOOD_FILE" exit 0 fi if [ "$LAST_STATUS" = "error" ] || [ "$LAST_STATUS" = "failed" ]; then # Failure — only alert if we haven't already reported this one if [ -f "$LAST_GOOD_FILE" ]; then PREV_GOOD=$(cat "$LAST_GOOD_FILE") if [ "$LAST_TS" != "$PREV_GOOD" ]; then echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details" echo "$LAST_TS" > "$LAST_GOOD_FILE" fi else echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details" echo "$LAST_TS" > "$LAST_GOOD_FILE" fi else # All good — mark this timestamp so we only alert on new failures echo "$LAST_TS" > "$LAST_GOOD_FILE" fi