Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README

2026-07-15 17:45:36 -04:00
commit ae056eaf83
197 changed files with 26127 additions and 0 deletions
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Watchdog for the email triage cron job (5929c5f1deff).
+# Checks if the most recent run ended with an error.
+# Silent (no output) when OK, alerts when failed.
+set -euo pipefail
+
+SCRIPT_DIR="/root/.hermes"
+DB="$SCRIPT_DIR/state.db"
+JOB_PREFIX="cron_5929c5f1deff"
+LAST_GOOD_FILE="$SCRIPT_DIR/email_triage/.watchdog_last_ok"
+mkdir -p "$(dirname "$LAST_GOOD_FILE")"
+
+# Query the most recent cron session for this job
+RESULT=$(python3 -c "
+import sqlite3, sys, json
+try:
+    con = sqlite3.connect('$DB')
+    cur = con.cursor()
+    row = cur.execute(\"\"\"select started_at, end_reason from sessions
+                         where id like '$JOB_PREFIX%' and end_reason is not null
+                         order by started_at desc limit 1\"\"\").fetchone()
+    con.close()
+    if row:
+        print(json.dumps({'started_at': row[0], 'end_reason': row[1]}))
+    else:
+        print('{}')
+except Exception as e:
+    print(json.dumps({'error': str(e)}))
+" 2>/dev/null)
+
+LAST_STATUS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('end_reason',''))" 2>/dev/null || echo "")
+LAST_TS=$(echo "$RESULT" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d.get('started_at',''))" 2>/dev/null || echo "")
+
+if [ -z "$LAST_STATUS" ]; then
+    # No data yet — first run, don't alert
+    echo "$LAST_TS" > "$LAST_GOOD_FILE"
+    exit 0
+fi
+
+if [ "$LAST_STATUS" = "error" ] || [ "$LAST_STATUS" = "failed" ]; then
+    # Failure — only alert if we haven't already reported this one
+    if [ -f "$LAST_GOOD_FILE" ]; then
+        PREV_GOOD=$(cat "$LAST_GOOD_FILE")
+        if [ "$LAST_TS" != "$PREV_GOOD" ]; then
+            echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
+            echo "$LAST_TS" > "$LAST_GOOD_FILE"
+        fi
+    else
+        echo "⚠️ Email triage job FAILED (run at timestamp $LAST_TS) — check hermes cron list for details"
+        echo "$LAST_TS" > "$LAST_GOOD_FILE"
+    fi
+else
+    # All good — mark this timestamp so we only alert on new failures
+    echo "$LAST_TS" > "$LAST_GOOD_FILE"
+fi