Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README

scripts/hermes-consolidate.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+# hermes-consolidate.sh — Auto-consolidate Hermes memory every 10 min.
+#
+# ORDER OF OPERATIONS (backup BEFORE prune is a hard guarantee):
+#   1. Dump current MEMORY.md entries to JSON.
+#   2. Upload that JSON to S3 (s3://hermes-vps-backups/live/memories/memory-backup.json).
+#      -> If the S3 upload fails, we ABORT before pruning. Nothing is deleted
+#         unless we have a fresh off-box backup.
+#   3. Run the pruner (pattern prune + size safety valve + 7-day fact-store prune).
+#   4. Keep a local timestamped pre-prune copy for fast restore.
+#
+# Target: keep MEMORY.md under ~70% of the 10,000-char limit (7,000 chars).
+# Silent on success, loud on failure (no_agent friendly).
+set -euo pipefail
+
+HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
+MEMORY_MD="$HERMES_HOME/memories/MEMORY.md"
+CONSOLIDATE_PY="$HERMES_HOME/scripts/hermes-consolidate.py"
+BACKUP_DIR="$HERMES_HOME/memories/.consolidate-backups"
+ENDPOINT="https://s3.us-east-1.wasabisys.com"
+S3_BUCKET="hermes-vps-backups"
+S3_KEY="live/memories/memory-backup.json"
+LOG="/var/log/hermes-consolidate.log"
+
+log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" >> "$LOG"; }
+
+# Source AWS CLI venv
+if [ -f /opt/awscli-venv/bin/activate ]; then
+    # shellcheck disable=SC1091
+    source /opt/awscli-venv/bin/activate
+fi
+
+mkdir -p "$BACKUP_DIR"
+
+if [ ! -f "$MEMORY_MD" ]; then
+    log "FAILED: MEMORY.md not found at $MEMORY_MD"
+    echo "RESULT:FAIL|MEMORY.md not found"
+    exit 1
+fi
+
+TS=$(date -u +%Y%m%d_%H%M%S)
+LOCAL_BACKUP="$BACKUP_DIR/memory-backup-${TS}.json"
+
+# --- Step 1: dump current entries to JSON (no mutation) ---
+if ! python3 "$CONSOLIDATE_PY" --dump-only > "$LOCAL_BACKUP" 2>>"$LOG"; then
+    log "FAILED: dump step failed"
+    echo "RESULT:FAIL|dump failed"
+    rm -f "$LOCAL_BACKUP"
+    exit 1
+fi
+
+# --- Step 2: upload backup to S3 BEFORE pruning ---
+if ! aws s3 cp "$LOCAL_BACKUP" "s3://$S3_BUCKET/$S3_KEY" \
+        --endpoint-url "$ENDPOINT" --quiet 2>>"$LOG"; then
+    log "FAILED: S3 backup upload failed — ABORTING before prune (nothing deleted)"
+    echo "RESULT:FAIL|S3 backup failed, prune aborted"
+    exit 1
+fi
+# Also keep a timestamped copy in S3 for point-in-time restore.
+aws s3 cp "$LOCAL_BACKUP" \
+    "s3://$S3_BUCKET/live/memories/history/memory-backup-${TS}.json" \
+    --endpoint-url "$ENDPOINT" --quiet 2>>"$LOG" || true
+
+log "Backup uploaded to s3://$S3_BUCKET/$S3_KEY ($(stat --printf='%s' "$LOCAL_BACKUP") bytes)"
+
+# --- Step 3: prune (safe now that backup is off-box) ---
+PRUNE_OUT=$(python3 "$CONSOLIDATE_PY" --prune-facts 2>&1 >/dev/null) || {
+    log "FAILED: prune step failed: $PRUNE_OUT"
+    echo "RESULT:FAIL|prune failed"
+    exit 1
+}
+log "$PRUNE_OUT"
+
+# --- Step 4: local retention — keep last 48h of pre-prune copies ---
+find "$BACKUP_DIR" -name "memory-backup-*.json" -mmin +2880 -delete 2>/dev/null || true
+
+# Echo the pruner RESULT line for cron visibility
+echo "$PRUNE_OUT"
+exit 0