Initial resurrection kit — 81 scripts, 62 references, configs, systemd units, crons, Docker compose files, Caddy config, master README
This commit is contained in:
root
Executable
+288
@@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env python3
|
||||
"""DRE unified inbox poller — checks dre@ and collections@ on MXroute every 60s.
|
||||
|
||||
Writes parsed emails to /var/www/internal/data/dre-mails.json for the DRE portal.
|
||||
Leaves messages as UNSEEN so the original mailbox user still sees them as new.
|
||||
"""
|
||||
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from email.header import decode_header, make_header
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────────
|
||||
|
||||
IMAP_HOST = "heracles.mxrouting.net"
|
||||
IMAP_PORT = 993
|
||||
|
||||
MAILBOXES = [
|
||||
{"label": "dre", "user": "dre@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
||||
{"label": "collections", "user": "collections@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
||||
]
|
||||
|
||||
OUTPUT_FILE = "/var/www/internal/data/dre-mails.json"
|
||||
MAX_ENTRIES = 500
|
||||
BODY_PREVIEW_CHARS = 500 # Full body stored, preview truncated for portal
|
||||
|
||||
# Claim pattern: DRE-XXXX-XXXX
|
||||
CLAIM_RE = re.compile(r"DRE-\d{4}-\d{4}")
|
||||
|
||||
# ── Logging ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def setup_logging():
|
||||
logger = logging.getLogger("dre-mail-poller")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
# Syslog
|
||||
try:
|
||||
syslog = logging.handlers.SysLogHandler(address="/dev/log")
|
||||
syslog.setFormatter(logging.Formatter("dre-mail-poller: %(message)s"))
|
||||
logger.addHandler(syslog)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# File log
|
||||
fh = logging.FileHandler("/var/log/dre-mail-poller.log")
|
||||
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
logger.addHandler(fh)
|
||||
|
||||
# Also stderr for cron visibility
|
||||
sh = logging.StreamHandler(sys.stderr)
|
||||
sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
logger.addHandler(sh)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
log = setup_logging()
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def decode_str(val: str | None) -> str:
|
||||
"""Decode an RFC2047-encoded header value to plain text."""
|
||||
if not val:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(val)))
|
||||
except Exception:
|
||||
return val.strip()
|
||||
|
||||
|
||||
def decode_body(part) -> str:
|
||||
"""Extract text from an email part, handling base64/quoted-printable."""
|
||||
try:
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload is None:
|
||||
return ""
|
||||
return payload.decode(charset, errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def get_body(msg) -> str:
|
||||
"""Extract the plain-text body of an email (prefer text/plain, fallback text/html)."""
|
||||
if msg.is_multipart():
|
||||
# Try text/plain first
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
body = decode_body(part)
|
||||
if body.strip():
|
||||
return body
|
||||
# Fallback to text/html
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/html":
|
||||
return decode_body(part)
|
||||
# Last resort: first text/* part
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_maintype()
|
||||
if ct == "text":
|
||||
return decode_body(part)
|
||||
else:
|
||||
return decode_body(msg)
|
||||
return ""
|
||||
|
||||
|
||||
def clean_body(text: str) -> str:
|
||||
"""Strip excessive whitespace from body text."""
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def parse_date(date_str: str | None) -> str:
|
||||
"""Parse email Date header to ISO 8601 string, fallback to now."""
|
||||
if not date_str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
dt = parsedate_to_datetime(date_str)
|
||||
return dt.isoformat()
|
||||
except Exception:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def check_claim(subject: str) -> str | None:
|
||||
"""Extract the DRE claim ID from subject if present."""
|
||||
m = CLAIM_RE.search(subject)
|
||||
return m.group(0) if m else None
|
||||
|
||||
|
||||
# ── JSON Database ──────────────────────────────────────────────────────────────
|
||||
|
||||
def load_db() -> list[dict]:
|
||||
"""Load existing email database from disk."""
|
||||
if not os.path.exists(OUTPUT_FILE):
|
||||
return []
|
||||
try:
|
||||
with open(OUTPUT_FILE, "r") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
log.warning(f"Could not load {OUTPUT_FILE}: {e} — starting fresh")
|
||||
return []
|
||||
|
||||
|
||||
def save_db(emails: list[dict]):
|
||||
"""Write email database to disk, keeping last MAX_ENTRIES."""
|
||||
emails.sort(key=lambda e: e.get("date", ""), reverse=True)
|
||||
trimmed = emails[:MAX_ENTRIES]
|
||||
# Ensure output directory exists
|
||||
os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
|
||||
with open(OUTPUT_FILE, "w") as f:
|
||||
json.dump(trimmed, f, indent=2, ensure_ascii=False)
|
||||
log.info(f"Written {len(trimmed)} emails to {OUTPUT_FILE}")
|
||||
|
||||
|
||||
# ── IMAP Polling ───────────────────────────────────────────────────────────────
|
||||
|
||||
def poll_mailbox(label: str, user: str, password: str, existing_ids: set) -> list[dict]:
|
||||
"""Connect to a mailbox, fetch UNSEEN messages, return parsed email records."""
|
||||
new_emails = []
|
||||
conn = None
|
||||
try:
|
||||
log.info(f"Connecting to {label} ({user})…")
|
||||
conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT)
|
||||
conn.login(user, password)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "UNSEEN")
|
||||
if status != "OK" or not data[0]:
|
||||
log.info(f"No UNSEEN messages in {label}")
|
||||
conn.logout()
|
||||
return []
|
||||
|
||||
uids = data[0].split()
|
||||
log.info(f"Found {len(uids)} UNSEEN message(s) in {label}")
|
||||
|
||||
for uid_bytes in uids:
|
||||
uid_str = uid_bytes.decode()
|
||||
# Fetch by UID to get stable identifiers
|
||||
status, fetch_data = conn.uid("fetch", uid_bytes, "(RFC822 UID)")
|
||||
if status != "OK":
|
||||
continue
|
||||
|
||||
raw_email = None
|
||||
uid_val = uid_str
|
||||
for part in fetch_data:
|
||||
if isinstance(part, tuple):
|
||||
raw_email = part[1]
|
||||
elif isinstance(part, bytes) and part.startswith(b"UID "):
|
||||
uid_val = part.decode().split("UID ")[-1].strip()
|
||||
|
||||
if raw_email is None:
|
||||
continue
|
||||
|
||||
# Build unique ID: mailbox label + UID
|
||||
unique_id = f"{label}:{uid_val}"
|
||||
|
||||
# Skip duplicates
|
||||
if unique_id in existing_ids:
|
||||
log.debug(f"Skipping duplicate: {unique_id}")
|
||||
continue
|
||||
|
||||
msg = email.message_from_bytes(raw_email)
|
||||
|
||||
subject = decode_str(msg.get("Subject"))
|
||||
from_addr = decode_str(msg.get("From"))
|
||||
to_addr = decode_str(msg.get("To"))
|
||||
date_str = parse_date(msg.get("Date"))
|
||||
body_full = clean_body(get_body(msg))
|
||||
body_preview = body_full[:BODY_PREVIEW_CHARS]
|
||||
claim_id = check_claim(subject)
|
||||
|
||||
record = {
|
||||
"id": unique_id,
|
||||
"mailbox": label,
|
||||
"from": from_addr,
|
||||
"to": to_addr,
|
||||
"subject": subject,
|
||||
"date": date_str,
|
||||
"body_preview": body_preview,
|
||||
"body": body_full,
|
||||
"is_read": False,
|
||||
"claim_match": claim_id,
|
||||
}
|
||||
|
||||
new_emails.append(record)
|
||||
existing_ids.add(unique_id)
|
||||
log.info(f"New email [{label}] From: {from_addr} | Subject: {subject[:80]}")
|
||||
|
||||
except imaplib.IMAP4.error as e:
|
||||
log.error(f"IMAP error for {label}: {e}")
|
||||
except (ConnectionRefusedError, TimeoutError, OSError) as e:
|
||||
log.error(f"Connection error for {label}: {e}")
|
||||
except Exception as e:
|
||||
log.error(f"Unexpected error for {label}: {e}")
|
||||
finally:
|
||||
if conn:
|
||||
try:
|
||||
conn.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return new_emails
|
||||
|
||||
|
||||
def main():
|
||||
log.info("=" * 60)
|
||||
log.info("DRE mail poller starting…")
|
||||
|
||||
# Load existing database
|
||||
emails = load_db()
|
||||
existing_ids = {e["id"] for e in emails if "id" in e}
|
||||
log.info(f"Loaded {len(emails)} existing records ({len(existing_ids)} unique IDs)")
|
||||
|
||||
fresh_count = 0
|
||||
for mb in MAILBOXES:
|
||||
new_msgs = poll_mailbox(mb["label"], mb["user"], mb["pass"], existing_ids)
|
||||
emails.extend(new_msgs)
|
||||
fresh_count += len(new_msgs)
|
||||
|
||||
if fresh_count > 0:
|
||||
log.info(f"Collected {fresh_count} new message(s), saving…")
|
||||
save_db(emails)
|
||||
else:
|
||||
log.info("No new messages — database unchanged")
|
||||
# Still ensure the file exists (first run with no mail)
|
||||
if not os.path.exists(OUTPUT_FILE):
|
||||
save_db(emails)
|
||||
|
||||
log.info("DRE mail poller complete.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user