289 lines
9.6 KiB
Python
Executable File
289 lines
9.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""DRE unified inbox poller — checks dre@ and collections@ on MXroute every 60s.
|
|
|
|
Writes parsed emails to /var/www/internal/data/dre-mails.json for the DRE portal.
|
|
Leaves messages as UNSEEN so the original mailbox user still sees them as new.
|
|
"""
|
|
|
|
import email
|
|
import imaplib
|
|
import json
|
|
import logging
|
|
import logging.handlers
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from email.header import decode_header, make_header
|
|
from email.utils import parsedate_to_datetime
|
|
|
|
# ── Configuration ──────────────────────────────────────────────────────────────
|
|
|
|
IMAP_HOST = "heracles.mxrouting.net"
|
|
IMAP_PORT = 993
|
|
|
|
MAILBOXES = [
|
|
{"label": "dre", "user": "dre@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
|
{"label": "collections", "user": "collections@debtrecoveryexperts.com", "pass": "M8ke.Money.Honey!"},
|
|
]
|
|
|
|
OUTPUT_FILE = "/var/www/internal/data/dre-mails.json"
|
|
MAX_ENTRIES = 500
|
|
BODY_PREVIEW_CHARS = 500 # Full body stored, preview truncated for portal
|
|
|
|
# Claim pattern: DRE-XXXX-XXXX
|
|
CLAIM_RE = re.compile(r"DRE-\d{4}-\d{4}")
|
|
|
|
# ── Logging ────────────────────────────────────────────────────────────────────
|
|
|
|
def setup_logging():
|
|
logger = logging.getLogger("dre-mail-poller")
|
|
logger.setLevel(logging.INFO)
|
|
|
|
# Syslog
|
|
try:
|
|
syslog = logging.handlers.SysLogHandler(address="/dev/log")
|
|
syslog.setFormatter(logging.Formatter("dre-mail-poller: %(message)s"))
|
|
logger.addHandler(syslog)
|
|
except Exception:
|
|
pass
|
|
|
|
# File log
|
|
fh = logging.FileHandler("/var/log/dre-mail-poller.log")
|
|
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
|
logger.addHandler(fh)
|
|
|
|
# Also stderr for cron visibility
|
|
sh = logging.StreamHandler(sys.stderr)
|
|
sh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
|
logger.addHandler(sh)
|
|
|
|
return logger
|
|
|
|
|
|
log = setup_logging()
|
|
|
|
# ── Helpers ────────────────────────────────────────────────────────────────────
|
|
|
|
def decode_str(val: str | None) -> str:
|
|
"""Decode an RFC2047-encoded header value to plain text."""
|
|
if not val:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(val)))
|
|
except Exception:
|
|
return val.strip()
|
|
|
|
|
|
def decode_body(part) -> str:
|
|
"""Extract text from an email part, handling base64/quoted-printable."""
|
|
try:
|
|
charset = part.get_content_charset() or "utf-8"
|
|
payload = part.get_payload(decode=True)
|
|
if payload is None:
|
|
return ""
|
|
return payload.decode(charset, errors="replace")
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def get_body(msg) -> str:
|
|
"""Extract the plain-text body of an email (prefer text/plain, fallback text/html)."""
|
|
if msg.is_multipart():
|
|
# Try text/plain first
|
|
for part in msg.walk():
|
|
ct = part.get_content_type()
|
|
if ct == "text/plain":
|
|
body = decode_body(part)
|
|
if body.strip():
|
|
return body
|
|
# Fallback to text/html
|
|
for part in msg.walk():
|
|
ct = part.get_content_type()
|
|
if ct == "text/html":
|
|
return decode_body(part)
|
|
# Last resort: first text/* part
|
|
for part in msg.walk():
|
|
ct = part.get_content_maintype()
|
|
if ct == "text":
|
|
return decode_body(part)
|
|
else:
|
|
return decode_body(msg)
|
|
return ""
|
|
|
|
|
|
def clean_body(text: str) -> str:
|
|
"""Strip excessive whitespace from body text."""
|
|
lines = []
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if line:
|
|
lines.append(line)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_date(date_str: str | None) -> str:
|
|
"""Parse email Date header to ISO 8601 string, fallback to now."""
|
|
if not date_str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
try:
|
|
dt = parsedate_to_datetime(date_str)
|
|
return dt.isoformat()
|
|
except Exception:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def check_claim(subject: str) -> str | None:
|
|
"""Extract the DRE claim ID from subject if present."""
|
|
m = CLAIM_RE.search(subject)
|
|
return m.group(0) if m else None
|
|
|
|
|
|
# ── JSON Database ──────────────────────────────────────────────────────────────
|
|
|
|
def load_db() -> list[dict]:
|
|
"""Load existing email database from disk."""
|
|
if not os.path.exists(OUTPUT_FILE):
|
|
return []
|
|
try:
|
|
with open(OUTPUT_FILE, "r") as f:
|
|
data = json.load(f)
|
|
if isinstance(data, list):
|
|
return data
|
|
except (json.JSONDecodeError, OSError) as e:
|
|
log.warning(f"Could not load {OUTPUT_FILE}: {e} — starting fresh")
|
|
return []
|
|
|
|
|
|
def save_db(emails: list[dict]):
|
|
"""Write email database to disk, keeping last MAX_ENTRIES."""
|
|
emails.sort(key=lambda e: e.get("date", ""), reverse=True)
|
|
trimmed = emails[:MAX_ENTRIES]
|
|
# Ensure output directory exists
|
|
os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
|
|
with open(OUTPUT_FILE, "w") as f:
|
|
json.dump(trimmed, f, indent=2, ensure_ascii=False)
|
|
log.info(f"Written {len(trimmed)} emails to {OUTPUT_FILE}")
|
|
|
|
|
|
# ── IMAP Polling ───────────────────────────────────────────────────────────────
|
|
|
|
def poll_mailbox(label: str, user: str, password: str, existing_ids: set) -> list[dict]:
|
|
"""Connect to a mailbox, fetch UNSEEN messages, return parsed email records."""
|
|
new_emails = []
|
|
conn = None
|
|
try:
|
|
log.info(f"Connecting to {label} ({user})…")
|
|
conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT)
|
|
conn.login(user, password)
|
|
conn.select("INBOX")
|
|
|
|
status, data = conn.search(None, "UNSEEN")
|
|
if status != "OK" or not data[0]:
|
|
log.info(f"No UNSEEN messages in {label}")
|
|
conn.logout()
|
|
return []
|
|
|
|
uids = data[0].split()
|
|
log.info(f"Found {len(uids)} UNSEEN message(s) in {label}")
|
|
|
|
for uid_bytes in uids:
|
|
uid_str = uid_bytes.decode()
|
|
# Fetch by UID to get stable identifiers
|
|
status, fetch_data = conn.uid("fetch", uid_bytes, "(RFC822 UID)")
|
|
if status != "OK":
|
|
continue
|
|
|
|
raw_email = None
|
|
uid_val = uid_str
|
|
for part in fetch_data:
|
|
if isinstance(part, tuple):
|
|
raw_email = part[1]
|
|
elif isinstance(part, bytes) and part.startswith(b"UID "):
|
|
uid_val = part.decode().split("UID ")[-1].strip()
|
|
|
|
if raw_email is None:
|
|
continue
|
|
|
|
# Build unique ID: mailbox label + UID
|
|
unique_id = f"{label}:{uid_val}"
|
|
|
|
# Skip duplicates
|
|
if unique_id in existing_ids:
|
|
log.debug(f"Skipping duplicate: {unique_id}")
|
|
continue
|
|
|
|
msg = email.message_from_bytes(raw_email)
|
|
|
|
subject = decode_str(msg.get("Subject"))
|
|
from_addr = decode_str(msg.get("From"))
|
|
to_addr = decode_str(msg.get("To"))
|
|
date_str = parse_date(msg.get("Date"))
|
|
body_full = clean_body(get_body(msg))
|
|
body_preview = body_full[:BODY_PREVIEW_CHARS]
|
|
claim_id = check_claim(subject)
|
|
|
|
record = {
|
|
"id": unique_id,
|
|
"mailbox": label,
|
|
"from": from_addr,
|
|
"to": to_addr,
|
|
"subject": subject,
|
|
"date": date_str,
|
|
"body_preview": body_preview,
|
|
"body": body_full,
|
|
"is_read": False,
|
|
"claim_match": claim_id,
|
|
}
|
|
|
|
new_emails.append(record)
|
|
existing_ids.add(unique_id)
|
|
log.info(f"New email [{label}] From: {from_addr} | Subject: {subject[:80]}")
|
|
|
|
except imaplib.IMAP4.error as e:
|
|
log.error(f"IMAP error for {label}: {e}")
|
|
except (ConnectionRefusedError, TimeoutError, OSError) as e:
|
|
log.error(f"Connection error for {label}: {e}")
|
|
except Exception as e:
|
|
log.error(f"Unexpected error for {label}: {e}")
|
|
finally:
|
|
if conn:
|
|
try:
|
|
conn.logout()
|
|
except Exception:
|
|
pass
|
|
|
|
return new_emails
|
|
|
|
|
|
def main():
|
|
log.info("=" * 60)
|
|
log.info("DRE mail poller starting…")
|
|
|
|
# Load existing database
|
|
emails = load_db()
|
|
existing_ids = {e["id"] for e in emails if "id" in e}
|
|
log.info(f"Loaded {len(emails)} existing records ({len(existing_ids)} unique IDs)")
|
|
|
|
fresh_count = 0
|
|
for mb in MAILBOXES:
|
|
new_msgs = poll_mailbox(mb["label"], mb["user"], mb["pass"], existing_ids)
|
|
emails.extend(new_msgs)
|
|
fresh_count += len(new_msgs)
|
|
|
|
if fresh_count > 0:
|
|
log.info(f"Collected {fresh_count} new message(s), saving…")
|
|
save_db(emails)
|
|
else:
|
|
log.info("No new messages — database unchanged")
|
|
# Still ensure the file exists (first run with no mail)
|
|
if not os.path.exists(OUTPUT_FILE):
|
|
save_db(emails)
|
|
|
|
log.info("DRE mail poller complete.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|