Initial skills documentation — 25 categories, all SKILL.md + references + scripts
This commit is contained in:
@@ -0,0 +1,255 @@
|
||||
---
|
||||
name: imap-email-search-and-extract
|
||||
description: "Search, extract, and process emails from IMAP inbox — flight itineraries, attachment PDFs, confirmation codes, and structured data from HTML/plain-text emails."
|
||||
version: 1.2.0
|
||||
author: ShoNuff
|
||||
platforms: [linux]
|
||||
metadata:
|
||||
hermes:
|
||||
tags: [email, imap, pdf, itinerary, flight, extraction]
|
||||
---
|
||||
|
||||
# IMAP Email Search and Extract
|
||||
|
||||
Search the user's IMAP inbox for specific emails and extract structured data.
|
||||
|
||||
## Standard approach\n\n### Connection setup\n\nCredentials typically come from Himalaya password files (`/root/.config/himalaya/*.pass`). For WordPress-hosted sites (Apex Track Experience, ITPP customer sites), SMTP/IMAP credentials may instead be in the WordPress database — see `references/wordpress-smtp-credential-discovery.md`.\n\n```python
|
||||
import imaplib, email
|
||||
HOST = "mail.germainebrown.com"
|
||||
PORT = 993
|
||||
USER = "g@germainebrown.com"
|
||||
PW = open("/root/.config/himalaya/g-germainebrown.pass").read().strip()
|
||||
|
||||
conn = imaplib.IMAP4_SSL(HOST, PORT)
|
||||
conn.login(USER, PW)
|
||||
conn.select("INBOX")
|
||||
```
|
||||
|
||||
### Searching strategies
|
||||
|
||||
**By sender and subject:**
|
||||
```python
|
||||
status, data = conn.search(None, 'FROM "Copa Airlines" SUBJECT "Cartagena"')
|
||||
```
|
||||
|
||||
**By date range:**
|
||||
```python
|
||||
status, data = conn.search(None, '(SINCE "28-May-2026" BEFORE "5-Jul-2026")')
|
||||
```
|
||||
|
||||
**Broad OR search across multiple keywords:**
|
||||
```python
|
||||
status, data = conn.search(None, 'OR OR SUBJECT "Cartagena" SUBJECT "flight" SUBJECT "itinerary"')
|
||||
```
|
||||
|
||||
**Body text match:**
|
||||
```python
|
||||
status, data = conn.search(None, 'BODY "Cartagena"')
|
||||
```
|
||||
|
||||
**Chain multiple filters:**
|
||||
```python
|
||||
# Sender + keyword across subject + body
|
||||
status, data = conn.search(None, 'FROM "Copa" OR SUBJECT "Cartagena" BODY "Cartagena"')
|
||||
```
|
||||
|
||||
### Decoding email content
|
||||
|
||||
**Get body (plain text or HTML):**
|
||||
```python
|
||||
def get_body(msg):
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct in ("text/plain", "text/html"):
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode('utf-8', errors='replace')
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
return payload.decode('utf-8', errors='replace')
|
||||
return ""
|
||||
```
|
||||
|
||||
**Strip HTML tags for readability:**
|
||||
```python
|
||||
import re
|
||||
clean = re.sub(r'<[^>]+>', ' ', body)
|
||||
clean = re.sub(r'\s+', ' ', clean).strip()
|
||||
```
|
||||
|
||||
### Extracting attachments (PDFs, images)
|
||||
|
||||
```python
|
||||
for part in raw.walk():
|
||||
fn = part.get_filename()
|
||||
if fn and fn.endswith('.pdf'):
|
||||
payload = part.get_payload(decode=True)
|
||||
with open("/tmp/output.pdf", 'wb') as f:
|
||||
f.write(payload)
|
||||
```
|
||||
|
||||
### Parsing ticket/PDF data
|
||||
|
||||
After extracting a PDF attachment, parse with `pdfminer`:
|
||||
|
||||
```python
|
||||
from pdfminer.high_level import extract_text
|
||||
from io import BytesIO
|
||||
text = extract_text(BytesIO(pdf_bytes))
|
||||
|
||||
# Extract structured info
|
||||
for line in text.split('\n'):
|
||||
if 'Flight Number' in line or 'CM ' in line:
|
||||
flights.append(line.strip())
|
||||
if 'Departure' in line or 'Arrival' in line:
|
||||
# next line has the date/time
|
||||
```
|
||||
|
||||
**Install:** `pip3 install pdfminer.six`
|
||||
|
||||
**See `references/pdf-attachment-extraction.md`** for Copa Airlines and other carrier-specific PDF extraction patterns, MIME multipart navigation, and pitfalls.
|
||||
|
||||
### Flight itinerary extraction patterns
|
||||
|
||||
Common fields to extract:
|
||||
|
||||
| Field | Pattern |
|
||||
|---|---|
|
||||
| Flight number | `CM XXX` (Copa), `AA XXXX` (American), `UA XXXX` (United) |
|
||||
| Confirmation code | 6-char alphanumeric (e.g. `A4GSFD`) |
|
||||
| Departure time | `\d{1,2}:\d{2} (AM|PM)` |
|
||||
| Airport codes | 3-letter codes inbound from text: `(MCO|PTY|CTG|ATL|SAV|MIA|IAD)` |
|
||||
| Dates | `July? \d{1,2},? 2026` |
|
||||
|
||||
### Travel itinerary assembly
|
||||
|
||||
For full trip planning from extracted email data (flights + rental cars + hotels + activity research), see `references/travel-itinerary-building.md`. Covers:
|
||||
|
||||
- Multi-sender flight extraction (Copa, Allegiant, Avis)
|
||||
- Account number hunt from HTML statements (Avis Wizard #)
|
||||
- Departure day timeline with drive time + traffic estimates
|
||||
- Markdown itinerary template
|
||||
- Pitfalls per carrier (Allegiant surveys, Copa MCO departure, AT&T SMS unreliability)
|
||||
- Real-time inbox monitoring during trip planning (Avis verification code → confirmation chain)
|
||||
- Travel plan adjustments by priority (same flight, PreCheck, no bags, gate arrival time)
|
||||
- Global Entry return procedure: customs clearance at MCO with GE kiosks (~10 min for 4 people, kiosk takes ~2 min/person, skip regular line, kids under 12 go through with parent)
|
||||
|
||||
### Verification code reader
|
||||
|
||||
When a service emails a security code (Avis login, banking), the agent can read the inbox faster than the user can find the email. See `references/avis-verification-code.md`.
|
||||
|
||||
### Avis rental reservation extraction (email approval chain)
|
||||
|
||||
A common pattern: user starts an Avis reservation, Avis sends a security code to their inbox, user asks the agent to check email. The agent finds the security code email, the user logs in, and shortly after Avis sends reservation confirmations.
|
||||
|
||||
**Detection flow:**
|
||||
|
||||
1. User says "check email for rental" — search `FROM "Avis" SINCE last-24h`
|
||||
2. Find the security code email (`Subject: "Your security code"`) — extract the code
|
||||
3. Moment after user logs in, Avis emails two confirmation emails (one per rental leg)
|
||||
4. Re-check inbox and extract reservation numbers, vehicle info, dates, costs
|
||||
|
||||
**Avis confirmation key fields:**
|
||||
- Reservation # pattern: `\d+US\d+`
|
||||
- Vehicle: "Ford Explorer or Similar"
|
||||
- Pickup/Dropoff locations and times
|
||||
- Estimated total (includes taxes/fees)
|
||||
|
||||
**Pitfalls:**
|
||||
- Avis sends security code FIRST — user needs code to log in before confirmations appear
|
||||
- Two separate confirmations arrive minutes apart (outbound leg + return leg)
|
||||
- Times in confirmation are local time zone
|
||||
- The estimated total includes taxes and fees
|
||||
|
||||
### Global Entry return procedure
|
||||
|
||||
A common pattern: user starts an Avis reservation, Avis sends a security code to their inbox, user asks the agent to check email. The agent finds the security code email, the user logs in, and shortly after Avis sends reservation confirmations.
|
||||
|
||||
**Detection flow:**
|
||||
|
||||
1. User says "check email for rental" — search `FROM "Avis" SINCE last-24h`
|
||||
2. Find the security code email (`Subject: "Your security code"`) — extract the 6-digit code
|
||||
3. Moment after user logs in, Avis emails **two confirmation emails** (one per rental)
|
||||
4. Re-check and find them — extract reservation numbers, vehicle info, dates, costs
|
||||
|
||||
**Avis confirmation key fields:**
|
||||
|
||||
```python
|
||||
# Extract from plain text:
|
||||
re.search(r'Reservation #(\d+US\d+)', body)
|
||||
re.search(r'Ford Explorer or Similar', body)
|
||||
re.search(r'Pick Up Location[^\n]*\n(.*?)\n', body, re.DOTALL)
|
||||
re.search(r'(\$[\d,]+\.\d{2})', body)
|
||||
```
|
||||
|
||||
**Pitfalls:**
|
||||
- Avis sends the security code email FIRST — the user needs the code to log in before confirmations come through
|
||||
- Two separate confirmations arrive minutes apart (SAV→MCO one-way + MCO→SAV return)
|
||||
- The estimated total includes taxes and fees — strip `$` and commas for clean display
|
||||
- Times in confirmation are in local time zone (Eastern for SAV/MCO)
|
||||
|
||||
### Global Entry return procedure
|
||||
|
||||
When the user returns to the US with Global Entry, see `references/global-entry-return-procedure.md` for customs clearance timelines, kiosk procedure, and timeline placement in the itinerary.
|
||||
|
||||
### Drive time estimation (Savannah → Orlando)
|
||||
|
||||
For Savannah → MCO (~285 mi, Saturday July):
|
||||
|
||||
| Condition | Time |
|
||||
|---|---|
|
||||
| No traffic | 4h 15m |
|
||||
| Light traffic | 4h 30m |
|
||||
| Moderate | 5h |
|
||||
| Heavy | 5h 30m |
|
||||
|
||||
Jacksonville I-95 southbound backs up around lunchtime (11 AM - 1 PM) on Saturdays. Best departure: 8:30 AM. Worst: after 9 AM hits Jacksonville lunch rush.
|
||||
|
||||
### Inbox cleaning — solicitor/trash triage
|
||||
|
||||
For bulk classification of inbox messages as solicitation vs. keep, see `references/inbox-solicitation-triaging.md` and the runnable script at `scripts/inbox-solicitation-cleaner.py`.
|
||||
|
||||
The script uses a multi-heuristic approach:
|
||||
|
||||
1. **Allowlist** — internal senders, known legit domains (never flagged)
|
||||
2. **Subject-line keywords** — immediate signals ("insurance quote", "factoring", "load board")
|
||||
3. **Body content by sender domain class** — free-email senders vs. business-domain senders get different treatment
|
||||
4. **Combined signal accumulation** — threshold-based (≥2 signals = flagged)
|
||||
|
||||
**Known false positives** include government registration notices (UCR.gov, login.gov), bank product emails with trucking keywords in boilerplate, and Slack onboarding templates. See the reference for the full false-positive table and mitigation strategies.
|
||||
|
||||
**Move pattern:** COPY to target folder → STORE +FLAGS \\Deleted → EXPUNGE. Always use BODY.PEEK[] to avoid marking messages as read.
|
||||
|
||||
### Headers-only fetch (efficient preview)
|
||||
|
||||
```python
|
||||
status, msg_data = conn.fetch(num, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])")
|
||||
hdr = msg_data[0][1].decode('utf-8', errors='replace')
|
||||
```
|
||||
|
||||
### WPForms form submission delivery verification
|
||||
|
||||
For WordPress sites using WPForms + WP Mail SMTP, see `references/wpforms-delivery-verification.md` for the complete pattern:
|
||||
1. Query `wp_wpforms_entries` in the WordPress DB for registrant names/emails
|
||||
2. Check the admin IMAP inbox for form notification emails (confirms pipeline triggered)
|
||||
3. Search for bounce messages via `FROM "mailer-daemon"` or `FROM "postmaster"`
|
||||
4. Cross-reference bounced recipient addresses with registrant emails
|
||||
5. Trace SPF/DKIM failures from bounce body back to DNS records
|
||||
6. Verify the SMTP relay IP is in the SPF record
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Multiple flight confirmation emails have the same content** — Copa Airlines sends one PDF attachment per passenger. Both Germaine's and Anita's PDFs contain identical flight details; only the passenger name differs. To confirm both are on the same booking, compare the confirmation code (A4GSFD) across both emails.
|
||||
- **Emails are HTML by default** — plain text version may be empty or missing. Check both `text/plain` and `text/html` parts.
|
||||
- **PDFs are `multipart/mixed` subparts** — iterate `raw.walk()`, check `part.get_filename()` for `.pdf` extension.
|
||||
- **IMAP search dates use `DD-Mon-YYYY` format** — e.g. `28-May-2026`, not `2026-05-28` or `05/28/2026`.
|
||||
- **IMAP date search is based on internal date, not header Date** — Internaldate is when IMAP received the message, which is usually close to the Date header but not identical.
|
||||
- **Email HTML may use inline styles** — stripping all tags removes styling but keeps content. CSS class names (`style-Xia0GT4t_-text`) are noise — filter them out.
|
||||
- **Copies to self may have the same subject** — When the user forwards or BCCs themselves, you get duplicate-looking results. Check the Date/From fields to distinguish.
|
||||
- **Mailer-Daemon bounces are multipart** — The original message is included as an attachment. The bounce notification is text/plain, the original is text/rfc822-headers. Parse the root part for the bounce reason, the attachment for the original subject/body.
|
||||
- **`SPAM` folder not `INBOX.spam`** — IMAP folder names are case-sensitive. Check both `SPAM`, `INBOX.spam`, `Spam`, `Junk`.
|
||||
- **Multipart with many image attachments** — Copa Airlines ticket receipts include 6-7 inline images (logos, QR codes) alongside the PDF. The PDF is the last or second-to-last part in the MIME tree.
|
||||
- **HTML emails from Copa use content blocker redirects** — Links in the email go through `sendgrid.net` tracking URLs, not direct to Copa.com. Don't try to open them — extract data from the visible text in the email body.
|
||||
- **`get_content_charset()` may return `None`** — Fall back to `utf-8` or `iso-8859-1` for encoded payloads.
|
||||
@@ -0,0 +1,26 @@
|
||||
# Avis Email Verification Code Workflow
|
||||
|
||||
When a service (Avis, bank, etc.) emails a one-time security code, the agent can read the inbox faster than the user can find the email.
|
||||
|
||||
## Pattern: Monitor inbox for verification code
|
||||
|
||||
1. User triggers login at avis.com → email goes out
|
||||
2. Agent checks inbox for "security code" from Avis
|
||||
3. Extracts the code and tells the user
|
||||
4. User enters it — logged in
|
||||
|
||||
The email content is HTML-only (no text/plain alternative). The code itself may be embedded in invisible HTML constructs — extract from body text after stripping tags.
|
||||
|
||||
## Confirmations: Avis reservation chain
|
||||
|
||||
After login, the user will click through to book. Avis sends immediate confirmations:
|
||||
1. Avis sends "Reservation Confirmation" email — extract reservation #, vehicle, pick-up/drop-off, total
|
||||
2. Agent reports the key details back to user
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- Avis uses email-based login — the security code goes to the user's inbox, making the agent's IMAP access essential
|
||||
- Avis For Business sends separate password reset emails — the password reset email from "Avis For Business" is a different flow from the personal account login. Distinguish by subject line
|
||||
- Verification code emails have near-identical subjects — "Avis: Your security code" multiple times. The last one (highest UID) is the current login attempt
|
||||
- Ford Explorer is the default mid-size SUV — Avis frequently assigns Ford Explorer or similar for mid-size SUV bookings
|
||||
- Confirmation numbers — Avis reservations starting with #1958XXXXUSX pattern. Store both reservation #s when booking a round-trip-two-reservation itinerary
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# Global Entry — Customs Return Procedure
|
||||
|
||||
When a user returns to the US from an international trip and has Global Entry, use this reference for the customs clearance section of the itinerary.
|
||||
|
||||
## What Global Entry does
|
||||
|
||||
- Bypasses the regular CBP line at US airports
|
||||
- Self-service kiosk: scan passport, fingerprint scan, answer customs declaration → printed receipt → hand to officer
|
||||
- Time: ~2 min per person, ~10-15 min total for family
|
||||
|
||||
## Key rules
|
||||
|
||||
- **First port of entry clears customs.** Even if the final destination is elsewhere (e.g. fly MCO, drive to Savannah), customs is at MCO. The itinerary timeline must account for this.
|
||||
- **Children under 12** go through Global Entry with a parent. No separate enrollment needed.
|
||||
- **Global Entry cards** are optional — passport scanning at the kiosk works.
|
||||
- **No pre-departure form required.** Unlike Mobile Passport Control (MPC), GE has no app to fill out beforehand.
|
||||
|
||||
## Timeline placement
|
||||
|
||||
In a return-day itinerary timeline:
|
||||
|
||||
```
|
||||
1:44 PM Arrive MCO
|
||||
1:44 PM Deplane, follow signs to US Customs
|
||||
1:50 PM At Global Entry kiosk area
|
||||
2:00 PM Through customs (~10 min total for 4)
|
||||
2:15 PM Shuttle/bus to rental car lot
|
||||
```
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **No app needed.** The user asked "is there anything we need to fill out before landing" — answer is no for Global Entry. MPC app is optional backup only.
|
||||
- **Duty-free limit:** $800/person. With carry-ons only, unlikely to be exceeded.
|
||||
- **No restricted agricultural products** from most standard tourist destinations. Advise user to check CBP.gov for destination-specific restrictions.
|
||||
- **GE + PreCheck = two separate programs.** PreCheck is airport security (TSA); GE is customs (CBP). Having both covers both sides of the trip.
|
||||
- **Known Traveler Number (KTN)** is needed on the flight reservation for PreCheck to print on the boarding pass. Global Entry members automatically get a KTN — it's the PASSID number on the back of the GE card.
|
||||
@@ -0,0 +1,66 @@
|
||||
# Inbox Solicitation Triaging — Design & Pitfalls
|
||||
|
||||
Pattern for cleaning an inbox by classifying and moving solicitations.
|
||||
|
||||
## Core Technique
|
||||
|
||||
1. **Connect via IMAP (imaplib)**, select INBOX, fetch ALL messages
|
||||
2. **Iterate each message** with `BODY.PEEK[]` (non-destructive read — doesn't mark as seen)
|
||||
3. **Run multi-heuristic solicitor detection** on sender domain, subject line, and body text
|
||||
4. **Copy flagged to target folder**, then `+FLAGS \\Deleted` + `EXPUNGE` from INBOX
|
||||
5. **Save detailed JSON report** and print a grouped summary
|
||||
|
||||
## Multi-Heuristic Detection Strategy
|
||||
|
||||
Four layered checks:
|
||||
|
||||
### 1. Allowlist Bypass (highest priority)
|
||||
- **Internal senders** (first-name patterns, company domain): skip entirely
|
||||
- **Legit domains** (`boxpilotlogistics.com`, etc.): skip entirely
|
||||
|
||||
### 2. Subject-Line Keywords (high signal)
|
||||
Immediate giveaways: "insurance quote", "free quote", "factoring", "compliance", "load board", "mc number", "advertisement"
|
||||
|
||||
### 3. Body Content Analysis (by sender domain class)
|
||||
|
||||
**Free/personal domains** (gmail, yahoo, outlook, etc. — defined in FREE_DOMAINS set):
|
||||
- If sender has free domain AND references an external business domain in the body → strong solicitation signal
|
||||
- Also scan for solicitation keywords + phrases in body text
|
||||
|
||||
**Business-domain senders** (not free, not internal):
|
||||
- Count multiple trucking-solicitation signals (keywords + phrases)
|
||||
- Threshold: 2+ signals = solicitation
|
||||
|
||||
**Transactional invoice guard:** If subject/body contains ≥2 invoice keywords ("invoice", "receipt", "payment confirmation") and no other signals triggered, keep in inbox.
|
||||
|
||||
### 4. Combined Signal Accumulation
|
||||
- Body keywords: +1 each
|
||||
- Solicitation phrases: +2 each (trucking-specific phrases are stronger)
|
||||
- Subject keyword match: +1
|
||||
- Threshold: 2+ total = flagged
|
||||
|
||||
## Known False-Positive Categories
|
||||
|
||||
| Category | Why flagged | Typical senders | Action |
|
||||
|---|---|---|---|
|
||||
| Government registration notices | Body has "fmcsa", "usdot", "authority" | ucr.gov, login.gov, dot.gov | Move back — official notices |
|
||||
| Bank product promos | Boilerplate contains "eld", "limited time" | americanexpress@blueprint.americanexpress.com | Review individually |
|
||||
| Slack onboarding | Template has "eld", "limited time" in footer | no-reply@email.slackhq.com | Keep transactional, move promos |
|
||||
| FMCSA process agent confirmations | Body has "boc-3", "process agent", "fmcsa" | fmcsaprocessagents.com | Keep if service was ordered |
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **HTML boilerplate noise**: Some email templates (Amex, Slack) contain trucking keywords like "eld" in their footer HTML. The heuristic catches them as false positives. Mitigation: whitelist known transactional templates or strip common boilerplate before matching.
|
||||
- **UCR.gov is legitimate** but its subject/body triggers "compliance", "authority", "usdot". These are official registration notices, not solicitations.
|
||||
- **BODY.PEEK[] vs BODY[]**: Always use PEEK to avoid marking messages as read during review.
|
||||
- **IMAP COPY + STORE \\Deleted** is the portable move pattern. Some servers support `MOVE` but not all.
|
||||
- **conn.expunge()** must be called after all deletions or the messages remain hidden but not freed.
|
||||
- **Character encodings**: Fall back through `utf-8` → `iso-8859-1` → `windows-1252` when `get_content_charset()` returns None.
|
||||
- **HTML-only emails**: Need regex tag stripping (`re.sub(r'<[^>]+>', ' ', body)`) before keyword matching. CSS class names (`style-Xia0GT4t_-text`) are noise — filter them out.
|
||||
|
||||
## Script
|
||||
|
||||
See `scripts/inbox-solicitation-cleaner.py` — a standalone runnable script with the full heuristic set. Adjust the Config block (host, credentials, folder name) and run:
|
||||
```bash
|
||||
python3 scripts/inbox-solicitation-cleaner.py
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
# PDF Attachment Extraction from Multipart/Mixed Emails
|
||||
|
||||
Airline ticket receipts (Copa, American, United, Delta) often embed PDF attachments inside `multipart/mixed` MIME messages alongside inline images (logos, QR codes).
|
||||
|
||||
## Key pattern
|
||||
|
||||
The PDF is typically the **last meaningful attachment** in the MIME tree, surrounded by 6-8 `image/jpeg` inline images. Don't assume the first `.pdf` match is the right one — iterate all parts and pick the one with the correct filename.
|
||||
|
||||
```python
|
||||
for part in raw.walk():
|
||||
fn = part.get_filename()
|
||||
if fn and fn.endswith('.pdf'):
|
||||
payload = part.get_payload(decode=True)
|
||||
# Save to disk
|
||||
with open("/tmp/ticket.pdf", 'wb') as f:
|
||||
f.write(payload)
|
||||
# Or parse directly from memory
|
||||
from pdfminer.high_level import extract_text
|
||||
from io import BytesIO
|
||||
text = extract_text(BytesIO(payload))
|
||||
```
|
||||
|
||||
## Parsing e-ticket PDFs with pdfminer
|
||||
|
||||
Install: `pip3 install pdfminer.six`
|
||||
|
||||
```python
|
||||
from pdfminer.high_level import extract_text
|
||||
from io import BytesIO
|
||||
|
||||
text = extract_text(BytesIO(pdf_bytes))
|
||||
|
||||
# Extract flight numbers (Copa uses CM###)
|
||||
for line in text.split('\n'):
|
||||
if 'Flight Number' in line or 'CM ' in line:
|
||||
flights.append(line.strip())
|
||||
```
|
||||
|
||||
## Common airline PDF fields
|
||||
|
||||
| Airline | Flight prefix | Confirmation length | PDF attachment name |
|
||||
|---|---|---|---|
|
||||
| Copa | CM | 6 alphanumeric (A4GSFD) | ETKT_*.pdf |
|
||||
| American | AA | 6 alphanumeric | e-ticket-*.pdf |
|
||||
| United | UA | 6 alphanumeric | itinerary-*.pdf |
|
||||
| Delta | DL | 6 alphanumeric | eTicket*.pdf |
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Copa tickets have 6-8 inline image parts** (logo, QR, barcode, social icons) before the PDF. The PDF is typically the second-to-last or last attachment.
|
||||
- **`get_content_charset()` may return None** on PDFs — use `utf-8` or `iso-8859-1` fallback for the PDF parsing step (the PDF itself is binary, the charset applies to the text portion of the email, not the attachment).
|
||||
- **Copa confirmation codes look like A4GSFD** — 6 chars, alphanumeric. Found in the email subject line and the PDF body. Also stored in the email's `X-Confirmation` header (if present).
|
||||
- **Header-only fetch** (`BODY.PEEK[HEADER.FIELDS]`) won't show attachment filenames. You need to fetch the full message body to find PDFs.
|
||||
- **Some airlines use `Content-Type: application/octet-stream`** instead of `application/pdf` for PDF attachments. Check `get_filename()` regardless of content type.
|
||||
@@ -0,0 +1,150 @@
|
||||
# Travel Itinerary Building from Emails
|
||||
|
||||
This pattern covers extracting flight/hotel/car rental data from IMAP emails and assembling a structured itinerary document.
|
||||
|
||||
## Typical sources
|
||||
|
||||
| Type | Senders | Data format |
|
||||
|---|---|---|
|
||||
| Flights | Copa Airlines, United, American, Delta, Allegiant | PDF attachment in multipart/mixed email |
|
||||
| Hotel/Airbnb | Airbnb, Booking.com, VRBO | HTML email with dates, address, confirmation |
|
||||
| Car rental | Avis, Hertz, Enterprise, Budget | HTML email with confirmation #, vehicle class |
|
||||
| Account numbers | Avis account statements | HTML email with Wizard/Preferred number |
|
||||
|
||||
## Extraction flow
|
||||
|
||||
1. **Search inbox** — broad OR search across senders and subject keywords
|
||||
2. **Filter** — skip survey/feedback/survey emails (Allegiant sends many post-flight surveys)
|
||||
3. **Extract PDFs** — for airline tickets, extract the PDF attachment from multipart/mixed (see `references/pdf-attachment-extraction.md`)
|
||||
4. **Parse with pdfminer** — extract flight numbers, times, dates, passenger names
|
||||
5. **Build structured data** — leg-by-leg table with date, time, airport codes, flight numbers
|
||||
6. **Search for ancillary bookings** — rental car, hotel, insurance in the same inbox
|
||||
|
||||
## Email search strategies for trip data
|
||||
|
||||
**Multiple sender search (one call per sender):**
|
||||
```python
|
||||
for sender in ["Copa", "Avis", "Allegiant", "Airbnb"]:
|
||||
status, data = conn.search(None, f'FROM "{sender}" SUBJECT "itinerary"')
|
||||
```
|
||||
|
||||
**Keyword OR across subject lines:**
|
||||
```python
|
||||
status, data = conn.search(None, 'OR OR SUBJECT "Cartagena" SUBJECT "flight" SUBJECT "itinerary"')
|
||||
```
|
||||
|
||||
**Chain multiple filters:**
|
||||
```python
|
||||
status, data = conn.search(None, 'FROM "Copa" OR SUBJECT "Cartagena" BODY "Cartagena"')
|
||||
```
|
||||
|
||||
## Account number extraction (Avis Wizard #, etc.)
|
||||
|
||||
Account numbers are often embedded in HTML email statements rather than PDFs. The Wizard number may be partially redacted in the HTML (`P72***`). If the user has past rentals, the confirmation email will show the full account number in the header or body.
|
||||
|
||||
**Method:** Search the raw HTML for the account section:
|
||||
```python
|
||||
idx = html.find('Wizard')
|
||||
if idx >= 0:
|
||||
# Extract text between Wizard #: and next HTML tag
|
||||
m = re.search(r'Wizard\s*#[:\\s]*<span[^>]*>(.*?)</span>', html[idx:idx+300], re.DOTALL)
|
||||
```
|
||||
|
||||
For Avis Wizard numbers specifically: the HTML statement email shows the number partially redacted (e.g. `P72***`). The full number appears on the Avis website after login. If the user needs to book, they'll need to get it from their Avis account or a past confirmation email.
|
||||
|
||||
## Itinerary document structure
|
||||
|
||||
Use a consistent markdown template:
|
||||
|
||||
```markdown
|
||||
# [Destination] Trip — [Date Range]
|
||||
|
||||
**Travelers:** [Names with ages if relevant]
|
||||
|
||||
---
|
||||
|
||||
## Flight Schedule
|
||||
|
||||
### [Traveler Names] — [Airline] (Confirmation: [CODE])
|
||||
|
||||
| Date | Leg | Flight | Departs | Arrives | Duration |
|
||||
|---|---|---|---|---|---|
|
||||
|
||||
---
|
||||
|
||||
## Departure Day — [Date]
|
||||
|
||||
### Time-based Agenda
|
||||
|
||||
| Time | Activity | Notes |
|
||||
|---|---|---|
|
||||
|
||||
---
|
||||
|
||||
## Activities
|
||||
|
||||
- Attractions categorized as Must-See, Kid-Friendly, Adult, Rainy Day
|
||||
- Estimated time per attraction
|
||||
- Location relative to accommodation
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
- [ ] Items specific to this trip
|
||||
```
|
||||
|
||||
## Travel plan adjustments
|
||||
|
||||
When the user provides corrections to the itinerary (all on same flight, TSA PreCheck, no checked bags, gate arrival 90 min before boarding), apply them in order:
|
||||
|
||||
1. **Same flight for all** — update the flight schedule table to list all travelers under one confirmation
|
||||
2. **TSA PreCheck** — adjust MCO arrival: PreCheck average wait ~10-15 min vs 30-45 min standard. Can shift airport arrival 30 min later.
|
||||
3. **No checked bags** — remove bag claim step from timeline. Carry-ons go straight to security after shuttle.
|
||||
4. **Gate 90min before departure** — work backward from boarding time (~30 min before departure). Add 30 min buffer for food/restroom. This sets the "at security" time, then subtract PreCheck estimate to set "arrive airport" time.
|
||||
5. **No rental car in destination** — note as "taxis/ride-share from airport" in the itinerary. Research Avis options as reference data only, document clearly that no booking is needed.
|
||||
6. **Car rental: monitor inbox for confirmations** — the user may start the Avis rental process themselves (email verification code login). Search inbox for "Avis" since last 24 hours for confirmation emails. Avis sends multiple emails per booking (security code, then reset password, then confirmation). The confirmation email contains the reservation number, vehicle class, pick-up/drop-off locations and times, and estimated total. Extract and add to the itinerary.
|
||||
|
||||
## Real-time inbox monitoring during trip planning
|
||||
|
||||
When the user says "check email now for [rental/flight/hotel]" during active trip planning:
|
||||
|
||||
1. Search inbox for the relevant sender since last 24h: `conn.search(None, f'(FROM "Avis" SINCE {since})')`
|
||||
2. Multiple emails from the same sender in the same session likely form a chain: security code → password reset → confirmation
|
||||
3. The confirmation email is the most important — extract reservation number, dates, vehicle class, and any cost
|
||||
4. Report findings concisely in a table
|
||||
5. Update the itinerary checklist to mark the booking as completed
|
||||
6. Remove the associated todo item from the trip planning checklist
|
||||
|
||||
## Drive time estimation
|
||||
|
||||
For Savannah → MCO (July 11, Saturday, ~285 miles):
|
||||
|
||||
| Condition | Time | Notes |
|
||||
|---|---|---|
|
||||
| No traffic | 4h 15m | Typical late-night / early morning |
|
||||
| Light traffic | 4h 30m | Leave before 9 AM Saturday |
|
||||
| Moderate | 5h | Jacksonvlille I-95 afternoon backup |
|
||||
| Heavy | 5h 30m | Holiday weekend, accident, or rain |
|
||||
|
||||
- Saturday I-95 southbound through Jacksonville typically backs up around lunchtime (11 AM - 1 PM)
|
||||
- Best departure: 8:30 AM from Savannah → arrives MCO ~1:30 PM with buffer
|
||||
- Worst case: leave later than 9 AM, hit Jacksonville lunch rush, arrive MCO 2:30+ PM
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Allegiant sends survey/feedback emails** for 48+ hours after each flight. Exclude these by filtering out subject lines containing "feedback" or "survey".
|
||||
- **Allegiant flight confirmation details are in HTML tables** — the FROM/TO airport codes appear inline (e.g. "SAV to CAK") rather than in structured fields. Parse these with regex.
|
||||
- **Copa confirmation emails are in two parts** — 1) a "Your reservation is processing" email, then 2) the "Electronic ticket receipt and confirmation" with the actual PDF. The processing email has the confirmation code but no flight details.
|
||||
- **Copa flights depart from MCO, not SAV** — Copa flies out of Orlando (MCO) for Cartagena. The user drives from Savannah to MCO. If searching for the user's flight and finding Copa, check the departure airport — it may not be their home city.
|
||||
- **Carrier gateways are unreliable for SMS delivery** — AT&T's `@txt.att.net` gateway was shut down. Don't use carrier gateways for automated messages unless confirmed working.
|
||||
- **Anita and Germaine were booked on Copa together** — same confirmation code (A4GSFD) produces two separate PDFs, one per passenger. Download both: the text extraction gives the same data repeated with different names.
|
||||
- **Avis Wizard number is partially redacted** in account statement emails. The last 3 characters show as `***`. For booking purposes, the user must retrieve the full number from their Avis account profile page.
|
||||
- **One-way rental from Savannah to MCO**: Avis allows one-way rentals. Pick up at Savannah location, drop at MCO airport. Need full Wizard # for Preferred Plus benefits.
|
||||
- **Copa flight PDFs contain 6-7 inline image attachments** alongside the ticket PDF. The PDF is the last or second-to-last MIME part in multipart/mixed.
|
||||
- **Global Entry on return: MCO is first port of entry.** Though final destination is Savannah, US customs is cleared at Orlando. Account for ~10-15 min GE kiosk time in the return timeline.
|
||||
- **Avis sends multiple emails per booking in a chain**: security code → (optional) password reset → confirmation. The confirmation email arrives last and contains the reservation number, vehicle class, and pricing. Search `FROM "Avis"` since the last 24h to find all of them.
|
||||
- **Avis confirmation has a "Ford Explorer or Similar"** as default for mid-size SUV. The reservation number format is `#XXXXXXXXUSX` (9 digits + US + 1 digit).
|
||||
- **Avis one-way rental allowed**: pick up at Avis Savannah (SAV), drop at MCO. No extra fee.
|
||||
- **Avis verification code emails** have subject "Avis: Your security code." The code appears in the email body. The agent can read it from the inbox before the user finds it.
|
||||
- **Avis confirmation email arrives last** in a chain: security code → (optional) password reset → confirmation. Search FROM "Avis" since last 24h to get all of them.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Verification Code Reader
|
||||
|
||||
When a service sends a verification code by email (Avis, banks, accounts), the agent can read it from the inbox before the user can find it manually.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. User triggers a login that sends a code email (e.g. entering email on avis.com)
|
||||
2. User says "check email for code" or the workflow expects the code
|
||||
3. Search IMAP for the security-code email from that sender
|
||||
4. Extract the code and relay it
|
||||
|
||||
## Finding the code in the email
|
||||
|
||||
Security-code emails are typically text-only or minimal HTML. The code itself can be:
|
||||
- In the subject line (rare)
|
||||
- In the body as a bold or large number after "security code" or "verification code"
|
||||
- In a table row labeled "Your Security Code"
|
||||
|
||||
## Searching strategy
|
||||
|
||||
```python
|
||||
# Search recent emails from the sender
|
||||
status, data = conn.search(None, f'(FROM "avis@e.avis.com" SINCE "{since}")')
|
||||
```
|
||||
|
||||
The sender is typically:
|
||||
- Avis: `avis@e.avis.com` — subject "Avis: Your security code"
|
||||
- Other services follow similar patterns
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Security codes expire fast** — most are valid for 5-15 minutes. Don't over-optimize search patterns, just grab the latest email from that sender.
|
||||
- **HTML emails may obfuscate the code** — if you can't find a numeric pattern, strip all tags and look for "code" context: `re.search(r'code[:\s]*(\d{4,8})', clean_body, re.IGNORECASE)`
|
||||
- **One-time passwords (OTP) vs link-based auth** — some services send a magic link instead of a numeric code. If no numeric pattern matches, check for a URL containing `token=` or `code=` in the body.
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
# WordPress SMTP Credential Discovery
|
||||
|
||||
When a site uses WP Mail SMTP (or similar) plugin, credentials are stored in the `wp_options` table as a serialized PHP array under `wp_mail_smtp`.
|
||||
|
||||
## Query Pattern
|
||||
|
||||
```sql
|
||||
SELECT option_value FROM wp_options WHERE option_name = 'wp_mail_smtp';
|
||||
```
|
||||
|
||||
The value is a PHP serialized array. Extract fields with:
|
||||
```bash
|
||||
mysql -h 127.0.0.1 -P <PORT> -u <USER> -p'<PASS>' <DB> \
|
||||
-e "SELECT option_value FROM wp_options WHERE option_name = 'wp_mail_smtp'" \
|
||||
--skip-column-names 2>/dev/null | python3 -c "
|
||||
import sys, re
|
||||
data = sys.stdin.read()
|
||||
# PHP serialized array — extract key fields via regex
|
||||
for key in ['mailer', 'host', 'port', 'user', 'pass']:
|
||||
match = re.search(r's:4:\"$key\";s:(\d+):\"(.*?)\"', data)
|
||||
if match: print(f'{key}: {match.group(2)}')
|
||||
"
|
||||
```
|
||||
|
||||
Or parse the PHP serialized format more robustly with Python (the `php-serialize` library, or use the regex approach above for the specific fields needed).
|
||||
|
||||
## Common Locations
|
||||
|
||||
| Site | DB Name | Host | Port | User | How to access |
|
||||
|---|---|---|---|---|---|
|
||||
| Apex Track Experience | `apextrackexperience_1781549652` | c1113726.sgvps.net:2525 | 2525 | `contact@apextrackexperience.com` | MySQL tunnel on Core (`127.0.0.1:33060`) |
|
||||
| Any WordPress site on wphost02 | varies | varies | varies | varies | SSH to wphost02, query local MySQL |
|
||||
|
||||
## MySQL Tunnel Access
|
||||
|
||||
On Core, the Apex Track Experience database is accessible via SSH tunnel through wphost02:
|
||||
```bash
|
||||
mysql -h 127.0.0.1 -P 33060 -u <USER> -p'<PASS>' <DB> -e "SELECT 1"
|
||||
```
|
||||
|
||||
The tunnel is maintained by `mysql-tunnel` systemd service (Core → wphost02:3306).
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- The `wp_mail_smtp` option stores the password as plaintext (not hashed) in the PHP serialized array
|
||||
- The `wp_mail_smtp_initial_version` and `wp_mail_smtp_version` options confirm the plugin version
|
||||
- WPForms uses the same WP Mail SMTP plugin for email — form notification emails are sent via these same credentials
|
||||
- The MySQL password in the connection string is also in plaintext in systemd unit files — keep unit files `chmod 600`
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
# WPForms Email Delivery Verification
|
||||
|
||||
Pattern for checking whether WPForms confirmation emails were actually delivered to form submitters, by cross-referencing IMAP inbox, bounce messages, WordPress DB entries, and DNS records.
|
||||
|
||||
## Overview
|
||||
|
||||
WPForms sends two categories of email on form submission:
|
||||
|
||||
1. **Admin notification** → sent to site admin/owner (e.g. `contact@domain.com`). These appear in the admin IMAP inbox.
|
||||
2. **Confirmation to registrant** → sent to the person who filled the form. These go to external email addresses and may bounce.
|
||||
|
||||
## Verification Flow
|
||||
|
||||
### Step 1: Identify form IDs
|
||||
|
||||
Query the WordPress DB to discover which forms are active and their IDs:
|
||||
|
||||
```sql
|
||||
SELECT ID, post_title FROM wp_posts
|
||||
WHERE post_type='wpforms' AND post_status='publish'
|
||||
ORDER BY ID;
|
||||
```
|
||||
|
||||
Common conventions: form ID 268 = waiver, 270 = paid registration (varies per site).
|
||||
|
||||
### Step 2: Get form entries from the DB
|
||||
|
||||
```sql
|
||||
SELECT entry_id, date, status, LEFT(fields,800)
|
||||
FROM wp_wpforms_entries WHERE form_id=270
|
||||
ORDER BY entry_id;
|
||||
```
|
||||
|
||||
Extract registrant names and emails from the `fields` JSON column. The JSON structure uses numbered field IDs as keys; each has a `value` and an `email` field.
|
||||
|
||||
### Step 3: Check admin notifications in IMAP inbox
|
||||
|
||||
Connect to the site's IMAP inbox and search for WPForms submission notifications:
|
||||
|
||||
```python
|
||||
conn.search(None, f'(SINCE "{yesterday}")')
|
||||
# Look for messages FROM the site's email with subjects containing
|
||||
# registration, waiver, or the form title
|
||||
```
|
||||
|
||||
Admin notifications confirm the form was submitted and the pipeline triggered.
|
||||
|
||||
### Step 4: Check for bounce messages
|
||||
|
||||
Search for mailer-daemon / postmaster bounces:
|
||||
|
||||
```python
|
||||
conn.search(None, 'OR FROM "mailer-daemon" FROM "postmaster"')
|
||||
```
|
||||
|
||||
For each bounce, extract the original recipient address from the body:
|
||||
|
||||
```
|
||||
Delivery to the following recipient failed permanently: user@gmail.com
|
||||
Technical details: 550-5.7.26 Sender unauthenticated (SPF/DKIM)
|
||||
```
|
||||
|
||||
### Step 5: Cross-reference bounces with registrants
|
||||
|
||||
Match bounce recipient addresses against registrant emails from Step 2. These are the registrants whose confirmations didn't arrive.
|
||||
|
||||
### Step 6: Trace SPF/DKIM root cause
|
||||
|
||||
From the bounce message body, extract the sending IP reported by the receiving MTA:
|
||||
|
||||
```
|
||||
SPF [domain.com] with ip: [X.X.X.X] = did not pass
|
||||
```
|
||||
|
||||
Then check:
|
||||
|
||||
```bash
|
||||
# Current SPF record
|
||||
dig +short TXT domain.com | grep spf
|
||||
|
||||
# Current DKIM
|
||||
dig +short TXT default._domainkey.domain.com
|
||||
|
||||
# Current DMARC
|
||||
dig +short TXT _dmarc.domain.com
|
||||
```
|
||||
|
||||
**Common failure:** WP Mail SMTP sends through a relay (e.g. SiteGround `c*.sgvps.net`) which resolves to an IP not in the SPF record. Check the SMTP relay IP:
|
||||
|
||||
```bash
|
||||
dig +short c1113726.sgvps.net
|
||||
```
|
||||
|
||||
Then verify that IP is in the SPF include chain. If not, add it.
|
||||
|
||||
### Step 7: Assess remaining registrants
|
||||
|
||||
If no bounce was received for a registrant, the confirmation likely reached them. Limitations:
|
||||
- Some MTAs silently drop (no bounce) if the mailbox is full
|
||||
- Some spam filters never bounce
|
||||
- Only Gmail/Outlook typically send proper bounce notifications
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- **Bounces from MXroute IPs** — If the WordPress site was migrated between hosting providers, old SPF records may still reference the previous provider's sending IPs. The bounce IP is the actual sending IP, not the SMTP relay hostname.
|
||||
- **WP Mail SMTP logs disabled** — The `wp_mail_smtp` option has `logs.enabled => false` by default in Lite mode. There are no sent-mail logs to check directly.
|
||||
- **Form entries include test/submitted-but-unpaid entries** — Early entries may be test submissions from the site owner with $1.00 amounts. Filter these out by checking the payment field (`amount_raw`).
|
||||
- **Admin notification vs registrant confirmation** — Just because the admin got a notification doesn't mean the registrant did. WPForms sends them separately through the same SMTP pipeline.
|
||||
- **Form field IDs vary** — The `fields` JSON uses numeric field IDs that differ per form configuration. Parse the JSON and check field names, not IDs.
|
||||
@@ -0,0 +1,320 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Inbox solicitor cleaner — standalone script.
|
||||
Reads all messages in INBOX, applies multi-heuristic detection,
|
||||
copies flagged to a Solicitation folder, deletes from INBOX.
|
||||
Adjust the Config block for your server.
|
||||
|
||||
Usage:
|
||||
python3 inbox-solicitation-cleaner.py
|
||||
|
||||
For the reference patterns and heuristic design notes, see
|
||||
references/inbox-solicitation-triaging.md in this skill.
|
||||
"""
|
||||
|
||||
import imaplib
|
||||
import email
|
||||
import re
|
||||
import json
|
||||
from email.header import decode_header, make_header
|
||||
from email.utils import parseaddr
|
||||
|
||||
# ═══════════ Config — ADAPT THESE ═══════════
|
||||
IMAP_HOST = "mail.boxpilotlogistics.com"
|
||||
IMAP_PORT = 993
|
||||
USER = "hello@boxpilotlogistics.com"
|
||||
PASS = "CHANGE_ME"
|
||||
SOLICITATION_FOLDER = "Solicitation"
|
||||
|
||||
# Senders / domains NEVER to flag
|
||||
INTERNAL_SENDERS = {"curt", "anita"}
|
||||
LEGIT_DOMAINS = {"boxpilotlogistics.com"}
|
||||
|
||||
# Free / personal email domains (solicitation signal)
|
||||
FREE_DOMAINS = {
|
||||
"gmail.com", "yahoo.com", "ymail.com", "outlook.com", "hotmail.com",
|
||||
"aol.com", "icloud.com", "me.com", "mac.com", "protonmail.com",
|
||||
"proton.me", "pm.me", "zoho.com", "mail.com", "yandex.com",
|
||||
"fastmail.com", "tutanota.com", "gmx.com", "live.com", "msn.com",
|
||||
"comcast.net", "verizon.net", "att.net", "sbcglobal.net", "bellsouth.net",
|
||||
"cox.net", "earthlink.net", "charter.net", "optonline.net",
|
||||
}
|
||||
|
||||
SOLICITATION_KEYWORDS = [
|
||||
"insurance", "coverage", "liability", "cargo insurance",
|
||||
"general liability", "motor truck cargo", "physical damage",
|
||||
"occupational accident", "workers comp", "bobtail", "non-trucking",
|
||||
"factoring", "freight factoring", "invoice factoring", "cash flow",
|
||||
"quick pay", "same day pay",
|
||||
"fuel card", "fuel discount", "fuel savings", "fuel program",
|
||||
"diesel discount", "fuel rewards",
|
||||
"compliance", "dot compliance", "fmcsa", "authority", "mc number",
|
||||
"usdot", "motor carrier authority", "boc-3", "boc 3", "process agent",
|
||||
"operating authority", "new authority", "authority filing",
|
||||
"eld", "electronic logging", "elog", "h.o.s.", "hours of service",
|
||||
"logbook", "electronic log",
|
||||
"dispatch", "dispatcher", "load board", "loadboard", "freight matching",
|
||||
"dat load", "truckstop", "find loads", "backhaul",
|
||||
"extended warranty", "vehicle warranty", "truck warranty",
|
||||
"warranty coverage", "warranty expir",
|
||||
"logo design", "website design", "seo service", "web development",
|
||||
"digital marketing", "social media marketing",
|
||||
"special offer", "limited time", "act now", "sign up today",
|
||||
"get started today", "free quote", "no obligation", "risk free",
|
||||
"exclusive offer", "grow your business", "take your business",
|
||||
]
|
||||
|
||||
SOLICITATION_PHRASES = [
|
||||
"we help trucking companies", "help your trucking",
|
||||
"trucking company", "new entrant", "new mc", "new authority",
|
||||
"carrier authority", "your mc", "trucking authority",
|
||||
"startup carrier", "new carrier", "motor carrier",
|
||||
"dot number", "safety audit", "new entrant safety",
|
||||
"new to trucking", "independent contractor",
|
||||
"owner operator", "small fleet",
|
||||
]
|
||||
|
||||
SOLICITATION_SUBJECT_KEYWORDS = [
|
||||
"get a quote", "free quote", "insurance quote", "fuel card",
|
||||
"factoring", "dispatch services", "load board", "eld mandate",
|
||||
"compliance", "authority filing",
|
||||
"your mc", "new mc", "mc number",
|
||||
"logo design", "website design", "seo",
|
||||
"extended warranty", "reduce your cost",
|
||||
"partner with us", "limited time offer",
|
||||
"trucking solution", "fleet solution",
|
||||
"lower your rate", "save on fuel",
|
||||
"solicitation", "advertisement", "ad:",
|
||||
]
|
||||
|
||||
|
||||
def decode_mime_header(val):
|
||||
if val is None:
|
||||
return ""
|
||||
try:
|
||||
return str(make_header(decode_header(val)))
|
||||
except Exception:
|
||||
return val
|
||||
|
||||
|
||||
def get_email_domain(email_addr):
|
||||
_, addr = parseaddr(email_addr)
|
||||
if "@" in addr:
|
||||
return addr.lower().split("@")[1]
|
||||
return ""
|
||||
|
||||
|
||||
def get_clean_sender(msg):
|
||||
from_hdr = decode_mime_header(msg.get("From", ""))
|
||||
name, addr = parseaddr(from_hdr)
|
||||
return name or addr.split("@")[0] if "@" in addr else from_hdr, addr.lower()
|
||||
|
||||
|
||||
def get_body_text(msg):
|
||||
body = ""
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
if ct == "text/plain":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
body += payload.decode(charset, errors="replace")
|
||||
except LookupError:
|
||||
body += payload.decode("utf-8", errors="replace")
|
||||
else:
|
||||
payload = msg.get_payload(decode=True)
|
||||
if payload:
|
||||
charset = msg.get_content_charset() or "utf-8"
|
||||
try:
|
||||
body = payload.decode(charset, errors="replace")
|
||||
except LookupError:
|
||||
body = payload.decode("utf-8", errors="replace")
|
||||
if not body:
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
if part.get_content_type() == "text/html":
|
||||
payload = part.get_payload(decode=True)
|
||||
if payload:
|
||||
charset = part.get_content_charset() or "utf-8"
|
||||
try:
|
||||
raw = payload.decode(charset, errors="replace")
|
||||
except LookupError:
|
||||
raw = payload.decode("utf-8", errors="replace")
|
||||
body = re.sub(r"<[^>]+>", " ", raw)
|
||||
body = re.sub(r"\s+", " ", body).strip()
|
||||
return body
|
||||
|
||||
|
||||
def has_business_domain_ref(body, sender_domain):
|
||||
refs = re.findall(
|
||||
r'(?:https?://)?(?:www\.)?([a-zA-Z0-9][a-zA-Z0-9-]*\.[a-zA-Z]{2,})',
|
||||
body
|
||||
)
|
||||
for d in refs:
|
||||
d_lower = d.lower()
|
||||
if d_lower not in FREE_DOMAINS and d_lower != sender_domain \
|
||||
and "google" not in d_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_solicitation(sender_name, sender_addr, subject, body, from_domain):
|
||||
reasons = []
|
||||
subj_l = subject.lower()
|
||||
body_l = body.lower()
|
||||
|
||||
sender_local = sender_addr.split("@")[0].lower() if "@" in sender_addr else ""
|
||||
for internal in INTERNAL_SENDERS:
|
||||
if internal in sender_addr or internal in sender_local:
|
||||
return False, []
|
||||
if from_domain in LEGIT_DOMAINS:
|
||||
return False, []
|
||||
|
||||
for kw in SOLICITATION_SUBJECT_KEYWORDS:
|
||||
if kw in subj_l:
|
||||
reasons.append(f"Subject contains solicitation keyword: '{kw}'")
|
||||
break
|
||||
|
||||
if from_domain in FREE_DOMAINS:
|
||||
if has_business_domain_ref(body, from_domain):
|
||||
reasons.append(f"Free-email sender ({from_domain}) referencing "
|
||||
f"business domain in body")
|
||||
for kw in SOLICITATION_KEYWORDS:
|
||||
if kw in body_l:
|
||||
reasons.append(f"Body contains solicitation keyword: '{kw}'")
|
||||
break
|
||||
for phrase in SOLICITATION_PHRASES:
|
||||
if phrase in body_l:
|
||||
reasons.append(f"Body contains solicitation phrase: '{phrase}'")
|
||||
break
|
||||
|
||||
if from_domain not in FREE_DOMAINS and from_domain not in LEGIT_DOMAINS:
|
||||
trucking_signals = 0
|
||||
seen = set()
|
||||
for phrase in SOLICITATION_PHRASES:
|
||||
if phrase in body_l:
|
||||
trucking_signals += 1
|
||||
if phrase not in seen:
|
||||
seen.add(phrase)
|
||||
reasons.append(f"Body contains trucking solicitation phrase: "
|
||||
f"'{phrase}'")
|
||||
for kw in SOLICITATION_KEYWORDS:
|
||||
if kw in body_l:
|
||||
trucking_signals += 1
|
||||
if kw not in seen:
|
||||
seen.add(kw)
|
||||
reasons.append(f"Body contains solicitation keyword: '{kw}'")
|
||||
for kw in SOLICITATION_SUBJECT_KEYWORDS:
|
||||
if kw in subj_l:
|
||||
trucking_signals += 1
|
||||
kw_rep = f"Subject keyword: '{kw}'"
|
||||
if kw_rep not in seen:
|
||||
seen.add(kw_rep)
|
||||
reasons.append(kw_rep)
|
||||
if trucking_signals >= 2:
|
||||
reasons.append(f"Multiple trucking-solicitation signals (count: "
|
||||
f"{trucking_signals})")
|
||||
|
||||
if not reasons:
|
||||
pitch_count = 0
|
||||
for kw in SOLICITATION_KEYWORDS:
|
||||
if kw in subj_l or kw in body_l:
|
||||
pitch_count += 1
|
||||
for phrase in SOLICITATION_PHRASES:
|
||||
if phrase in body_l:
|
||||
pitch_count += 2
|
||||
if pitch_count >= 2:
|
||||
reasons.append(f"Combined solicitation signal strength: {pitch_count}")
|
||||
|
||||
return bool(reasons), reasons
|
||||
|
||||
|
||||
def main():
|
||||
conn = imaplib.IMAP4_SSL(IMAP_HOST, IMAP_PORT)
|
||||
conn.login(USER, PASS)
|
||||
conn.select("INBOX")
|
||||
|
||||
status, data = conn.search(None, "ALL")
|
||||
msg_ids = data[0].split() if data[0] else []
|
||||
print(f"Total in INBOX: {len(msg_ids)}")
|
||||
|
||||
# Ensure target folder exists
|
||||
status, folders = conn.list()
|
||||
folder_pat = re.compile(r'.*"([^"]+)"$')
|
||||
folder_names = []
|
||||
for f in folders:
|
||||
m = folder_pat.search(f.decode("utf-8", errors="replace"))
|
||||
if m:
|
||||
folder_names.append(m.group(1))
|
||||
if not any(SOLICITATION_FOLDER in fn for fn in folder_names):
|
||||
conn.create(SOLICITATION_FOLDER)
|
||||
|
||||
flagged, kept, errors = [], [], []
|
||||
|
||||
for idx, msg_id in enumerate(msg_ids, 1):
|
||||
try:
|
||||
status, data = conn.fetch(msg_id, "(BODY.PEEK[])")
|
||||
if status != "OK":
|
||||
errors.append((msg_id, "FETCH failed"))
|
||||
continue
|
||||
raw = data[0][1]
|
||||
msg = email.message_from_bytes(raw) \
|
||||
if isinstance(raw, bytes) \
|
||||
else email.message_from_string(raw)
|
||||
|
||||
subject = decode_mime_header(msg.get("Subject", ""))
|
||||
sname, saddr = get_clean_sender(msg)
|
||||
domain = get_email_domain(saddr)
|
||||
body = get_body_text(msg)
|
||||
|
||||
solicit, reasons = is_solicitation(sname, saddr, subject, body, domain)
|
||||
entry = {
|
||||
"id": int(msg_id), "from": saddr, "from_name": sname or saddr,
|
||||
"subject": subject, "domain": domain,
|
||||
}
|
||||
if solicit:
|
||||
entry["reasons"] = reasons
|
||||
flagged.append(entry)
|
||||
else:
|
||||
kept.append(entry)
|
||||
except Exception as e:
|
||||
errors.append((msg_id, str(e)))
|
||||
|
||||
print(f"\nRESULTS: {len(flagged)} flagged, {len(kept)} kept, "
|
||||
f"{len(errors)} errors")
|
||||
|
||||
moved = 0
|
||||
for entry in flagged:
|
||||
mid = str(entry["id"]).encode()
|
||||
try:
|
||||
status, _ = conn.copy(mid, SOLICITATION_FOLDER)
|
||||
if status == "OK":
|
||||
conn.store(mid, "+FLAGS", "\\Deleted")
|
||||
moved += 1
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
conn.expunge()
|
||||
conn.logout()
|
||||
|
||||
report = {
|
||||
"total_inbox": len(msg_ids), "flagged_count": len(flagged),
|
||||
"kept_count": len(kept), "moved_count": moved,
|
||||
"errors": [{"msg_id": str(e[0]), "error": e[1]} for e in errors],
|
||||
"flagged": flagged,
|
||||
"kept_summary": [
|
||||
{"id": e["id"], "from": e["from"], "subject": e["subject"][:80]}
|
||||
for e in kept
|
||||
],
|
||||
}
|
||||
with open("inbox_review_report.json", "w") as f:
|
||||
json.dump(report, f, indent=2, default=str)
|
||||
|
||||
print(f"\nSummary: {moved} moved, {len(kept)} kept, report → "
|
||||
f"inbox_review_report.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user