268 lines
9.3 KiB
Python
Executable File
268 lines
9.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
daily-feed-summary.py — Collect RSS feeds, summarize top articles, deliver digest.
|
|
|
|
Feeds:
|
|
- XDA Developers: https://www.xda-developers.com/feed/
|
|
- MakeUseOf: https://www.makeuseof.com/feed/
|
|
- VirtualizationHowTo: https://www.virtualizationhowto.com/feed/
|
|
- Top Gear: No RSS — scraped from front page
|
|
- 9to5Mac: https://9to5mac.com/feed/
|
|
|
|
Output: Markdown summary with top 10-15 articles across all feeds.
|
|
"""
|
|
|
|
import feedparser, json, sys, os, re, subprocess
|
|
from datetime import datetime, timezone, timedelta
|
|
from email.mime.text import MIMEText
|
|
import smtplib
|
|
import urllib.request
|
|
from html.parser import HTMLParser
|
|
|
|
FEEDS = {
|
|
"XDA Developers": "https://www.xda-developers.com/feed/",
|
|
"MakeUseOf": "https://www.makeuseof.com/feed/",
|
|
"Virtualization HowTo": "https://www.virtualizationhowto.com/feed/",
|
|
"9to5Mac": "https://9to5mac.com/feed/",
|
|
"Gizmodo": "https://gizmodo.com/feed/",
|
|
"How-To Geek": "https://www.howtogeek.com/feed/",
|
|
}
|
|
|
|
# Top Gear — no RSS, try scraping car-news section
|
|
TOPGEAR_URL = "https://www.topgear.com/car-news"
|
|
|
|
class HHLinkParser(HTMLParser):
|
|
"""Parse HotHardware /news page for article links."""
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.links = []
|
|
self._in_link = False
|
|
self._href = None
|
|
self._text = ""
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = dict(attrs)
|
|
if tag == 'a' and 'href' in attrs:
|
|
href = attrs['href']
|
|
if href.startswith('/news/') and len(href) > 7:
|
|
self._in_link = True
|
|
self._href = href
|
|
self._text = ""
|
|
|
|
def handle_data(self, data):
|
|
if self._in_link:
|
|
self._text += data.strip()
|
|
|
|
def handle_endtag(self, tag):
|
|
if self._in_link and tag == 'a':
|
|
self._in_link = False
|
|
txt = self._text.strip()
|
|
if txt and len(txt) > 10:
|
|
self.links.append((txt, self._href))
|
|
|
|
def fetch_hothardware():
|
|
"""Scrape HotHardware news page for article links."""
|
|
try:
|
|
req = urllib.request.Request(
|
|
"https://hothardware.com/news",
|
|
headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'}
|
|
)
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
html = resp.read().decode('utf-8', errors='replace')
|
|
|
|
parser = HHLinkParser()
|
|
parser.feed(html)
|
|
|
|
seen = set()
|
|
articles = []
|
|
for title, href in parser.links:
|
|
if href not in seen:
|
|
seen.add(href)
|
|
url = f"https://hothardware.com{href}" if href.startswith('/') else href
|
|
articles.append((title, url))
|
|
|
|
return articles[:10]
|
|
except Exception as e:
|
|
return [("HotHardware scrape failed", f"Error: {e}")]
|
|
|
|
|
|
class TGLinkParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.links = []
|
|
self._capture = False
|
|
self._href = None
|
|
self._depth = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
attrs = dict(attrs)
|
|
if tag == 'a' and 'href' in attrs:
|
|
href = attrs['href']
|
|
if href.startswith('/car-news/') and not href.startswith('/car-news/?'):
|
|
title = attrs.get('title', '') or ''
|
|
text = ''
|
|
if title:
|
|
self.links.append((href, title))
|
|
self._href = href
|
|
|
|
def fetch_topgear():
|
|
"""Scrape Top Gear front page for article links."""
|
|
try:
|
|
req = urllib.request.Request(
|
|
TOPGEAR_URL,
|
|
headers={'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36'}
|
|
)
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
html = resp.read().decode('utf-8', errors='replace')
|
|
|
|
parser = TGLinkParser()
|
|
parser.feed(html)
|
|
|
|
# Deduplicate by URL
|
|
seen = set()
|
|
articles = []
|
|
for href, title in parser.links:
|
|
if href not in seen:
|
|
seen.add(href)
|
|
url = f"https://www.topgear.com{href}" if href.startswith('/') else href
|
|
articles.append((title, url))
|
|
|
|
return articles[:15]
|
|
except Exception as e:
|
|
return [("Top Gear scrape failed", f"Error: {e}")]
|
|
|
|
def fetch_feed(url, limit=10):
|
|
"""Parse RSS feed, return list of (title, link, summary)."""
|
|
try:
|
|
feed = feedparser.parse(url)
|
|
entries = []
|
|
for entry in feed.entries[:limit]:
|
|
title = entry.get('title', 'No title')
|
|
link = entry.get('link', '')
|
|
# Get summary/description
|
|
summary = ''
|
|
if 'summary' in entry:
|
|
summary = entry.summary
|
|
elif 'description' in entry:
|
|
summary = entry.description
|
|
# Strip HTML
|
|
summary = re.sub(r'<[^>]+>', '', summary)[:300]
|
|
entries.append((title, link, summary))
|
|
return entries
|
|
except Exception as e:
|
|
return [(f"Feed error: {e}", "", "")]
|
|
|
|
def build_digest():
|
|
"""Collect all feeds and build markdown digest."""
|
|
lines = []
|
|
now = datetime.now(timezone.utc)
|
|
eastern = now - timedelta(hours=4) # approximate ET
|
|
lines.append(f"# 📰 Daily Tech Digest — {eastern.strftime('%A, %B %d, %Y')}")
|
|
lines.append("")
|
|
|
|
all_articles = []
|
|
|
|
# RSS feeds
|
|
for name, url in FEEDS.items():
|
|
entries = fetch_feed(url, 10)
|
|
lines.append(f"## {name}")
|
|
lines.append("")
|
|
for title, link, summary in entries:
|
|
if link:
|
|
lines.append(f"- **[{title}]({link})**")
|
|
else:
|
|
lines.append(f"- **{title}**")
|
|
if summary:
|
|
lines.append(f" _{summary}_")
|
|
lines.append("")
|
|
all_articles.extend(entries)
|
|
|
|
# Top Gear
|
|
lines.append("## Top Gear")
|
|
lines.append("")
|
|
tg = fetch_topgear()
|
|
for title, url in tg:
|
|
if url.startswith("https://"):
|
|
lines.append(f"- **[{title}]({url})**")
|
|
else:
|
|
lines.append(f"- *{title}*")
|
|
lines.append("")
|
|
all_articles.extend([(t, u, "") for t, u in tg])
|
|
|
|
# HotHardware (scraped — no RSS)
|
|
lines.append("## HotHardware")
|
|
lines.append("")
|
|
hh = fetch_hothardware()
|
|
for title, url in hh:
|
|
if url.startswith("https://"):
|
|
lines.append(f"- **[{title}]({url})**")
|
|
else:
|
|
lines.append(f"- *{title}*")
|
|
lines.append("")
|
|
all_articles.extend([(t, u, "") for t, u in hh])
|
|
|
|
lines.append("---")
|
|
lines.append(f"*{len(all_articles)} articles from {len(FEEDS) + 2} sources*")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def send_email(body):
|
|
"""Send digest via SMTP to g@germainebrown.com as HTML."""
|
|
sender = "g@germainebrown.com"
|
|
pwd_file = "/root/.config/himalaya/g-germainebrown.pass"
|
|
with open(pwd_file) as f:
|
|
password = f.read().strip()
|
|
|
|
# Convert markdown links [text](url) to HTML <a href="url">text</a>
|
|
import re
|
|
html_body = re.sub(
|
|
r'\[([^\]]+)\]\(([^)]+)\)',
|
|
r'<a href="\2" style="color:#2563eb;text-decoration:underline;">\1</a>',
|
|
body
|
|
)
|
|
# Convert markdown bold **text** to <strong>text</strong>
|
|
html_body = re.sub(r'\*\*([^*]+)\*\*', r'<strong>\1</strong>', html_body)
|
|
# Convert markdown italic _text_ to <em>text</em>
|
|
html_body = re.sub(r'_([^_]+)_', r'<em>\1</em>', html_body)
|
|
# Convert ## headers to <h2>
|
|
html_body = re.sub(r'^## (.+)$', r'<h2 style="font-size:16px;margin:16px 0 4px;color:#1a1a2e;">\1</h2>', html_body, flags=re.MULTILINE)
|
|
# Convert # header to <h1>
|
|
html_body = re.sub(r'^# (.+)$', r'<h1 style="font-size:20px;margin:0 0 8px;color:#1a1a2e;">\1</h1>', html_body, flags=re.MULTILINE)
|
|
# Convert --- to <hr>
|
|
html_body = re.sub(r'^---+$', r'<hr style="border:none;border-top:1px solid #e2e8f0;margin:16px 0;">', html_body, flags=re.MULTILINE)
|
|
# Wrap in proper HTML
|
|
html = f'''<!DOCTYPE html>
|
|
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
|
|
<style>body{{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;font-size:13px;line-height:1.6;color:#333;padding:20px;max-width:640px;margin:0 auto;}}
|
|
a{{color:#2563eb;}}p{{margin:4px 0;}}</style>
|
|
</head><body>
|
|
<pre style="font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;font-size:13px;line-height:1.6;white-space:pre-wrap;word-wrap:break-word;">{body}</pre>
|
|
</body></html>'''
|
|
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
|
|
msg = MIMEMultipart('alternative')
|
|
msg['Subject'] = f"Daily Tech Digest — {datetime.now().strftime('%b %d')}"
|
|
msg['From'] = sender
|
|
msg['To'] = sender
|
|
|
|
msg.attach(MIMEText(body, 'plain', 'utf-8'))
|
|
msg.attach(MIMEText(html, 'html', 'utf-8'))
|
|
|
|
with smtplib.SMTP('mail.germainebrown.com', 2525) as s:
|
|
s.starttls()
|
|
s.login(sender, password)
|
|
s.send_message(msg)
|
|
|
|
print(f"Digest sent to {sender}")
|
|
|
|
if __name__ == "__main__":
|
|
digest = build_digest()
|
|
# Output to stdout (captured by cron) and also email
|
|
print(digest)
|
|
|
|
# If run as standalone, also email
|
|
if len(sys.argv) > 1 and sys.argv[1] == '--email':
|
|
send_email(digest)
|