hjess/mmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First commit

Browse Source
This commit is contained in:
Henrik Jess Nielsen
2026-05-26 22:21:27 +02:00
parent 2743a236b2
commit 05eed51e7d
90 changed files with 8690 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

559
ground_news.py Normal file
View File

@@ -0,0 +1,559 @@
"""
ground_news.py — Ground News article fetcher + local SQLite store
Key design:
- RSC payload trick: send RSC: 1 header to get Next.js App Router data
- page_cache table: raw RSC payloads with TTL (don't re-fetch fresh pages)
- articles table: all extracted fields, categories merged across pages
- fetch_article(slug) — single article, rich data
- fetch_category(slug) — all stories on an interest page (~15 stories)
- fetch_all() — all known interest categories in parallel
- top_articles(n, days)— query DB for top-N by source_count
"""
import re
import json
import time
import sqlite3
import httpx
import concurrent.futures
from pathlib import Path
from db import get_conn, DBConn
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
DB_PATH = Path(__file__).parent / "ground_news.db"
BASE_URL = "https://ground.news"
CACHE_TTL = {
"interest": 30 * 60, # category pages: 30 min
"article": 6 * 60 * 60, # single articles: 6 h
}
HEADERS = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
"Accept": "text/html,application/xhtml+xml",
"RSC": "1",
"Next-Router-State-Tree": (
"%5B%22%22%2C%7B%22children%22%3A%5B%22__PAGE__%22%2C%7B%7D%5D%7D%2Cnull%2Cnull%2Ctrue%5D"
),
}
# All known interest slugs (auto-discovered from ground.news homepage 2026-05-24)
KNOWN_INTERESTS: dict[str, str] = {
"europe": "Europe",
"europe-economy": "Europe Economy",
"european-politics": "European Politics",
"european-union": "European Union",
"european-security-and-nato": "European Security & NATO",
"uk-politics": "UK Politics",
"united-kingdom": "United Kingdom",
"international": "International",
"north-america": "North America",
"south-america": "South America",
"africa": "Africa",
"asia": "Asia",
"australia": "Australia",
"us-politics": "US Politics",
"united-states": "United States",
"donald-trump": "Donald Trump",
"trump-administration": "Trump Administration",
"israeli-palestinian-conflict": "Israeli-Palestinian Conflict",
"business-and-markets": "Business & Markets",
"premier-league": "Premier League",
"soccer": "Soccer",
"memorial-day": "Memorial Day",
# Financial / C25 relevant categories
"pharma": "Pharmaceuticals",
"energy": "Energy",
"renewable-energy": "Renewable Energy",
"denmark": "Denmark",
"finance": "Finance",
"corporate": "Corporate",
"technology": "Technology",
"climate-change": "Climate Change",
"shipping": "Shipping",
# Danish/Nordic specific
"biotech": "Biotech",
"healthcare": "Healthcare",
"pharmaceutical": "Pharmaceutical",
"nordic": "Nordic",
"scandinavia": "Scandinavia",
"denmark-economy": "Denmark Economy",
"danish-economy": "Danish Economy",
"global-economy": "Global Economy",
"global-markets": "Global Markets",
"stock-market": "Stock Market",
"investing": "Investing",
"clean-energy": "Clean Energy",
"logistics": "Logistics",
"diabetes": "Diabetes",
}
# ---------------------------------------------------------------------------
# Database
# ---------------------------------------------------------------------------
def get_db() -> DBConn:
"""Return a DBConn wrapper (Postgres or SQLite). Schema is managed by db.py."""
return get_conn()
# ---------------------------------------------------------------------------
# HTTP fetch with cache
# ---------------------------------------------------------------------------
def fetch_cached(db: DBConn, url: str, page_type: str = "interest") -> tuple[str, bool]:
"""Return (content, from_cache). Re-fetches if stale per CACHE_TTL."""
row = db.execute(
"SELECT content, fetched_at FROM page_cache WHERE url=?", (url,)
).fetchone()
ttl = CACHE_TTL.get(page_type, 1800)
now = int(time.time())
if row and (now - row["fetched_at"]) < ttl:
return row["content"], True
r = httpx.get(url, headers=HEADERS, follow_redirects=True, timeout=20)
r.raise_for_status()
db.upsert(
"page_cache", "url",
["url", "page_type", "fetched_at", "content"],
(url, page_type, now, r.text),
)
db.commit()
return r.text, False
# ---------------------------------------------------------------------------
# RSC payload parsers
# ---------------------------------------------------------------------------
# UUID v4 pattern
_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")
# blindspotData — has coverageProfileStatement + coverageProfileType before the numbers
_BLIND = re.compile(
r'"blindspotData":\{[^}]{0,400}' # skip coverageProfileStatement, coverageProfileType
r'"leftPercent":([\d.]+),"rightPercent":([\d.]+),"centerPercent":([\d.]+),'
r'"leftSrcCount":(\d+),"rightSrcCount":(\d+),"cntrSrcCount":(\d+)'
)
# Story anchor: start + title + slug + factuality (field order confirmed from RSC)
_STORY = re.compile(
r'"start":"(20\d{2}-[^"]+)",'
r'"title":"([^"]{10,200})",'
r'"slug":"([a-z0-9][a-z0-9_-]{15,})",'
r'"factuality":\{([^}]+)\}'
)
# Escaped JSON string value
_JSON_STR = re.compile(r'"((?:[^"\\]|\\.)*)"')
def _decode(s: str) -> str:
"""Decode a JSON-escaped string value."""
try:
return json.loads(f'"{s}"')
except Exception:
return s
def parse_stories(data: str, category: str) -> list[dict]:
"""Extract all story objects from an RSC payload."""
stories = []
for m in _STORY.finditer(data):
start, title, slug, fact_raw = m.group(1), m.group(2), m.group(3), m.group(4)
before = data[max(0, m.start() - 8000): m.start()]
after = data[m.end(): m.end() + 6000]
# UUID — last v4 UUID found before the story anchor (the story's own id)
uuids = _UUID.findall(before[-4000:])
story_id = uuids[-1] if uuids else None
# blindspotData (comes before the anchor)
blind = _BLIND.search(before[-8000:])
left_pct = right_pct = ctr_pct = None
left_cnt = right_cnt = ctr_cnt = None
if blind:
left_pct = float(blind.group(1)) # already 0-100
right_pct = float(blind.group(2))
ctr_pct = float(blind.group(3))
left_cnt = int(blind.group(4))
right_cnt = int(blind.group(5))
ctr_cnt = int(blind.group(6))
# biasSourceCount
bsc = re.search(r'"biasSourceCount":(\d+)', before[-8000:])
bias_src_count = int(bsc.group(1)) if bsc else 0
# overallBias score (-1 .. +1)
ob = re.search(r'"overallBias":([-\d.]+)', before[-8000:])
overall_bias = float(ob.group(1)) if ob else None
# blindspot label ("left"/"right"/"center")
bs = re.search(r'"blindspot":"(left|right|center|none)"', before[-8000:])
blindspot = bs.group(1) if bs else None
# description — allow JSON-escaped content
desc_m = re.search(r'"description":"((?:[^"\\]|\\.){0,600})"', before[-3000:])
description = _decode(desc_m.group(1)) if desc_m else None
# sourceCount (comes after the anchor in sources:[...])
sc = re.search(r'"sourceCount":(\d+)', after)
source_count = int(sc.group(1)) if sc else 0
# factuality
factuality = {k: int(v) for k, v in re.findall(r'"(\w+)":(\d+)', fact_raw)}
# Ground News interest UUIDs this story belongs to
int_m = re.search(r'"interests":\[([^\]]*)\]', before[-2000:])
interests = _UUID.findall(int_m.group(1)) if int_m else []
stories.append({
"slug": slug,
"story_id": story_id,
"title": _decode(title),
"description": description,
"start_date": start[:10],
"source_count": source_count,
"bias_src_count": bias_src_count,
"left_pct": left_pct,
"ctr_pct": ctr_pct,
"right_pct": right_pct,
"left_src_count": left_cnt,
"ctr_src_count": ctr_cnt,
"right_src_count":right_cnt,
"overall_bias": overall_bias,
"blindspot": blindspot,
"factuality": factuality,
"interests": interests,
"category": category,
})
return stories
def parse_single_article(data: str, slug: str) -> dict:
"""Richer extraction for a single article page (has wireStoryRefs etc)."""
def get(pattern, cast=str):
m = re.search(pattern, data)
try:
return cast(m.group(1)) if m else None
except Exception:
return None
# story_id: UUID before the slug
id_m = re.search(r'"id":"([0-9a-f-]{36})"[^}]{0,200}"slug":"' + re.escape(slug), data, re.DOTALL)
story_id = id_m.group(1) if id_m else get(r'"id":"([0-9a-f-]{36})"')
# Title — must come before wireStoryRefs
title_m = re.search(r'"title":"([^"]{10,200})"[^}]{0,100}"wireStoryRefs"', data, re.DOTALL)
title = _decode(title_m.group(1)) if title_m else get(r'"title":"([^"]{10,200})"')
# blindspotData
blind = _BLIND.search(data)
# Bias side breakdown
bias_breakdown = {}
for side in ("left", "center", "right"):
bm = re.search(
rf'"id":"{side}".*?"sourceCount":(\d+).*?"percent":(\d+)',
data, re.DOTALL
)
if bm:
bias_breakdown[side] = {"sources": int(bm.group(1)), "percent": int(bm.group(2))}
# factuality
fm = re.search(r'"factuality":\{([^}]+)\}', data)
factuality = {k: int(v) for k, v in re.findall(r'"(\w+)":(\d+)', fm.group(1))} if fm else {}
desc_m = re.search(r'"description":"((?:[^"\\]|\\.){20,600})"', data)
return {
"slug": slug,
"story_id": story_id,
"title": title,
"description": _decode(desc_m.group(1)) if desc_m else None,
"start_date": get(r'"start":"(20\d{2}-[^"]+)"'),
"source_count": get(r'"sourceCount":(\d+)', int),
"bias_src_count": get(r'"biasSourceCount":(\d+)', int),
"overall_bias": get(r'"overallBias":([-\d.]+)', float),
"blindspot": get(r'"blindspot":"(left|right|center|none)"'),
"left_pct": float(blind.group(1)) if blind else None,
"right_pct": float(blind.group(2)) if blind else None,
"ctr_pct": float(blind.group(3)) if blind else None,
"left_src_count": int(blind.group(4)) if blind else None,
"right_src_count": int(blind.group(5)) if blind else None,
"ctr_src_count": int(blind.group(6)) if blind else None,
"factuality": factuality,
"bias_breakdown": bias_breakdown,
}
# ---------------------------------------------------------------------------
# DB upsert
# ---------------------------------------------------------------------------
def upsert_articles(db: DBConn, stories: list[dict]) -> int:
"""Insert new / update existing articles. Returns count of new rows."""
now = int(time.time())
new = 0
for s in stories:
row = db.execute(
"SELECT categories, first_seen FROM articles WHERE slug=?", (s["slug"],)
).fetchone()
cats = set((row["categories"] or "").split(",")) if row else set()
cats.discard("")
cats.add(s["category"])
if row:
db.execute(
"""UPDATE articles SET
story_id=COALESCE(story_id, ?),
source_count=?, bias_src_count=?,
left_pct=?, ctr_pct=?, right_pct=?,
left_src_count=?, ctr_src_count=?, right_src_count=?,
overall_bias=?, blindspot=?,
description=COALESCE(description, ?),
categories=?, last_seen=?
WHERE slug=?""",
(s["story_id"],
s["source_count"], s["bias_src_count"],
s["left_pct"], s["ctr_pct"], s["right_pct"],
s["left_src_count"], s["ctr_src_count"], s["right_src_count"],
s["overall_bias"], s["blindspot"],
s["description"],
",".join(sorted(cats)), now,
s["slug"]),
)
else:
db.execute(
"""INSERT INTO articles
(slug, story_id, title, description, start_date,
source_count, bias_src_count,
left_pct, ctr_pct, right_pct,
left_src_count, ctr_src_count, right_src_count,
overall_bias, blindspot,
factuality_json, interests_json,
categories, first_seen, last_seen)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
(s["slug"], s["story_id"], s["title"], s["description"], s["start_date"],
s["source_count"], s["bias_src_count"],
s["left_pct"], s["ctr_pct"], s["right_pct"],
s["left_src_count"], s["ctr_src_count"], s["right_src_count"],
s["overall_bias"], s["blindspot"],
json.dumps(s["factuality"]), json.dumps(s["interests"]),
s["category"], now, now),
)
new += 1
db.commit()
return new
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
def fetch_article_text(slug: str, db: DBConn | None = None) -> str:
"""
Fetch full article RSC payload and return a clean text blob for NLP.
Extracts: main title + description + all source article headlines.
"""
own_db = db is None
if own_db:
db = get_db()
url = f"{BASE_URL}/article/{slug}"
data, _ = fetch_cached(db, url, "article")
if own_db:
db.close()
parts: list[str] = []
seen: set[str] = set()
def add(text: str) -> None:
if text and len(text) > 20 and text not in seen:
seen.add(text)
parts.append(text)
# Main title
for m in re.finditer(r'"title":"((?:[^"\\]|\\.){10,300})"', data):
t = _decode(m.group(1))
if not re.search(r'Getty|AFP|\/AFP|PHOTO-TAG', t, re.I):
add(t)
# Descriptions / excerpts
for pattern in [
r'"description":"((?:[^"\\]|\\.){20,600})"',
r'"excerpt":"((?:[^"\\]|\\.){20,400})"',
r'"summary":"((?:[^"\\]|\\.){20,400})"',
]:
for m in re.finditer(pattern, data):
t = _decode(m.group(1))
if not re.search(r'Getty|AFP|PHOTO-TAG|Author:', t, re.I):
add(t)
# Wire story / source article headlines
for m in re.finditer(r'"headline":"((?:[^"\\]|\\.){10,300})"', data):
add(_decode(m.group(1)))
return " ".join(parts)
def fetch_article(slug: str, db: DBConn | None = None) -> dict:
"""Fetch a single article page; optionally cache + store in DB."""
own_db = db is None
if own_db:
db = get_db()
url = f"{BASE_URL}/article/{slug}"
data, _ = fetch_cached(db, url, "article")
result = parse_single_article(data, slug)
if own_db:
db.close()
return result
def _http_fetch_category(
category_slug: str,
*,
force: bool = False,
) -> tuple[str, list[dict], bool]:
"""
Fetch one category page via HTTP only.
Uses a per-thread DB connection (psycopg2 connections are not thread-safe).
Returns (slug, stories, from_cache).
"""
db = get_conn()
url = f"{BASE_URL}/interest/{category_slug}"
if force:
db.execute("DELETE FROM page_cache WHERE url=?", (url,))
db.commit()
data, from_cache = fetch_cached(db, url, "interest")
db.close()
stories = parse_stories(data, category_slug)
return category_slug, stories, from_cache
def fetch_category(
category_slug: str,
db: DBConn,
*,
force: bool = False,
) -> tuple[list[dict], bool]:
"""
Fetch an interest category page.
Returns (stories, from_cache).
"""
_, stories, from_cache = _http_fetch_category(category_slug, force=force)
upsert_articles(db, stories)
return stories, from_cache
def fetch_all(
db: DBConn,
slugs: list[str] | None = None,
*,
force: bool = False,
workers: int = 12,
) -> dict[str, list[dict]]:
"""
Fetch all (or given) interest categories in parallel (HTTP only),
then upsert results serially into DB from the calling thread.
Returns {slug: [story, ...]} mapping.
"""
targets = slugs or list(KNOWN_INTERESTS.keys())
results: dict[str, list[dict]] = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
futs = {ex.submit(_http_fetch_category, s, force=force): s for s in targets}
for f in concurrent.futures.as_completed(futs):
slug = futs[f]
try:
_, stories, cached = f.result()
upsert_articles(db, stories) # DB write in main thread
results[slug] = stories
icon = "💾" if cached else "🌐"
print(f" {icon} {slug:<38} {len(stories):2} stories")
except Exception as e:
print(f" ✗ {slug:<38} ERROR: {e}")
results[slug] = []
return results
def top_articles(
db: DBConn,
limit: int = 30,
days: int | None = 2,
min_sources: int = 0,
) -> list[sqlite3.Row]:
"""Query DB for top articles by source_count."""
where = "WHERE source_count >= ?"
params: list = [min_sources]
if days is not None:
where += " AND start_date >= date('now', ?)"
params.append(f"-{days} days")
return db.execute(
f"SELECT * FROM articles {where} ORDER BY source_count DESC LIMIT ?",
(*params, limit),
).fetchall()
# ---------------------------------------------------------------------------
# Display
# ---------------------------------------------------------------------------
def print_top(rows: list[sqlite3.Row], header: str = "Top artikler") -> None:
print(f"\n{'='*76}")
print(f" {header} ({len(rows)} artikler)")
print(f"{'='*76}\n")
for i, a in enumerate(rows, 1):
bias = ""
if a["left_pct"] is not None:
bias = f" L{a['left_pct']:.0f}% C{a['ctr_pct']:.0f}% R{a['right_pct']:.0f}%"
cats = (a["categories"] or "").replace(",", " · ")
ob = f" bias={a['overall_bias']:+.2f}" if a["overall_bias"] is not None else ""
bs = f" blindspot={a['blindspot']}" if a["blindspot"] else ""
print(f"{i:2}. [{a['source_count']:4} src{bias}{ob}{bs}] [{a['start_date']}]")
print(f" {a['title'][:80]}")
if a["description"]:
print(f" {a['description'][:90]}")
print(f" [{cats}]")
print(f" /article/{a['slug']}")
print()
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
if __name__ == "__main__":
import sys
db = get_db()
if len(sys.argv) >= 2 and sys.argv[1] == "article":
slug = sys.argv[2]
url = f"{BASE_URL}/article/{slug}"
data, cached = fetch_cached(db, url, "article")
result = parse_single_article(data, slug)
print(f"({'cached' if cached else 'fetched'})")
print(json.dumps(result, indent=2, ensure_ascii=False))
elif len(sys.argv) >= 2 and sys.argv[1] == "category":
slug = sys.argv[2]
stories, cached = fetch_category(slug, db)
print(f"({'cached' if cached else 'fetched'}) {len(stories)} stories\n")
for s in sorted(stories, key=lambda x: x["source_count"], reverse=True):
print(f" [{s['source_count']:4} src] {s['title'][:70]}")
else:
force = "--force" in sys.argv
days = 3
print(f"Fetching all {len(KNOWN_INTERESTS)} categories (force={force})…\n")
fetch_all(db, force=force)
rows = top_articles(db, limit=30, days=days)
print_top(rows, f"Top 30 – seneste {days} dage")
db.close()