Ai News Aggregator

__pycache__/
*.py[cod]
*.so
.cache/
.rss_cache.json
venv/
*.log
.DS_Store

MIT License

Copyright (c) 2026 OpenClaw Contributors

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

#!/usr/bin/env python3
"""ArXiv 论文筛选脚本 - 获取 AI/Agent/Memory/Workflow 相关论文"""

import json
import sys
import os
import urllib.request
import urllib.parse
import xml.etree.ElementTree as ET
from datetime import datetime, timedelta

ARXIV_CATEGORIES = ["cs.AI", "cs.CL", "cs.LG", "cs.MA"]

SEARCH_QUERIES = [
    "LLM agent",
    "autonomous agent",
    "memory augmented language model",
    "workflow automation AI",
    "retrieval augmented generation",
    "chain of thought reasoning",
    "tool use language model",
    "multi agent system",
    "AI planning",
    "prompt engineering",
    "model context protocol",
]


def fetch_arxiv(query, max_results=20, categories=None):
    """从 ArXiv API 获取论文"""
    cat_filter = ""
    if categories:
        cat_parts = [f"cat:{c}" for c in categories]
        cat_filter = "+AND+(" + "+OR+".join(cat_parts) + ")"

    search_query = urllib.parse.quote(query)
    url = (
        f"http://export.arxiv.org/api/query?"
        f"search_query=all:{search_query}{cat_filter}"
        f"&sortBy=submittedDate&sortOrder=descending"
        f"&max_results={max_results}"
    )

    proxy = None
    http_proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy")
    if http_proxy:
        proxy = urllib.request.ProxyHandler({"http": http_proxy, "https": http_proxy})

    opener = urllib.request.build_opener(proxy) if proxy else urllib.request.build_opener()
    req = urllib.request.Request(url, headers={"User-Agent": "AI-News-Aggregator/1.0"})

    try:
        resp = opener.open(req, timeout=15)
        return resp.read().decode("utf-8")
    except Exception as e:
        print(f"Error fetching arxiv: {e}", file=sys.stderr)
        return None


def parse_arxiv(xml_content):
    """解析 ArXiv Atom XML"""
    ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
    papers = []

    try:
        root = ET.fromstring(xml_content)
        for entry in root.findall("atom:entry", ns):
            title = entry.findtext("atom:title", "", ns).strip().replace("\n", " ")
            summary = entry.findtext("atom:summary", "", ns).strip().replace("\n", " ")

            arxiv_id = entry.findtext("atom:id", "", ns).strip()
            published = entry.findtext("atom:published", "", ns).strip()

            authors = []
            for author in entry.findall("atom:author", ns):
                name = author.findtext("atom:name", "", ns).strip()
                if name:
                    authors.append(name)

            categories = []
            for cat in entry.findall("atom:category", ns):
                term = cat.get("term", "")
                if term:
                    categories.append(term)

            link = ""
            for l in entry.findall("atom:link", ns):
                if l.get("type") == "text/html":
                    link = l.get("href", "")
                    break
            if not link:
                link = arxiv_id

            papers.append({
                "title": title,
                "summary": summary[:300],
                "authors": authors[:5],
                "categories": categories,
                "url": link,
                "published": published[:10],
                "arxiv_id": arxiv_id.split("/")[-1] if "/" in arxiv_id else arxiv_id,
            })
    except ET.ParseError as e:
        print(f"XML parse error: {e}", file=sys.stderr)

    return papers


def score_paper(paper):
    """对论文进行相关性评分"""
    high_value_keywords = [
        "agent", "memory", "workflow", "tool use", "planning",
        "reasoning", "chain-of-thought", "rag", "retrieval",
        "autonomous", "multi-agent", "self-reflection", "mcp",
    ]
    medium_value_keywords = [
        "llm", "large language model", "transformer", "prompt",
        "instruction", "fine-tune", "alignment", "evaluation",
        "benchmark", "context window", "long context",
    ]

    text = (paper["title"] + " " + paper["summary"]).lower()
    score = 0
    for kw in high_value_keywords:
        if kw in text:
            score += 3
    for kw in medium_value_keywords:
        if kw in text:
            score += 1
    return score


def main():
    import argparse
    parser = argparse.ArgumentParser(description="ArXiv AI 论文筛选")
    parser.add_argument("--query", default=None, help="自定义搜索词")
    parser.add_argument("--limit", type=int, default=10, help="每个查询的最大结果数")
    parser.add_argument("--top", type=int, default=10, help="最终显示前 N 篇")
    parser.add_argument("--json", action="store_true", help="JSON 格式输出")
    args = parser.parse_args()

    all_papers = {}
    queries = [args.query] if args.query else SEARCH_QUERIES

    for q in queries:
        print(f"  搜索: {q}...", file=sys.stderr)
        content = fetch_arxiv(q, max_results=args.limit, categories=ARXIV_CATEGORIES)
        if content:
            papers = parse_arxiv(content)
            for p in papers:
                key = p["arxiv_id"]
                if key not in all_papers:
                    all_papers[key] = p
                    all_papers[key]["score"] = score_paper(p)
            print(f"  ✓ {q}: {len(papers)} 篇", file=sys.stderr)
        else:
            print(f"  ✗ {q}: 获取失败", file=sys.stderr)

    ranked = sorted(all_papers.values(), key=lambda x: x["score"], reverse=True)
    top_papers = ranked[:args.top]

    if args.json:
        print(json.dumps(top_papers, ensure_ascii=False, indent=2))
    else:
        print(f"\n📄 ArXiv AI 论文精选 - 从 {len(all_papers)} 篇中选出 Top {len(top_papers)}")
        print("=" * 60)
        for i, p in enumerate(top_papers, 1):
            stars = "⭐" * min(p["score"] // 3, 5)
            print(f"\n{i}. **{p['title']}** {stars}")
            print(f"   作者: {', '.join(p['authors'][:3])}")
            print(f"   分类: {', '.join(p['categories'][:3])}")
            print(f"   日期: {p['published']}")
            print(f"   摘要: {p['summary'][:150]}...")
            print(f"   链接: {p['url']}")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Build compact input for daily AI digest.

Pipeline:
1) Batch fetch RSS with ai-news-aggregator concurrent mode.
2) Clean and dedupe entries.
3) Attach category metadata.
4) Keep a bounded candidate set for LLM summarization.
5) Optionally fetch GitHub AI trending.
"""

from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from pathlib import Path
from typing import Any

ROOT = Path(__file__).resolve().parent
RSS_SCRIPT = ROOT / "rss_aggregator.py"
GH_SCRIPT = ROOT / "github_trending.py"
SOURCES_FILE = ROOT / "rss_sources.json"


def run_json_command(cmd: list[str], timeout: int) -> Any:
    proc = subprocess.run(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        timeout=timeout,
        env=os.environ.copy(),
        cwd=str(ROOT.parent.parent.parent),
    )
    if proc.returncode != 0:
        raise RuntimeError(f"command failed: {' '.join(cmd)}\n{proc.stderr.strip()}")
    try:
        return json.loads(proc.stdout)
    except json.JSONDecodeError as e:
        raise RuntimeError(f"json parse failed: {e}\nstdout[:500]={proc.stdout[:500]}") from e


def parse_dt(value: str) -> datetime | None:
    if not value:
        return None
    value = value.strip()
    if not value:
        return None

    try:
        dt = parsedate_to_datetime(value)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        return dt.astimezone(timezone.utc)
    except Exception:
        pass

    iso = value.replace("Z", "+00:00")
    try:
        dt = datetime.fromisoformat(iso)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        return dt.astimezone(timezone.utc)
    except Exception:
        return None


def load_source_categories() -> dict[str, str]:
    with open(SOURCES_FILE, "r", encoding="utf-8") as f:
        sources = json.load(f)
    mapping: dict[str, str] = {}
    for src in sources:
        mapping[src.get("name", "")] = src.get("category", "unknown")
    return mapping


def clean_items(items: list[dict[str, Any]], source_to_category: dict[str, str]) -> list[dict[str, Any]]:
    cleaned: list[dict[str, Any]] = []
    seen_urls: set[str] = set()

    for it in items:
        title = (it.get("title") or "").strip()
        url = (it.get("url") or "").strip()
        if not title or not url or not url.startswith("http"):
            continue
        if url in seen_urls:
            continue

        source = (it.get("source") or "").strip() or "unknown"
        category = source_to_category.get(source, "unknown")
        dt = parse_dt(it.get("date") or "")

        cleaned.append(
            {
                "title": title,
                "url": url,
                "desc": (it.get("desc") or "").strip(),
                "date": (it.get("date") or "").strip(),
                "source": source,
                "category": category,
                "ts": dt.timestamp() if dt else 0,
            }
        )
        seen_urls.add(url)

    return cleaned




# User interest keywords for scoring - higher weight = more interesting
INTEREST_KEYWORDS = {
    # Core interests (weight 3)
    "agent": 3, "agents": 3, "agentic": 3, "multi-agent": 3,
    "mcp": 3, "model context protocol": 3,
    "skill": 3, "skills": 3,
    "openclaw": 3, "claude code": 3, "codex": 3, "cursor": 3,
    "workflow": 3, "orchestration": 3,
    # Engineering practice (weight 2)
    "rag": 2, "retrieval": 2, "embedding": 2, "vector": 2,
    "memory": 2, "knowledge graph": 2, "ontology": 2,
    "langchain": 2, "llamaindex": 2, "crewai": 2, "autogen": 2,
    "tool use": 2, "function calling": 2,
    "prompt engineering": 2, "prompt": 2,
    "fine-tuning": 2, "fine tuning": 2,
    "deployment": 2, "production": 2, "observability": 2,
    "evaluation": 2, "benchmark": 2,
    # Secondary interests (weight 1)
    "llm": 1, "gpt": 1, "claude": 1, "gemini": 1,
    "transformer": 1, "diffusion": 1,
    "quantitative": 1, "trading": 1, "finance": 1,
    "open source": 1, "open-source": 1, "github": 1,
}

DEDUP_FILE = "/tmp/ainews_prefetch/pushed_urls.json"
DEDUP_DAYS = 7


def score_interest(item: dict[str, Any]) -> int:
    """Score item based on user interest keywords."""
    text = (item.get("title", "") + " " + item.get("desc", "")).lower()
    score = 0
    for keyword, weight in INTEREST_KEYWORDS.items():
        if keyword in text:
            score += weight
    return score


def load_pushed_urls() -> set[str]:
    """Load previously pushed URLs for cross-day dedup."""
    if not os.path.exists(DEDUP_FILE):
        return set()
    try:
        with open(DEDUP_FILE, "r") as f:
            data = json.load(f)
        from datetime import datetime, timedelta
        cutoff = (datetime.now() - timedelta(days=DEDUP_DAYS)).timestamp()
        return {url for url, ts in data.items() if ts > cutoff}
    except Exception:
        return set()


def save_pushed_urls(urls: set[str], existing: set[str]) -> None:
    """Save pushed URLs with timestamps."""
    from datetime import datetime, timedelta
    data = {}
    if os.path.exists(DEDUP_FILE):
        try:
            with open(DEDUP_FILE, "r") as f:
                data = json.load(f)
        except Exception:
            pass
    cutoff = (datetime.now() - timedelta(days=DEDUP_DAYS)).timestamp()
    data = {url: ts for url, ts in data.items() if ts > cutoff}
    now = datetime.now().timestamp()
    for url in urls:
        data[url] = now
    with open(DEDUP_FILE, "w") as f:
        json.dump(data, f)


def cap_candidates(items: list[dict[str, Any]], per_source: int, max_total: int) -> list[dict[str, Any]]:
    """Select candidates by interest score, not per-source hard cap.
    per_source is only used as anti-spam floor (max from single source = max_total//3).
    """
    pushed = load_pushed_urls()
    items = [it for it in items if it.get("url", "") not in pushed]

    for it in items:
        it["_interest"] = score_interest(it)

    items.sort(key=lambda x: (-x.get("_interest", 0), -x.get("ts", 0)))

    source_count: dict[str, int] = defaultdict(int)
    max_per_source = max(per_source, max_total // 3)
    result: list[dict[str, Any]] = []
    for it in items:
        src = it.get("source", "unknown")
        if source_count[src] >= max_per_source:
            continue
        result.append(it)
        source_count[src] += 1
        if len(result) >= max_total:
            break

    save_pushed_urls({it.get("url", "") for it in result}, pushed)

    for it in result:
        it.pop("_interest", None)
    return result


def main() -> int:
    parser = argparse.ArgumentParser(description="Build compact input for AI digest")
    parser.add_argument("--workers", type=int, default=20)
    parser.add_argument("--timeout", type=int, default=10)
    parser.add_argument("--days", type=int, default=3)
    parser.add_argument("--limit", type=int, default=2, help="per-source fetch cap")
    parser.add_argument("--per-source", type=int, default=1, help="per-source keep cap")
    parser.add_argument("--max-total", type=int, default=45)
    parser.add_argument("--out", default="/tmp/ainews_digest_input.json")
    parser.add_argument("--skip-github", action="store_true")
    args = parser.parse_args()

    source_to_category = load_source_categories()

    rss_cmd = [
        sys.executable,
        str(RSS_SCRIPT),
        "--category",
        "all",
        "--workers",
        str(args.workers),
        "--timeout",
        str(args.timeout),
        "--days",
        str(args.days),
        "--limit",
        str(args.limit),
        "--json",
    ]

    try:
        rss_raw = run_json_command(rss_cmd, timeout=max(30, args.timeout * 20))
    except Exception as e:
        print(json.dumps({"ok": False, "error": str(e)}, ensure_ascii=False))
        return 1

    cleaned = clean_items(rss_raw, source_to_category)
    candidates = cap_candidates(cleaned, per_source=args.per_source, max_total=args.max_total)

    gh_items: list[dict[str, Any]] = []
    gh_error = ""
    if not args.skip_github:
        gh_cmd = [sys.executable, str(GH_SCRIPT), "--ai-only", "--limit", "10", "--json"]
        try:
            gh_items = run_json_command(gh_cmd, timeout=40)
        except Exception as e:
            gh_error = str(e)

    stats_raw = Counter(it.get("category", "unknown") for it in cleaned)
    stats_candidates = Counter(it.get("category", "unknown") for it in candidates)

    payload = {
        "ok": True,
        "generatedAt": datetime.now(timezone.utc).isoformat(),
        "params": {
            "workers": args.workers,
            "timeout": args.timeout,
            "days": args.days,
            "limit": args.limit,
            "perSource": args.per_source,
            "maxTotal": args.max_total,
        },
        "stats": {
            "rawItems": len(rss_raw),
            "cleanItems": len(cleaned),
            "candidateItems": len(candidates),
            "byCategoryRaw": dict(stats_raw),
            "byCategoryCandidates": dict(stats_candidates),
            "githubTrendingCount": len(gh_items),
            "githubTrendingError": gh_error,
        },
        "candidates": [{k: v for k, v in it.items() if k != "ts"} for it in candidates],
        "githubTrending": gh_items,
    }

    out_path = Path(args.out)
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(payload, f, ensure_ascii=False, indent=2)

    print(json.dumps({"ok": True, "out": str(out_path), "stats": payload["stats"]}, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Build weekly reflection index for ai-news agent.

Input:
- memory/YYYY-MM-DD.md
- MEMORY.md (long-term lessons)

Output:
- knowledge/weekly/reflection-index-YYYY-Www.md
"""

from __future__ import annotations

from datetime import date, datetime, timedelta
from pathlib import Path
import re

ROOT = Path(__file__).resolve().parent.parent
MEMORY_DIR = ROOT / "memory"
LONG_MEMORY = ROOT / "MEMORY.md"
OUT_DIR = ROOT / "knowledge" / "weekly"


def week_dates(today: date) -> list[date]:
    start = today - timedelta(days=today.weekday())
    return [start + timedelta(days=i) for i in range(7)]


def read_text(path: Path) -> str:
    if not path.exists():
        return ""
    return path.read_text(encoding="utf-8", errors="ignore")


def pick_lines(text: str, limit: int = 6) -> list[str]:
    picks: list[str] = []
    for line in text.splitlines():
        s = line.strip()
        if not s:
            continue
        if s.startswith("#"):
            continue
        if len(s) < 8:
            continue
        if s.startswith("- ") or s.startswith("* ") or re.match(r"^\d+[\.)]", s):
            picks.append(s)
        if len(picks) >= limit:
            break
    if not picks:
        fallback = [ln.strip() for ln in text.splitlines() if ln.strip()][:limit]
        picks.extend(fallback)
    return picks[:limit]


def build() -> Path:
    today = date.today()
    dates = week_dates(today)
    iso_year, iso_week, _ = today.isocalendar()

    lines: list[str] = []
    lines.append(f"# Reflection Index - {iso_year}-W{iso_week:02d}")
    lines.append("")
    lines.append("## This Week Daily Reflections")

    for d in dates:
        p = MEMORY_DIR / f"{d.isoformat()}.md"
        text = read_text(p)
        if not text:
            lines.append(f"- {d.isoformat()}: (no file)")
            continue
        picks = pick_lines(text, limit=4)
        lines.append(f"- {d.isoformat()}:")
        for item in picks:
            lines.append(f"  - {item}")

    lines.append("")
    lines.append("## Long-Term Lessons Snapshot")
    long_text = read_text(LONG_MEMORY)
    lessons = []
    in_section = False
    for line in long_text.splitlines():
        s = line.strip()
        if s.startswith("## 经验教训") or s.lower().startswith("## lessons"):
            in_section = True
            continue
        if in_section and s.startswith("## "):
            break
        if in_section and (s.startswith("- ") or s.startswith("* ")):
            lessons.append(s)
    for item in lessons[-10:]:
        lines.append(f"- {item}")

    lines.append("")
    lines.append("## Next Actions")
    lines.append("- Keep daily reflection concise: issue -> fix -> learning.")
    lines.append("- Promote repeated failures into MEMORY.md lessons.")
    lines.append("- Review this index every Sunday.")

    OUT_DIR.mkdir(parents=True, exist_ok=True)
    out = OUT_DIR / f"reflection-index-{iso_year}-W{iso_week:02d}.md"
    out.write_text("\n".join(lines) + "\n", encoding="utf-8")
    return out


if __name__ == "__main__":
    out = build()
    print(out)

import json, sys, os
from datetime import datetime

out_dir = sys.argv[1]
digest_path = os.path.join(out_dir, "digest_latest.json")
gh_path = os.path.join(out_dir, "github_trending.json")

digest = {}
gh = []
try:
    with open(digest_path) as f: digest = json.load(f)
except: pass
try:
    with open(gh_path) as f: gh = json.load(f)
except: pass

summary = {
    "fetchedAt": datetime.now().isoformat(),
    "candidateCount": len(digest.get("candidates", [])),
    "githubCount": len(gh),
    "categories": digest.get("stats", {}).get("byCategoryCandidates", {}),
    "ready": True
}
with open(os.path.join(out_dir, "status.json"), "w") as f:
    json.dump(summary, f, ensure_ascii=False)
print(json.dumps(summary, ensure_ascii=False))

#!/usr/bin/env python3
"""GitHub Trending 抓取脚本 - 获取每日趋势项目"""

import json
import sys
import urllib.request
import urllib.error
import re


def _parse_trending_html(html: str) -> list[dict]:
    """用正则解析 GitHub Trending 页面（HTMLParser 因 GitHub 改版不再可靠）"""
    articles = re.findall(r"<article class=.*?Box-row.*?</article>", html, re.DOTALL)
    repos = []
    for article in articles:
        h2_link = re.search(r'<h2[^>]*>\s*<a[^>]*href="(/[^"]+)"', article, re.DOTALL)
        if not h2_link:
            continue
        href = h2_link.group(1)
        name = href.strip("/")

        desc_match = re.search(r'<p class="[^"]*col-9[^"]*">(.*?)</p>', article, re.DOTALL)
        desc = desc_match.group(1).strip() if desc_match else ""

        stars_match = re.search(r'([\d,]+)\s+stars\s+today', article)
        stars = stars_match.group(1) if stars_match else ""

        lang_match = re.search(r'itemprop="programmingLanguage">(.*?)</span>', article)
        lang = lang_match.group(1).strip() if lang_match else ""

        repos.append({
            "name": name,
            "desc": desc,
            "stars": stars,
            "lang": lang,
            "url": f"https://github.com{href}",
        })
    return repos


def fetch_trending(language="", since="daily"):
    """从 GitHub Trending 页面抓取趋势项目"""
    url = f"https://github.com/trending/{language}?since={since}"

    proxy = None
    import os
    http_proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy")
    if http_proxy:
        proxy = urllib.request.ProxyHandler({"http": http_proxy, "https": http_proxy})

    opener = urllib.request.build_opener(proxy) if proxy else urllib.request.build_opener()
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
        "Accept": "text/html",
        "Accept-Language": "en-US,en;q=0.9",
    })

    try:
        resp = opener.open(req, timeout=15)
        html = resp.read().decode("utf-8")
    except Exception as e:
        print(f"Error fetching trending: {e}", file=sys.stderr)
        return []

    return _parse_trending_html(html)


def filter_ai_repos(repos):
    """筛选 AI/ML 相关的项目"""
    ai_keywords = [
        "ai", "ml", "llm", "gpt", "agent", "transformer", "diffusion",
        "neural", "deep-learning", "machine-learning", "nlp", "cv",
        "rag", "embedding", "vector", "langchain", "autogen", "crewai",
        "openai", "anthropic", "gemini", "claude", "ollama", "llama",
        "stable-diffusion", "midjourney", "workflow", "memory", "mcp",
        "copilot", "cursor", "agentic", "reasoning", "cot", "chain-of-thought",
    ]
    filtered = []
    for repo in repos:
        text = (repo["name"] + " " + repo["desc"]).lower()
        if any(kw in text for kw in ai_keywords):
            filtered.append(repo)
    return filtered


def main():
    import argparse
    parser = argparse.ArgumentParser(description="GitHub Trending 抓取")
    parser.add_argument("--language", default="", help="编程语言筛选")
    parser.add_argument("--since", default="daily", choices=["daily", "weekly", "monthly"])
    parser.add_argument("--ai-only", action="store_true", help="仅显示 AI 相关项目")
    parser.add_argument("--limit", type=int, default=15, help="最多显示数量")
    parser.add_argument("--json", action="store_true", help="JSON 格式输出")
    args = parser.parse_args()

    repos = fetch_trending(args.language, args.since)

    if args.ai_only:
        repos = filter_ai_repos(repos)

    repos = repos[:args.limit]

    if args.json:
        print(json.dumps(repos, ensure_ascii=False, indent=2))
    else:
        if not repos:
            print("未获取到趋势项目（可能是网络问题或页面结构变化）")
            return

        print(f"🔥 GitHub Trending ({args.since}) - {'AI/ML 相关' if args.ai_only else '全部'}")
        print("=" * 60)
        for i, repo in enumerate(repos, 1):
            stars_str = f" ⭐ {repo['stars']} today" if repo.get("stars") else ""
            print(f"\n{i}. **{repo['name']}**{stars_str}")
            if repo["desc"]:
                print(f"   {repo['desc']}")
            print(f"   {repo['url']}")


if __name__ == "__main__":
    main()

#!/bin/bash
# ainews 数据预取脚本 - 统一拉取所有数据源
# 定时执行: 早(07:50) + 中(11:30) + 晚(19:30)

PYTHON=/opt/homebrew/bin/python3.12
FALLBACK_PYTHON=/Users/study/.pyenv/versions/3.11.5/bin/python3
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
OUT_DIR="/tmp/ainews_prefetch"
mkdir -p "$OUT_DIR"

# Use available Python
if [ -x "$PYTHON" ]; then
  PY=$PYTHON
elif [ -x "$FALLBACK_PYTHON" ]; then
  PY=$FALLBACK_PYTHON
else
  PY=python3
fi

export HTTP_PROXY="http://127.0.0.1:1087"
export HTTPS_PROXY="http://127.0.0.1:1087"

echo "[$(date)] Starting prefetch (Python: $PY)..."

# 1. RSS 聚合（100+ 源，并发抓取）
echo "[$(date)] Fetching RSS..."
$PY "$SCRIPT_DIR/build_digest_input.py" \
  --out "$OUT_DIR/digest_latest.json" \
  --workers 20 --timeout 10 --days 2 --max-total 40 \
  2>"$OUT_DIR/rss.log"

# 2. GitHub AI Trending
echo "[$(date)] Fetching GitHub Trending..."
$PY "$SCRIPT_DIR/github_trending.py" --ai-only --limit 10 --json \
  > "$OUT_DIR/github_trending.json" 2>>"$OUT_DIR/github.log" \
  || echo "[]" > "$OUT_DIR/github_trending.json"

# 3. ArXiv 论文（为午间 paper-digest 预取）
echo "[$(date)] Fetching ArXiv papers..."
$PY "$SCRIPT_DIR/arxiv_papers.py" --top 8 --json \
  > "$OUT_DIR/arxiv_papers.json" 2>>"$OUT_DIR/arxiv.log" \
  || echo "[]" > "$OUT_DIR/arxiv_papers.json"

# 4. 生成状态文件
$PY "$SCRIPT_DIR/gen_status.py" "$OUT_DIR"

echo "[$(date)] Prefetch done."

#!/usr/bin/env python3
"""RSS 聚合脚本 - 高性能并发版

优化点：
1. 并发抓取 (10线程) - 10倍速度提升
2. ETag/Last-Modified 缓存 - 避免重复下载
3. 智能超时 (10秒) - 快速失败
4. 日期过滤 - 支持只抓取最近 N 天的新闻
"""

import json
import sys
import os
import xml.etree.ElementTree as ET
import urllib.request
from html import unescape
import re
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone, timedelta
import time
import logging
import argparse

# 配置
MAX_WORKERS = 10
TIMEOUT = 15  # 适度超时
CACHE_PATH = Path("/tmp/.ainews_rss_cache.json")
CACHE_TTL_HOURS = 1

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    datefmt='%H:%M:%S'
)
logger = logging.getLogger(__name__)


def load_sources(category=None):
    """从 rss_sources.json 加载源列表"""
    sources_file = Path(__file__).parent / "rss_sources.json"
    if not sources_file.exists():
        logger.error(f"No sources file at {sources_file}")
        return []
    with open(sources_file) as f:
        sources = json.load(f)
    if category and category != "all":
        sources = [s for s in sources if s.get("category") == category]
    return sources


def load_cache():
    """加载缓存"""
    try:
        if CACHE_PATH.exists():
            with open(CACHE_PATH) as f:
                cache = json.load(f)
            # 清理过期缓存
            now = time.time()
            ttl = CACHE_TTL_HOURS * 3600
            return {k: v for k, v in cache.items() if now - v.get("ts", 0) < ttl}
    except Exception:
        pass
    return {}


def save_cache(cache):
    """保存缓存"""
    try:
        with open(CACHE_PATH, 'w') as f:
            json.dump(cache, f)
    except Exception as e:
        logger.debug(f"Cache save error: {e}")


def fetch_rss_concurrent(source, cache, timeout=TIMEOUT):
    """并发抓取单个 RSS 源"""
    name = source["name"]
    url = source["url"]

    req_headers = {
        "User-Agent": "AI-News-Aggregator/2.0",
        "Accept": "application/rss+xml, application/xml, text/xml, application/atom+xml, */*",
    }

    # 添加条件请求头
    cache_entry = cache.get(url, {})
    if cache_entry.get("etag"):
        req_headers["If-None-Match"] = cache_entry["etag"]
    if cache_entry.get("last_modified"):
        req_headers["If-Modified-Since"] = cache_entry["last_modified"]

    proxy = None
    http_proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy")
    if http_proxy:
        proxy = urllib.request.ProxyHandler({"http": http_proxy, "https": http_proxy})
    opener = urllib.request.build_opener(proxy) if proxy else urllib.request.build_opener()

    req = urllib.request.Request(url, headers=req_headers)

    try:
        with opener.open(req, timeout=timeout) as resp:
            content = resp.read().decode("utf-8", errors="replace")

            # 更新缓存（保存内容和元数据）
            cache[url] = {
                "content": content,
                "etag": resp.headers.get("ETag"),
                "last_modified": resp.headers.get("Last-Modified"),
                "ts": time.time()
            }

            return {"name": name, "status": "ok", "content": content}

    except urllib.error.HTTPError as e:
        if e.code == 304:  # Not Modified - 从缓存返回
            if cache_entry.get("content"):
                logger.debug(f"{name}: 304, using cached content")
                return {"name": name, "status": "ok", "content": cache_entry["content"]}
            return {"name": name, "status": "error", "error": "304 but no cache"}
        return {"name": name, "status": "error", "error": f"HTTP {e.code}"}
    except Exception as e:
        return {"name": name, "status": "error", "error": str(e)}


def parse_date(date_str):
    """解析各种 RSS 日期格式，返回 UTC datetime"""
    if not date_str:
        return None

    date_str = date_str.strip()

    # 格式1: RFC 2822 (RSS 2.0) - Thu, 05 Feb 2026 00:00:00 +0000
    rfc_patterns = [
        '%a, %d %b %Y %H:%M:%S %z',
        '%a, %d %b %Y %H:%M:%S %Z',
        '%d %b %Y %H:%M:%S %z',
        '%d %b %Y %H:%M:%S %Z',
    ]

    for pattern in rfc_patterns:
        try:
            return datetime.strptime(date_str, pattern)
        except ValueError:
            continue

    # 格式2: ISO 8601 (Atom) - 2026-02-05T00:00:00Z 或 2026-02-05T00:00:00+00:00
    iso_variants = [
        date_str.replace('Z', '+00:00'),
        date_str.replace('z', '+00:00'),
    ]
    if '+' not in date_str and '-' in date_str[10:]:
        # 可能没有时区，假设 UTC
        try:
            return datetime.fromisoformat(date_str.replace('Z', ''))
        except ValueError:
            pass

    for iso_str in iso_variants:
        try:
            return datetime.fromisoformat(iso_str)
        except ValueError:
            continue

    # 格式3: 简单格式 - 2026-02-05 或 Feb 25, 2026
    simple_patterns = [
        '%Y-%m-%d',
        '%Y-%m-%d %H:%M:%S',
        '%b %d, %Y',
        '%B %d, %Y',
    ]
    for pattern in simple_patterns:
        try:
            dt = datetime.strptime(date_str, pattern)
            return dt.replace(tzinfo=timezone.utc)
        except ValueError:
            continue

    return None


def parse_rss(xml_content, source_name):
    """解析 RSS"""
    items = []

    try:
        root = ET.fromstring(xml_content)
        ns = {"atom": "http://www.w3.org/2005/Atom"}

        # RSS 2.0
        for item in root.findall(".//item"):
            title = (item.findtext("title") or "").strip()
            link = (item.findtext("link") or "").strip()
            desc = (item.findtext("description") or "").strip()
            pub_date = (item.findtext("pubDate") or "").strip()
            desc = unescape(re.sub(r"<[^>]+>", "", desc))[:200]
            items.append({"title": title, "url": link, "desc": desc, "date": pub_date, "source": source_name})

        # Atom
        for entry in root.findall("atom:entry", ns) + root.findall("entry"):
            title = ""
            t = entry.find("atom:title", ns) or entry.find("title")
            if t is not None and getattr(t, 'text', None):
                title = t.text.strip()

            link = ""
            l = entry.find("atom:link", ns) or entry.find("link")
            if l is not None:
                link = l.get("href", "").strip()

            desc = ""
            s = entry.find("atom:summary", ns) or entry.find("summary")
            c = entry.find("atom:content", ns) or entry.find("content")
            content_text = ""
            if s is not None and getattr(s, 'text', None):
                content_text = s.text
            elif c is not None and getattr(c, 'text', None):
                content_text = c.text
            if content_text:
                desc = unescape(re.sub(r"<[^>]+>", "", content_text))[:200]

            pub_date = ""
            p = entry.find("atom:published", ns) or entry.find("published") or entry.find("atom:updated", ns) or entry.find("updated")
            if p is not None and getattr(p, 'text', None):
                pub_date = p.text.strip()

            items.append({"title": title, "url": link, "desc": desc, "date": pub_date, "source": source_name})

    except ET.ParseError as e:
        logger.debug(f"Parse error for {source_name}: {e}")

    return items


def filter_by_date(items, days):
    """过滤最近 N 天的文章"""
    if not days or days <= 0:
        return items

    cutoff = datetime.now(timezone.utc) - timedelta(days=days)
    filtered = []

    for item in items:
        pub_date = parse_date(item.get('date', ''))
        if pub_date:
            # 确保有时区信息
            if pub_date.tzinfo is None:
                pub_date = pub_date.replace(tzinfo=timezone.utc)
            if pub_date >= cutoff:
                filtered.append(item)
        else:
            # 无法解析日期时，默认保留（可能是最新的）
            filtered.append(item)

    return filtered


def main():
    parser = argparse.ArgumentParser(description="RSS 聚合 - 高性能并发版")
    parser.add_argument("--category", default="all", help="分类筛选")
    parser.add_argument("--limit", type=int, default=10, help="每源最多条目")
    parser.add_argument("--json", action="store_true")
    parser.add_argument("--workers", type=int, default=MAX_WORKERS, help="并发线程数")
    parser.add_argument("--timeout", type=int, default=TIMEOUT, help="超时秒数")
    parser.add_argument("--days", type=int, default=None, help="只抓取最近 N 天的新闻")
    args = parser.parse_args()

    sources = load_sources(args.category)
    if not sources:
        logger.error("No sources found")
        sys.exit(1)

    logger.info(f"Fetching {len(sources)} sources with {args.workers} workers...")

    cache = load_cache()
    all_items = []
    errors = []
    from_cache = 0

    # 并发抓取
    start_time = time.time()
    with ThreadPoolExecutor(max_workers=args.workers) as executor:
        futures = {
            executor.submit(fetch_rss_concurrent, src, cache, args.timeout): src
            for src in sources
        }

        for future in as_completed(futures):
            result = future.result()
            name = result["name"]

            if result["status"] == "ok":
                items = parse_rss(result["content"], name)

                # 日期过滤
                if args.days:
                    items = filter_by_date(items, args.days)

                all_items.extend(items[:args.limit])
                # 判断是否来自缓存（根据内容匹配）
                url = next((s["url"] for s in sources if s["name"] == name), "")
                if cache.get(url, {}).get("content") == result["content"]:
                    from_cache += 1
                logger.info(f"✅ {name}: {len(items)} items")
            else:
                errors.append(name)
                logger.warning(f"❌ {name}: {result.get('error', 'failed')}")

    # 保存缓存
    save_cache(cache)

    elapsed = time.time() - start_time
    logger.info(f"\nCompleted in {elapsed:.1f}s - {len(all_items)} articles from {len(sources) - len(errors)}/{len(sources)} sources")

    if args.json:
        print(json.dumps(all_items, ensure_ascii=False, indent=2))
    else:
        by_source = {}
        for item in all_items:
            by_source.setdefault(item["source"], []).append(item)

        filter_info = f" | 最近{args.days}天" if args.days else ""
        print(f"\n{'='*60}")
        print(f"RSS 聚合结果 ({len(all_items)} 条){filter_info}")
        print(f"耗时: {elapsed:.1f}秒 | 来自缓存: {from_cache} | 失败: {len(errors)}")
        if errors:
            print(f"失败源: {', '.join(errors[:5])}{'...' if len(errors) > 5 else ''}")
        print("="*60)

        for source, items in by_source.items():
            print(f"\n📌 {source} ({len(items)})")
            for item in items[:3]:
                title = item["title"][:60] + "..." if len(item["title"]) > 60 else item["title"]
                print(f"  • {title}")


if __name__ == "__main__":
    main()

[
  {
    "name": "OpenAI Blog",
    "url": "https://openai.com/blog/rss.xml",
    "category": "company"
  },
  {
    "name": "OpenAI News",
    "url": "https://openai.com/news/rss.xml",
    "category": "company"
  },
  {
    "name": "Google Research Blog",
    "url": "https://blog.research.google/feeds/posts/default?alt=rss",
    "category": "company"
  },
  {
    "name": "Google DeepMind Blog",
    "url": "https://deepmind.google/blog/rss.xml",
    "category": "company"
  },
  {
    "name": "Google AI Blog",
    "url": "https://ai.googleblog.com/feeds/posts/default?alt=rss",
    "category": "company"
  },
  {
    "name": "Anthropic News",
    "url": "https://raw.githubusercontent.com/Olshansk/rss-feeds/main/feeds/feed_anthropic_news.xml",
    "category": "company"
  },
  {
    "name": "Anthropic Research",
    "url": "https://raw.githubusercontent.com/Olshansk/rss-feeds/main/feeds/feed_anthropic_research.xml",
    "category": "company"
  },
  {
    "name": "Anthropic Engineering",
    "url": "https://raw.githubusercontent.com/Olshansk/rss-feeds/main/feeds/feed_anthropic_engineering.xml",
    "category": "company"
  },
  {
    "name": "Hugging Face Blog",
    "url": "https://huggingface.co/blog/feed.xml",
    "category": "company"
  },
  {
    "name": "Microsoft AI Blog",
    "url": "https://blogs.microsoft.com/ai/feed/",
    "category": "company"
  },
  {
    "name": "Meta AI Blog",
    "url": "https://ai.meta.com/blog/rss/",
    "category": "company"
  },
  {
    "name": "NVIDIA AI Blog",
    "url": "https://blogs.nvidia.com/feed/",
    "category": "company"
  },
  {
    "name": "AWS ML Blog",
    "url": "https://aws.amazon.com/blogs/machine-learning/feed/",
    "category": "company"
  },
  {
    "name": "Apple ML Research",
    "url": "https://machinelearning.apple.com/rss.xml",
    "category": "company"
  },
  {
    "name": "Stability AI News",
    "url": "https://stability.ai/news?format=rss",
    "category": "company"
  },
  {
    "name": "HuggingFace Daily Papers",
    "url": "https://papers.takara.ai/api/rss",
    "category": "papers"
  },
  {
    "name": "BAIR Blog",
    "url": "https://bair.berkeley.edu/blog/feed.xml",
    "category": "papers"
  },
  {
    "name": "arXiv - AI",
    "url": "https://rss.arxiv.org/rss/cs.AI",
    "category": "papers"
  },
  {
    "name": "arXiv - ML",
    "url": "https://rss.arxiv.org/rss/cs.LG",
    "category": "papers"
  },
  {
    "name": "arXiv - NLP",
    "url": "https://rss.arxiv.org/rss/cs.CL",
    "category": "papers"
  },
  {
    "name": "arXiv - CV",
    "url": "https://rss.arxiv.org/rss/cs.CV",
    "category": "papers"
  },
  {
    "name": "The Verge - AI",
    "url": "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml",
    "category": "media"
  },
  {
    "name": "TechCrunch - AI",
    "url": "https://techcrunch.com/category/artificial-intelligence/feed/",
    "category": "media"
  },
  {
    "name": "MIT Technology Review",
    "url": "https://www.technologyreview.com/feed/",
    "category": "media"
  },
  {
    "name": "MIT Tech Review JP",
    "url": "https://www.technologyreview.jp/feed/",
    "category": "media"
  },
  {
    "name": "Ars Technica - AI",
    "url": "https://feeds.arstechnica.com/arstechnica/technology-lab",
    "category": "media"
  },
  {
    "name": "Wired - AI",
    "url": "https://www.wired.com/feed/tag/ai/latest/rss",
    "category": "media"
  },
  {
    "name": "The Decoder",
    "url": "https://the-decoder.com/feed/",
    "category": "media"
  },
  {
    "name": "AI News",
    "url": "https://www.artificialintelligence-news.com/feed/",
    "category": "media"
  },
  {
    "name": "AI Trends",
    "url": "https://www.aitrends.com/feed/",
    "category": "media"
  },
  {
    "name": "Science Magazine",
    "url": "https://www.science.org/rss/news_current.xml",
    "category": "media"
  },
  {
    "name": "Semafor AI",
    "url": "https://www.semafor.com/newsletters/ai/rss.xml",
    "category": "media"
  },
  {
    "name": "Financial Times - AI",
    "url": "https://www.ft.com/artificial-intelligence?format=rss",
    "category": "media"
  },
  {
    "name": "Synced (AI Review)",
    "url": "https://syncedreview.com/feed/",
    "category": "media"
  },
  {
    "name": "Hacker News - AI",
    "url": "https://hnrss.org/newest?q=AI",
    "category": "tech_news"
  },
  {
    "name": "Hacker News - LLM",
    "url": "https://hnrss.org/newest?q=LLM",
    "category": "tech_news"
  },
  {
    "name": "Hacker News - Machine Learning",
    "url": "https://hnrss.org/newest?q=machine+learning",
    "category": "tech_news"
  },
  {
    "name": "HN Best",
    "url": "https://hnrss.org/frontpage",
    "category": "community"
  },
  {
    "name": "GitHub Trending Daily",
    "url": "https://mshibanami.github.io/GitHubTrendingRSS/daily/all.xml",
    "category": "community"
  },
  {
    "name": "GitHub Trending Python",
    "url": "https://mshibanami.github.io/GitHubTrendingRSS/daily/python.xml",
    "category": "community"
  },
  {
    "name": "GitHub Trending Weekly",
    "url": "https://mshibanami.github.io/GitHubTrendingRSS/weekly/all.xml",
    "category": "community"
  },
  {
    "name": "Lobsters",
    "url": "https://lobste.rs/rss",
    "category": "community"
  },
  {
    "name": "Product Hunt",
    "url": "https://www.producthunt.com/feed",
    "category": "community"
  },
  {
    "name": "HelloGitHub",
    "url": "https://hellogithub.com/rss",
    "category": "community"
  },
  {
    "name": "V2EX Hot",
    "url": "https://rsshub.rssforever.com/v2ex/topics/hot",
    "category": "community"
  },
  {
    "name": "LangChain Releases",
    "url": "https://github.com/langchain-ai/langchain/releases.atom",
    "category": "community"
  },
  {
    "name": "LlamaIndex Releases",
    "url": "https://github.com/run-llama/llama_index/releases.atom",
    "category": "community"
  },
  {
    "name": "Microsoft AutoGen Releases",
    "url": "https://github.com/microsoft/autogen/releases.atom",
    "category": "community"
  },
  {
    "name": "crewAI Releases",
    "url": "https://github.com/crewAIInc/crewAI/releases.atom",
    "category": "community"
  },
  {
    "name": "OpenHands Releases",
    "url": "https://github.com/All-Hands-AI/OpenHands/releases.atom",
    "category": "community"
  },
  {
    "name": "mem0 Releases",
    "url": "https://github.com/mem0ai/mem0/releases.atom",
    "category": "community"
  },
  {
    "name": "MCP Spec Releases",
    "url": "https://github.com/modelcontextprotocol/spec/releases.atom",
    "category": "community"
  },
  {
    "name": "awesome-mcp-servers",
    "url": "https://github.com/punkpeye/awesome-mcp-servers/commits/main.atom",
    "category": "community"
  },
  {
    "name": "Simon Willison",
    "url": "https://simonwillison.net/atom/everything/",
    "category": "newsletter"
  },
  {
    "name": "Lilian Weng (Lil'Log)",
    "url": "https://lilianweng.github.io/index.xml",
    "category": "newsletter"
  },
  {
    "name": "The Batch (Andrew Ng)",
    "url": "https://www.deeplearning.ai/the-batch/feed/",
    "category": "newsletter"
  },
  {
    "name": "Machine Learning Mastery",
    "url": "https://machinelearningmastery.com/feed/",
    "category": "newsletter"
  },
  {
    "name": "Towards Data Science",
    "url": "https://towardsdatascience.com/feed",
    "category": "newsletter"
  },
  {
    "name": "KDnuggets",
    "url": "https://www.kdnuggets.com/feed",
    "category": "newsletter"
  },
  {
    "name": "MarkTechPost",
    "url": "https://www.marktechpost.com/feed/",
    "category": "newsletter"
  },
  {
    "name": "Ben's Bites",
    "url": "https://www.bensbites.com/feed",
    "category": "newsletter"
  },
  {
    "name": "Latent Space",
    "url": "https://www.latent.space/feed",
    "category": "newsletter"
  },
  {
    "name": "Import AI",
    "url": "https://importai.substack.com/feed",
    "category": "newsletter"
  },
  {
    "name": "机器之心",
    "url": "https://www.jiqizhixin.com/rss",
    "category": "cn_media"
  },
  {
    "name": "36氪快讯",
    "url": "https://36kr.com/newsflashes",
    "category": "cn_media"
  },
  {
    "name": "少数派",
    "url": "https://sspai.com/feed",
    "category": "cn_media"
  },
  {
    "name": "InfoQ 中文",
    "url": "https://www.infoq.cn/feed",
    "category": "cn_media"
  },
  {
    "name": "阮一峰周刊",
    "url": "https://www.ruanyifeng.com/blog/atom.xml",
    "category": "tech_general"
  },
  {
    "name": "Swift Blog",
    "url": "https://www.swift.org/atom.xml",
    "category": "tech_general"
  },
  {
    "name": "Kotlin Blog",
    "url": "https://blog.jetbrains.com/kotlin/feed/",
    "category": "tech_general"
  },
  {
    "name": "Apple Newsroom",
    "url": "https://www.apple.com/newsroom/rss-feed.rss",
    "category": "tech_general"
  },
  {
    "name": "AI Act",
    "url": "https://artificialintelligenceact.com/feed/",
    "category": "newsletter"
  },
  {
    "name": "LangChain Blog",
    "url": "https://blog.langchain.dev/rss/",
    "category": "ai-agent"
  },
  {
    "name": "LlamaIndex Blog",
    "url": "https://blog.llamaindex.ai/feed",
    "category": "ai-agent"
  },
  {
    "name": "Mem0 Blog",
    "url": "https://blog.mem0.ai/rss",
    "category": "ai-agent"
  },
  {
    "name": "IEEE Spectrum AI",
    "url": "https://spectrum.ieee.org/rss/topic/artificial-intelligence",
    "category": "media"
  },
  {
    "name": "VentureBeat AI",
    "url": "https://venturebeat.com/category/ai/feed/",
    "category": "media"
  },
  {
    "name": "量子位",
    "url": "https://www.qbitai.com/feed",
    "category": "cn_media"
  },
  {
    "name": "Andrej Karpathy",
    "url": "https://karpathy.ai/feed.xml",
    "category": "newsletter"
  },
  {
    "name": "Ollama Blog",
    "url": "https://ollama.com/blog/rss.xml",
    "category": "ai-agent"
  },
  {
    "name": "vLLM Blog",
    "url": "https://blog.vllm.ai/feed",
    "category": "ai-agent"
  },
  {
    "name": "Mistral AI",
    "url": "https://mistral.ai/news/rss.xml",
    "category": "company"
  },
  {
    "name": "Cohere Blog",
    "url": "https://cohere.com/blog/rss.xml",
    "category": "company"
  },
  {
    "name": "Pinecone Blog",
    "url": "https://www.pinecone.io/blog/rss.xml",
    "category": "ai-agent"
  },
  {
    "name": "Weights & Biases",
    "url": "https://wandb.ai/site/rss.xml",
    "category": "newsletter"
  },
  {
    "name": "Sam Altman (Twitter)",
    "url": "https://nitter.net/sama/rss",
    "category": "twitter"
  },
  {
    "name": "Andrej Karpathy (Twitter)",
    "url": "https://nitter.net/karpathy/rss",
    "category": "twitter"
  },
  {
    "name": "Yann LeCun (Twitter)",
    "url": "https://nitter.net/ylecun/rss",
    "category": "twitter"
  },
  {
    "name": "Demis Hassabis (Twitter)",
    "url": "https://nitter.net/demishassabis/rss",
    "category": "twitter"
  },
  {
    "name": "Jim Fan (Twitter)",
    "url": "https://nitter.net/DrJimFan/rss",
    "category": "twitter"
  },
  {
    "name": "Dario Amodei (Twitter)",
    "url": "https://nitter.net/DarioAmodei/rss",
    "category": "twitter"
  },
  {
    "name": "Greg Brockman (Twitter)",
    "url": "https://nitter.net/gdb/rss",
    "category": "twitter"
  },
  {
    "name": "Harrison Chase (Twitter)",
    "url": "https://nitter.net/hwchase17/rss",
    "category": "twitter"
  },
  {
    "name": "Marc Andreessen (Twitter)",
    "url": "https://nitter.net/pmarca/rss",
    "category": "twitter"
  },
  {
    "name": "Paul Graham (Twitter)",
    "url": "https://nitter.net/paulg/rss",
    "category": "twitter"
  },
  {
    "name": "Sebastian Raschka",
    "url": "https://magazine.sebastianraschka.com/feed",
    "category": "newsletter"
  },
  {
    "name": "Eugene Yan",
    "url": "https://eugeneyan.com/rss/",
    "category": "newsletter"
  },
  {
    "name": "Stephen Wolfram",
    "url": "https://writings.stephenwolfram.com/feed/",
    "category": "newsletter"
  },
  {
    "name": "Gwern",
    "url": "https://gwern.substack.com/feed",
    "category": "newsletter"
  },
  {
    "name": "Minimaxir (Max Woolf)",
    "url": "https://minimaxir.com/index.xml",
    "category": "newsletter"
  },
  {
    "name": "Dwarkesh Patel",
    "url": "https://www.dwarkeshpatel.com/feed",
    "category": "newsletter"
  }
]