Technews

__pycache__/
*.pyc
*.pyo
*.pyd
.Python
*.so
.Python build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
.env
.venv
env/
venv/
ENV/
.stored_cache/

requests>=2.28.0
beautifulsoup4>=4.11.0

#!/usr/bin/env python3
"""
Article Fetcher - Fetches and extracts content from article URLs
"""

import json
import random
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import List, Dict, Optional
import requests
from bs4 import BeautifulSoup

# Common user agents to rotate
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
]


def get_random_ua() -> str:
    """Get a random user agent for requests."""
    return random.choice(USER_AGENTS)


def fetch_article(url: str, timeout: int = 15) -> Dict:
    """Fetch article content from URL."""
    headers = {
        "User-Agent": get_random_ua(),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate",
        "Connection": "keep-alive",
    }
    
    result = {
        "url": url,
        "success": False,
        "title": "",
        "content": "",
        "word_count": 0,
        "error": None
    }
    
    try:
        response = requests.get(url, headers=headers, timeout=timeout)
        response.raise_for_status()
        
        soup = BeautifulSoup(response.text, "html.parser")
        
        # Extract title
        title_elem = soup.select_one("title")
        result["title"] = title_elem.get_text(strip=True) if title_elem else ""
        
        # Try to find main content - common selectors
        content_selectors = [
            "article",
            "[role=main]",
            "main",
            "div.post-content",
            "div.article-content",
            "div.entry-content",
            "div.content",
            "div.story-body",
        ]
        
        content = None
        for selector in content_selectors:
            elem = soup.select_one(selector)
            if elem:
                content = elem
                break
        
        if not content:
            # Fallback: find largest text block
            paragraphs = soup.select("p")
            if paragraphs:
                content = paragraphs[0].parent
        
        if content:
            # Extract text from paragraphs
            text_parts = []
            for p in content.find_all(["p", "h1", "h2", "h3", "li"]):
                text = p.get_text(strip=True)
                if len(text) > 20:  # Filter out nav elements
                    text_parts.append(text)
            
            result["content"] = " ".join(text_parts)
            result["word_count"] = len(result["content"].split())
            result["success"] = True
            
    except requests.exceptions.Timeout:
        result["error"] = "timeout"
    except requests.exceptions.RequestException as e:
        result["error"] = str(e)
    except Exception as e:
        result["error"] = f"parse_error: {str(e)}"
    
    return result


def fetch_multiple(urls: List[str], max_workers: int = 5) -> List[Dict]:
    """Fetch multiple articles in parallel."""
    results = []
    
    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        futures = {executor.submit(fetch_article, url): url for url in urls}
        
        for future in as_completed(futures):
            result = future.result()
            results.append(result)
    
    return results


def summarize_content(content: str, max_words: int = 100) -> str:
    """Generate a brief summary of article content."""
    if not content:
        return ""
    
    words = content.split()
    if len(words) <= max_words:
        return content
    
    # Simple extractive summary: take first N words
    return " ".join(words[:max_words]) + "..."


def main():
    """Main entry point - reads URLs from stdin or args."""
    import sys
    
    if len(sys.argv) > 1:
        urls = sys.argv[1:]
    else:
        # Read from stdin
        input_data = sys.stdin.read()
        urls = json.loads(input_data).get("urls", [])
    
    if not urls:
        print(json.dumps({"error": "No URLs provided"}))
        return
    
    results = fetch_multiple(urls)
    
    # Add summaries
    for r in results:
        if r.get("success"):
            r["summary"] = summarize_content(r.get("content", ""))
    
    print(json.dumps({"articles": results}))


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Social Reactions - Finds and highlights social media reactions to articles
"""

import json
import re
from urllib.parse import quote
from typing import List, Dict, Optional
import requests

# Twitter/X search (uses their public API - limited but works)
TWITTER_SEARCH = "https://nitter.net/search"


def find_twitter_reactions(article_title: str, article_url: str, max_results: int = 3) -> List[Dict]:
    """
    Search for Twitter reactions to an article.
    Note: This is a simplified implementation using nitter (privacy-friendly Twitter frontend)
    """
    reactions = []
    
    try:
        # Try to search for the article title on Twitter
        query = f'"{article_title[:50]}"' if len(article_title) > 50 else f'"{article_title}"'
        search_url = f"{TWITTER_SEARCH}?q={quote(query)}&f=live"
        
        headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
        }
        
        # For now, return a placeholder - full Twitter scraping requires API access
        # This demonstrates the architecture
        reactions.append({
            "platform": "twitter",
            "query": query,
            "note": "Full Twitter integration requires API keys",
            "search_url": search_url
        })
        
    except Exception as e:
        reactions.append({
            "platform": "twitter",
            "error": str(e)
        })
    
    return reactions


def find_hacker_news(article_url: str) -> Optional[Dict]:
    """Check if article was posted on Hacker News."""
    # HN Algolia API
    hn_api = "https://hn.algolia.com/api/v1/search"
    
    try:
        # Extract domain for searching
        domain_match = re.search(r'https?://([^/]+)', article_url)
        if not domain_match:
            return None
        
        domain = domain_match.group(1)
        response = requests.get(
            hn_api,
            params={"query": domain, "tags": "story", "hitsPerPage": 1},
            timeout=10
        )
        
        if response.status_code == 200:
            data = response.json()
            if data.get("hits"):
                hit = data["hits"][0]
                return {
                    "hn_url": f"https://news.ycombinator.com/item?id={hit.get('objectID')}",
                    "title": hit.get("title"),
                    "points": hit.get("points"),
                    "comment_count": hit.get("numComments")
                }
    except Exception:
        pass
    
    return None


def extract_spicy_tweets(content: str) -> List[str]:
    """Extract potential hot takes from article content."""
    # Look for quotes, controversial statements in article
    spicy_bits = []
    
    # Find quoted text
    quotes = re.findall(r'"([^"]+)"', content)
    
    # Look for strong opinion words
    opinion_words = ["criticized", "praised", "controversial", "scandal", "lawsuit", 
                     "breakthrough", "revolutionary", "disaster", "failure", "success"]
    
    for quote in quotes[:5]:  # Limit to 5 quotes
        if any(word in quote.lower() for word in opinion_words):
            spicy_bits.append(f'"{quote}"')
    
    return spicy_bits


def analyze_reactions(articles: List[Dict]) -> List[Dict]:
    """Analyze social reactions for a list of articles."""
    analyzed = []
    
    for article in articles:
        if not article.get("success"):
            analyzed.append(article)
            continue
        
        title = article.get("title", "")
        url = article.get("url", "")
        content = article.get("content", "")
        
        # Find HN posts
        hn_data = find_hacker_news(url)
        
        # Extract spicy quotes
        spicy = extract_spicy_tweets(content)
        
        article["reactions"] = {
            "hacker_news": hn_data,
            "spicy_quotes": spicy,
            "twitter_search": find_twitter_reactions(title, url)
        }
        
        analyzed.append(article)
    
    return analyzed


def main():
    """Main entry point."""
    import sys
    
    input_data = json.loads(sys.stdin.read())
    articles = input_data.get("articles", [])
    
    if not articles:
        print(json.dumps({"error": "No articles provided"}))
        return
    
    analyzed = analyze_reactions(articles)
    print(json.dumps({"analyzed": analyzed}))


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
TechMeme Scraper - Fetches top stories from techmeme.com via RSS
"""

import json
import time
from pathlib import Path
from typing import List, Dict, Optional
import requests
import xml.etree.ElementTree as ET

TECHMEME_RSS = "https://www.techmeme.com/feed.xml"
CACHE_FILE = Path.home() / ".cache/technews/stories.json"


def parse_rss(xml_content: str, num_stories: int = 10) -> List[Dict]:
    """Parse TechMeme RSS feed and extract stories."""
    stories = []
    
    try:
        root = ET.fromstring(xml_content)
        
        for item in root.findall(".//item")[:num_stories]:
            title_elem = item.find("title")
            link_elem = item.find("link")
            desc_elem = item.find("description")
            pubdate_elem = item.find("pubDate")
            
            title = title_elem.text if title_elem is not None else ""
            link = link_elem.text if link_elem is not None else ""
            
            # Extract summary from description (strip HTML)
            description = ""
            if desc_elem is not None and desc_elem.text:
                # Simple HTML strip - remove tags
                desc_text = desc_elem.text
                import re
                desc_text = re.sub(r'<[^>]+>', ' ', desc_text)
                description = ' '.join(desc_text.split())[:300]
            
            pubdate = pubdate_elem.text if pubdate_elem is not None else ""
            
            stories.append({
                "title": title,
                "url": link,
                "summary": description,
                "timestamp": pubdate,
                "source": "techmeme"
            })
            
    except ET.ParseError as e:
        # Fallback: try regex parsing
        stories = parse_rss_fallback(xml_content, num_stories)
    
    return stories


def parse_rss_fallback(xml_content: str, num_stories: int = 10) -> List[Dict]:
    """Fallback RSS parser using regex."""
    import re
    stories = []
    
    # Match items
    item_pattern = r'<item>(.*?)</item>'
    items = re.findall(item_pattern, xml_content, re.DOTALL)[:num_stories]
    
    for item in items:
        title_match = re.search(r'<title>(.*?)</title>', item, re.DOTALL)
        link_match = re.search(r'<link>(.*?)</link>', item)
        desc_match = re.search(r'<description>(.*?)</description>', item, re.DOTALL)
        date_match = re.search(r'<pubDate>(.*?)</pubDate>', item)
        
        title = title_match.group(1) if title_match else ""
        link = link_match.group(1) if link_match else ""
        
        description = ""
        if desc_match:
            desc_text = re.sub(r'<[^>]+>', ' ', desc_match.group(1))
            description = ' '.join(desc_text.split())[:300]
        
        pubdate = date_match.group(1) if date_match else ""
        
        stories.append({
            "title": title,
            "url": link,
            "summary": description,
            "timestamp": pubdate,
            "source": "techmeme"
        })
    
    return stories


def fetch_techmeme(num_stories: int = 10) -> List[Dict]:
    """Fetch top stories from TechMeme RSS feed."""
    headers = {
        "User-Agent": "Mozilla/5.0 (compatible; TechNewsBot/1.0)",
        "Accept": "application/rss+xml, application/xml, text/xml, */*",
    }
    
    response = requests.get(TECHMEME_RSS, headers=headers, timeout=30)
    response.raise_for_status()
    
    return parse_rss(response.text, num_stories)


def save_cache(stories: List[Dict]):
    """Cache fetched stories."""
    cache_dir = CACHE_FILE.parent
    cache_dir.mkdir(parents=True, exist_ok=True)
    with open(CACHE_FILE, "w") as f:
        json.dump({"cached_at": time.time(), "stories": stories}, f, indent=2)


def load_cache(max_age_hours: int = 2) -> Optional[List[Dict]]:
    """Load cached stories if recent enough."""
    if not CACHE_FILE.exists():
        return None
    
    try:
        with open(CACHE_FILE) as f:
            data = json.load(f)
        
        age = (time.time() - data.get("cached_at", 0)) / 3600
        if age < max_age_hours:
            return data.get("stories", [])
    except (json.JSONDecodeError, OSError):
        pass
    
    return None


def main(num_stories: int = 10, use_cache: bool = True) -> str:
    """Main entry point - returns JSON output."""
    # Try cache first
    if use_cache:
        cached = load_cache()
        if cached:
            return json.dumps({"cached": True, "stories": cached[:num_stories]})
    
    stories = fetch_techmeme(num_stories)
    save_cache(stories)
    
    return json.dumps({"cached": False, "stories": stories})


if __name__ == "__main__":
    import sys
    num = int(sys.argv[1]) if len(sys.argv) > 1 else 10
    print(main(num_stories=num))

#!/usr/bin/env python3
"""
TechNews Orchestrator - Main entry point for the technews skill
"""

import json
import sys
from pathlib import Path
from typing import List, Dict

# Add scripts to path
SCRIPT_DIR = Path(__file__).parent

# Import our modules
sys.path.insert(0, str(SCRIPT_DIR))

from techmeme_scraper import fetch_techmeme, load_cache, save_cache
from article_fetcher import fetch_multiple, summarize_content
from social_reactions import analyze_reactions


def format_output(stories: List[Dict]) -> str:
    """Format stories for display."""
    output = []
    output.append("📰 **Tech News Briefing**")
    output.append("")
    
    for i, story in enumerate(stories, 1):
        output.append(f"**{i}. {story['title']}**")
        # Use markdown inline link for Telegram
        output.append(f"🔗 [{story['url']}]({story['url']})")
        
        if story.get("summary"):
            output.append(f"📝 {story['summary'][:200]}...")
        
        # Show reactions if available
        if story.get("reactions"):
            reactions = story["reactions"]
            
            if reactions.get("hacker_news"):
                hn = reactions["hacker_news"]
                hn_url = hn.get("hn_url", "")
                points = hn.get("points", 0)
                comments = hn.get("comment_count", 0)
                output.append(f"💬 [HN: {points}pts, {comments} comments]({hn_url})")
            
            if reactions.get("spicy_quotes"):
                output.append(f"🔥 \"{reactions['spicy_quotes'][0][:100]}...\"")
        
        output.append("")
    
    return "\n".join(output)


def run_technews(num_stories: int = 10) -> str:
    """Main technews workflow."""
    try:
        # Step 1: Fetch from TechMeme
        print("Fetching TechMeme stories...")
        stories = fetch_techmeme(num_stories)
        
        if not stories:
            return "❌ Could not fetch stories from TechMeme"
        
        # Step 2: Fetch article content
        print(f"Fetching {len(stories)} articles...")
        urls = [s["url"] for s in stories]
        articles = fetch_multiple(urls)
        
        # Step 3: Merge content into stories
        for story, article in zip(stories, articles):
            if article.get("success"):
                story["content"] = article.get("content", "")
                story["summary"] = article.get("summary", "")
        
        # Step 4: Analyze reactions
        print("Analyzing social reactions...")
        analyzed = analyze_reactions(stories)
        
        # Step 5: Format output
        return format_output(analyzed)
        
    except requests.exceptions.RequestException as e:
        return f"❌ Network error: {str(e)}"
    except Exception as e:
        return f"❌ Unexpected error: {str(e)}"


def main():
    """CLI entry point."""
    num = int(sys.argv[1]) if len(sys.argv) > 1 else 10
    
    result = run_technews(num_stories=num)
    print(result)


if __name__ == "__main__":
    main()

/home/art/projects/skills/shared/technews

0.1.0