Aggregating Crypto News

# Aggregating Crypto News - Configuration
# Version: 2.0.0
# Author: Jeremy Longshore <jeremy@intentsolutions.io>

# =============================================================================
# Default Settings
# =============================================================================
defaults:
  # Default time period for news scan
  period: 24h

  # Number of articles to return
  top: 20

  # Sort order (relevance or recency)
  sort_by: relevance

  # Output format (table, json, csv)
  format: table

# =============================================================================
# Fetch Settings
# =============================================================================
fetch:
  # Request timeout in seconds
  timeout: 10

  # Maximum parallel requests
  max_workers: 10

  # User agent string
  user_agent: "CryptoNewsAggregator/2.0 (RSS Feed Reader)"

# =============================================================================
# Cache Settings
# =============================================================================
cache:
  # Enable response caching
  enabled: true

  # Cache time-to-live in seconds (5 minutes)
  ttl: 300

  # Cache directory (relative to skill root)
  directory: ./data/cache

# =============================================================================
# Scoring Weights
# =============================================================================
scoring:
  # Source quality weight (0-30 points)
  source_quality_multiplier: 3

  # Maximum keyword score
  max_keyword_score: 50

  # Recency bonuses by age
  recency_bonus:
    1h: 20
    4h: 15
    12h: 10
    24h: 5
    3d: 2

# =============================================================================
# Categories
# =============================================================================
categories:
  market:
    description: "General market news and price movements"
    keywords: []

  defi:
    description: "DeFi protocols, yield farming, DEXes"
    keywords:
      - defi
      - yield farming
      - liquidity pool
      - dex
      - lending protocol

  nft:
    description: "NFT projects, marketplaces, collections"
    keywords:
      - nft
      - opensea
      - blur
      - collection
      - mint

  regulatory:
    description: "Government, SEC, legal developments"
    keywords:
      - sec
      - cftc
      - regulation
      - congress
      - compliance

  layer1:
    description: "L1 blockchain news"
    keywords:
      - ethereum
      - solana
      - bitcoin
      - cardano
      - avalanche

  layer2:
    description: "L2 scaling solutions"
    keywords:
      - arbitrum
      - optimism
      - polygon
      - zksync
      - base
      - rollup

  exchange:
    description: "Exchange news, listings, delistings"
    keywords:
      - binance
      - coinbase
      - kraken
      - listing
      - delist

  security:
    description: "Hacks, exploits, vulnerabilities"
    keywords:
      - hack
      - exploit
      - vulnerability
      - audit
      - breach
      - stolen

# =============================================================================
# Deduplication
# =============================================================================
deduplication:
  # Enable title-based deduplication
  enabled: true

  # Similarity threshold (0-1)
  threshold: 0.8

# RSS News Sources Registry
# Version: 2.0.0
# Author: Jeremy Longshore <jeremy@intentsolutions.io>
#
# Quality score (1-10):
#   9-10: Tier 1 - Major outlets, fast, reliable
#   7-8:  Tier 2 - Good coverage, some delay
#   5-6:  Tier 3 - Niche or aggregated content
#   1-4:  Tier 4 - Lower quality, promotional

sources:
  # ==========================================================================
  # Tier 1: Major Crypto News Outlets
  # ==========================================================================
  - name: CoinDesk
    url: https://www.coindesk.com/arc/outboundfeeds/rss/
    category: market
    quality: 9
    description: Leading crypto news, fast on breaking stories

  - name: The Block
    url: https://www.theblock.co/rss.xml
    category: market
    quality: 9
    description: Institutional-grade crypto journalism

  - name: Decrypt
    url: https://decrypt.co/feed
    category: market
    quality: 8
    description: Accessible crypto news for all levels

  - name: CoinTelegraph
    url: https://cointelegraph.com/rss
    category: market
    quality: 8
    description: High-volume crypto news coverage

  - name: Blockworks
    url: https://blockworks.co/feed/
    category: market
    quality: 8
    description: Institutional crypto and DeFi news

  # ==========================================================================
  # Tier 2: Quality Crypto Publications
  # ==========================================================================
  - name: Bitcoin Magazine
    url: https://bitcoinmagazine.com/feed
    category: market
    quality: 8
    description: Bitcoin-focused coverage

  - name: CryptoSlate
    url: https://cryptoslate.com/feed/
    category: market
    quality: 7
    description: Comprehensive crypto coverage

  - name: DL News
    url: https://www.dlnews.com/feed/
    category: market
    quality: 7
    description: DeFi and web3 focused

  - name: Unchained
    url: https://unchainedcrypto.com/feed/
    category: market
    quality: 7
    description: In-depth crypto analysis

  # ==========================================================================
  # Tier 3: Aggregators and Niche Sites
  # ==========================================================================
  - name: NewsBTC
    url: https://www.newsbtc.com/feed/
    category: market
    quality: 6
    description: General crypto news

  - name: CryptoPotato
    url: https://cryptopotato.com/feed/
    category: market
    quality: 6
    description: Trading and market news

  - name: U.Today
    url: https://u.today/rss
    category: market
    quality: 6
    description: High-volume crypto news

  - name: Bitcoinist
    url: https://bitcoinist.com/feed/
    category: market
    quality: 6
    description: Bitcoin and altcoin news

  - name: AMBCrypto
    url: https://ambcrypto.com/feed/
    category: market
    quality: 5
    description: Altcoin coverage

  # ==========================================================================
  # DeFi Specific
  # ==========================================================================
  - name: DeFi Pulse
    url: https://defipulse.com/blog/feed/
    category: defi
    quality: 8
    description: DeFi protocol analysis

  - name: The Defiant
    url: https://thedefiant.io/feed/
    category: defi
    quality: 8
    description: DeFi news and research

  # ==========================================================================
  # NFT Specific
  # ==========================================================================
  - name: NFT Now
    url: https://nftnow.com/feed/
    category: nft
    quality: 7
    description: NFT news and culture

  # ==========================================================================
  # Layer 2 and Ethereum
  # ==========================================================================
  - name: Week in Ethereum
    url: https://weekinethereumnews.com/feed/
    category: layer1
    quality: 8
    description: Weekly Ethereum digest

  # ==========================================================================
  # Regulatory and Institutional
  # ==========================================================================
  - name: Crypto Law Insider
    url: https://cryptolawinsider.com/feed/
    category: regulatory
    quality: 7
    description: Crypto legal developments

# =============================================================================
# Source Categories Reference
# =============================================================================
# market:     General market news, price movements
# defi:       DeFi protocols, yield farming, DEXes
# nft:        NFT projects, marketplaces
# regulatory: Government, SEC, legal
# layer1:     L1 blockchain news
# layer2:     L2 scaling solutions
# exchange:   Exchange news
# security:   Hacks, exploits

#!/usr/bin/env python3
"""
Feed Fetcher - Parallel RSS Feed Fetching

Fetch multiple RSS feeds in parallel with timeout handling and caching.

Author: Jeremy Longshore <jeremy@intentsolutions.io>
Version: 2.0.0
License: MIT
"""

import sys
import hashlib
import json
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta

try:
    import requests
except ImportError:
    print("Error: requests library required. Install with: pip install requests", file=sys.stderr)
    sys.exit(1)


class FeedFetcher:
    """Fetch multiple RSS feeds in parallel with caching."""

    def __init__(
        self,
        timeout: int = 10,
        max_workers: int = 10,
        cache_ttl: int = 300,  # 5 minutes
        verbose: bool = False,
    ):
        """
        Initialize feed fetcher.

        Args:
            timeout: Request timeout in seconds
            max_workers: Max parallel requests
            cache_ttl: Cache time-to-live in seconds
            verbose: Enable verbose logging
        """
        self.timeout = timeout
        self.max_workers = max_workers
        self.cache_ttl = cache_ttl
        self.verbose = verbose
        self.cache_dir = Path(__file__).parent.parent / "data" / "cache"
        self.cache_dir.mkdir(parents=True, exist_ok=True)

    def fetch_all(self, sources: List[Dict[str, Any]]) -> Dict[str, str]:
        """
        Fetch all RSS feeds in parallel.

        Args:
            sources: List of source dictionaries with 'name' and 'url'

        Returns:
            Dictionary mapping source name to feed content
        """
        results = {}

        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
            future_to_source = {executor.submit(self._fetch_one, source): source for source in sources}

            for future in as_completed(future_to_source):
                source = future_to_source[future]
                try:
                    content = future.result()
                    if content:
                        results[source["name"]] = content
                except Exception as e:
                    if self.verbose:
                        print(f"Error fetching {source['name']}: {e}", file=sys.stderr)

        return results

    def _fetch_one(self, source: Dict[str, Any]) -> Optional[str]:
        """
        Fetch a single RSS feed.

        Args:
            source: Source dictionary with 'name' and 'url'

        Returns:
            Feed content string or None if failed
        """
        name = source.get("name", "unknown")
        url = source.get("url", "")

        if not url:
            return None

        # Check cache first
        cached = self._get_cached(url)
        if cached:
            if self.verbose:
                print(f"  Cache hit: {name}", file=sys.stderr)
            return cached

        # Fetch from network
        try:
            if self.verbose:
                print(f"  Fetching: {name}", file=sys.stderr)

            headers = {
                "User-Agent": "CryptoNewsAggregator/2.0 (RSS Feed Reader)",
                "Accept": "application/rss+xml, application/xml, text/xml, */*",
            }

            response = requests.get(url, timeout=self.timeout, headers=headers)
            response.raise_for_status()

            content = response.text

            # Cache the response
            self._set_cached(url, content)

            return content

        except requests.exceptions.Timeout:
            if self.verbose:
                print(f"  Timeout: {name}", file=sys.stderr)
            return None
        except requests.exceptions.RequestException as e:
            if self.verbose:
                print(f"  Error: {name} - {e}", file=sys.stderr)
            return None

    def _get_cache_path(self, url: str) -> Path:
        """Get cache file path for URL."""
        url_hash = hashlib.md5(url.encode()).hexdigest()
        return self.cache_dir / f"{url_hash}.json"

    def _get_cached(self, url: str) -> Optional[str]:
        """Get cached content if valid."""
        cache_path = self._get_cache_path(url)

        if not cache_path.exists():
            return None

        try:
            with open(cache_path, "r") as f:
                cached = json.load(f)

            # Check if expired
            cached_time = datetime.fromisoformat(cached.get("timestamp", ""))
            if datetime.utcnow() - cached_time > timedelta(seconds=self.cache_ttl):
                return None

            return cached.get("content")

        except Exception:
            return None

    def _set_cached(self, url: str, content: str) -> None:
        """Cache content for URL."""
        cache_path = self._get_cache_path(url)

        try:
            with open(cache_path, "w") as f:
                json.dump({"url": url, "timestamp": datetime.utcnow().isoformat(), "content": content}, f)
        except Exception:
            pass

    def clear_cache(self) -> int:
        """Clear all cached feeds. Returns count of files deleted."""
        count = 0
        for cache_file in self.cache_dir.glob("*.json"):
            try:
                cache_file.unlink()
                count += 1
            except Exception:
                pass
        return count

#!/usr/bin/env python3
"""
Feed Parser - RSS Feed Parsing and Normalization

Parse RSS/Atom feeds and normalize entries to common schema.

Author: Jeremy Longshore <jeremy@intentsolutions.io>
Version: 2.0.0
License: MIT
"""

import sys
import re
import html
from datetime import datetime
from typing import List, Dict, Any, Optional
from difflib import SequenceMatcher

try:
    import feedparser
except ImportError:
    print("Error: feedparser library required. Install with: pip install feedparser", file=sys.stderr)
    sys.exit(1)


class FeedParser:
    """Parse RSS/Atom feeds and normalize to common schema."""

    def __init__(self):
        """Initialize feed parser."""
        self.html_tag_pattern = re.compile(r"<[^>]+>")

    def parse_feed(
        self, content: str, source_name: str = "Unknown", source_category: str = "market", source_quality: int = 5
    ) -> List[Dict[str, Any]]:
        """
        Parse RSS/Atom feed content into article list.

        Args:
            content: Raw feed XML content
            source_name: Name of the source
            source_category: Default category for this source
            source_quality: Quality score (1-10)

        Returns:
            List of normalized article dictionaries
        """
        articles = []

        try:
            feed = feedparser.parse(content)

            for entry in feed.entries:
                article = self._parse_entry(
                    entry, source_name=source_name, source_category=source_category, source_quality=source_quality
                )
                if article:
                    articles.append(article)

        except Exception:
            pass

        return articles

    def _parse_entry(
        self, entry: Any, source_name: str, source_category: str, source_quality: int
    ) -> Optional[Dict[str, Any]]:
        """
        Parse a single feed entry.

        Args:
            entry: feedparser entry object
            source_name: Name of the source
            source_category: Default category
            source_quality: Quality score

        Returns:
            Normalized article dictionary or None if invalid
        """
        # Extract title
        title = entry.get("title", "")
        if not title:
            return None

        title = self._clean_text(title)

        # Extract URL
        url = entry.get("link", "")
        if not url:
            return None

        # Extract summary
        summary = ""
        if "summary" in entry:
            summary = self._clean_text(entry.summary)
        elif "description" in entry:
            summary = self._clean_text(entry.description)
        elif "content" in entry and entry.content:
            summary = self._clean_text(entry.content[0].get("value", ""))

        # Truncate summary if too long
        if len(summary) > 500:
            summary = summary[:500] + "..."

        # Extract published date
        published = self._parse_date(entry)

        # Detect coins mentioned
        coins_mentioned = self._detect_coins(f"{title} {summary}")

        # Detect category from content
        category = self._detect_category(f"{title} {summary}", source_category)

        return {
            "title": title,
            "url": url,
            "source": source_name,
            "source_quality": source_quality,
            "published": published,
            "summary": summary,
            "category": category,
            "coins_mentioned": coins_mentioned,
        }

    def _clean_text(self, text: str) -> str:
        """Remove HTML tags and clean whitespace."""
        if not text:
            return ""

        # Decode HTML entities
        text = html.unescape(text)

        # Remove HTML tags
        text = self.html_tag_pattern.sub("", text)

        # Normalize whitespace
        text = " ".join(text.split())

        return text.strip()

    def _parse_date(self, entry: Any) -> Optional[datetime]:
        """Parse published date from entry."""
        # Try published_parsed first
        if hasattr(entry, "published_parsed") and entry.published_parsed:
            try:
                return datetime(*entry.published_parsed[:6])
            except Exception:
                pass

        # Try updated_parsed
        if hasattr(entry, "updated_parsed") and entry.updated_parsed:
            try:
                return datetime(*entry.updated_parsed[:6])
            except Exception:
                pass

        # Try parsing string dates
        for field in ["published", "updated", "date"]:
            date_str = entry.get(field)
            if date_str:
                for fmt in [
                    "%Y-%m-%dT%H:%M:%SZ",
                    "%Y-%m-%dT%H:%M:%S%z",
                    "%a, %d %b %Y %H:%M:%S %z",
                    "%a, %d %b %Y %H:%M:%S GMT",
                    "%Y-%m-%d %H:%M:%S",
                ]:
                    try:
                        return datetime.strptime(date_str[:24], fmt[:24])
                    except Exception:
                        continue

        return None

    def _detect_coins(self, text: str) -> List[str]:
        """Detect cryptocurrency symbols mentioned in text."""
        text_upper = text.upper()

        # Common coin patterns
        coins = []
        coin_patterns = [
            ("BTC", ["BITCOIN", "BTC"]),
            ("ETH", ["ETHEREUM", "ETH", "ETHER"]),
            ("SOL", ["SOLANA", "SOL"]),
            ("BNB", ["BINANCE", "BNB"]),
            ("XRP", ["RIPPLE", "XRP"]),
            ("ADA", ["CARDANO", "ADA"]),
            ("DOGE", ["DOGECOIN", "DOGE"]),
            ("DOT", ["POLKADOT", "DOT"]),
            ("AVAX", ["AVALANCHE", "AVAX"]),
            ("LINK", ["CHAINLINK", "LINK"]),
            ("MATIC", ["POLYGON", "MATIC"]),
            ("UNI", ["UNISWAP", "UNI"]),
            ("AAVE", ["AAVE"]),
            ("MKR", ["MAKER", "MKR"]),
            ("CRV", ["CURVE", "CRV"]),
            ("LDO", ["LIDO", "LDO"]),
            ("ARB", ["ARBITRUM", "ARB"]),
            ("OP", ["OPTIMISM", " OP "]),
        ]

        for symbol, patterns in coin_patterns:
            for pattern in patterns:
                if pattern in text_upper:
                    if symbol not in coins:
                        coins.append(symbol)
                    break

        return coins

    def _detect_category(self, text: str, default: str) -> str:
        """Detect news category from content."""
        text_lower = text.lower()

        category_keywords = {
            "defi": ["defi", "yield farming", "liquidity pool", "dex", "lending protocol", "aave", "uniswap", "curve"],
            "nft": ["nft", "opensea", "blur", "collection", "mint", "pfp", "digital art"],
            "regulatory": ["sec", "cftc", "regulation", "congress", "senator", "compliance", "lawsuit", "settlement"],
            "exchange": ["binance", "coinbase", "kraken", "listing", "delist", "exchange", "trading pair"],
            "security": ["hack", "exploit", "vulnerability", "audit", "breach", "stolen", "attack", "rug pull"],
            "layer2": ["arbitrum", "optimism", "polygon", "zksync", "base", "layer 2", "l2", "rollup"],
        }

        for category, keywords in category_keywords.items():
            if any(kw in text_lower for kw in keywords):
                return category

        return default

    def deduplicate(self, articles: List[Dict[str, Any]], threshold: float = 0.8) -> List[Dict[str, Any]]:
        """
        Remove duplicate articles based on title similarity.

        Args:
            articles: List of article dictionaries
            threshold: Similarity threshold (0-1) for considering duplicates

        Returns:
            Deduplicated list of articles
        """
        if not articles:
            return []

        unique = []
        seen_titles = []

        for article in articles:
            title = article.get("title", "").lower()

            # Check similarity against all seen titles
            is_duplicate = False
            for seen in seen_titles:
                similarity = SequenceMatcher(None, title, seen).ratio()
                if similarity >= threshold:
                    is_duplicate = True
                    break

            if not is_duplicate:
                unique.append(article)
                seen_titles.append(title)

        return unique

#!/usr/bin/env python3
"""
News Formatters - Output Formatting for Crypto News

Format news results for table, JSON, and CSV output.

Author: Jeremy Longshore <jeremy@intentsolutions.io>
Version: 2.0.0
License: MIT
"""

import json
import csv
import io
from datetime import datetime, timedelta
from typing import Dict, Any


class NewsFormatter:
    """Format crypto news results for various output types."""

    def format(self, result: Dict[str, Any], format_type: str = "table") -> str:
        """
        Format news results.

        Args:
            result: Dictionary with articles and meta
            format_type: Output format (table, json, csv)

        Returns:
            Formatted output string
        """
        if format_type == "json":
            return self._format_json(result)
        elif format_type == "csv":
            return self._format_csv(result)
        else:
            return self._format_table(result)

    def _format_table(self, result: Dict[str, Any]) -> str:
        """Format results as aligned table."""
        lines = []
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        meta = result.get("meta", {})

        # Header
        lines.append("=" * 90)
        lines.append(f"  CRYPTO NEWS AGGREGATOR{' ' * 42}Updated: {timestamp}")
        lines.append("=" * 90)

        articles = result.get("articles", [])
        if articles:
            lines.append("")
            period = meta.get("period", "24h")
            lines.append(f"  TOP CRYPTO NEWS ({period})")
            lines.append("-" * 90)
            lines.append(f"  {'Rank':<6}{'Source':<16}{'Title':<44}{'Age':<10}{'Score':>8}")
            lines.append("-" * 90)

            for article in articles:
                rank = article.get("rank", "-")
                source = article.get("source", "Unknown")[:14]
                title = article.get("title", "")[:42]
                age = self._format_age(article.get("published"))
                score = f"{article.get('relevance_score', 0):.1f}"

                lines.append(f"  {rank:<6}{source:<16}{title:<44}{age:<10}{score:>8}")

            lines.append("-" * 90)

        else:
            lines.append("")
            lines.append("  No articles found matching your criteria.")
            lines.append("")

        # Summary
        lines.append("")
        shown = meta.get("shown", 0)
        sources = meta.get("sources_checked", 0)
        total = meta.get("after_filter", 0)

        lines.append(f"  Summary: {shown} articles shown | Scanned: {sources} sources | Matched: {total}")

        # Filters applied
        filters = meta.get("filters", {})
        filter_parts = []
        if filters.get("coins"):
            filter_parts.append(f"coins={','.join(filters['coins'])}")
        if filters.get("category"):
            filter_parts.append(f"category={filters['category']}")
        if filters.get("min_score"):
            filter_parts.append(f"min_score={filters['min_score']}")

        if filter_parts:
            lines.append(f"  Filters: {', '.join(filter_parts)}")

        lines.append("=" * 90)

        return "\n".join(lines)

    def _format_json(self, result: Dict[str, Any]) -> str:
        """Format results as JSON."""
        # Prepare articles for JSON serialization
        output = {"articles": [], "meta": result.get("meta", {})}

        for article in result.get("articles", []):
            serializable = {
                "rank": article.get("rank"),
                "title": article.get("title"),
                "url": article.get("url"),
                "source": article.get("source"),
                "published": None,
                "age": self._format_age(article.get("published")),
                "category": article.get("category"),
                "relevance_score": article.get("relevance_score"),
                "coins_mentioned": article.get("coins_mentioned", []),
                "summary": article.get("summary", "")[:200],
            }

            # Convert datetime to ISO format
            pub = article.get("published")
            if pub:
                serializable["published"] = pub.isoformat() + "Z"

            output["articles"].append(serializable)

        # Add timestamp
        output["meta"]["generated_at"] = datetime.utcnow().isoformat() + "Z"

        return json.dumps(output, indent=2)

    def _format_csv(self, result: Dict[str, Any]) -> str:
        """Format results as CSV."""
        output = io.StringIO()

        fieldnames = [
            "rank",
            "title",
            "url",
            "source",
            "published",
            "age",
            "category",
            "relevance_score",
            "coins_mentioned",
        ]

        writer = csv.DictWriter(output, fieldnames=fieldnames)
        writer.writeheader()

        for article in result.get("articles", []):
            pub = article.get("published")
            published_str = pub.isoformat() if pub else ""

            writer.writerow(
                {
                    "rank": article.get("rank", ""),
                    "title": article.get("title", ""),
                    "url": article.get("url", ""),
                    "source": article.get("source", ""),
                    "published": published_str,
                    "age": self._format_age(pub),
                    "category": article.get("category", ""),
                    "relevance_score": article.get("relevance_score", ""),
                    "coins_mentioned": ",".join(article.get("coins_mentioned", [])),
                }
            )

        return output.getvalue()

    def _format_age(self, published: datetime) -> str:
        """Format datetime as human-readable age."""
        if not published:
            return "unknown"

        now = datetime.utcnow()
        age = now - published

        if age < timedelta(minutes=1):
            return "just now"
        elif age < timedelta(hours=1):
            minutes = int(age.total_seconds() / 60)
            return f"{minutes}m ago"
        elif age < timedelta(hours=24):
            hours = int(age.total_seconds() / 3600)
            return f"{hours}h ago"
        elif age < timedelta(days=7):
            days = int(age.total_seconds() / 86400)
            return f"{days}d ago"
        else:
            return published.strftime("%Y-%m-%d")


def format_article_detail(article: Dict[str, Any]) -> str:
    """Format a single article with full details."""
    lines = []

    lines.append(f"Title: {article.get('title', 'Unknown')}")
    lines.append(f"Source: {article.get('source', 'Unknown')}")
    lines.append(f"URL: {article.get('url', '')}")

    pub = article.get("published")
    if pub:
        lines.append(f"Published: {pub.strftime('%Y-%m-%d %H:%M:%S UTC')}")

    lines.append(f"Category: {article.get('category', 'Unknown')}")
    lines.append(f"Score: {article.get('relevance_score', 0):.1f}")

    coins = article.get("coins_mentioned", [])
    if coins:
        lines.append(f"Coins: {', '.join(coins)}")

    summary = article.get("summary", "")
    if summary:
        lines.append("")
        lines.append("Summary:")
        lines.append(summary)

    return "\n".join(lines)

#!/usr/bin/env python3
"""
Crypto News Aggregator - Main CLI Entry Point

Aggregate breaking cryptocurrency news from multiple RSS sources.

Author: Jeremy Longshore <jeremy@intentsolutions.io>
Version: 2.0.0
License: MIT
"""

import argparse
import sys
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Dict, Any

# Add scripts directory to path for local imports
SCRIPT_DIR = Path(__file__).parent
sys.path.insert(0, str(SCRIPT_DIR))

from feed_fetcher import FeedFetcher
from feed_parser import FeedParser
from scorer import NewsScorer
from formatters import NewsFormatter


def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        description="Aggregate breaking cryptocurrency news from multiple sources",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  %(prog)s                                # Default scan (24h, top 20)
  %(prog)s --coin BTC --period 4h         # Bitcoin news, past 4 hours
  %(prog)s --category defi                # DeFi category only
  %(prog)s --format json --output news.json  # Export to JSON
        """,
    )

    # Filtering options
    parser.add_argument("--coin", type=str, help="Filter by single coin symbol (e.g., BTC, ETH)")
    parser.add_argument("--coins", type=str, help="Filter by multiple coins (comma-separated, e.g., BTC,ETH,SOL)")
    parser.add_argument(
        "--category",
        type=str,
        choices=["market", "defi", "nft", "regulatory", "layer1", "layer2", "exchange", "security"],
        help="Filter by news category",
    )
    parser.add_argument(
        "--period",
        type=str,
        choices=["1h", "4h", "24h", "7d"],
        default="24h",
        help="Time window for news (default: 24h)",
    )

    # Output options
    parser.add_argument("--top", type=int, default=20, help="Number of results to return (default: 20)")
    parser.add_argument("--min-score", type=float, default=0, help="Minimum relevance score (default: 0)")
    parser.add_argument(
        "--sort-by",
        type=str,
        choices=["relevance", "recency"],
        default="relevance",
        help="Sort results by (default: relevance)",
    )

    # Format and export
    parser.add_argument(
        "--format",
        "-f",
        type=str,
        choices=["table", "json", "csv"],
        default="table",
        help="Output format (default: table)",
    )
    parser.add_argument("--output", "-o", type=str, help="Output file path (default: stdout)")

    # Debug options
    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose output")
    parser.add_argument("--version", action="version", version="%(prog)s 2.0.0")

    return parser.parse_args()


def get_time_threshold(period: str) -> datetime:
    """Convert period string to datetime threshold."""
    now = datetime.utcnow()
    if period == "1h":
        return now - timedelta(hours=1)
    elif period == "4h":
        return now - timedelta(hours=4)
    elif period == "24h":
        return now - timedelta(hours=24)
    elif period == "7d":
        return now - timedelta(days=7)
    else:
        return now - timedelta(hours=24)


def load_sources() -> List[Dict[str, Any]]:
    """Load RSS sources from config."""
    config_path = SCRIPT_DIR.parent / "config" / "sources.yaml"

    if config_path.exists():
        try:
            import yaml

            with open(config_path, "r") as f:
                config = yaml.safe_load(f)
                return config.get("sources", [])
        except ImportError:
            pass
        except Exception:
            pass

    # Default sources if config not available
    return [
        {
            "name": "CoinDesk",
            "url": "https://www.coindesk.com/arc/outboundfeeds/rss/",
            "category": "market",
            "quality": 9,
        },
        {"name": "CoinTelegraph", "url": "https://cointelegraph.com/rss", "category": "market", "quality": 8},
        {"name": "The Block", "url": "https://www.theblock.co/rss.xml", "category": "market", "quality": 9},
        {"name": "Decrypt", "url": "https://decrypt.co/feed", "category": "market", "quality": 8},
        {"name": "Bitcoin Magazine", "url": "https://bitcoinmagazine.com/feed", "category": "market", "quality": 8},
        {"name": "CryptoSlate", "url": "https://cryptoslate.com/feed/", "category": "market", "quality": 7},
        {"name": "NewsBTC", "url": "https://www.newsbtc.com/feed/", "category": "market", "quality": 6},
        {"name": "CryptoPotato", "url": "https://cryptopotato.com/feed/", "category": "market", "quality": 6},
        {"name": "U.Today", "url": "https://u.today/rss", "category": "market", "quality": 6},
        {"name": "Blockworks", "url": "https://blockworks.co/feed/", "category": "market", "quality": 8},
        {"name": "DeFi Pulse", "url": "https://defipulse.com/blog/feed/", "category": "defi", "quality": 8},
        {"name": "DL News", "url": "https://www.dlnews.com/feed/", "category": "market", "quality": 7},
    ]


def main() -> None:
    """Main entry point."""
    args = parse_args()

    # Parse coin filters
    coins = []
    if args.coin:
        coins = [args.coin.upper()]
    elif args.coins:
        coins = [c.strip().upper() for c in args.coins.split(",")]

    # Get time threshold
    time_threshold = get_time_threshold(args.period)

    # Load sources
    sources = load_sources()

    if args.verbose:
        print(f"Loaded {len(sources)} sources", file=sys.stderr)

    # Initialize components
    fetcher = FeedFetcher(timeout=10, verbose=args.verbose)
    parser = FeedParser()
    scorer = NewsScorer()
    formatter = NewsFormatter()

    # Fetch feeds
    if args.verbose:
        print("Fetching feeds...", file=sys.stderr)

    raw_feeds = fetcher.fetch_all(sources)

    if args.verbose:
        print(f"Fetched {len(raw_feeds)} feeds successfully", file=sys.stderr)

    # Parse feeds into articles
    all_articles = []
    for source_name, feed_content in raw_feeds.items():
        source_info = next((s for s in sources if s["name"] == source_name), {})
        articles = parser.parse_feed(
            feed_content,
            source_name=source_name,
            source_category=source_info.get("category", "market"),
            source_quality=source_info.get("quality", 5),
        )
        all_articles.extend(articles)

    if args.verbose:
        print(f"Parsed {len(all_articles)} total articles", file=sys.stderr)

    # Deduplicate
    unique_articles = parser.deduplicate(all_articles)

    if args.verbose:
        print(f"After deduplication: {len(unique_articles)} articles", file=sys.stderr)

    # Score articles
    for article in unique_articles:
        article["relevance_score"] = scorer.calculate_score(
            title=article.get("title", ""),
            summary=article.get("summary", ""),
            source_quality=article.get("source_quality", 5),
            published=article.get("published"),
        )

    # Apply filters
    filtered = []
    for article in unique_articles:
        # Time filter
        pub_date = article.get("published")
        if pub_date and pub_date < time_threshold:
            continue

        # Coin filter
        if coins:
            text = f"{article.get('title', '')} {article.get('summary', '')}".upper()
            if not any(coin in text for coin in coins):
                continue

        # Category filter
        if args.category and article.get("category") != args.category:
            # Also check for category keywords in content
            category_keywords = {
                "defi": ["defi", "yield", "lending", "dex", "liquidity"],
                "nft": ["nft", "opensea", "blur", "collection", "mint"],
                "regulatory": ["sec", "regulation", "congress", "law", "compliance"],
                "exchange": ["binance", "coinbase", "kraken", "listing", "delist"],
                "security": ["hack", "exploit", "vulnerability", "audit", "breach"],
                "layer1": ["ethereum", "solana", "bitcoin", "cardano", "avalanche"],
                "layer2": ["arbitrum", "optimism", "polygon", "zksync", "base"],
            }
            keywords = category_keywords.get(args.category, [])
            text = f"{article.get('title', '')} {article.get('summary', '')}".lower()
            if not any(kw in text for kw in keywords):
                continue

        # Score filter
        if article.get("relevance_score", 0) < args.min_score:
            continue

        filtered.append(article)

    if args.verbose:
        print(f"After filtering: {len(filtered)} articles", file=sys.stderr)

    # Sort
    if args.sort_by == "relevance":
        filtered.sort(key=lambda x: x.get("relevance_score", 0), reverse=True)
    else:
        filtered.sort(key=lambda x: x.get("published") or datetime.min, reverse=True)

    # Limit results
    result_articles = filtered[: args.top]

    # Add ranks
    for i, article in enumerate(result_articles, 1):
        article["rank"] = i

    # Prepare result
    result = {
        "articles": result_articles,
        "meta": {
            "period": args.period,
            "sources_checked": len(sources),
            "total_fetched": len(all_articles),
            "after_dedup": len(unique_articles),
            "after_filter": len(filtered),
            "shown": len(result_articles),
            "filters": {"coins": coins if coins else None, "category": args.category, "min_score": args.min_score},
        },
    }

    # Format output
    output = formatter.format(result, format_type=args.format)

    # Write output
    if args.output:
        output_path = Path(args.output)
        output_path.parent.mkdir(parents=True, exist_ok=True)
        with open(output_path, "w") as f:
            f.write(output)
        print(f"Output written to {output_path}", file=sys.stderr)
    else:
        print(output)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
News Scorer - Relevance Scoring for Crypto News

Calculate relevance scores based on keywords, source quality, and recency.

Author: Jeremy Longshore <jeremy@intentsolutions.io>
Version: 2.0.0
License: MIT
"""

from datetime import datetime, timedelta
from typing import Optional, List, Dict


class NewsScorer:
    """Calculate relevance scores for crypto news articles."""

    def __init__(self):
        """Initialize news scorer with keyword weights."""
        # High-impact keywords (market-moving)
        self.high_impact_keywords = {
            "hack": 15,
            "exploit": 15,
            "breach": 12,
            "stolen": 12,
            "listing": 10,
            "delist": 12,
            "sec": 12,
            "lawsuit": 10,
            "settlement": 10,
            "partnership": 8,
            "acquisition": 10,
            "merger": 10,
            "etf": 12,
            "approval": 10,
            "rejection": 10,
            "all-time high": 10,
            "ath": 8,
            "crash": 10,
            "surge": 8,
            "plunge": 8,
            "rally": 8,
            "dump": 8,
            "pump": 8,
            "airdrop": 10,
            "fork": 10,
            "upgrade": 8,
            "mainnet": 8,
            "launch": 8,
            "bankruptcy": 12,
            "insolvent": 12,
            "freeze": 10,
            "halting": 10,
        }

        # Medium-impact keywords
        self.medium_impact_keywords = {
            "bitcoin": 5,
            "ethereum": 5,
            "defi": 4,
            "nft": 4,
            "regulation": 6,
            "institutional": 5,
            "whale": 6,
            "binance": 4,
            "coinbase": 4,
            "stablecoin": 5,
            "usdt": 4,
            "usdc": 4,
            "layer 2": 4,
            "rollup": 4,
            "yield": 4,
            "staking": 4,
            "validator": 4,
            "governance": 4,
            "proposal": 4,
            "vote": 4,
        }

        # Negative keywords (reduce score for promotional content)
        self.negative_keywords = {
            "sponsored": -10,
            "partner content": -10,
            "advertisement": -15,
            "promotion": -5,
            "giveaway": -5,
            "free": -3,
            "sign up": -3,
            "discount": -5,
        }

    def calculate_score(
        self, title: str, summary: str = "", source_quality: int = 5, published: Optional[datetime] = None
    ) -> float:
        """
        Calculate relevance score for an article.

        Args:
            title: Article title
            summary: Article summary/description
            source_quality: Source quality rating (1-10)
            published: Publication datetime

        Returns:
            Relevance score (0-100)
        """
        text = f"{title} {summary}".lower()
        score = 0.0

        # Base score from source quality (0-30 points)
        score += source_quality * 3

        # Keyword scoring (0-50 points)
        keyword_score = 0.0

        # High-impact keywords
        for keyword, weight in self.high_impact_keywords.items():
            if keyword in text:
                keyword_score += weight

        # Medium-impact keywords
        for keyword, weight in self.medium_impact_keywords.items():
            if keyword in text:
                keyword_score += weight

        # Negative keywords
        for keyword, weight in self.negative_keywords.items():
            if keyword in text:
                keyword_score += weight  # weight is negative

        # Cap keyword score
        keyword_score = min(50, max(0, keyword_score))
        score += keyword_score

        # Recency bonus (0-20 points)
        if published:
            recency_score = self._calculate_recency_score(published)
            score += recency_score

        # Ensure score is in range
        score = min(100, max(0, score))

        return round(score, 1)

    def _calculate_recency_score(self, published: datetime) -> float:
        """Calculate recency bonus based on publication time."""
        now = datetime.utcnow()
        age = now - published

        if age < timedelta(hours=1):
            return 20  # Very fresh
        elif age < timedelta(hours=4):
            return 15
        elif age < timedelta(hours=12):
            return 10
        elif age < timedelta(hours=24):
            return 5
        elif age < timedelta(days=3):
            return 2
        else:
            return 0

    def explain_score(
        self, title: str, summary: str = "", source_quality: int = 5, published: Optional[datetime] = None
    ) -> Dict[str, any]:
        """
        Generate explanation of score components.

        Args:
            title: Article title
            summary: Article summary
            source_quality: Source quality rating
            published: Publication datetime

        Returns:
            Dictionary with score breakdown
        """
        text = f"{title} {summary}".lower()

        explanation = {
            "base_score": source_quality * 3,
            "source_quality": source_quality,
            "keywords_found": [],
            "negative_keywords_found": [],
            "keyword_score": 0,
            "recency_score": 0,
            "final_score": 0,
        }

        # Find matching keywords
        keyword_score = 0.0

        for keyword, weight in self.high_impact_keywords.items():
            if keyword in text:
                explanation["keywords_found"].append({"keyword": keyword, "weight": weight})
                keyword_score += weight

        for keyword, weight in self.medium_impact_keywords.items():
            if keyword in text:
                explanation["keywords_found"].append({"keyword": keyword, "weight": weight})
                keyword_score += weight

        for keyword, weight in self.negative_keywords.items():
            if keyword in text:
                explanation["negative_keywords_found"].append({"keyword": keyword, "weight": weight})
                keyword_score += weight

        explanation["keyword_score"] = min(50, max(0, keyword_score))

        if published:
            explanation["recency_score"] = self._calculate_recency_score(published)

        explanation["final_score"] = self.calculate_score(title, summary, source_quality, published)

        return explanation


def get_market_moving_keywords() -> List[str]:
    """Return list of market-moving keywords for highlighting."""
    return [
        "hack",
        "exploit",
        "breach",
        "stolen",
        "listing",
        "delist",
        "sec",
        "lawsuit",
        "etf",
        "approval",
        "rejection",
        "all-time high",
        "ath",
        "crash",
        "surge",
        "airdrop",
        "fork",
        "upgrade",
        "mainnet",
        "bankruptcy",
        "insolvent",
        "freeze",
        "halting",
    ]