Content Analysis

"""
Basic Content Analysis Example

This example demonstrates how to use the Content Analysis skill
to perform traditional NLP analysis on text data.
"""

import pandas as pd
import numpy as np
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from scripts.text_analyzer import TextAnalyzer, SocialMediaAnalyzer
from scripts.sentiment_analyzer import SentimentAnalyzer
from scripts.topic_analyzer import TopicAnalyzer
from scripts.content_visualizer import ContentVisualizer

def create_sample_data():
    """
    Create sample text data for analysis demonstration.
    """
    sample_texts = [
        "I absolutely love this new product! It's amazing and works perfectly.",
        "The customer service was terrible. I'm very disappointed with the experience.",
        "This is an okay product. Nothing special but it does the job.",
        "Outstanding quality and great value for money. Highly recommended!",
        "Poor packaging and the item arrived damaged. Not happy at all.",
        "Average quality, meets expectations but doesn't exceed them.",
        "Fantastic experience from start to finish. Will definitely buy again!",
        "The product broke after just one week. Complete waste of money.",
        "Good product for the price. Some minor issues but overall satisfied.",
        "Excellent build quality and attention to detail. Worth every penny!",
        "Slow shipping and the product doesn't match the description.",
        "Decent product, but could be improved with better features.",
        "Absolutely love it! Best purchase I've made this year.",
        "Mediocre quality, expected more for this price point.",
        "Great customer support and fast resolution of my issues.",
        "Product stopped working after a few days. Very frustrated.",
        "Solid product with good performance. Meets my needs well.",
        "Terrible user manual and confusing setup process.",
        "Impressed with the quality and fast delivery. Excellent service!",
        "Not worth the price. There are better alternatives available."
    ]

    # Create DataFrame
    df = pd.DataFrame({
        'text': sample_texts,
        'category': np.random.choice(['Product', 'Service', 'Shipping'], len(sample_texts)),
        'source': np.random.choice(['Twitter', 'Review', 'Email'], len(sample_texts)),
        'date': pd.date_range('2024-01-01', periods=len(sample_texts), freq='D')
    })

    return df

def demonstrate_text_analysis():
    """
    Demonstrate basic text analysis capabilities.
    """
    print("=== Basic Text Analysis Demo ===\n")

    # Sample text
    sample_text = "I absolutely love this new product! It's amazing and works perfectly. The customer service was excellent too."

    # Initialize analyzer
    analyzer = TextAnalyzer()

    # Clean text
    cleaned = analyzer.clean_text(sample_text)
    print(f"Original text: {sample_text}")
    print(f"Cleaned text: {cleaned}")

    # Tokenize
    tokens = analyzer.tokenize_text(sample_text)
    print(f"\nTokens: {tokens[:10]}...")  # Show first 10 tokens

    # Extract keywords
    keywords = analyzer.extract_keywords(sample_text, top_k=5)
    print(f"\nTop keywords: {keywords}")

    # Calculate readability
    readability = analyzer.calculate_readability(sample_text)
    print(f"\nReadability metrics: {readability}")

    # Get text statistics
    stats = analyzer.get_text_statistics(sample_text)
    print(f"\nText statistics: {stats}")

def demonstrate_sentiment_analysis():
    """
    Demonstrate sentiment analysis capabilities.
    """
    print("\n=== Sentiment Analysis Demo ===\n")

    # Sample texts with different sentiments
    texts = [
        "I love this product! It's absolutely amazing.",
        "This is terrible. I hate it and want my money back.",
        "The product is okay. Nothing special but does the job.",
        "Outstanding quality and excellent customer service!",
        "Poor quality and terrible customer support."
    ]

    # Initialize sentiment analyzer
    analyzer = SentimentAnalyzer()

    # Analyze each text
    results = []
    for text in texts:
        result = analyzer.analyze_sentiment_vader(text)
        result['text'] = text
        results.append(result)

        print(f"Text: {text}")
        print(f"Sentiment: {result['label']} (Compound: {result['compound']:.3f})")
        print(f"Positive: {result['positive']:.3f}, Negative: {result['negative']:.3f}, Neutral: {result['neutral']:.3f}")
        print()

    # Get sentiment distribution
    distribution = analyzer.classify_sentiment_distribution(texts)
    print("Sentiment Distribution:")
    print(f"Positive: {distribution.get('positive_percent', 0):.1f}% ({distribution.get('positive_count', 0)} texts)")
    print(f"Negative: {distribution.get('negative_percent', 0):.1f}% ({distribution.get('negative_count', 0)} texts)")
    print(f"Neutral: {distribution.get('neutral_percent', 0):.1f}% ({distribution.get('neutral_count', 0)} texts)")

def demonstrate_topic_analysis():
    """
    Demonstrate topic analysis capabilities.
    """
    print("\n=== Topic Analysis Demo ===\n")

    # Sample texts about different topics
    texts = [
        "The new smartphone has amazing camera quality and fast performance.",
        "Customer service was excellent and resolved my issue quickly.",
        "Fast shipping and great packaging. Product arrived in perfect condition.",
        "The laptop battery life is impressive and the screen is very clear.",
        "Technical support was very helpful and knowledgeable.",
        "The delivery was delayed but the product quality is good.",
        "Great value for money and the features are exactly what I needed.",
        "Had to return the item due to manufacturing defects.",
        "The tablet is lightweight and perfect for travel.",
        "Prices are competitive but the warranty coverage is limited."
    ]

    # Initialize topic analyzer
    analyzer = TopicAnalyzer()

    # Extract keywords using TF-IDF
    tfidf_keywords = analyzer.extract_keywords_tfidf(texts, top_k=10)
    print("TF-IDF Keywords:")
    for keyword, score in tfidf_keywords:
        print(f"  {keyword}: {score:.4f}")

    # Extract keywords using frequency
    freq_keywords = analyzer.extract_keywords_frequency(texts, top_k=10)
    print(f"\nFrequency Keywords:")
    for keyword, freq in freq_keywords:
        print(f"  {keyword}: {freq}")

    # Perform topic modeling
    print(f"\nPerforming topic modeling...")
    topic_results = analyzer.perform_lda_topic_modeling(texts, num_topics=3)

    if topic_results and 'topics' in topic_results:
        print(f"Discovered {topic_results['num_topics']} topics:")
        for topic_key, topic_data in topic_results['topics'].items():
            print(f"\n{topic_key.upper()}:")
            print(f"  Top words: {', '.join(topic_data['words'][:8])}")

def demonstrate_social_media_analysis():
    """
    Demonstrate social media specific analysis.
    """
    print("\n=== Social Media Analysis Demo ===\n")

    # Sample social media posts
    social_posts = [
        "Just tried the new #iPhone15 and it's amazing! 📱 #Apple #Tech",
        "Terrible customer service @CompanyXYZ. Waited 2 hours for support! 😤",
        "Check out our latest sale! 50% off everything. #Shopping #Deals",
        "@JohnDoe thanks for the recommendation! The product works great 👍",
        "Retweet if you agree that we need better environmental policies! 🌍 #Climate",
        "New blog post: 10 tips for better productivity. Link in bio! ✨",
        "Can't believe they cancelled my order without notification! So frustrated 😠",
        "Happy holidays everyone! 🎄 Wishing you joy and peace this season."
    ]

    # Initialize social media analyzer
    analyzer = SocialMediaAnalyzer()

    # Analyze each post
    print("Social Media Post Analysis:")
    for i, post in enumerate(social_posts, 1):
        hashtags = analyzer.extract_hashtags(post)
        mentions = analyzer.extract_mentions(post)
        engagement = analyzer.count_engagement_indicators(post)
        platform = analyzer.detect_social_media_platform(post)

        print(f"\nPost {i}:")
        print(f"  Text: {post}")
        print(f"  Platform: {platform}")
        print(f"  Hashtags: {hashtags}")
        print(f"  Mentions: {mentions}")
        print(f"  Engagement indicators: {engagement}")

def demonstrate_content_clustering():
    """
    Demonstrate content clustering capabilities.
    """
    print("\n=== Content Clustering Demo ===\n")

    # Sample texts for clustering
    texts = [
        "The smartphone has excellent camera and battery life.",
        "Great customer service and fast response time.",
        "The tablet offers good value for money.",
        "Technical support team was very helpful.",
        "Laptop performance is outstanding for gaming.",
        "Shipping was fast and packaging was secure.",
        "The phone's user interface is intuitive.",
        "Had issues with product registration.",
        "Great warranty coverage and after-sales service.",
        "The device is lightweight and portable."
    ]

    # Initialize analyzer
    analyzer = TopicAnalyzer()

    # Perform clustering
    print("Performing text clustering...")
    cluster_results = analyzer.cluster_texts(texts, num_clusters=3)

    if cluster_results and 'cluster_terms' in cluster_results:
        print(f"Cluster Analysis Results:")
        print(f"Number of clusters: {cluster_results['num_clusters']}")
        print(f"Inertia (within-cluster sum of squares): {cluster_results['inertia']:.2f}")

        print(f"\nTop terms by cluster:")
        for cluster_id, terms in cluster_results['cluster_terms'].items():
            print(f"  Cluster {cluster_id}: {', '.join(terms[:5])}")

def create_visualizations():
    """
    Create sample visualizations.
    """
    print("\n=== Creating Visualizations ===\n")

    # Sample sentiment data
    sentiment_data = [
        {'label': 'Positive', 'compound': 0.8},
        {'label': 'Negative', 'compound': -0.6},
        {'label': 'Neutral', 'compound': 0.1},
        {'label': 'Positive', 'compound': 0.9},
        {'label': 'Positive', 'compound': 0.7},
        {'label': 'Negative', 'compound': -0.8},
        {'label': 'Neutral', 'compound': 0.0},
        {'label': 'Positive', 'compound': 0.6}
    ]

    # Sample topic results
    topic_results = {
        'num_topics': 3,
        'topics': {
            'topic_0': {
                'words': ['product', 'quality', 'excellent', 'great', 'value'],
                'weights': [0.15, 0.12, 0.10, 0.09, 0.08]
            },
            'topic_1': {
                'words': ['customer', 'service', 'support', 'help', 'response'],
                'weights': [0.14, 0.13, 0.11, 0.09, 0.08]
            },
            'topic_2': {
                'words': ['shipping', 'delivery', 'fast', 'packaging', 'condition'],
                'weights': [0.16, 0.14, 0.11, 0.09, 0.07]
            }
        },
        'doc_topic_distribution': [
            [0.8, 0.1, 0.1],
            [0.1, 0.8, 0.1],
            [0.2, 0.2, 0.6],
            [0.7, 0.2, 0.1],
            [0.1, 0.7, 0.2],
            [0.2, 0.1, 0.7]
        ]
    }

    # Initialize visualizer
    visualizer = ContentVisualizer()

    # Create sentiment distribution plot
    print("Creating sentiment distribution visualization...")
    sentiment_fig = visualizer.plot_sentiment_distribution(sentiment_data)
    if sentiment_fig:
        sentiment_fig.savefig('sentiment_distribution.png', dpi=300, bbox_inches='tight')
        print("Saved: sentiment_distribution.png")

    # Create topic modeling visualization
    print("Creating topic modeling visualization...")
    topic_fig = visualizer.plot_topic_modeling_results(topic_results)
    if topic_fig:
        topic_fig.savefig('topic_modeling_results.png', dpi=300, bbox_inches='tight')
        print("Saved: topic_modeling_results.png")

    # Create word cloud
    print("Creating word cloud...")
    sample_text = """
    product quality excellent value customer service great support
    shipping delivery fast packaging condition warranty coverage
    technical helpful responsive resolution satisfaction recommendation
    """
    wordcloud_fig = visualizer.create_word_cloud(sample_text, title="Content Analysis Word Cloud")
    if wordcloud_fig:
        wordcloud_fig.savefig('content_wordcloud.png', dpi=300, bbox_inches='tight')
        print("Saved: content_wordcloud.png")

def main():
    """
    Run complete basic content analysis example.
    """
    print("Content Analysis Skill - Basic Analysis Example")
    print("=" * 50)

    # Create sample data
    print("\n1. Creating sample data...")
    df = create_sample_data()
    print(f"   Created dataset with {len(df)} text samples")

    # Demonstrate text analysis
    print("\n2. Demonstrating text analysis...")
    demonstrate_text_analysis()

    # Demonstrate sentiment analysis
    print("\n3. Demonstrating sentiment analysis...")
    demonstrate_sentiment_analysis()

    # Demonstrate topic analysis
    print("\n4. Demonstrating topic analysis...")
    demonstrate_topic_analysis()

    # Demonstrate social media analysis
    print("\n5. Demonstrating social media analysis...")
    demonstrate_social_media_analysis()

    # Demonstrate content clustering
    print("\n6. Demonstrating content clustering...")
    demonstrate_content_clustering()

    # Create visualizations
    print("\n7. Creating visualizations...")
    create_visualizations()

    print("\n" + "=" * 50)
    print("Basic Content Analysis Example Complete!")
    print("\nGenerated files:")
    print("  - sentiment_distribution.png")
    print("  - topic_modeling_results.png")
    print("  - content_wordcloud.png")

if __name__ == "__main__":
    main()

text,sentiment,category,source,date
"I absolutely love this new smartphone! The camera quality is amazing and the battery life is incredible.",positive,Product,Review,2024-01-15
"The customer service was excellent. They resolved my issue quickly and professionally.",positive,Service,Review,2024-01-16
"This product is terrible. It broke after just one week of use.",negative,Product,Review,2024-01-17
"Good value for money. The product meets my expectations but doesn't exceed them.",neutral,Product,Review,2024-01-18
"Outstanding quality and attention to detail. Highly recommend this brand!",positive,Product,Review,2024-01-19
"Poor packaging and the item arrived damaged. Very disappointed with the experience.",negative,Shipping,Review,2024-01-20
"The user interface is intuitive and easy to navigate. Great user experience!",positive,Product,Review,2024-01-21
"Average quality at best. There are better alternatives available in this price range.",negative,Product,Review,2024-01-22
"Fast shipping and secure packaging. Product arrived in perfect condition.",positive,Shipping,Review,2024-01-23
"Terrible customer support. Waited on hold for over an hour and then got disconnected.",negative,Service,Review,2024-01-24
"The laptop performance is outstanding for gaming. No lag whatsoever!",positive,Product,Review,2024-01-25
"Mediocre build quality. Expected more for this premium price point.",negative,Product,Review,2024-01-26
"Great warranty coverage and excellent after-sales support. Peace of mind purchase.",positive,Service,Review,2024-01-27
"The delivery was delayed by a week without any communication.",negative,Shipping,Review,2024-01-28
"Solid product with good features. Meets my daily needs perfectly.",positive,Product,Review,2024-01-29
"Difficult setup process and unclear instructions. Frustrating experience.",negative,Product,Review,2024-01-30
"Responsive customer service team that really cares about customer satisfaction.",positive,Service,Review,2024-01-31
"The product quality is okay but the price is too high for what you get.",negative,Product,Review,2024-02-01
"Eco-friendly packaging and fast delivery. Love the sustainable approach!",positive,Shipping,Review,2024-02-02
"Technical support was very knowledgeable and patient. Solved my complex issue.",positive,Service,Review,2024-02-03

[
  {
    "id": 1,
    "platform": "Twitter",
    "text": "Just tried the new #iPhone15 and it's absolutely amazing! The camera quality is stunning 📱 @Apple",
    "timestamp": "2024-01-15T10:30:00Z",
    "likes": 245,
    "retweets": 42,
    "hashtags": ["iPhone15", "Apple"],
    "mentions": ["Apple"],
    "user": "tech_enthusiast"
  },
  {
    "id": 2,
    "platform": "Twitter",
    "text": "Terrible customer service @CompanyXYZ. Waited 2 hours for support only to be disconnected! 😤 #Fail",
    "timestamp": "2024-01-15T14:22:00Z",
    "likes": 89,
    "retweets": 15,
    "hashtags": ["Fail"],
    "mentions": ["CompanyXYZ"],
    "user": "frustrated_customer"
  },
  {
    "id": 3,
    "platform": "Instagram",
    "text": "Check out our latest summer collection! 50% off everything in store. #Shopping #Deals #SummerSale",
    "timestamp": "2024-01-16T09:15:00Z",
    "likes": 567,
    "hashtags": ["Shopping", "Deals", "SummerSale"],
    "mentions": [],
    "user": "fashion_brand"
  },
  {
    "id": 4,
    "platform": "Twitter",
    "text": "@JohnDoe thanks for the recommendation! The noise-cancelling headphones work perfectly 👍 #HappyCustomer",
    "timestamp": "2024-01-16T16:45:00Z",
    "likes": 23,
    "retweets": 3,
    "hashtags": ["HappyCustomer"],
    "mentions": ["JohnDoe"],
    "user": "satisfied_buyer"
  },
  {
    "id": 5,
    "platform": "Facebook",
    "text": "New blog post: 10 Productivity Tips That Changed My Life. Link in bio! ✨ #Productivity #SelfImprovement",
    "timestamp": "2024-01-17T08:00:00Z",
    "likes": 124,
    "shares": 28,
    "hashtags": ["Productivity", "SelfImprovement"],
    "mentions": [],
    "user": "productivity_guru"
  },
  {
    "id": 6,
    "platform": "Twitter",
    "text": "Can't believe they cancelled my order without any notification! So frustrated with this company 😠",
    "timestamp": "2024-01-17T19:30:00Z",
    "likes": 67,
    "retweets": 8,
    "hashtags": [],
    "mentions": [],
    "user": "angry_shopper"
  },
  {
    "id": 7,
    "platform": "Instagram",
    "text": "Beautiful sunset from my balcony today. Nature never ceases to amaze me! 🌅 #Nature #Sunset #Blessed",
    "timestamp": "2024-01-18T18:15:00Z",
    "likes": 892,
    "hashtags": ["Nature", "Sunset", "Blessed"],
    "mentions": [],
    "user": "nature_lover"
  },
  {
    "id": 8,
    "platform": "Twitter",
    "text": "Just launched my new podcast about entrepreneurship! First episode is live 🎙️ #Startup #Business",
    "timestamp": "2024-01-19T11:00:00Z",
    "likes": 156,
    "retweets": 34,
    "hashtags": ["Startup", "Business"],
    "mentions": [],
    "user": "entrepreneur_pod"
  },
  {
    "id": 9,
    "platform": "Facebook",
    "text": "Retweet if you agree that we need better environmental policies! 🌍 Our planet deserves better #ClimateAction",
    "timestamp": "2024-01-20T07:45:00Z",
    "likes": 423,
    "shares": 189,
    "hashtags": ["ClimateAction"],
    "mentions": [],
    "user": "environmental_activist"
  },
  {
    "id": 10,
    "platform": "Twitter",
    "text": "The new software update is amazing! So many useful features and the UI is much cleaner now @TechCorp",
    "timestamp": "2024-01-20T15:20:00Z",
    "likes": 78,
    "retweets": 12,
    "hashtags": [],
    "mentions": ["TechCorp"],
    "user": "software_developer"
  }
]

"""
Content Visualization Tools

This module provides comprehensive visualization capabilities for content analysis
including sentiment plots, topic visualizations, and interactive dashboards.
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots
import plotly.offline as pyo
from typing import List, Dict, Optional, Tuple, Union
from wordcloud import WordCloud
import matplotlib.patches as mpatches
from matplotlib.gridspec import GridSpec
import warnings

# Set matplotlib parameters for better Chinese font support
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans', 'Arial Unicode MS']
plt.rcParams['axes.unicode_minus'] = False

class ContentVisualizer:
    """
    Comprehensive content visualization toolkit.

    Provides various visualization types for content analysis results
    including sentiment analysis, topic modeling, and performance metrics.
    """

    def __init__(self, style: str = 'seaborn-v0_8', figsize: Tuple[int, int] = (12, 8)):
        """
        Initialize the ContentVisualizer.

        Args:
            style: Matplotlib style to use
            figsize: Default figure size
        """
        self.style = style
        self.figsize = figsize
        self.color_palette = sns.color_palette("husl", 10)

        # Set matplotlib style
        try:
            plt.style.use(style)
        except:
            plt.style.use('default')

        # Configure seaborn
        sns.set_palette("husl")

    def plot_sentiment_distribution(self, sentiment_data: List[Dict],
                                   title: str = "Sentiment Analysis Results",
                                   interactive: bool = False) -> Union[plt.Figure, go.Figure]:
        """
        Plot sentiment distribution analysis.

        Args:
            sentiment_data: List of sentiment analysis results
            title: Plot title
            interactive: Whether to create interactive plot

        Returns:
            Matplotlib or Plotly figure
        """
        if not sentiment_data:
            return None

        # Extract sentiment labels
        sentiments = [result.get('label', result.get('sentiment', 'Neutral'))
                     for result in sentiment_data]

        # Count sentiments
        sentiment_counts = pd.Series(sentiments).value_counts()

        if interactive:
            # Create interactive plot with Plotly
            fig = make_subplots(
                rows=1, cols=2,
                specs=[[{"type": "pie"}, {"type": "bar"}]],
                subplot_titles=("Sentiment Distribution", "Sentiment Counts"),
                horizontal_spacing=0.1
            )

            # Pie chart
            colors = ['#2E8B57', '#DC143C', '#708090']  # Green, Red, Gray
            fig.add_trace(
                go.Pie(
                    labels=sentiment_counts.index,
                    values=sentiment_counts.values,
                    name="Sentiment",
                    marker_colors=colors[:len(sentiment_counts)]
                ),
                row=1, col=1
            )

            # Bar chart
            fig.add_trace(
                go.Bar(
                    x=sentiment_counts.index,
                    y=sentiment_counts.values,
                    marker_color=colors[:len(sentiment_counts)],
                    name="Count"
                ),
                row=1, col=2
            )

            fig.update_layout(
                title=title,
                showlegend=False,
                height=500
            )

            return fig

        else:
            # Create static plot with Matplotlib
            fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))

            # Pie chart
            colors = ['#2E8B57', '#DC143C', '#708090']
            wedges, texts, autotexts = ax1.pie(
                sentiment_counts.values,
                labels=sentiment_counts.index,
                colors=colors[:len(sentiment_counts)],
                autopct='%1.1f%%',
                startangle=90
            )
            ax1.set_title('Sentiment Distribution', fontweight='bold')

            # Bar chart
            bars = ax2.bar(sentiment_counts.index, sentiment_counts.values,
                          color=colors[:len(sentiment_counts)])
            ax2.set_title('Sentiment Counts', fontweight='bold')
            ax2.set_ylabel('Number of Texts')

            # Add value labels on bars
            for bar, count in zip(bars, sentiment_counts.values):
                ax2.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5,
                        str(count), ha='center', va='bottom')

            plt.suptitle(title, fontsize=16, fontweight='bold')
            plt.tight_layout()

            return fig

    def plot_sentiment_timeline(self, sentiment_data: List[Dict],
                              timestamps: List[str] = None,
                              time_window: str = 'daily',
                              title: str = "Sentiment Trends Over Time") -> Union[plt.Figure, go.Figure]:
        """
        Plot sentiment trends over time.

        Args:
            sentiment_data: List of sentiment analysis results
            timestamps: List of timestamps corresponding to data
            time_window: Time aggregation window ('daily', 'weekly', 'monthly')
            title: Plot title

        Returns:
            Matplotlib or Plotly figure
        """
        if not sentiment_data:
            return None

        # Create DataFrame
        df = pd.DataFrame(sentiment_data)

        # Extract sentiment scores
        if 'compound' in df.columns:
            df['sentiment_score'] = df['compound']
        elif 'confidence' in df.columns:
            # Convert sentiment to numeric
            sentiment_map = {'positive': 1, 'negative': -1, 'neutral': 0}
            df['sentiment_score'] = df['sentiment'].map(sentiment_map) * df['confidence']
        else:
            # Simple sentiment mapping
            sentiment_map = {'positive': 1, 'negative': -1, 'neutral': 0}
            df['sentiment_score'] = df['sentiment'].map(sentiment_map)

        # Add timestamps if provided
        if timestamps:
            df['timestamp'] = pd.to_datetime(timestamps)
            df.set_index('timestamp', inplace=True)

            # Aggregate by time window
            if time_window == 'daily':
                df_resampled = df.resample('D')
            elif time_window == 'weekly':
                df_resampled = df.resample('W')
            elif time_window == 'monthly':
                df_resampled = df.resample('M')
            else:
                df_resampled = df.resample('D')

            sentiment_trends = df_resampled['sentiment_score'].mean()

            # Create interactive plot
            fig = go.Figure()

            fig.add_trace(go.Scatter(
                x=sentiment_trends.index,
                y=sentiment_trends.values,
                mode='lines+markers',
                name='Average Sentiment',
                line=dict(color='blue', width=2),
                marker=dict(size=6)
            ))

            # Add zero line
            fig.add_hline(y=0, line_dash="dash", line_color="gray")

            fig.update_layout(
                title=title,
                xaxis_title="Date",
                yaxis_title="Sentiment Score",
                hovermode='x unified'
            )

            return fig

        else:
            # Simple line plot
            fig, ax = plt.subplots(figsize=self.figsize)

            ax.plot(range(len(df)), df['sentiment_score'], marker='o', linewidth=2, markersize=4)
            ax.axhline(y=0, color='gray', linestyle='--', alpha=0.7)
            ax.set_title(title, fontweight='bold')
            ax.set_xlabel('Text Index')
            ax.set_ylabel('Sentiment Score')
            ax.grid(True, alpha=0.3)

            return fig

    def plot_topic_modeling_results(self, topic_results: Dict,
                                   title: str = "Topic Modeling Analysis") -> plt.Figure:
        """
        Plot comprehensive topic modeling results.

        Args:
            topic_results: Results from topic modeling
            title: Plot title

        Returns:
            Matplotlib figure
        """
        if not topic_results:
            return None

        num_topics = topic_results.get('num_topics', 0)
        if num_topics == 0:
            return None

        # Create figure with subplots
        fig = plt.figure(figsize=(20, 15))
        gs = GridSpec(3, 3, figure=fig, hspace=0.3, wspace=0.3)

        # 1. Topic prevalence (top left)
        ax1 = fig.add_subplot(gs[0, 0])
        if 'doc_topic_distribution' in topic_results:
            doc_topic_dist = pd.DataFrame(topic_results['doc_topic_distribution'])
            avg_topic_dist = doc_topic_dist.mean()

            ax1.bar(range(len(avg_topic_dist)), avg_topic_dist.values, color=self.color_palette[:len(avg_topic_dist)])
            ax1.set_title('Average Topic Prevalence', fontweight='bold')
            ax1.set_xlabel('Topic ID')
            ax1.set_ylabel('Average Probability')
            ax1.set_xticks(range(len(avg_topic_dist)))
            ax1.set_xticklabels([f'T{i}' for i in range(len(avg_topic_dist))])

        # 2. Dominant topic distribution (top middle)
        ax2 = fig.add_subplot(gs[0, 1])
        if 'doc_topic_distribution' in topic_results:
            doc_topic_dist = pd.DataFrame(topic_results['doc_topic_distribution'])
            dominant_topics = doc_topic_dist.idxmax(axis=1)
            topic_counts = dominant_topics.value_counts().sort_index()

            ax2.bar(topic_counts.index, topic_counts.values, color=self.color_palette[:len(topic_counts)])
            ax2.set_title('Dominant Topic Distribution', fontweight='bold')
            ax2.set_xlabel('Topic ID')
            ax2.set_ylabel('Number of Documents')

        # 3. Topic correlation heatmap (top right)
        ax3 = fig.add_subplot(gs[0, 2])
        if 'doc_topic_distribution' in topic_results:
            doc_topic_dist = pd.DataFrame(topic_results['doc_topic_distribution'])
            topic_corr = doc_topic_dist.corr()

            im = ax3.imshow(topic_corr.values, cmap='coolwarm', aspect='auto', vmin=-1, vmax=1)
            ax3.set_xticks(range(len(topic_corr.columns)))
            ax3.set_yticks(range(len(topic_corr.columns)))
            ax3.set_xticklabels([f'T{i}' for i in range(len(topic_corr.columns))])
            ax3.set_yticklabels([f'T{i}' for i in range(len(topic_corr.columns))])
            ax3.set_title('Topic Correlation Matrix', fontweight='bold')

            # Add colorbar
            plt.colorbar(im, ax=ax3, shrink=0.8)

        # 4-6. Top words for topics (middle row)
        topics = topic_results.get('topics', {})
        for i in range(min(3, num_topics)):
            ax = fig.add_subplot(gs[1, i])
            topic_key = f'topic_{i}'
            if topic_key in topics:
                topic_data = topics[topic_key]
                words = topic_data.get('words', [])[:10]
                weights = topic_data.get('weights', [])[:10]

                if words and weights:
                    y_pos = range(len(words))
                    bars = ax.barh(y_pos, weights, color=self.color_palette[i])
                    ax.set_yticks(y_pos)
                    ax.set_yticklabels(words)
                    ax.set_title(f'Topic {i} Top Words', fontweight='bold')
                    ax.set_xlabel('Weight')
                    ax.invert_yaxis()

        # 7-9. Topic word clouds (bottom row)
        for i in range(min(3, num_topics)):
            ax = fig.add_subplot(gs[2, i])
            topic_key = f'topic_{i}'
            if topic_key in topics:
                topic_data = topics[topic_key]
                words = topic_data.get('words', [])
                weights = topic_data.get('weights', [])

                if words and weights:
                    # Create word frequency dictionary
                    word_freq = {word: weight for word, weight in zip(words, weights)}

                    # Create word cloud
                    wordcloud = WordCloud(
                        width=400, height=300,
                        background_color='white',
                        colormap='viridis',
                        max_words=20
                    ).generate_from_frequencies(word_freq)

                    ax.imshow(wordcloud, interpolation='bilinear')
                    ax.axis('off')
                    ax.set_title(f'Topic {i} Word Cloud', fontweight='bold')

        plt.suptitle(title, fontsize=18, fontweight='bold')
        return fig

    def plot_content_quality_analysis(self, quality_data: List[Dict],
                                    title: str = "Content Quality Analysis") -> plt.Figure:
        """
        Plot content quality analysis results.

        Args:
            quality_data: List of quality analysis results
            title: Plot title

        Returns:
            Matplotlib figure
        """
        if not quality_data:
            return None

        # Extract quality metrics
        df = pd.DataFrame(quality_data)

        # Create figure
        fig, axes = plt.subplots(2, 3, figsize=(18, 12))
        axes = axes.flatten()

        # 1. Overall quality distribution
        ax1 = axes[0]
        if 'overall_score' in df.columns:
            ax1.hist(df['overall_score'], bins=20, alpha=0.7, color='skyblue', edgecolor='black')
            ax1.set_title('Overall Quality Score Distribution', fontweight='bold')
            ax1.set_xlabel('Quality Score')
            ax1.set_ylabel('Frequency')
            ax1.axvline(df['overall_score'].mean(), color='red', linestyle='--',
                       label=f'Mean: {df["overall_score"].mean():.3f}')
            ax1.legend()

        # 2. Criteria scores radar chart
        ax2 = axes[1]
        if 'criteria_scores' in df.columns and not df['criteria_scores'].empty:
            # Extract criteria names and average scores
            all_criteria = set()
            for scores in df['criteria_scores']:
                all_criteria.update(scores.keys())

            if all_criteria:
                avg_scores = {}
                for criterion in all_criteria:
                    scores = [item.get(criterion, {}).get('score', 0)
                             for item in df['criteria_scores'] if criterion in item]
                    avg_scores[criterion] = np.mean(scores) if scores else 0

                # Create radar chart
                criteria = list(avg_scores.keys())
                scores = list(avg_scores.values())

                # Add first point to close the circle
                criteria += [criteria[0]]
                scores += [scores[0]]

                angles = np.linspace(0, 2 * np.pi, len(criteria))

                ax2.plot(angles, scores, 'o-', linewidth=2, color='green')
                ax2.fill(angles, scores, alpha=0.25, color='green')
                ax2.set_xticks(angles[:-1])
                ax2.set_xticklabels(criteria[:-1], rotation=45, ha='right')
                ax2.set_title('Average Criteria Scores', fontweight='bold')
                ax2.set_ylim(0, 1)
                ax2.grid(True)

        # 3. Quality vs Length scatter plot
        ax3 = axes[2]
        if 'overall_score' in df.columns:
            # Calculate text lengths if available
            text_lengths = []
            for item in quality_data:
                if 'text' in item:
                    text_lengths.append(len(item['text']))
                else:
                    text_lengths.append(0)

            if text_lengths:
                ax3.scatter(text_lengths, df['overall_score'], alpha=0.6, color='purple')
                ax3.set_title('Quality vs Text Length', fontweight='bold')
                ax3.set_xlabel('Text Length (characters)')
                ax3.set_ylabel('Quality Score')

                # Add trend line
                z = np.polyfit(text_lengths, df['overall_score'], 1)
                p = np.poly1d(z)
                ax3.plot(text_lengths, p(text_lengths), "r--", alpha=0.8)

        # 4. Quality categories
        ax4 = axes[3]
        if 'overall_score' in df.columns:
            # Categorize quality scores
            def categorize_score(score):
                if score >= 0.8:
                    return 'Excellent'
                elif score >= 0.6:
                    return 'Good'
                elif score >= 0.4:
                    return 'Fair'
                else:
                    return 'Poor'

            quality_categories = df['overall_score'].apply(categorize_score)
            category_counts = quality_categories.value_counts()

            colors = ['#2E8B57', '#FFD700', '#FF8C00', '#DC143C']
            bars = ax4.bar(category_counts.index, category_counts.values,
                           color=colors[:len(category_counts)])
            ax4.set_title('Quality Categories Distribution', fontweight='bold')
            ax4.set_ylabel('Number of Documents')

            # Add value labels
            for bar, count in zip(bars, category_counts.values):
                ax4.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.5,
                        str(count), ha='center', va='bottom')

        # 5. Improvement suggestions word cloud
        ax5 = axes[4]
        all_suggestions = []
        for item in quality_data:
            if 'suggestions' in item:
                all_suggestions.extend(item['suggestions'])

        if all_suggestions:
            # Join suggestions and create word cloud
            suggestions_text = ' '.join(all_suggestions)
            wordcloud = WordCloud(
                width=400, height=300,
                background_color='white',
                colormap='plasma',
                max_words=50
            ).generate(suggestions_text)

            ax5.imshow(wordcloud, interpolation='bilinear')
            ax5.axis('off')
            ax5.set_title('Common Improvement Suggestions', fontweight='bold')

        # 6. Quality trend over time (if timestamps available)
        ax6 = axes[5]
        if 'overall_score' in df.columns:
            # Simple index-based trend
            ax6.plot(range(len(df)), df['overall_score'], marker='o', linewidth=2, markersize=4)
            ax6.set_title('Quality Trend (Document Order)', fontweight='bold')
            ax6.set_xlabel('Document Index')
            ax6.set_ylabel('Quality Score')
            ax6.grid(True, alpha=0.3)

            # Add moving average
            window = min(5, len(df) // 4)
            if window > 1:
                moving_avg = df['overall_score'].rolling(window=window).mean()
                ax6.plot(range(len(df)), moving_avg, 'r-', linewidth=2,
                        label=f'{window}-point Moving Average')
                ax6.legend()

        plt.suptitle(title, fontsize=16, fontweight='bold')
        plt.tight_layout()
        return fig

    def plot_content_performance_metrics(self, performance_data: Dict,
                                       title: str = "Content Performance Metrics") -> plt.Figure:
        """
        Plot content performance metrics dashboard.

        Args:
            performance_data: Dictionary with performance metrics
            title: Plot title

        Returns:
            Matplotlib figure
        """
        if not performance_data:
            return None

        fig, axes = plt.subplots(2, 2, figsize=(15, 12))
        axes = axes.flatten()

        # 1. Engagement metrics
        ax1 = axes[0]
        if 'engagement' in performance_data:
            engagement_data = performance_data['engagement']
            metrics = list(engagement_data.keys())
            values = list(engagement_data.values())

            bars = ax1.bar(metrics, values, color=self.color_palette[:len(metrics)])
            ax1.set_title('Engagement Metrics', fontweight='bold')
            ax1.set_ylabel('Value')
            ax1.tick_params(axis='x', rotation=45)

            # Add value labels
            for bar, value in zip(bars, values):
                ax1.text(bar.get_x() + bar.get_width()/2, bar.get_height() + max(values)*0.01,
                        f'{value:.2f}', ha='center', va='bottom')

        # 2. Content categories performance
        ax2 = axes[1]
        if 'category_performance' in performance_data:
            cat_data = performance_data['category_performance']
            categories = list(cat_data.keys())
            avg_scores = [item.get('avg_score', 0) for item in cat_data.values()]

            bars = ax2.bar(categories, avg_scores, color=self.color_palette[:len(categories)])
            ax2.set_title('Performance by Category', fontweight='bold')
            ax2.set_ylabel('Average Score')
            ax2.tick_params(axis='x', rotation=45)

        # 3. Temporal performance trends
        ax3 = axes[2]
        if 'temporal_trends' in performance_data:
            trends = performance_data['temporal_trends']
            dates = list(trends.keys())
            scores = list(trends.values())

            ax3.plot(dates, scores, marker='o', linewidth=2, markersize=6)
            ax3.set_title('Performance Trends Over Time', fontweight='bold')
            ax3.set_xlabel('Date')
            ax3.set_ylabel('Performance Score')
            ax3.tick_params(axis='x', rotation=45)
            ax3.grid(True, alpha=0.3)

        # 4. Top performing content
        ax4 = axes[3]
        if 'top_content' in performance_data:
            top_content = performance_data['top_content']
            titles = [item.get('title', f'Content {i+1}') for i, item in enumerate(top_content)]
            scores = [item.get('score', 0) for item in top_content]

            # Truncate long titles
            titles = [title[:20] + '...' if len(title) > 20 else title for title in titles]

            bars = ax4.barh(titles, scores, color=self.color_palette[:len(titles)])
            ax4.set_title('Top Performing Content', fontweight='bold')
            ax4.set_xlabel('Performance Score')
            ax4.invert_yaxis()

        plt.suptitle(title, fontsize=16, fontweight='bold')
        plt.tight_layout()
        return fig

    def create_interactive_dashboard(self, analysis_results: Dict,
                                   title: str = "Content Analysis Dashboard") -> go.Figure:
        """
        Create interactive dashboard with multiple analysis views.

        Args:
            analysis_results: Comprehensive analysis results
            title: Dashboard title

        Returns:
            Plotly figure with subplots
        """
        # Create subplot layout
        fig = make_subplots(
            rows=3, cols=2,
            specs=[
                [{"type": "pie"}, {"type": "bar"}],
                [{"type": "scatter"}, {"type": "bar"}],
                [{"type": "table"}, {"type": "indicator"}]
            ],
            subplot_titles=[
                "Sentiment Distribution", "Topic Prevalence",
                "Sentiment Timeline", "Quality Metrics",
                "Top Content", "Overall Performance"
            ],
            vertical_spacing=0.08,
            horizontal_spacing=0.1
        )

        # 1. Sentiment distribution pie chart
        if 'sentiment_analysis' in analysis_results:
            sentiment_data = analysis_results['sentiment_analysis']
            if 'distribution' in sentiment_data:
                labels = list(sentiment_data['distribution'].keys())
                values = list(sentiment_data['distribution'].values())

                fig.add_trace(
                    go.Pie(
                        labels=labels,
                        values=values,
                        name="Sentiment"
                    ),
                    row=1, col=1
                )

        # 2. Topic prevalence bar chart
        if 'topic_analysis' in analysis_results:
            topic_data = analysis_results['topic_analysis']
            if 'topic_prevalence' in topic_data:
                topics = list(topic_data['topic_prevalence'].keys())
                prevalence = list(topic_data['topic_prevalence'].values())

                fig.add_trace(
                    go.Bar(
                        x=topics,
                        y=prevalence,
                        name="Topic Prevalence"
                    ),
                    row=1, col=2
                )

        # 3. Sentiment timeline
        if 'sentiment_timeline' in analysis_results:
            timeline_data = analysis_results['sentiment_timeline']
            dates = timeline_data.get('dates', [])
            scores = timeline_data.get('scores', [])

            if dates and scores:
                fig.add_trace(
                    go.Scatter(
                        x=dates,
                        y=scores,
                        mode='lines+markers',
                        name="Sentiment Trend"
                    ),
                    row=2, col=1
                )

        # 4. Quality metrics
        if 'quality_analysis' in analysis_results:
            quality_data = analysis_results['quality_analysis']
            if 'metrics' in quality_data:
                metrics = list(quality_data['metrics'].keys())
                values = list(quality_data['metrics'].values())

                fig.add_trace(
                    go.Bar(
                        x=metrics,
                        y=values,
                        name="Quality Metrics"
                    ),
                    row=2, col=2
                )

        # 5. Top content table
        if 'top_content' in analysis_results:
            top_data = analysis_results['top_content'][:5]  # Top 5 items
            headers = ["Title", "Score", "Category"]

            # Prepare table data
            table_data = []
            for item in top_data:
                row = [
                    item.get('title', 'N/A')[:30],  # Truncate long titles
                    f"{item.get('score', 0):.3f}",
                    item.get('category', 'N/A')
                ]
                table_data.append(row)

            fig.add_trace(
                go.Table(
                    header=dict(values=headers, fill_color='lightblue'),
                    cells=dict(values=list(zip(*table_data)) if table_data else [[], [], []],
                              fill_color='lightgray')
                ),
                row=3, col=1
            )

        # 6. Overall performance indicator
        if 'overall_performance' in analysis_results:
            overall_score = analysis_results['overall_performance'].get('score', 0)

            fig.add_trace(
                go.Indicator(
                    mode="gauge+number+delta",
                    value=overall_score,
                    domain={'x': [0, 1], 'y': [0, 1]},
                    title={'text': "Overall Performance"},
                    gauge={
                        'axis': {'range': [None, 1]},
                        'bar': {'color': "darkblue"},
                        'steps': [
                            {'range': [0, 0.3], 'color': "lightgray"},
                            {'range': [0.3, 0.7], 'color': "gray"},
                            {'range': [0.7, 1], 'color': "lightgreen"}
                        ],
                        'threshold': {
                            'line': {'color': "red", 'width': 4},
                            'thickness': 0.75,
                            'value': 0.8
                        }
                    }
                ),
                row=3, col=2
            )

        # Update layout
        fig.update_layout(
            title=title,
            height=1200,
            showlegend=False
        )

        return fig

    def create_word_cloud(self, text_data: Union[str, List[str]],
                         title: str = "Word Cloud",
                         width: int = 800,
                         height: int = 400,
                         colormap: str = 'viridis') -> plt.Figure:
        """
        Create word cloud visualization.

        Args:
            text_data: Text or list of texts to visualize
            title: Plot title
            width: Word cloud width
            height: Word cloud height
            colormap: Matplotlib colormap name

        Returns:
            Matplotlib figure
        """
        if not text_data:
            return None

        # Combine text if list provided
        if isinstance(text_data, list):
            text = ' '.join(text_data)
        else:
            text = text_data

        # Create word cloud
        wordcloud = WordCloud(
            width=width,
            height=height,
            background_color='white',
            colormap=colormap,
            max_words=100,
            relative_scaling=0.5,
            random_state=42
        ).generate(text)

        # Create figure
        fig, ax = plt.subplots(figsize=(width/100, height/100))
        ax.imshow(wordcloud, interpolation='bilinear')
        ax.axis('off')
        ax.set_title(title, fontweight='bold', fontsize=16)

        return fig

    def save_visualization(self, fig: Union[plt.Figure, go.Figure],
                          filename: str,
                          format: str = 'png',
                          dpi: int = 300):
        """
        Save visualization to file.

        Args:
            fig: Figure to save
            filename: Output filename
            format: Output format ('png', 'jpg', 'svg', 'html')
            dpi: Resolution for raster formats
        """
        try:
            if isinstance(fig, go.Figure):
                if format.lower() == 'html':
                    fig.write_html(filename)
                else:
                    fig.write_image(filename, format=format, width=1200, height=800)
            else:
                fig.savefig(filename, format=format, dpi=dpi, bbox_inches='tight')
            print(f"Visualization saved to {filename}")
        except Exception as e:
            print(f"Error saving visualization: {e}")

"""
LLM-Enhanced Content Analysis

This module provides LLM-powered content analysis capabilities for
advanced text understanding, sentiment analysis, and topic extraction
using models like OpenAI GPT and Qwen (通义千问).
"""

import pandas as pd
import numpy as np
import json
import time
from typing import List, Dict, Optional, Union, Tuple
import requests
import warnings
from concurrent.futures import ThreadPoolExecutor, as_completed
import threading

class LLMAnalyzer:
    """
    LLM-powered content analysis toolkit.

    Provides advanced text analysis capabilities using Large Language Models
    for nuanced understanding, emotion detection, and semantic analysis.
    """

    def __init__(self, provider: str = 'openai', api_key: str = None, model: str = None):
        """
        Initialize the LLMAnalyzer.

        Args:
            provider: LLM provider ('openai', 'qwen', 'local')
            api_key: API key for the LLM service
            model: Model name to use
        """
        self.provider = provider
        self.api_key = api_key
        self.model = model
        self.rate_limit_delay = 1.0  # Delay between API calls
        self.max_retries = 3

        # Configure default models
        if provider == 'openai':
            self.model = model or 'gpt-3.5-turbo'
            self.api_base = 'https://api.openai.com/v1'
            self.headers = {
                'Authorization': f'Bearer {api_key}',
                'Content-Type': 'application/json'
            }
        elif provider == 'qwen':
            self.model = model or 'qwen-turbo'
            self.api_base = 'https://dashscope.aliyuncs.com/api/v1'
            self.headers = {
                'Authorization': f'Bearer {api_key}',
                'Content-Type': 'application/json'
            }
        else:
            raise ValueError("Provider must be 'openai' or 'qwen'")

    def _make_api_request(self, messages: List[Dict], **kwargs) -> Dict:
        """
        Make API request to the LLM service.

        Args:
            messages: List of messages for the conversation
            **kwargs: Additional parameters

        Returns:
            API response dictionary
        """
        for attempt in range(self.max_retries):
            try:
                payload = {
                    'model': self.model,
                    'messages': messages,
                    **kwargs
                }

                response = requests.post(
                    f"{self.api_base}/chat/completions",
                    headers=self.headers,
                    json=payload,
                    timeout=30
                )

                if response.status_code == 200:
                    return response.json()
                elif response.status_code == 429:  # Rate limit
                    time.sleep(self.rate_limit_delay * (2 ** attempt))
                    continue
                else:
                    response.raise_for_status()

            except Exception as e:
                if attempt == self.max_retries - 1:
                    raise e
                time.sleep(self.rate_limit_delay * (2 ** attempt))

        return {}

    def _extract_content_from_response(self, response: Dict) -> str:
        """Extract content from LLM response."""
        try:
            if self.provider == 'openai':
                return response['choices'][0]['message']['content']
            elif self.provider == 'qwen':
                return response['output']['choices'][0]['message']['content']
        except (KeyError, IndexError):
            return ""

    def analyze_sentiment_llm(self, text: str,
                             context: str = None,
                             detailed: bool = False) -> Dict:
        """
        Analyze sentiment using LLM for nuanced understanding.

        Args:
            text: Text to analyze
            context: Additional context for analysis
            detailed: Whether to return detailed analysis

        Returns:
            Dictionary with sentiment analysis results
        """
        if not text or not text.strip():
            return {
                'sentiment': 'neutral',
                'confidence': 0.0,
                'emotions': [],
                'explanation': 'No text provided'
            }

        # Build prompt
        if detailed:
            prompt = f"""
Analyze the sentiment of the following text in detail. Consider:
1. Overall sentiment (positive, negative, neutral)
2. Specific emotions present (joy, anger, fear, sadness, surprise, etc.)
3. Confidence level in your analysis
4. Brief explanation for your assessment

Text: {text}

Context: {context or 'No additional context provided'}

Please respond in JSON format:
{{
    "sentiment": "positive/negative/neutral",
    "confidence": 0.0-1.0,
    "emotions": ["emotion1", "emotion2", ...],
    "explanation": "Brief explanation"
}}
"""
        else:
            prompt = f"""
Analyze the sentiment of this text and respond with only "positive", "negative", or "neutral":

Text: {text}

Sentiment:"""

        messages = [
            {"role": "system", "content": "You are a sentiment analysis expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.1)
            content = self._extract_content_from_response(response)

            if detailed:
                try:
                    # Parse JSON response
                    result = json.loads(content.strip())
                    return result
                except json.JSONDecodeError:
                    # Fallback parsing
                    return {
                        'sentiment': 'neutral',
                        'confidence': 0.5,
                        'emotions': [],
                        'explanation': content
                    }
            else:
                # Simple sentiment classification
                sentiment = content.strip().lower()
                if sentiment in ['positive', 'negative', 'neutral']:
                    return {
                        'sentiment': sentiment,
                        'confidence': 0.8,  # Default confidence
                        'text': text
                    }
                else:
                    return {
                        'sentiment': 'neutral',
                        'confidence': 0.0,
                        'text': text
                    }

        except Exception as e:
            print(f"Error in LLM sentiment analysis: {e}")
            return {
                'sentiment': 'neutral',
                'confidence': 0.0,
                'error': str(e),
                'text': text
            }

    def analyze_batch_sentiment_llm(self, texts: List[str],
                                  context: str = None,
                                  detailed: bool = False,
                                  max_workers: int = 3) -> List[Dict]:
        """
        Analyze sentiment for a batch of texts using LLM.

        Args:
            texts: List of texts to analyze
            context: Additional context for analysis
            detailed: Whether to return detailed analysis
            max_workers: Maximum concurrent API calls

        Returns:
            List of sentiment analysis results
        """
        results = []

        # Use thread pool for concurrent processing
        with ThreadPoolExecutor(max_workers=max_workers) as executor:
            # Submit all tasks
            future_to_text = {
                executor.submit(self.analyze_sentiment_llm, text, context, detailed): text
                for text in texts
            }

            # Collect results as they complete
            for future in as_completed(future_to_text):
                text = future_to_text[future]
                try:
                    result = future.result()
                    result['text'] = text
                    results.append(result)

                    # Rate limiting
                    time.sleep(self.rate_limit_delay)

                except Exception as e:
                    results.append({
                        'text': text,
                        'sentiment': 'neutral',
                        'confidence': 0.0,
                        'error': str(e)
                    })

        return results

    def extract_topics_llm(self, texts: List[str],
                          topics_per_text: int = 3,
                          context: str = None) -> Dict:
        """
        Extract topics using LLM for semantic understanding.

        Args:
            texts: List of texts to analyze
            topics_per_text: Number of topics to extract per text
            context: Additional context for analysis

        Returns:
            Dictionary with topic extraction results
        """
        if not texts:
            return {'topics': [], 'text_topics': {}}

        # Build prompt
        prompt = f"""
Analyze the following texts and extract the main topics and themes.
For each text, identify the top {topics_per_text} most relevant topics.
Consider the semantic meaning and context.

Context: {context or 'No additional context provided'}

Texts to analyze:
{chr(10).join([f"{i+1}. {text}" for i, text in enumerate(texts)])}

Please respond in JSON format:
{{
    "overall_topics": ["topic1", "topic2", ...],
    "text_topics": {{
        "text_1": ["topic1", "topic2", ...],
        "text_2": ["topic1", "topic2", ...],
        ...
    }},
    "topic_descriptions": {{
        "topic1": "description of topic1",
        ...
    }}
}}
"""

        messages = [
            {"role": "system", "content": "You are a topic analysis expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.3)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())

                # Map text indices to original texts
                if 'text_topics' in result:
                    mapped_text_topics = {}
                    for i, text in enumerate(texts):
                        text_key = f"text_{i+1}"
                        if text_key in result['text_topics']:
                            mapped_text_topics[i] = result['text_topics'][text_key]

                    result['text_topics'] = mapped_text_topics

                return result

            except json.JSONDecodeError:
                # Fallback - try simple parsing
                return {
                    'overall_topics': [],
                    'text_topics': {},
                    'raw_response': content
                }

        except Exception as e:
            print(f"Error in LLM topic extraction: {e}")
            return {'error': str(e), 'overall_topics': [], 'text_topics': {}}

    def classify_content_llm(self, texts: List[str],
                           categories: List[str],
                           context: str = None) -> Dict:
        """
        Classify content into predefined categories using LLM.

        Args:
            texts: List of texts to classify
            categories: List of possible categories
            context: Additional context for classification

        Returns:
            Dictionary with classification results
        """
        if not texts or not categories:
            return {'classifications': [], 'uncategorized': []}

        # Build prompt
        categories_str = ", ".join(categories)
        prompt = f"""
Classify the following texts into one of these categories: {categories_str}

Context: {context or 'No additional context provided'}

Texts to classify:
{chr(10).join([f"{i+1}. {text}" for i, text in enumerate(texts)])}

Please respond in JSON format:
{{
    "classifications": [
        {{"text_index": 1, "category": "category_name", "confidence": 0.95, "reasoning": "brief explanation"}},
        ...
    ]
}}
"""

        messages = [
            {"role": "system", "content": "You are a content classification expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.1)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())
                return result

            except json.JSONDecodeError:
                # Fallback - return empty classifications
                return {'classifications': [], 'raw_response': content}

        except Exception as e:
            print(f"Error in LLM content classification: {e}")
            return {'error': str(e), 'classifications': []}

    def summarize_text_llm(self, text: str,
                         max_length: int = 150,
                         style: str = 'neutral') -> str:
        """
        Generate a summary of text using LLM.

        Args:
            text: Text to summarize
            max_length: Maximum length of summary
            style: Summary style ('neutral', 'formal', 'casual')

        Returns:
            Generated summary text
        """
        if not text or not text.strip():
            return ""

        style_instructions = {
            'neutral': "Provide a balanced, objective summary.",
            'formal': "Provide a professional, formal summary.",
            'casual': "Provide a friendly, conversational summary."
        }

        prompt = f"""
Summarize the following text in no more than {max_length} words.
{style_instructions.get(style, style_instructions['neutral'])}

Text: {text}

Summary:"""

        messages = [
            {"role": "system", "content": "You are a text summarization expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.3)
            return self._extract_content_from_response(response).strip()

        except Exception as e:
            print(f"Error in LLM text summarization: {e}")
            return ""

    def analyze_content_quality_llm(self, text: str,
                                  criteria: List[str] = None) -> Dict:
        """
        Analyze content quality using LLM.

        Args:
            text: Text to analyze
            criteria: Quality criteria to evaluate

        Returns:
            Dictionary with quality analysis results
        """
        if not text or not text.strip():
            return {'overall_score': 0.0, 'criteria_scores': {}, 'suggestions': []}

        if not criteria:
            criteria = ['clarity', 'engagement', 'accuracy', 'relevance', 'structure']

        criteria_str = ", ".join(criteria)
        prompt = f"""
Analyze the quality of the following text based on these criteria: {criteria_str}

For each criterion, provide a score from 0.0 to 1.0 and brief feedback.
Also provide an overall quality score and suggestions for improvement.

Text: {text}

Please respond in JSON format:
{{
    "overall_score": 0.0-1.0,
    "criteria_scores": {{
        "criterion1": {{"score": 0.0-1.0, "feedback": "brief feedback"}},
        ...
    }},
    "suggestions": ["suggestion1", "suggestion2", ...]
}}
"""

        messages = [
            {"role": "system", "content": "You are a content quality analysis expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.2)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())
                return result

            except json.JSONDecodeError:
                # Fallback - return basic structure
                return {
                    'overall_score': 0.5,
                    'criteria_scores': {},
                    'raw_response': content
                }

        except Exception as e:
            print(f"Error in LLM quality analysis: {e}")
            return {'error': str(e), 'overall_score': 0.0}

    def generate_content_insights_llm(self, texts: List[str],
                                    analysis_type: str = 'general') -> Dict:
        """
        Generate comprehensive content insights using LLM.

        Args:
            texts: List of texts to analyze
            analysis_type: Type of analysis ('general', 'marketing', 'sentiment', 'engagement')

        Returns:
            Dictionary with insights and recommendations
        """
        if not texts:
            return {'insights': [], 'recommendations': [], 'patterns': []}

        # Build analysis-specific prompt
        analysis_prompts = {
            'general': "Analyze these texts and provide general insights about content themes, patterns, and characteristics.",
            'marketing': "Analyze these marketing texts and provide insights about messaging effectiveness, audience appeal, and optimization opportunities.",
            'sentiment': "Analyze these texts focusing on emotional patterns, sentiment trends, and psychological insights.",
            'engagement': "Analyze these texts for engagement potential, viral characteristics, and audience interaction factors."
        }

        prompt = f"""
{analysis_prompts.get(analysis_type, analysis_prompts['general'])}

Provide:
1. Key insights about the content
2. Notable patterns and trends
3. Specific recommendations for improvement
4. Target audience characteristics

Texts to analyze:
{chr(10).join([f"{i+1}. {text}" for i, text in enumerate(texts)])}

Please respond in JSON format:
{{
    "insights": ["insight1", "insight2", ...],
    "patterns": ["pattern1", "pattern2", ...],
    "recommendations": ["recommendation1", "recommendation2", ...],
    "target_audience": ["characteristic1", "characteristic2", ...]
}}
"""

        messages = [
            {"role": "system", "content": "You are a content analysis and strategy expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.3)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())
                return result

            except json.JSONDecodeError:
                # Fallback - try simple parsing
                return {
                    'insights': [],
                    'patterns': [],
                    'recommendations': [],
                    'raw_response': content
                }

        except Exception as e:
            print(f"Error in LLM insights generation: {e}")
            return {'error': str(e), 'insights': [], 'recommendations': []}

    def detect_viral_potential_llm(self, text: str) -> Dict:
        """
        Detect viral potential of content using LLM.

        Args:
            text: Text to analyze for viral potential

        Returns:
            Dictionary with viral potential analysis
        """
        if not text or not text.strip():
            return {'viral_score': 0.0, 'factors': [], 'recommendations': []}

        prompt = f"""
Analyze this text for viral potential. Consider factors like:
- Emotional appeal
- Shareability
- Novelty
- Controversy
- Practical value
- Entertainment value

Provide a viral potential score (0.0-1.0) and specific factors.

Text: {text}

Please respond in JSON format:
{{
    "viral_score": 0.0-1.0,
    "factors": [
        {{"factor": "emotional_appeal", "score": 0.0-1.0, "explanation": "brief explanation"}},
        ...
    ],
    "recommendations": ["recommendation1", "recommendation2", ...]
}}
"""

        messages = [
            {"role": "system", "content": "You are a viral content analysis expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.3)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())
                return result

            except json.JSONDecodeError:
                # Fallback - return basic structure
                return {
                    'viral_score': 0.5,
                    'factors': [],
                    'raw_response': content
                }

        except Exception as e:
            print(f"Error in viral potential detection: {e}")
            return {'error': str(e), 'viral_score': 0.0}

    def translate_content_llm(self, text: str,
                            target_language: str,
                            preserve_style: bool = True) -> Dict:
        """
        Translate content using LLM.

        Args:
            text: Text to translate
            target_language: Target language for translation
            preserve_style: Whether to preserve original style

        Returns:
            Dictionary with translation results
        """
        if not text or not text.strip():
            return {'translated_text': '', 'confidence': 0.0}

        style_instruction = " and preserve the original tone and style" if preserve_style else ""
        prompt = f"""
Translate the following text to {target_language}{style_instruction}.

Text: {text}

Translation:"""

        messages = [
            {"role": "system", "content": f"You are a professional translator specializing in translation to {target_language}."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.1)
            translated_text = self._extract_content_from_response(response).strip()

            return {
                'translated_text': translated_text,
                'original_text': text,
                'target_language': target_language,
                'confidence': 0.9  # Default confidence
            }

        except Exception as e:
            print(f"Error in LLM translation: {e}")
            return {
                'translated_text': '',
                'error': str(e),
                'original_text': text
            }

    def analyze_content_llm_comprehensive(self, text: str,
                                        context: str = None) -> Dict:
        """
        Perform comprehensive content analysis using LLM.

        Args:
            text: Text to analyze
            context: Additional context for analysis

        Returns:
            Dictionary with comprehensive analysis results
        """
        if not text or not text.strip():
            return {}

        prompt = f"""
Perform a comprehensive analysis of this text. Include:
1. Sentiment analysis with emotion detection
2. Key topics and themes
3. Content quality assessment
4. Audience identification
5. Engagement potential
6. Improvement recommendations

Context: {context or 'No additional context provided'}

Text: {text}

Please respond in JSON format:
{{
    "sentiment": {{
        "overall": "positive/negative/neutral",
        "confidence": 0.0-1.0,
        "emotions": ["emotion1", "emotion2", ...]
    }},
    "topics": ["topic1", "topic2", ...],
    "quality": {{
        "score": 0.0-1.0,
        "clarity": 0.0-1.0,
        "engagement": 0.0-1.0
    }},
    "audience": ["characteristic1", "characteristic2", ...],
    "engagement_potential": {{
        "score": 0.0-1.0,
        "factors": ["factor1", "factor2", ...]
    }},
    "recommendations": ["recommendation1", "recommendation2", ...]
}}
"""

        messages = [
            {"role": "system", "content": "You are a comprehensive content analysis expert."},
            {"role": "user", "content": prompt}
        ]

        try:
            response = self._make_api_request(messages, temperature=0.3)
            content = self._extract_content_from_response(response)

            # Parse JSON response
            try:
                result = json.loads(content.strip())
                return result

            except json.JSONDecodeError:
                # Fallback - perform individual analyses
                return {
                    'sentiment': self.analyze_sentiment_llm(text, context, detailed=True),
                    'topics': self.extract_topics_llm([text], context=context).get('overall_topics', []),
                    'raw_response': content
                }

        except Exception as e:
            print(f"Error in comprehensive LLM analysis: {e}")
            return {'error': str(e)}