Skool Monitor

#!/usr/bin/env python3
"""
Skool browser-based client that uses Playwright to auto-generate AWS WAF tokens.
This solves the single-use token problem by maintaining a real browser session.
"""

import os
import sys
import json
import time
from dotenv import load_dotenv
from playwright.sync_api import sync_playwright, Browser, BrowserContext, Page

load_dotenv()


def format_for_skool(text: str) -> str:
    """Convert plain text to Skool's TipTap/ProseMirror HTML format."""
    paragraphs = text.split('\n\n')  # Split on blank lines
    html_parts = []
    for p in paragraphs:
        p = p.replace('\n', '<br>')  # Single newlines become <br>
        html_parts.append(f'<p>{p}</p>')
    # Join paragraphs with blank line marker
    return '<p><br class="ProseMirror-trailingBreak"></p>'.join(html_parts)


class SkoolBrowserClient:
    """
    Skool client that uses a real browser to bypass WAF token limitations.
    Maintains a persistent browser session and extracts fresh tokens on demand.
    """

    def __init__(self, auth_token=None, client_id=None, headless=True):
        self.base_url = "https://www.skool.com"
        self.api_base_url = "https://api2.skool.com"

        self.auth_token = auth_token or os.getenv("SKOOL_AUTH_TOKEN")
        self.client_id = client_id or os.getenv("SKOOL_CLIENT_ID")

        if not self.auth_token:
            raise ValueError("SKOOL_AUTH_TOKEN not found in environment")

        self.headless = headless
        self.playwright = None
        self.browser = None
        self.context = None
        self.page = None

        self._start_browser()

    def _start_browser(self):
        """Start Playwright browser with authentication cookies."""
        print("Starting browser session...")

        self.playwright = sync_playwright().start()
        self.browser = self.playwright.chromium.launch(headless=self.headless)

        # Create context with cookies
        self.context = self.browser.new_context(
            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36"
        )

        # Add authentication cookies
        self.context.add_cookies([
            {
                'name': 'auth_token',
                'value': self.auth_token,
                'domain': '.skool.com',
                'path': '/'
            },
            {
                'name': 'client_id',
                'value': self.client_id,
                'domain': '.skool.com',
                'path': '/'
            }
        ])

        self.page = self.context.new_page()

        # Navigate to Skool to establish session
        print("Loading Skool...")
        self.page.goto(f"{self.base_url}/makerschool", wait_until="domcontentloaded")
        time.sleep(2)  # Wait for any background JS to execute

        print("✓ Browser session established")

    def _get_fresh_waf_token(self):
        """
        Get a fresh AWS WAF token from the browser cookies.
        This is automatically generated by Skool's JavaScript.
        """
        cookies = self.context.cookies()
        for cookie in cookies:
            if cookie['name'] == 'aws-waf-token':
                return cookie['value']

        # If no token yet, trigger it by navigating
        print("No WAF token found, triggering generation...")
        self.page.reload(wait_until="domcontentloaded")
        time.sleep(2)

        cookies = self.context.cookies()
        for cookie in cookies:
            if cookie['name'] == 'aws-waf-token':
                return cookie['value']

        raise Exception("Could not obtain AWS WAF token from browser")

    def _make_api_request(self, method, endpoint, json_data=None, params=None):
        """
        Make an API request using fresh WAF token from browser.
        """
        waf_token = self._get_fresh_waf_token()

        # Build the full URL
        url = f"{self.api_base_url}{endpoint}"
        if params:
            query_string = '&'.join(f"{k}={v}" for k, v in params.items())
            url = f"{url}?{query_string}"

        # Make request using Playwright's request context
        response = self.context.request.fetch(
            url,
            method=method,
            headers={
                'accept': '*/*',
                'content-type': 'application/json',
                'origin': 'https://www.skool.com',
                'referer': 'https://www.skool.com/',
                'x-aws-waf-token': waf_token
            },
            data=json.dumps(json_data) if json_data else None
        )

        if response.status == 403:
            # Token might have been used, try once more with fresh token
            print("Got 403, retrying with fresh token...")
            time.sleep(1)
            waf_token = self._get_fresh_waf_token()

            response = self.context.request.fetch(
                url,
                method=method,
                headers={
                    'accept': '*/*',
                    'content-type': 'application/json',
                    'origin': 'https://www.skool.com',
                    'referer': 'https://www.skool.com/',
                    'x-aws-waf-token': waf_token
                },
                data=json.dumps(json_data) if json_data else None
            )

        if not response.ok:
            raise Exception(f"API request failed: {response.status} - {response.text()}")

        # Some endpoints return empty body on success
        try:
            return response.json()
        except Exception:
            return {"status": response.status, "success": True}

    def get_group_id(self, community_slug):
        """Extract group_id from a community page."""
        url = f"{self.base_url}/{community_slug}"
        self.page.goto(url, wait_until="domcontentloaded")

        # Extract from __NEXT_DATA__
        next_data = self.page.evaluate("""
            () => {
                const script = document.getElementById('__NEXT_DATA__');
                return script ? JSON.parse(script.textContent) : null;
            }
        """)

        if not next_data:
            raise Exception("Could not find __NEXT_DATA__")

        group = next_data.get('props', {}).get('pageProps', {}).get('currentGroup', {})
        group_id = group.get('id')

        if not group_id:
            raise Exception("Could not extract group_id")

        return group_id

    def create_post(self, group_id, title, content, labels=None, follow=True):
        """Create a new post in a community."""
        payload = {
            "post_type": "generic",
            "group_id": group_id,
            "metadata": {
                "title": title,
                "content": content,
                "attachments": "",
                "labels": labels or "",
                "action": 0,
                "video_ids": ""
            }
        }

        params = {'follow': 'true' if follow else 'false'}
        return self._make_api_request('POST', '/posts', json_data=payload, params=params)

    def reply_to_post(self, group_id, post_id, content, follow=False):
        """Reply to an existing post (create a comment)."""
        payload = {
            "post_type": "comment",
            "group_id": group_id,
            "root_id": post_id,
            "parent_id": post_id,
            "metadata": {
                "title": "",
                "content": content,
                "attachments": "",
                "action": 0,
                "video_ids": ""
            }
        }

        params = {'follow': 'true' if follow else 'false'}
        return self._make_api_request('POST', '/posts', json_data=payload, params=params)

    def like_post(self, post_id):
        """Like a post or comment."""
        payload = {"old": "", "new": "up"}
        return self._make_api_request('PUT', f'/posts/{post_id}/vote', json_data=payload)

    def unlike_post(self, post_id):
        """Remove like from a post or comment."""
        payload = {"old": "up", "new": ""}
        return self._make_api_request('PUT', f'/posts/{post_id}/vote', json_data=payload)

    def close(self):
        """Clean up browser resources."""
        if self.page:
            self.page.close()
        if self.context:
            self.context.close()
        if self.browser:
            self.browser.close()
        if self.playwright:
            self.playwright.stop()
        print("Browser session closed")

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Skool browser client (auto WAF tokens)")
    parser.add_argument("command",
                       choices=['create', 'reply', 'like', 'unlike', 'group-id'],
                       help="Command to run")
    parser.add_argument("--community", default="makerschool", help="Community slug")
    parser.add_argument("--title", help="Post title (for create)")
    parser.add_argument("--content", help="Post/comment content")
    parser.add_argument("--post-id", help="Post ID (for reply/like/unlike)")
    parser.add_argument("--group-id", help="Group ID (for create/reply)")
    parser.add_argument("--labels", help="Label ID for post category")
    parser.add_argument("--headless", action="store_true", default=True, help="Run browser in headless mode")
    parser.add_argument("--visible", action="store_true", help="Show browser window (debug)")

    args = parser.parse_args()

    try:
        headless = args.headless and not args.visible

        with SkoolBrowserClient(headless=headless) as client:
            if args.command == 'group-id':
                print(f"Getting group_id for {args.community}...")
                group_id = client.get_group_id(args.community)
                print(f"Group ID: {group_id}")

            elif args.command == 'create':
                if not args.title or not args.content:
                    print("Error: --title and --content are required for create")
                    sys.exit(1)

                if not args.group_id:
                    print(f"Group ID not provided, fetching from {args.community}...")
                    group_id = client.get_group_id(args.community)
                else:
                    group_id = args.group_id

                print(f"Creating post in {args.community}...")
                result = client.create_post(group_id, args.title, args.content, labels=args.labels)
                print("✓ Post created successfully!")
                print(json.dumps(result, indent=2))

            elif args.command == 'reply':
                if not args.content or not args.post_id:
                    print("Error: --content and --post-id are required for reply")
                    sys.exit(1)

                if not args.group_id:
                    print(f"Group ID not provided, fetching from {args.community}...")
                    group_id = client.get_group_id(args.community)
                else:
                    group_id = args.group_id

                print(f"Replying to post {args.post_id}...")
                result = client.reply_to_post(group_id, args.post_id, args.content)
                print("✓ Reply posted successfully!")
                print(json.dumps(result, indent=2))

            elif args.command == 'like':
                if not args.post_id:
                    print("Error: --post-id is required for like")
                    sys.exit(1)

                print(f"Liking post {args.post_id}...")
                result = client.like_post(args.post_id)
                print("✓ Post liked successfully!")
                print(json.dumps(result, indent=2))

            elif args.command == 'unlike':
                if not args.post_id:
                    print("Error: --post-id is required for unlike")
                    sys.exit(1)

                print(f"Unliking post {args.post_id}...")
                result = client.unlike_post(args.post_id)
                print("✓ Post unliked successfully!")
                print(json.dumps(result, indent=2))

    except Exception as e:
        print(f"Error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Skool API client for interacting with Skool communities via their internal API.
Uses browser cookies for authentication.
Supports read and write operations: create posts, reply, like, search.
"""

import os
import sys
import json
import requests
import re
from datetime import datetime
from dotenv import load_dotenv
from bs4 import BeautifulSoup

load_dotenv()

class SkoolClient:
    def __init__(self, auth_token=None, client_id=None, aws_waf_token=None):
        self.base_url = "https://www.skool.com"
        self.api_base_url = "https://api2.skool.com"

        self.auth_token = auth_token or os.getenv("SKOOL_AUTH_TOKEN")
        self.client_id = client_id or os.getenv("SKOOL_CLIENT_ID")
        self.aws_waf_token = aws_waf_token or os.getenv("SKOOL_AWS_WAF_TOKEN")

        if not self.auth_token:
            raise ValueError("SKOOL_AUTH_TOKEN not found in environment")

        self.session = requests.Session()
        self._update_headers()

        # Set cookies
        self.session.cookies.set('auth_token', self.auth_token, domain='.skool.com')
        if self.client_id:
            self.session.cookies.set('client_id', self.client_id, domain='.skool.com')
        if self.aws_waf_token:
            self.session.cookies.set('aws-waf-token', self.aws_waf_token, domain='.skool.com')

    def _update_headers(self):
        """Update session headers for API requests."""
        self.session.headers.update({
            'accept': '*/*',
            'accept-language': 'en-US,en;q=0.9',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
            'origin': 'https://www.skool.com',
            'referer': 'https://www.skool.com/',
            'sec-fetch-dest': 'empty',
            'sec-fetch-mode': 'cors',
            'sec-fetch-site': 'same-site',
        })

        # Add AWS WAF token to headers if available
        if self.aws_waf_token:
            self.session.headers.update({
                'x-aws-waf-token': self.aws_waf_token
            })

    def get_community_info(self, community_slug):
        """Get basic info about a community."""
        url = f"{self.base_url}/{community_slug}"
        response = self.session.get(url)
        response.raise_for_status()
        return response.text

    def get_group_id(self, community_slug):
        """
        Extract group_id from a community page.
        This is required for creating posts.
        """
        html = self.get_community_info(community_slug)
        soup = BeautifulSoup(html, 'html.parser')
        next_data_script = soup.find('script', {'id': '__NEXT_DATA__'})

        if not next_data_script:
            raise Exception("Could not find __NEXT_DATA__ in page")

        try:
            data = json.loads(next_data_script.string)
            group = data.get('props', {}).get('pageProps', {}).get('currentGroup', {})
            group_id = group.get('id')

            if not group_id:
                raise Exception("Could not extract group_id from page data")

            return group_id
        except json.JSONDecodeError:
            raise Exception("Could not parse __NEXT_DATA__ JSON")

    def create_post(self, group_id, title, content, labels=None, follow=True):
        """
        Create a new post in a community.

        Args:
            group_id: Community group ID (get via get_group_id())
            title: Post title
            content: Post content (markdown supported)
            labels: Optional label ID (category/tag)
            follow: Whether to follow the post (default True)

        Returns:
            dict: API response with post data
        """
        url = f"{self.api_base_url}/posts"
        params = {'follow': 'true' if follow else 'false'}

        payload = {
            "post_type": "generic",
            "group_id": group_id,
            "metadata": {
                "title": title,
                "content": content,
                "attachments": "",
                "labels": labels or "",
                "action": 0,
                "video_ids": ""
            }
        }

        headers = {'content-type': 'application/json'}
        response = self.session.post(url, params=params, json=payload, headers=headers)
        response.raise_for_status()

        return response.json()

    def reply_to_post(self, group_id, post_id, content, follow=False):
        """
        Reply to an existing post (create a comment).

        Args:
            group_id: Community group ID
            post_id: ID of post to reply to (root_id and parent_id)
            content: Comment content
            follow: Whether to follow the post (default False)

        Returns:
            dict: API response with comment data
        """
        url = f"{self.api_base_url}/posts"
        params = {'follow': 'true' if follow else 'false'}

        payload = {
            "post_type": "comment",
            "group_id": group_id,
            "root_id": post_id,
            "parent_id": post_id,
            "metadata": {
                "title": "",
                "content": content,
                "attachments": "",
                "action": 0,
                "video_ids": ""
            }
        }

        headers = {'content-type': 'application/json'}
        response = self.session.post(url, params=params, json=payload, headers=headers)
        response.raise_for_status()

        return response.json()

    def like_post(self, post_id):
        """
        Like a post or comment.

        Args:
            post_id: ID of post to like

        Returns:
            dict: API response
        """
        url = f"{self.api_base_url}/posts/{post_id}/vote"
        payload = {"old": "", "new": "up"}

        headers = {'content-type': 'application/json'}
        response = self.session.put(url, json=payload, headers=headers)
        response.raise_for_status()

        return response.json()

    def unlike_post(self, post_id):
        """
        Remove like from a post or comment.

        Args:
            post_id: ID of post to unlike

        Returns:
            dict: API response
        """
        url = f"{self.api_base_url}/posts/{post_id}/vote"
        payload = {"old": "up", "new": ""}

        headers = {'content-type': 'application/json'}
        response = self.session.put(url, json=payload, headers=headers)
        response.raise_for_status()

        return response.json()

    def search_posts(self, community_slug, query, build_id="1763672899107"):
        """
        Search for posts in a community.

        Args:
            community_slug: Community slug (e.g., "makerschool")
            query: Search query string
            build_id: Next.js build ID (can usually stay static)

        Returns:
            dict: Search results
        """
        url = f"{self.base_url}/_next/data/{build_id}/{community_slug}/-/search.json"
        params = {
            'q': query,
            'group': community_slug
        }

        headers = {'x-nextjs-data': '1'}
        response = self.session.get(url, params=params, headers=headers)
        response.raise_for_status()

        return response.json()


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Skool API client with write operations")
    parser.add_argument("command",
                       choices=['test', 'create', 'reply', 'like', 'unlike', 'search', 'group-id'],
                       help="Command to run")
    parser.add_argument("--community", default="makerschool", help="Community slug")
    parser.add_argument("--title", help="Post title (for create)")
    parser.add_argument("--content", help="Post/comment content")
    parser.add_argument("--post-id", help="Post ID (for reply/like/unlike)")
    parser.add_argument("--group-id", help="Group ID (for create/reply)")
    parser.add_argument("--query", help="Search query")
    parser.add_argument("--labels", help="Label ID for post category")

    args = parser.parse_args()

    try:
        client = SkoolClient()

        if args.command == 'test':
            print("Testing connection to Skool...")
            html = client.get_community_info(args.community)
            print(f"✓ Successfully connected to {args.community}")
            print(f"Response length: {len(html)} characters")

        elif args.command == 'group-id':
            print(f"Getting group_id for {args.community}...")
            group_id = client.get_group_id(args.community)
            print(f"Group ID: {group_id}")

        elif args.command == 'create':
            if not args.title or not args.content:
                print("Error: --title and --content are required for create")
                sys.exit(1)

            if not args.group_id:
                print(f"Group ID not provided, fetching from {args.community}...")
                group_id = client.get_group_id(args.community)
            else:
                group_id = args.group_id

            print(f"Creating post in {args.community}...")
            result = client.create_post(group_id, args.title, args.content, labels=args.labels)
            print("✓ Post created successfully!")
            print(json.dumps(result, indent=2))

        elif args.command == 'reply':
            if not args.content or not args.post_id:
                print("Error: --content and --post-id are required for reply")
                sys.exit(1)

            if not args.group_id:
                print(f"Group ID not provided, fetching from {args.community}...")
                group_id = client.get_group_id(args.community)
            else:
                group_id = args.group_id

            print(f"Replying to post {args.post_id}...")
            result = client.reply_to_post(group_id, args.post_id, args.content)
            print("✓ Reply posted successfully!")
            print(json.dumps(result, indent=2))

        elif args.command == 'like':
            if not args.post_id:
                print("Error: --post-id is required for like")
                sys.exit(1)

            print(f"Liking post {args.post_id}...")
            result = client.like_post(args.post_id)
            print("✓ Post liked successfully!")
            print(json.dumps(result, indent=2))

        elif args.command == 'unlike':
            if not args.post_id:
                print("Error: --post-id is required for unlike")
                sys.exit(1)

            print(f"Unliking post {args.post_id}...")
            result = client.unlike_post(args.post_id)
            print("✓ Post unliked successfully!")
            print(json.dumps(result, indent=2))

        elif args.command == 'search':
            if not args.query:
                print("Error: --query is required for search")
                sys.exit(1)

            print(f"Searching for '{args.query}' in {args.community}...")
            result = client.search_posts(args.community, args.query)
            print(json.dumps(result, indent=2))

    except Exception as e:
        print(f"Error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Skool comment scraper - fetches comments for posts using browser-based WAF token generation.
Designed for large-scale scraping with rate limiting and incremental saves.
"""

import os
import sys
import json
import time
import argparse
from datetime import datetime
from playwright.sync_api import sync_playwright
import requests

# Load environment variables
def load_env():
    env_path = os.path.join(os.path.dirname(__file__), '..', '.env')
    if os.path.exists(env_path):
        with open(env_path) as f:
            for line in f:
                if '=' in line and not line.startswith('#'):
                    k, v = line.strip().split('=', 1)
                    os.environ[k] = v


class SkoolCommentScraper:
    """
    Scrapes comments from Skool posts using Playwright for WAF token generation.
    """

    def __init__(self, auth_token=None, client_id=None, headless=True):
        self.base_url = "https://www.skool.com"
        self.api_url = "https://api2.skool.com"
        self.group_id = "e256cd9ef1ac4dbe9197634db46e9e3b"  # makerschool

        self.auth_token = auth_token or os.getenv("SKOOL_AUTH_TOKEN")
        self.client_id = client_id or os.getenv("SKOOL_CLIENT_ID")

        if not self.auth_token:
            raise ValueError("SKOOL_AUTH_TOKEN not found")

        self.headless = headless
        self.playwright = None
        self.browser = None
        self.context = None
        self.page = None
        self.session = requests.Session()

        self._waf_token = None
        self._waf_token_uses = 0
        self._max_waf_uses = 50  # Refresh WAF token every N requests

    def start(self):
        """Start the browser session."""
        print("Starting browser session...", file=sys.stderr)

        self.playwright = sync_playwright().start()
        self.browser = self.playwright.chromium.launch(headless=self.headless)
        self.context = self.browser.new_context(
            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
        )

        self.context.add_cookies([
            {'name': 'auth_token', 'value': self.auth_token, 'domain': '.skool.com', 'path': '/'},
            {'name': 'client_id', 'value': self.client_id, 'domain': '.skool.com', 'path': '/'}
        ])

        self.page = self.context.new_page()

        # Navigate to establish session
        print("Loading Skool to generate WAF token...", file=sys.stderr)
        self.page.goto(f"{self.base_url}/makerschool", wait_until="domcontentloaded", timeout=60000)
        time.sleep(3)

        self._refresh_waf_token()
        print("Browser session ready", file=sys.stderr)

    def stop(self):
        """Stop the browser session."""
        if self.browser:
            self.browser.close()
        if self.playwright:
            self.playwright.stop()

    def _refresh_waf_token(self):
        """Get a fresh WAF token from browser cookies."""
        cookies = self.context.cookies()
        for cookie in cookies:
            if cookie['name'] == 'aws-waf-token':
                self._waf_token = cookie['value']
                self._waf_token_uses = 0
                return

        # Trigger token generation by reloading
        print("Refreshing WAF token...", file=sys.stderr)
        self.page.reload(wait_until="domcontentloaded")
        time.sleep(2)

        cookies = self.context.cookies()
        for cookie in cookies:
            if cookie['name'] == 'aws-waf-token':
                self._waf_token = cookie['value']
                self._waf_token_uses = 0
                return

        raise Exception("Could not obtain WAF token")

    def _get_waf_token(self):
        """Get WAF token, refreshing if needed."""
        if self._waf_token_uses >= self._max_waf_uses:
            self._refresh_waf_token()
        self._waf_token_uses += 1
        return self._waf_token

    def fetch_comments(self, post_id, limit=20):
        """
        Fetch all comments for a post using browser-based request with pagination.

        Args:
            post_id: The post ID
            limit: Max comments per request (API max is 20)

        Returns:
            List of comment dictionaries
        """
        all_comments = []
        cursor = None
        page_num = 0

        while True:
            # Build URL with pagination cursor if we have one
            url = f"{self.api_url}/posts/{post_id}/comments?group-id={self.group_id}&limit={limit}"
            if cursor:
                url += f"&last={cursor}"

            try:
                # Use Playwright to make the request (inherits browser's WAF token)
                response = self.page.evaluate(f'''
                    async () => {{
                        const response = await fetch("{url}", {{
                            method: "GET",
                            headers: {{
                                "accept": "application/json",
                            }},
                            credentials: "include"
                        }});
                        if (!response.ok) {{
                            return {{ error: response.status, text: await response.text() }};
                        }}
                        return await response.json();
                    }}
                ''')

                if isinstance(response, dict) and 'error' in response:
                    print(f"  API error {response['error']}", file=sys.stderr)
                    break

                # Parse comments from post_tree.children structure
                post_tree = response.get('post_tree', {})
                children = post_tree.get('children', [])

                if not children:
                    break

                parsed = self._parse_comments(children, post_id)
                all_comments.extend(parsed)
                page_num += 1

                # Check for pagination cursor
                next_cursor = response.get('last')
                if not next_cursor or next_cursor == cursor:
                    break  # No more pages

                cursor = next_cursor

                # Small delay between pagination requests
                if cursor:
                    time.sleep(0.5)

            except Exception as e:
                print(f"  Error fetching comments for {post_id}: {e}", file=sys.stderr)
                break

        return all_comments

    def _parse_comments(self, raw_comments, post_id, parent_id=None):
        """Parse raw comment data into clean format with proper parent tracking."""
        parsed = []

        for c in raw_comments:
            # Handle nested structure
            if 'post' in c:
                comment_data = c['post']
            else:
                comment_data = c

            meta = comment_data.get('metadata', {})
            user = comment_data.get('user', {})
            comment_id = comment_data.get('id')

            parsed.append({
                'id': comment_id,
                'post_id': post_id,
                'parent_id': parent_id,  # Use passed-in parent, not API field
                'content': meta.get('content', ''),
                'author': user.get('name', 'Unknown'),
                'author_id': user.get('id'),
                'author_name': f"{user.get('firstName', '')} {user.get('lastName', '')}".strip(),
                'created_at': comment_data.get('createdAt'),
                'likes': meta.get('upvotes', 0),
            })

            # Process nested replies - pass current comment's ID as parent
            nested = c.get('children', [])
            if nested:
                parsed.extend(self._parse_comments(nested, post_id, parent_id=comment_id))

        return parsed

    def scrape_comments_for_posts(self, posts, output_file, delay=3.0, skip_existing=True):
        """
        Scrape comments for multiple posts with incremental saves.

        Args:
            posts: List of post dicts (must have 'id' and 'comments' count)
            output_file: Path to save results (JSON)
            delay: Seconds between requests
            skip_existing: Skip posts that already have comments in output
        """
        # Load existing data
        existing_data = {}
        if os.path.exists(output_file) and skip_existing:
            try:
                with open(output_file, 'r') as f:
                    existing_data = json.load(f)
                print(f"Loaded {len(existing_data)} existing posts with comments", file=sys.stderr)
            except:
                existing_data = {}

        # Filter posts with comments
        posts_with_comments = [p for p in posts if p.get('comments', 0) > 0]
        print(f"Posts with comments: {len(posts_with_comments)}", file=sys.stderr)

        # Skip already scraped
        if skip_existing:
            posts_to_scrape = [p for p in posts_with_comments if p['id'] not in existing_data]
        else:
            posts_to_scrape = posts_with_comments

        print(f"Posts to scrape: {len(posts_to_scrape)}", file=sys.stderr)

        total_comments = 0

        for i, post in enumerate(posts_to_scrape, 1):
            post_id = post['id']
            expected_comments = post.get('comments', 0)

            print(f"[{i}/{len(posts_to_scrape)}] Fetching {expected_comments} comments for: {post.get('title', '')[:50]}...", file=sys.stderr)

            comments = self.fetch_comments(post_id)

            if comments:
                existing_data[post_id] = {
                    'post_id': post_id,
                    'post_title': post.get('title', ''),
                    'post_url': post.get('url', ''),
                    'expected_comments': expected_comments,
                    'fetched_comments': len(comments),
                    'comments': comments,
                    'scraped_at': datetime.utcnow().isoformat() + 'Z'
                }
                total_comments += len(comments)
                print(f"  Got {len(comments)} comments", file=sys.stderr)
            else:
                print(f"  No comments returned", file=sys.stderr)

            # Save incrementally every 10 posts
            if i % 10 == 0:
                with open(output_file, 'w') as f:
                    json.dump(existing_data, f, indent=2)
                print(f"  [Saved checkpoint: {len(existing_data)} posts, {total_comments} comments]", file=sys.stderr)

            # Rate limiting
            if i < len(posts_to_scrape):
                time.sleep(delay)

        # Final save
        with open(output_file, 'w') as f:
            json.dump(existing_data, f, indent=2)

        print(f"\nComplete! Scraped {total_comments} comments from {len(posts_to_scrape)} posts", file=sys.stderr)
        print(f"Total posts with comments: {len(existing_data)}", file=sys.stderr)
        print(f"Saved to: {output_file}", file=sys.stderr)

        return existing_data


def main():
    load_env()

    parser = argparse.ArgumentParser(description="Scrape comments from Skool posts")
    parser.add_argument("--posts-file", required=True, help="JSON file with posts to scrape")
    parser.add_argument("--output", required=True, help="Output JSON file for comments")
    parser.add_argument("--delay", type=float, default=3.0, help="Seconds between requests (default: 3.0)")
    parser.add_argument("--limit", type=int, help="Max posts to process")
    parser.add_argument("--no-skip", action="store_true", help="Don't skip already scraped posts")
    parser.add_argument("--visible", action="store_true", help="Show browser window")

    args = parser.parse_args()

    # Load posts
    with open(args.posts_file, 'r') as f:
        posts = json.load(f)

    if args.limit:
        posts = posts[:args.limit]

    print(f"Loaded {len(posts)} posts from {args.posts_file}", file=sys.stderr)

    # Initialize scraper
    scraper = SkoolCommentScraper(headless=not args.visible)

    try:
        scraper.start()
        scraper.scrape_comments_for_posts(
            posts,
            args.output,
            delay=args.delay,
            skip_existing=not args.no_skip
        )
    finally:
        scraper.stop()


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Skool community scraper - extracts posts, members, and activity from Skool communities.
Uses reverse-engineered internal API via browser cookies.
"""

import os
import sys
import json
import re
import time
import requests
from bs4 import BeautifulSoup
from datetime import datetime
from dotenv import load_dotenv

load_dotenv()

class SkoolScraper:
    def __init__(self, auth_token=None, client_id=None):
        self.base_url = "https://www.skool.com"
        self.auth_token = auth_token or os.getenv("SKOOL_AUTH_TOKEN")
        self.client_id = client_id or os.getenv("SKOOL_CLIENT_ID")

        if not self.auth_token:
            raise ValueError("SKOOL_AUTH_TOKEN not found in environment")

        self.session = requests.Session()
        self.session.headers.update({
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'accept-language': 'en-US,en;q=0.9',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        })

        # Set cookies
        self.session.cookies.set('auth_token', self.auth_token, domain='.skool.com')
        if self.client_id:
            self.session.cookies.set('client_id', self.client_id, domain='.skool.com')

    def _extract_next_data(self, html):
        """Extract Next.js data from HTML."""
        soup = BeautifulSoup(html, 'html.parser')
        next_data_script = soup.find('script', {'id': '__NEXT_DATA__'})

        if not next_data_script:
            return None

        try:
            return json.loads(next_data_script.string)
        except json.JSONDecodeError:
            return None

    def get_community_posts(self, community_slug, max_posts=50, start_page=1, output_file=None, delay=1.0):
        """
        Get recent posts from a community with pagination support.
        Returns a list of post dictionaries with title, author, content, etc.

        Args:
            community_slug: The community identifier (e.g., 'makerschool')
            max_posts: Maximum number of posts to retrieve (default: 50)
            start_page: Page number to start from (default: 1)
            output_file: Optional file path to write posts incrementally (safer for large scrapes)
            delay: Seconds to wait between requests (default: 1.0) - be respectful to servers
        """
        posts = []
        seen_ids = set()
        current_page = start_page

        # If output_file specified, initialize or load existing data
        if output_file:
            import os
            if os.path.exists(output_file):
                try:
                    with open(output_file, 'r') as f:
                        posts = json.load(f)
                    # Deduplicate existing posts
                    unique_posts = []
                    for p in posts:
                        if p['id'] not in seen_ids:
                            seen_ids.add(p['id'])
                            unique_posts.append(p)
                    posts = unique_posts
                    print(f"Loaded {len(posts)} unique posts from {output_file}", file=sys.stderr)
                except:
                    posts = []

        while len(posts) < max_posts:
            # Build URL with pagination
            url = f"{self.base_url}/{community_slug}"
            if current_page > 1:
                url += f"?p={current_page}"

            print(f"Fetching page {current_page}... ({len(posts)} unique posts so far)", file=sys.stderr)

            try:
                response = self.session.get(url)
                response.raise_for_status()
            except requests.exceptions.HTTPError as e:
                print(f"HTTP error on page {current_page}: {e}", file=sys.stderr)
                # Save progress and exit gracefully
                if output_file and posts:
                    with open(output_file, 'w') as f:
                        json.dump(posts, f, indent=2)
                    print(f"Saved {len(posts)} posts before error", file=sys.stderr)
                raise

            next_data = self._extract_next_data(response.text)
            if not next_data:
                raise Exception("Could not extract Next.js data from page")

            new_posts_this_page = 0
            try:
                page_props = next_data.get('props', {}).get('pageProps', {})

                # Skool uses postTrees structure
                post_trees = page_props.get('postTrees', [])

                # If no posts on this page, we've reached the end
                if not post_trees:
                    print(f"No more posts found at page {current_page}", file=sys.stderr)
                    break

                for tree in post_trees:
                    if len(posts) >= max_posts:
                        break

                    post = tree.get('post', {})
                    post_id = post.get('id')

                    # Skip duplicates
                    if post_id in seen_ids:
                        continue

                    seen_ids.add(post_id)
                    metadata = post.get('metadata', {})
                    user = post.get('user', {})

                    posts.append({
                        'id': post_id,
                        'title': metadata.get('title', 'No title'),
                        'slug': post.get('name'),
                        'content': metadata.get('content', ''),
                        'author': user.get('name', 'Unknown'),
                        'author_id': user.get('id'),
                        'created_at': post.get('createdAt'),
                        'likes': metadata.get('upvotes', 0),
                        'comments': metadata.get('comments', 0),
                        'pinned': metadata.get('pinned', False),
                        'my_vote': metadata.get('myVote'),
                        'url': f"{self.base_url}/{community_slug}/{post.get('name')}",
                        'raw': post
                    })
                    new_posts_this_page += 1

                # Write incrementally to file after each page
                if output_file and new_posts_this_page > 0:
                    with open(output_file, 'w') as f:
                        json.dump(posts, f, indent=2)

            except Exception as e:
                print(f"Error parsing posts: {e}")
                with open('.tmp/skool_debug_data.json', 'w') as f:
                    json.dump(next_data, f, indent=2)
                print("Saved raw data to .tmp/skool_debug_data.json for inspection")
                raise

            current_page += 1

            # Rate limiting - wait between requests
            if len(posts) < max_posts:
                time.sleep(delay)

        return posts

    def _find_posts_in_data(self, data, max_depth=5):
        """Recursively search for post-like data structures."""
        if max_depth == 0:
            return []

        posts = []

        if isinstance(data, dict):
            # Check if this dict looks like a post
            if ('title' in data or 'content' in data or 'body' in data) and ('id' in data or 'slug' in data):
                posts.append(data)

            # Recursively search all dict values
            for value in data.values():
                posts.extend(self._find_posts_in_data(value, max_depth - 1))

        elif isinstance(data, list):
            # Search all list items
            for item in data:
                posts.extend(self._find_posts_in_data(item, max_depth - 1))

        return posts

    def get_new_posts(self, community_slug, since_timestamp=None):
        """
        Get posts created after a specific timestamp.
        If since_timestamp is None, returns all recent posts.
        """
        all_posts = self.get_community_posts(community_slug, max_posts=100)

        if not since_timestamp:
            return all_posts

        # Filter posts by timestamp
        new_posts = []
        for post in all_posts:
            created_at = post.get('created_at')
            if created_at:
                # Parse timestamp (format may vary)
                try:
                    post_time = datetime.fromisoformat(created_at.replace('Z', '+00:00'))
                    since_time = datetime.fromisoformat(since_timestamp.replace('Z', '+00:00'))
                    if post_time > since_time:
                        new_posts.append(post)
                except:
                    # If parsing fails, include the post
                    new_posts.append(post)

        return new_posts


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Skool community scraper")
    parser.add_argument("command", choices=['posts', 'new'], help="Command to run")
    parser.add_argument("--community", default="makerschool", help="Community slug")
    parser.add_argument("--limit", type=int, default=20, help="Number of posts to fetch")
    parser.add_argument("--start-page", type=int, default=1, help="Page to start from (skip already-fetched pages)")
    parser.add_argument("--delay", type=float, default=1.0, help="Seconds between requests (default: 1.0)")
    parser.add_argument("--since", help="ISO timestamp for filtering new posts")
    parser.add_argument("--output", help="Output JSON file")

    args = parser.parse_args()

    try:
        scraper = SkoolScraper()

        if args.command == 'posts':
            print(f"Fetching posts from {args.community} starting at page {args.start_page}...", file=sys.stderr)
            posts = scraper.get_community_posts(
                args.community,
                max_posts=args.limit,
                start_page=args.start_page,
                output_file=args.output,
                delay=args.delay
            )

            print(f"Found {len(posts)} posts:\n")
            for i, post in enumerate(posts, 1):
                # Format timestamp
                created = post['created_at']
                if created:
                    try:
                        dt = datetime.fromisoformat(created.replace('Z', '+00:00'))
                        created_str = dt.strftime('%Y-%m-%d %H:%M UTC')
                    except:
                        created_str = created
                else:
                    created_str = 'Unknown'

                print(f"{'='*80}")
                print(f"{i}. {post['title']}")
                print(f"By: {post['author']} | {created_str}")
                print(f"Likes: {post['likes']} | Comments: {post['comments']}")
                print(f"URL: {post['url']}")
                print(f"\n{post['content']}")
                print()

            if args.output:
                with open(args.output, 'w') as f:
                    json.dump(posts, f, indent=2)
                print(f"\nSaved {len(posts)} posts to {args.output}")

        elif args.command == 'new':
            print(f"Fetching new posts from {args.community} since {args.since}...")
            posts = scraper.get_new_posts(args.community, since_timestamp=args.since)

            print(f"Found {len(posts)} new posts")

            if args.output:
                with open(args.output, 'w') as f:
                    json.dump(posts, f, indent=2)
                print(f"Saved to {args.output}")

    except Exception as e:
        print(f"Error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Skool unread posts fetcher - retrieves posts with new activity you haven't seen.

Uses Skool's native ?filter=unread parameter to fetch posts where:
- You've never viewed the post (myView is None)
- There are new comments since you last viewed (lastComment > myView)
"""

import os
import sys
import json
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timezone, timedelta
from dotenv import load_dotenv

load_dotenv()


class SkoolUnreads:
    def __init__(self, auth_token=None, client_id=None):
        self.base_url = "https://www.skool.com"
        self.auth_token = auth_token or os.getenv("SKOOL_AUTH_TOKEN")
        self.client_id = client_id or os.getenv("SKOOL_CLIENT_ID")

        if not self.auth_token:
            raise ValueError("SKOOL_AUTH_TOKEN not found in environment")

        self.session = requests.Session()
        self.session.headers.update({
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'accept-language': 'en-US,en;q=0.9',
            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        })
        self.session.cookies.set('auth_token', self.auth_token, domain='.skool.com')
        if self.client_id:
            self.session.cookies.set('client_id', self.client_id, domain='.skool.com')

    def _extract_next_data(self, html):
        """Extract Next.js data from HTML."""
        soup = BeautifulSoup(html, 'html.parser')
        next_data_script = soup.find('script', {'id': '__NEXT_DATA__'})
        if not next_data_script:
            return None
        try:
            return json.loads(next_data_script.string)
        except json.JSONDecodeError:
            return None

    def get_unreads(self, community_slug, max_posts=100, since_hours=None):
        """
        Get all unread posts from a community.

        Returns posts where you either:
        - Have never viewed the post
        - Have new comments since your last view

        Args:
            community_slug: The community identifier (e.g., 'makerschool')
            max_posts: Maximum posts to return (default: 100)
            since_hours: Only return posts created within this many hours (default: None = all)

        Returns:
            List of post dicts with unread metadata
        """
        posts = []
        current_page = 1
        seen_ids = set()
        label_map = {}  # Cache label ID -> display name

        # Calculate cutoff time if since_hours specified
        cutoff = None
        if since_hours:
            cutoff = datetime.now(timezone.utc) - timedelta(hours=since_hours)
            print(f"Filtering to posts since: {cutoff.strftime('%Y-%m-%d %H:%M UTC')}", file=sys.stderr)

        while len(posts) < max_posts:
            # Use fl=unr (the correct Skool unread filter)
            url = f"{self.base_url}/{community_slug}?c=&s=newest&fl=unr"
            if current_page > 1:
                url += f"&p={current_page}"

            print(f"Fetching unread page {current_page}...", file=sys.stderr)

            try:
                response = self.session.get(url)
                response.raise_for_status()
            except requests.exceptions.HTTPError as e:
                print(f"HTTP error: {e}", file=sys.stderr)
                break

            next_data = self._extract_next_data(response.text)
            if not next_data:
                print("Could not extract Next.js data", file=sys.stderr)
                break

            page_props = next_data.get('props', {}).get('pageProps', {})
            post_trees = page_props.get('postTrees', [])

            # Build label map from group data (only on first page)
            if not label_map:
                current_group = page_props.get('currentGroup', {})
                for label in current_group.get('labels', []):
                    label_id = label.get('id')
                    display_name = label.get('metadata', {}).get('displayName', 'Unknown')
                    label_map[label_id] = display_name

            if not post_trees:
                break

            new_this_page = 0
            stop_pagination = False

            for tree in post_trees:
                if len(posts) >= max_posts:
                    break

                post = tree.get('post', {})
                post_id = post.get('id')

                if post_id in seen_ids:
                    continue
                seen_ids.add(post_id)

                # Check time filter
                created_at = post.get('createdAt')
                if cutoff and created_at:
                    try:
                        post_time = datetime.fromisoformat(created_at.replace('Z', '+00:00'))
                        if post_time < cutoff:
                            stop_pagination = True
                            continue  # Skip this post but check remaining on page
                    except:
                        pass

                metadata = post.get('metadata', {})
                user = post.get('user', {})

                # Determine unread reason
                my_view = metadata.get('myView')
                last_comment = metadata.get('lastComment')

                if my_view is None:
                    unread_reason = "never_viewed"
                elif last_comment and last_comment > my_view:
                    unread_reason = "new_comments"
                else:
                    unread_reason = "unknown"

                # Get category/label
                label_id = post.get('labelId')
                category = label_map.get(label_id, 'Uncategorized') if label_id else 'Uncategorized'

                posts.append({
                    'id': post_id,
                    'title': metadata.get('title', 'No title'),
                    'slug': post.get('name'),
                    'content': metadata.get('content', ''),
                    'author': user.get('name', 'Unknown'),
                    'author_id': user.get('id'),
                    'created_at': created_at,
                    'category': category,
                    'category_id': label_id,
                    'likes': metadata.get('upvotes', 0),
                    'comments': metadata.get('comments', 0),
                    'pinned': metadata.get('pinned', False),
                    'my_vote': metadata.get('myVote'),
                    'my_view': my_view,
                    'last_comment': last_comment,
                    'unread_reason': unread_reason,
                    'url': f"{self.base_url}/{community_slug}/{post.get('name')}",
                })
                new_this_page += 1

            if new_this_page == 0 or stop_pagination:
                break

            current_page += 1

        return posts

    def get_unread_count(self, community_slug):
        """Get total unread count without fetching all posts."""
        url = f"{self.base_url}/{community_slug}?c=&s=newest&fl=unr"
        response = self.session.get(url)
        response.raise_for_status()
        next_data = self._extract_next_data(response.text)
        if next_data:
            page_props = next_data.get('props', {}).get('pageProps', {})
            return page_props.get('total', 0)
        return 0

    def get_unread_summary(self, community_slug, since_hours=None, max_posts=500):
        """
        Get a categorized summary of unread posts.

        Args:
            community_slug: The community identifier
            since_hours: Only include posts from last N hours
            max_posts: Maximum posts to fetch (default: 500)

        Returns:
            Dict with 'total_all_time', 'never_viewed' and 'new_comments' lists
        """
        total_all_time = self.get_unread_count(community_slug)
        unreads = self.get_unreads(community_slug, max_posts=max_posts, since_hours=since_hours)

        summary = {
            'total_all_time': total_all_time,
            'total_filtered': len(unreads),
            'never_viewed': [],
            'new_comments': [],
        }

        for post in unreads:
            if post['unread_reason'] == 'never_viewed':
                summary['never_viewed'].append(post)
            else:
                summary['new_comments'].append(post)

        return summary


def format_post(post, index):
    """Format a single post for display."""
    created = post['created_at']
    if created:
        try:
            dt = datetime.fromisoformat(created.replace('Z', '+00:00'))
            created_str = dt.strftime('%Y-%m-%d %H:%M UTC')
        except:
            created_str = created
    else:
        created_str = 'Unknown'

    reason_emoji = "🆕" if post['unread_reason'] == 'never_viewed' else "💬"
    category = post.get('category', 'Uncategorized')

    lines = [
        f"{index}. {reason_emoji} {post['title']}",
        f"   By: {post['author']} | {created_str}",
        f"   Category: {category}",
        f"   Likes: {post['likes']} | Comments: {post['comments']}",
        f"   URL: {post['url']}",
    ]
    return '\n'.join(lines)


def main():
    import argparse
    parser = argparse.ArgumentParser(description="Fetch unread Skool posts")
    parser.add_argument("--community", default="makerschool", help="Community slug")
    parser.add_argument("--limit", type=int, default=500, help="Max posts to fetch")
    parser.add_argument("--since", type=int, help="Only posts from last N hours (e.g., --since 48)")
    parser.add_argument("--output", help="Output JSON file")
    parser.add_argument("--summary", action="store_true", help="Show categorized summary")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument("--count-only", action="store_true", help="Just show total unread count")

    args = parser.parse_args()

    try:
        fetcher = SkoolUnreads()

        if args.count_only:
            count = fetcher.get_unread_count(args.community)
            if args.json:
                print(json.dumps({'total_unread': count}))
            else:
                print(f"📬 Total unread: {count}")
            return

        if args.summary:
            summary = fetcher.get_unread_summary(args.community, since_hours=args.since, max_posts=args.limit)

            if args.json:
                print(json.dumps(summary, indent=2))
            else:
                print(f"\n📬 Unread Posts Summary for {args.community}")
                print(f"{'='*60}")
                print(f"Total unread (all time): {summary['total_all_time']}")
                if args.since:
                    print(f"Unread in last {args.since}h: {summary['total_filtered']}")
                print(f"  🆕 Never viewed: {len(summary['never_viewed'])}")
                print(f"  💬 New comments: {len(summary['new_comments'])}")

                if summary['never_viewed']:
                    print(f"\n🆕 NEVER VIEWED ({len(summary['never_viewed'])})")
                    print("-" * 40)
                    for i, post in enumerate(summary['never_viewed'][:20], 1):
                        print(format_post(post, i))
                        print()

                if summary['new_comments']:
                    print(f"\n💬 NEW COMMENTS ({len(summary['new_comments'])})")
                    print("-" * 40)
                    for i, post in enumerate(summary['new_comments'][:20], 1):
                        print(format_post(post, i))
                        print()
        else:
            posts = fetcher.get_unreads(args.community, max_posts=args.limit, since_hours=args.since)

            if args.json:
                print(json.dumps(posts, indent=2))
            else:
                time_label = f" (last {args.since}h)" if args.since else ""
                print(f"\n📬 {len(posts)} Unread Posts in {args.community}{time_label}")
                print("=" * 60)

                for i, post in enumerate(posts, 1):
                    print(format_post(post, i))
                    print()

        if args.output:
            if args.summary:
                data = fetcher.get_unread_summary(args.community, since_hours=args.since, max_posts=args.limit)
            else:
                data = fetcher.get_unreads(args.community, max_posts=args.limit, since_hours=args.since)
            with open(args.output, 'w') as f:
                json.dump(data, f, indent=2)
            print(f"\nSaved to {args.output}", file=sys.stderr)

    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        import traceback
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()