Deep Research

# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python

# Virtual environments
venv/
ENV/
env/

# IDE
.vscode/
.idea/
*.swp
*.swo
*~

# OS
.DS_Store
Thumbs.db

# Research output (kept local)
*.json
!schemas/*.json

# Test output
.pytest_cache/
.coverage
htmlcov/

# Deep Research Skill Dependencies
#
# Core: Python 3.9+ standard library only. No pip install needed.
#
# Optional tools (not Python packages):
#
# search-cli — multi-provider search aggregation (Brave, Serper, Exa, Jina, Firecrawl)
#   Install: brew tap 199-biotechnologies/tap && brew install search-cli
#   Config:  search config set keys.[provider] YOUR_KEY
#   Repo:    https://github.com/199-biotechnologies/search-cli
#
# weasyprint — PDF generation from HTML reports
#   Install: pip install weasyprint
#   Used by: reference/html-generation.md (Phase 8 PDF output)

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "Claim",
  "description": "An atomic claim extracted from the report. claim_id = sha256(section_id + sentence_text)[:16].",
  "type": "object",
  "required": ["claim_id", "section_id", "text", "claim_type", "support_status"],
  "properties": {
    "claim_id": {
      "type": "string",
      "pattern": "^[0-9a-f]{16}$",
      "description": "sha256(section_id + normalized_text)[:16]"
    },
    "section_id": {
      "type": "string",
      "description": "Section identifier (e.g. executive_summary, finding_1, synthesis)"
    },
    "text": {
      "type": "string",
      "description": "The atomic claim sentence"
    },
    "claim_type": {
      "type": "string",
      "enum": ["factual", "synthesis", "recommendation", "speculation"],
      "description": "Only factual claims hard-fail on lack of support"
    },
    "cited_source_ids": {
      "type": "array",
      "items": { "type": "string", "pattern": "^[0-9a-f]{16}$" },
      "default": [],
      "description": "Stable source_ids cited for this claim"
    },
    "evidence_ids": {
      "type": "array",
      "items": { "type": "string", "pattern": "^[0-9a-f]{16}$" },
      "default": [],
      "description": "Evidence rows that support this claim"
    },
    "support_status": {
      "type": "string",
      "enum": ["unverified", "supported", "partial", "unsupported", "needs_review"],
      "description": "Set by verify_claim_support.py (PR5)"
    },
    "extracted_at": {
      "type": "string",
      "format": "date-time"
    }
  },
  "additionalProperties": false
}

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "Evidence",
  "description": "A piece of evidence extracted from a source. evidence_id = sha256(source_id + normalized_quote + locator)[:16].",
  "type": "object",
  "required": ["evidence_id", "source_id", "quote", "evidence_type", "captured_at"],
  "properties": {
    "evidence_id": {
      "type": "string",
      "pattern": "^[0-9a-f]{16}$",
      "description": "sha256(source_id + normalized_quote + locator)[:16]"
    },
    "source_id": {
      "type": "string",
      "pattern": "^[0-9a-f]{16}$",
      "description": "References a source in sources.jsonl"
    },
    "retrieval_query": {
      "type": ["string", "null"],
      "description": "The search query or prompt that led to this evidence",
      "default": null
    },
    "locator": {
      "type": ["string", "null"],
      "description": "Page number, section heading, URL fragment, or timestamp within the source",
      "default": null
    },
    "quote": {
      "type": "string",
      "description": "Exact or near-exact text extracted from the source"
    },
    "evidence_type": {
      "type": "string",
      "enum": ["direct_quote", "paraphrase", "data_point", "figure_reference", "methodology"],
      "description": "How the evidence was captured"
    },
    "captured_at": {
      "type": "string",
      "format": "date-time"
    }
  },
  "additionalProperties": false
}

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "RunManifest",
  "description": "Manifest for a single research run. Created at init, updated throughout.",
  "type": "object",
  "required": ["version", "query", "mode", "started_at", "report_dir", "artifact_paths"],
  "properties": {
    "version": {
      "type": "string",
      "const": "3.0.0"
    },
    "query": {
      "type": "string",
      "description": "Original research question"
    },
    "mode": {
      "type": "string",
      "enum": ["quick", "standard", "deep", "ultradeep"]
    },
    "started_at": {
      "type": "string",
      "format": "date-time"
    },
    "finished_at": {
      "type": ["string", "null"],
      "format": "date-time",
      "default": null
    },
    "assumptions": {
      "type": "array",
      "items": {
        "type": "object",
        "required": ["assumption_id", "text", "materiality", "status"],
        "properties": {
          "assumption_id": {
            "type": "string",
            "pattern": "^asm_[0-9a-f]{8}$"
          },
          "text": { "type": "string" },
          "materiality": {
            "type": "string",
            "enum": ["low", "medium", "high"]
          },
          "status": {
            "type": "string",
            "enum": ["implicit", "user_confirmed", "evidence_validated"]
          }
        },
        "additionalProperties": false
      },
      "default": []
    },
    "provider_config": {
      "type": "object",
      "properties": {
        "primary": {
          "type": "string",
          "description": "Primary search provider (e.g. WebSearch)"
        },
        "scholarly": {
          "type": ["string", "null"],
          "description": "Scholarly API provider if configured (e.g. openalex, semantic_scholar)"
        }
      },
      "default": { "primary": "search-cli", "scholarly": null }
    },
    "report_dir": {
      "type": "string",
      "description": "Absolute path to the report directory"
    },
    "artifact_paths": {
      "type": "object",
      "required": ["sources", "evidence", "claims", "report"],
      "properties": {
        "sources": { "type": "string", "default": "sources.jsonl" },
        "evidence": { "type": "string", "default": "evidence.jsonl" },
        "claims": { "type": "string", "default": "claims.jsonl" },
        "report": { "type": "string", "default": "report.md" }
      },
      "additionalProperties": false
    },
    "continuation": {
      "type": ["object", "null"],
      "description": "Populated when resuming a previous run",
      "properties": {
        "previous_run_manifest": { "type": "string" },
        "resumed_at": { "type": "string", "format": "date-time" },
        "sections_completed": {
          "type": "array",
          "items": { "type": "string" }
        }
      },
      "default": null
    }
  },
  "additionalProperties": false
}

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "Source",
  "description": "A research source with stable identity. source_id = sha256(canonical_locator)[:16].",
  "type": "object",
  "required": ["source_id", "canonical_locator", "raw_url", "title", "source_type", "metadata_status", "registered_at"],
  "properties": {
    "source_id": {
      "type": "string",
      "pattern": "^[0-9a-f]{16}$",
      "description": "sha256(canonical_locator)[:16] — stable across edits and continuation"
    },
    "canonical_locator": {
      "type": "string",
      "description": "Canonical identifier: doi:10.1038/..., arxiv:2305.14251, or normalized URL (scheme+host+path, no fragment/tracking params)"
    },
    "raw_url": {
      "type": "string",
      "description": "Original URL as retrieved, before normalization"
    },
    "title": {
      "type": "string"
    },
    "authors": {
      "type": ["array", "null"],
      "items": { "type": "string" },
      "default": null
    },
    "year": {
      "type": ["string", "null"],
      "default": null
    },
    "source_type": {
      "type": "string",
      "enum": ["web", "academic", "documentation", "code", "news", "government", "book"]
    },
    "metadata_status": {
      "type": "string",
      "enum": ["unverified", "doi_verified", "url_verified", "title_matched"],
      "description": "How far metadata has been verified"
    },
    "registered_at": {
      "type": "string",
      "format": "date-time",
      "description": "ISO 8601 timestamp when source was registered"
    }
  },
  "additionalProperties": false
}

#!/usr/bin/env python3
"""
Citation Manager — stable source identity and run manifest management.

CLI subcommands:
  init-run             Create run_manifest.json + empty artifact JSONL files
  register-source      Append a source to sources.jsonl, return source_id
  assign-display-numbers  Generate stable_id -> display_number mapping
  export-bibliography   Render bibliography from sources.jsonl

Source identity:
  source_id = sha256(canonical_locator)[:16]
  canonical_locator = doi:..., arxiv:..., or normalized URL

All state is append-only JSONL. No mutable citation numbers in state files.
"""

import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime, timezone
from urllib.parse import urlparse, urlunparse


# ---------------------------------------------------------------------------
# Canonical locator normalization
# ---------------------------------------------------------------------------

DOI_RE = re.compile(r'(?:https?://(?:dx\.)?doi\.org/|doi:)(10\.\d{4,}/\S+)', re.IGNORECASE)
ARXIV_RE = re.compile(r'(?:https?://arxiv\.org/abs/|arxiv:)(\d{4}\.\d{4,}(?:v\d+)?)', re.IGNORECASE)

# URL query params that are tracking noise, not content identifiers
TRACKING_PARAMS = frozenset([
    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
    'ref', 'source', 'fbclid', 'gclid', 'mc_cid', 'mc_eid',
])


def canonicalize_locator(raw_url: str) -> str:
    """Derive a canonical locator from a raw URL or identifier string.

    Priority: DOI > arXiv > normalized URL.
    """
    # DOI
    m = DOI_RE.search(raw_url)
    if m:
        return f'doi:{m.group(1).rstrip(".")}'

    # arXiv
    m = ARXIV_RE.search(raw_url)
    if m:
        return f'arxiv:{m.group(1)}'

    # Normalized URL: lowercase scheme+host, strip fragment and tracking params
    parsed = urlparse(raw_url)
    scheme = (parsed.scheme or 'https').lower()
    host = (parsed.hostname or '').lower()
    path = parsed.path.rstrip('/')
    # Filter query params
    if parsed.query:
        pairs = []
        for part in parsed.query.split('&'):
            kv = part.split('=', 1)
            if kv[0].lower() not in TRACKING_PARAMS:
                pairs.append(part)
        query = '&'.join(sorted(pairs))
    else:
        query = ''
    return urlunparse((scheme, host, path, '', query, ''))


def compute_source_id(canonical_locator: str) -> str:
    """sha256(canonical_locator)[:16] hex."""
    return hashlib.sha256(canonical_locator.encode('utf-8')).hexdigest()[:16]


# ---------------------------------------------------------------------------
# JSONL helpers
# ---------------------------------------------------------------------------

def append_jsonl(path: str, obj: dict) -> None:
    with open(path, 'a') as f:
        f.write(json.dumps(obj, ensure_ascii=False) + '\n')


def read_jsonl(path: str) -> list[dict]:
    rows = []
    if not os.path.exists(path):
        return rows
    with open(path) as f:
        for line in f:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------

def cmd_init_run(args: argparse.Namespace) -> None:
    """Create run_manifest.json and empty JSONL artifact files."""
    out_dir = os.path.abspath(args.out_dir)
    os.makedirs(out_dir, exist_ok=True)

    artifact_paths = {
        'sources': 'sources.jsonl',
        'evidence': 'evidence.jsonl',
        'claims': 'claims.jsonl',
        'report': 'report.md',
    }

    manifest = {
        'version': '3.0.0',
        'query': args.query or '',
        'mode': args.mode,
        'started_at': datetime.now(timezone.utc).isoformat(),
        'finished_at': None,
        'assumptions': [],
        'provider_config': {
            'primary': 'search-cli',
            'scholarly': None,
        },
        'report_dir': out_dir,
        'artifact_paths': artifact_paths,
        'continuation': None,
    }

    manifest_path = os.path.join(out_dir, 'run_manifest.json')
    with open(manifest_path, 'w') as f:
        json.dump(manifest, f, indent=2, ensure_ascii=False)
        f.write('\n')

    # Create empty artifact files
    for name in ('sources', 'evidence', 'claims'):
        p = os.path.join(out_dir, artifact_paths[name])
        if not os.path.exists(p):
            open(p, 'w').close()

    print(json.dumps({'status': 'ok', 'manifest': manifest_path, 'dir': out_dir}))


def cmd_register_source(args: argparse.Namespace) -> None:
    """Register a source, append to sources.jsonl, print source_id."""
    data = json.loads(args.json)
    raw_url = data.get('raw_url', data.get('url', ''))
    if not raw_url:
        print(json.dumps({'error': 'raw_url is required'}), file=sys.stderr)
        sys.exit(1)

    canonical = data.get('canonical_locator') or canonicalize_locator(raw_url)
    source_id = compute_source_id(canonical)

    sources_path = os.path.join(args.dir, 'sources.jsonl')

    # Check for duplicate
    existing = read_jsonl(sources_path)
    for row in existing:
        if row.get('source_id') == source_id:
            print(json.dumps({
                'status': 'duplicate',
                'source_id': source_id,
                'canonical_locator': canonical,
            }))
            return

    source = {
        'source_id': source_id,
        'canonical_locator': canonical,
        'raw_url': raw_url,
        'title': data.get('title', ''),
        'authors': data.get('authors'),
        'year': data.get('year'),
        'source_type': data.get('source_type', 'web'),
        'metadata_status': data.get('metadata_status', 'unverified'),
        'registered_at': datetime.now(timezone.utc).isoformat(),
    }
    append_jsonl(sources_path, source)
    print(json.dumps({
        'status': 'registered',
        'source_id': source_id,
        'canonical_locator': canonical,
    }))


def cmd_assign_display_numbers(args: argparse.Namespace) -> None:
    """Read sources.jsonl, assign stable display numbers in registration order."""
    sources_path = os.path.join(args.dir, 'sources.jsonl')
    sources = read_jsonl(sources_path)

    mapping = {}
    for i, src in enumerate(sources, 1):
        sid = src['source_id']
        if sid not in mapping:
            mapping[sid] = i

    print(json.dumps(mapping, indent=2))


def cmd_export_bibliography(args: argparse.Namespace) -> None:
    """Generate bibliography from sources.jsonl."""
    sources_path = os.path.join(args.dir, 'sources.jsonl')
    sources = read_jsonl(sources_path)

    # Deduplicate by source_id, preserve order
    seen = set()
    unique = []
    for src in sources:
        if src['source_id'] not in seen:
            seen.add(src['source_id'])
            unique.append(src)

    style = args.style

    if style == 'markdown':
        lines = ['## Bibliography', '']
        for i, src in enumerate(unique, 1):
            author_str = ''
            if src.get('authors'):
                authors = src['authors']
                if len(authors) == 1:
                    author_str = f'{authors[0]}. '
                elif len(authors) == 2:
                    author_str = f'{authors[0]} & {authors[1]}. '
                else:
                    author_str = f'{authors[0]} et al. '

            year_str = f'({src["year"]})' if src.get('year') else '(n.d.)'
            title = src.get('title', 'Untitled')
            url = src.get('raw_url', '')
            lines.append(f'[{i}] {author_str}{year_str}. [{title}]({url})')
        print('\n'.join(lines))

    elif style == 'json':
        out = []
        for i, src in enumerate(unique, 1):
            out.append({
                'display_number': i,
                'source_id': src['source_id'],
                'canonical_locator': src['canonical_locator'],
                'title': src.get('title', ''),
                'authors': src.get('authors'),
                'year': src.get('year'),
                'raw_url': src.get('raw_url', ''),
            })
        print(json.dumps(out, indent=2, ensure_ascii=False))

    else:
        print(f'Unknown style: {style}', file=sys.stderr)
        sys.exit(1)


# ---------------------------------------------------------------------------
# CLI entry point
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        prog='citation_manager',
        description='Stable source identity and run manifest management for deep-research v3.0',
    )
    sub = parser.add_subparsers(dest='command', required=True)

    # init-run
    p_init = sub.add_parser('init-run', help='Create run manifest and empty artifact files')
    p_init.add_argument('--out-dir', required=True, help='Output directory for the research run')
    p_init.add_argument('--query', default='', help='Original research question')
    p_init.add_argument('--mode', default='standard', choices=['quick', 'standard', 'deep', 'ultradeep'])

    # register-source
    p_reg = sub.add_parser('register-source', help='Register a source and return its stable ID')
    p_reg.add_argument('--json', required=True, help='JSON object with at least raw_url and title')
    p_reg.add_argument('--dir', required=True, help='Run directory containing sources.jsonl')

    # assign-display-numbers
    p_num = sub.add_parser('assign-display-numbers', help='Map stable source IDs to display numbers')
    p_num.add_argument('--dir', required=True, help='Run directory containing sources.jsonl')

    # export-bibliography
    p_bib = sub.add_parser('export-bibliography', help='Generate bibliography from sources')
    p_bib.add_argument('--dir', required=True, help='Run directory containing sources.jsonl')
    p_bib.add_argument('--style', default='markdown', choices=['markdown', 'json'])

    args = parser.parse_args()

    dispatch = {
        'init-run': cmd_init_run,
        'register-source': cmd_register_source,
        'assign-display-numbers': cmd_assign_display_numbers,
        'export-bibliography': cmd_export_bibliography,
    }
    dispatch[args.command](args)


if __name__ == '__main__':
    main()

#!/usr/bin/env python3
"""
Evidence Store — append-only evidence persistence for deep-research v3.0.

CLI subcommands:
  init         Create empty evidence.jsonl in a run directory
  add          Append an evidence row, return evidence_id
  list         List evidence rows, optionally filtered by source_id
  export       Export evidence as JSON array

Evidence identity:
  evidence_id = sha256(source_id + normalized_quote + locator)[:16]

All state is append-only JSONL. Evidence is never modified after capture.
"""

import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime, timezone


# ---------------------------------------------------------------------------
# Evidence ID computation
# ---------------------------------------------------------------------------

_WHITESPACE_RE = re.compile(r'\s+')


def normalize_quote(quote: str) -> str:
    """Normalize whitespace for stable hashing."""
    return _WHITESPACE_RE.sub(' ', quote.strip()).lower()


def compute_evidence_id(source_id: str, quote: str, locator: str | None) -> str:
    """sha256(source_id + normalized_quote + locator)[:16] hex."""
    payload = source_id + normalize_quote(quote) + (locator or '')
    return hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]


# ---------------------------------------------------------------------------
# JSONL helpers (shared pattern with citation_manager)
# ---------------------------------------------------------------------------

def append_jsonl(path: str, obj: dict) -> None:
    with open(path, 'a') as f:
        f.write(json.dumps(obj, ensure_ascii=False) + '\n')


def read_jsonl(path: str) -> list[dict]:
    rows = []
    if not os.path.exists(path):
        return rows
    with open(path) as f:
        for line in f:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------

def cmd_init(args: argparse.Namespace) -> None:
    """Create empty evidence.jsonl if it doesn't exist."""
    out_dir = os.path.abspath(args.dir)
    path = os.path.join(out_dir, 'evidence.jsonl')
    if not os.path.exists(path):
        os.makedirs(out_dir, exist_ok=True)
        open(path, 'w').close()
    print(json.dumps({'status': 'ok', 'path': path}))


def cmd_add(args: argparse.Namespace) -> None:
    """Append evidence row, print evidence_id."""
    data = json.loads(args.json)
    source_id = data.get('source_id', '')
    quote = data.get('quote', '')
    if not source_id or not quote:
        print(json.dumps({'error': 'source_id and quote are required'}), file=sys.stderr)
        sys.exit(1)

    locator = data.get('locator')
    evidence_id = compute_evidence_id(source_id, quote, locator)
    evidence_path = os.path.join(args.dir, 'evidence.jsonl')

    # Check for duplicate
    existing = read_jsonl(evidence_path)
    for row in existing:
        if row.get('evidence_id') == evidence_id:
            print(json.dumps({
                'status': 'duplicate',
                'evidence_id': evidence_id,
            }))
            return

    valid_types = {'direct_quote', 'paraphrase', 'data_point', 'figure_reference', 'methodology'}
    evidence_type = data.get('evidence_type', 'direct_quote')
    if evidence_type not in valid_types:
        evidence_type = 'direct_quote'

    row = {
        'evidence_id': evidence_id,
        'source_id': source_id,
        'retrieval_query': data.get('retrieval_query'),
        'locator': locator,
        'quote': quote,
        'evidence_type': evidence_type,
        'captured_at': datetime.now(timezone.utc).isoformat(),
    }
    append_jsonl(evidence_path, row)
    print(json.dumps({
        'status': 'added',
        'evidence_id': evidence_id,
        'source_id': source_id,
    }))


def cmd_list(args: argparse.Namespace) -> None:
    """List evidence rows, optionally filtered."""
    evidence_path = os.path.join(args.dir, 'evidence.jsonl')
    rows = read_jsonl(evidence_path)

    if args.source_id:
        rows = [r for r in rows if r.get('source_id') == args.source_id]

    # Deduplicate by evidence_id
    seen = set()
    unique = []
    for r in rows:
        eid = r.get('evidence_id')
        if eid not in seen:
            seen.add(eid)
            unique.append(r)

    print(json.dumps({
        'count': len(unique),
        'evidence': unique,
    }, indent=2, ensure_ascii=False))


def cmd_export(args: argparse.Namespace) -> None:
    """Export all evidence as JSON array."""
    evidence_path = os.path.join(args.dir, 'evidence.jsonl')
    rows = read_jsonl(evidence_path)

    # Deduplicate
    seen = set()
    unique = []
    for r in rows:
        eid = r.get('evidence_id')
        if eid not in seen:
            seen.add(eid)
            unique.append(r)

    print(json.dumps(unique, indent=2, ensure_ascii=False))


# ---------------------------------------------------------------------------
# CLI entry point
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        prog='evidence_store',
        description='Append-only evidence persistence for deep-research v3.0',
    )
    sub = parser.add_subparsers(dest='command', required=True)

    # init
    p_init = sub.add_parser('init', help='Create empty evidence.jsonl')
    p_init.add_argument('--dir', required=True, help='Run directory')

    # add
    p_add = sub.add_parser('add', help='Append evidence row')
    p_add.add_argument('--json', required=True, help='JSON with source_id, quote, locator, evidence_type, retrieval_query')
    p_add.add_argument('--dir', required=True, help='Run directory containing evidence.jsonl')

    # list
    p_list = sub.add_parser('list', help='List evidence rows')
    p_list.add_argument('--dir', required=True, help='Run directory')
    p_list.add_argument('--source-id', default=None, help='Filter by source_id')

    # export
    p_export = sub.add_parser('export', help='Export all evidence as JSON array')
    p_export.add_argument('--dir', required=True, help='Run directory')

    args = parser.parse_args()

    dispatch = {
        'init': cmd_init,
        'add': cmd_add,
        'list': cmd_list,
        'export': cmd_export,
    }
    dispatch[args.command](args)


if __name__ == '__main__':
    main()

#!/usr/bin/env python3
"""
Atomic Claim Extractor — decomposes report sections into typed claims.

CLI subcommands:
  extract      Parse a markdown report into atomic claims (claims.jsonl)
  add          Manually add a single claim
  list         List claims, optionally filtered by section or type
  stats        Show claim statistics (counts by type/status)

Claim identity:
  claim_id = sha256(section_id + normalized_text)[:16]

Claim types (per GPT Pro's refinement of Codex's proposal):
  - factual: hard-fails on lack of support
  - synthesis: needs traceability, softer threshold
  - recommendation: needs traceability, softer threshold
  - speculation: labeled, no support gate
"""

import argparse
import hashlib
import json
import os
import re
import sys
from datetime import datetime, timezone


# ---------------------------------------------------------------------------
# Claim ID computation
# ---------------------------------------------------------------------------

_WHITESPACE_RE = re.compile(r'\s+')


def normalize_text(text: str) -> str:
    """Normalize for stable hashing."""
    return _WHITESPACE_RE.sub(' ', text.strip()).lower()


def compute_claim_id(section_id: str, text: str) -> str:
    """sha256(section_id + normalized_text)[:16] hex."""
    payload = section_id + normalize_text(text)
    return hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]


# ---------------------------------------------------------------------------
# JSONL helpers
# ---------------------------------------------------------------------------

def append_jsonl(path: str, obj: dict) -> None:
    with open(path, 'a') as f:
        f.write(json.dumps(obj, ensure_ascii=False) + '\n')


def read_jsonl(path: str) -> list[dict]:
    rows = []
    if not os.path.exists(path):
        return rows
    with open(path) as f:
        for line in f:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


# ---------------------------------------------------------------------------
# Report parsing helpers
# ---------------------------------------------------------------------------

# Section header patterns
SECTION_PATTERNS = [
    (re.compile(r'^##\s+Executive\s+Summary', re.I), 'executive_summary'),
    (re.compile(r'^##\s+Introduction', re.I), 'introduction'),
    (re.compile(r'^##\s+Finding\s+(\d+)', re.I), lambda m: f'finding_{m.group(1)}'),
    (re.compile(r'^##\s+Synthesis', re.I), 'synthesis'),
    (re.compile(r'^##\s+Limitations', re.I), 'limitations'),
    (re.compile(r'^##\s+Recommendations', re.I), 'recommendations'),
    (re.compile(r'^##\s+Conclusion', re.I), 'conclusion'),
    (re.compile(r'^##\s+(.+)', re.I), lambda m: re.sub(r'\W+', '_', m.group(1).strip().lower())[:30]),
]

# Citation pattern [N] or [N, M]
CITATION_RE = re.compile(r'\[(\d+(?:,\s*\d+)*)\]')

# Sentence splitting (basic but handles abbreviations)
SENTENCE_RE = re.compile(r'(?<=[.!?])\s+(?=[A-Z])')


def classify_claim(text: str, section_id: str) -> str:
    """Heuristic claim type classification."""
    lower = text.lower()

    # Recommendation indicators
    if any(w in lower for w in ['should', 'recommend', 'suggest', 'advise', 'consider']):
        if section_id == 'recommendations':
            return 'recommendation'
        return 'recommendation'

    # Speculation indicators
    if any(w in lower for w in ['might', 'could potentially', 'it is possible', 'may eventually',
                                 'hypothetically', 'speculatively']):
        return 'speculation'

    # Synthesis indicators (often in synthesis/conclusion sections)
    if section_id in ('synthesis', 'conclusion', 'limitations'):
        if any(w in lower for w in ['overall', 'taken together', 'collectively',
                                     'the evidence suggests', 'this implies']):
            return 'synthesis'

    # Default: factual
    return 'factual'


def parse_sections(markdown: str) -> list[tuple[str, str]]:
    """Parse markdown into (section_id, content) pairs."""
    lines = markdown.split('\n')
    sections = []
    current_id = 'preamble'
    current_lines = []

    for line in lines:
        matched = False
        for pattern, id_or_fn in SECTION_PATTERNS:
            m = pattern.match(line)
            if m:
                if current_lines:
                    sections.append((current_id, '\n'.join(current_lines)))
                current_id = id_or_fn(m) if callable(id_or_fn) else id_or_fn
                current_lines = []
                matched = True
                break
        if not matched:
            current_lines.append(line)

    if current_lines:
        sections.append((current_id, '\n'.join(current_lines)))

    return sections


def extract_sentences(text: str) -> list[str]:
    """Split text into sentences, filtering noise."""
    # Remove markdown formatting noise
    text = re.sub(r'^[-*]\s+', '', text, flags=re.M)  # bullet points
    text = re.sub(r'\*\*([^*]+)\*\*', r'\1', text)  # bold
    text = re.sub(r'\*([^*]+)\*', r'\1', text)  # italic

    sentences = SENTENCE_RE.split(text)
    result = []
    for s in sentences:
        s = s.strip()
        # Filter out very short fragments, headings, empty lines
        if len(s) > 30 and not s.startswith('#') and not s.startswith('|'):
            result.append(s)
    return result


# ---------------------------------------------------------------------------
# Subcommands
# ---------------------------------------------------------------------------

def cmd_extract(args: argparse.Namespace) -> None:
    """Extract atomic claims from a markdown report."""
    report_path = args.report
    if not os.path.exists(report_path):
        print(json.dumps({'error': f'Report not found: {report_path}'}), file=sys.stderr)
        sys.exit(1)

    with open(report_path) as f:
        markdown = f.read()

    claims_path = os.path.join(args.dir, 'claims.jsonl')
    existing_ids = {r['claim_id'] for r in read_jsonl(claims_path)}

    sections = parse_sections(markdown)
    added = 0
    skipped = 0

    for section_id, content in sections:
        if section_id == 'preamble':
            continue
        sentences = extract_sentences(content)
        for sentence in sentences:
            claim_id = compute_claim_id(section_id, sentence)
            if claim_id in existing_ids:
                skipped += 1
                continue

            # Extract citation numbers from sentence
            citation_nums = []
            for m in CITATION_RE.finditer(sentence):
                nums = [int(n.strip()) for n in m.group(1).split(',')]
                citation_nums.extend(nums)

            claim = {
                'claim_id': claim_id,
                'section_id': section_id,
                'text': sentence,
                'claim_type': classify_claim(sentence, section_id),
                'cited_source_ids': [],  # Populated by linking step
                'evidence_ids': [],  # Populated by verify_claim_support
                'support_status': 'unverified',
                'extracted_at': datetime.now(timezone.utc).isoformat(),
                '_citation_numbers': citation_nums,  # Temporary, for linking
            }
            append_jsonl(claims_path, claim)
            existing_ids.add(claim_id)
            added += 1

    print(json.dumps({
        'status': 'ok',
        'claims_added': added,
        'claims_skipped': skipped,
        'total_claims': len(existing_ids),
    }))


def cmd_add(args: argparse.Namespace) -> None:
    """Manually add a single claim."""
    data = json.loads(args.json)
    section_id = data.get('section_id', 'unknown')
    text = data.get('text', '')
    if not text:
        print(json.dumps({'error': 'text is required'}), file=sys.stderr)
        sys.exit(1)

    claim_id = compute_claim_id(section_id, text)
    claims_path = os.path.join(args.dir, 'claims.jsonl')

    existing = read_jsonl(claims_path)
    for row in existing:
        if row.get('claim_id') == claim_id:
            print(json.dumps({'status': 'duplicate', 'claim_id': claim_id}))
            return

    valid_types = {'factual', 'synthesis', 'recommendation', 'speculation'}
    claim_type = data.get('claim_type', 'factual')
    if claim_type not in valid_types:
        claim_type = 'factual'

    claim = {
        'claim_id': claim_id,
        'section_id': section_id,
        'text': text,
        'claim_type': claim_type,
        'cited_source_ids': data.get('cited_source_ids', []),
        'evidence_ids': data.get('evidence_ids', []),
        'support_status': 'unverified',
        'extracted_at': datetime.now(timezone.utc).isoformat(),
    }
    append_jsonl(claims_path, claim)
    print(json.dumps({'status': 'added', 'claim_id': claim_id}))


def cmd_list(args: argparse.Namespace) -> None:
    """List claims with optional filters."""
    claims_path = os.path.join(args.dir, 'claims.jsonl')
    rows = read_jsonl(claims_path)

    if args.section:
        rows = [r for r in rows if r.get('section_id') == args.section]
    if args.type:
        rows = [r for r in rows if r.get('claim_type') == args.type]
    if args.status:
        rows = [r for r in rows if r.get('support_status') == args.status]

    # Deduplicate
    seen = set()
    unique = []
    for r in rows:
        cid = r.get('claim_id')
        if cid not in seen:
            seen.add(cid)
            unique.append(r)

    print(json.dumps({'count': len(unique), 'claims': unique}, indent=2, ensure_ascii=False))


def cmd_stats(args: argparse.Namespace) -> None:
    """Show claim statistics."""
    claims_path = os.path.join(args.dir, 'claims.jsonl')
    rows = read_jsonl(claims_path)

    # Deduplicate
    seen = set()
    unique = []
    for r in rows:
        cid = r.get('claim_id')
        if cid not in seen:
            seen.add(cid)
            unique.append(r)

    by_type = {}
    by_status = {}
    by_section = {}
    for r in unique:
        t = r.get('claim_type', 'unknown')
        s = r.get('support_status', 'unknown')
        sec = r.get('section_id', 'unknown')
        by_type[t] = by_type.get(t, 0) + 1
        by_status[s] = by_status.get(s, 0) + 1
        by_section[sec] = by_section.get(sec, 0) + 1

    print(json.dumps({
        'total': len(unique),
        'by_type': by_type,
        'by_status': by_status,
        'by_section': by_section,
    }, indent=2))


# ---------------------------------------------------------------------------
# CLI entry point
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        prog='extract_claims',
        description='Atomic claim extraction and ledger for deep-research v3.0',
    )
    sub = parser.add_subparsers(dest='command', required=True)

    # extract
    p_ext = sub.add_parser('extract', help='Extract claims from markdown report')
    p_ext.add_argument('--report', required=True, help='Path to report.md')
    p_ext.add_argument('--dir', required=True, help='Run directory containing claims.jsonl')

    # add
    p_add = sub.add_parser('add', help='Manually add a single claim')
    p_add.add_argument('--json', required=True, help='JSON with section_id, text, claim_type')
    p_add.add_argument('--dir', required=True, help='Run directory')

    # list
    p_list = sub.add_parser('list', help='List claims')
    p_list.add_argument('--dir', required=True, help='Run directory')
    p_list.add_argument('--section', default=None, help='Filter by section_id')
    p_list.add_argument('--type', default=None, help='Filter by claim_type')
    p_list.add_argument('--status', default=None, help='Filter by support_status')

    # stats
    p_stats = sub.add_parser('stats', help='Claim statistics')
    p_stats.add_argument('--dir', required=True, help='Run directory')

    args = parser.parse_args()
    dispatch = {
        'extract': cmd_extract,
        'add': cmd_add,
        'list': cmd_list,
        'stats': cmd_stats,
    }
    dispatch[args.command](args)


if __name__ == '__main__':
    main()

#!/usr/bin/env python3
"""
Markdown to HTML converter for research reports
Properly converts markdown sections to HTML while preserving structure and formatting
"""

import re
from typing import Tuple
from pathlib import Path


def convert_markdown_to_html(markdown_text: str) -> Tuple[str, str]:
    """
    Convert markdown to HTML in two parts: content and bibliography

    Args:
        markdown_text: Full markdown report text

    Returns:
        Tuple of (content_html, bibliography_html)
    """
    # Split content and bibliography
    parts = markdown_text.split('## Bibliography')
    content_md = parts[0]
    bibliography_md = parts[1] if len(parts) > 1 else ""

    # Convert content (everything except bibliography)
    content_html = _convert_content_section(content_md)

    # Convert bibliography separately
    bibliography_html = _convert_bibliography_section(bibliography_md)

    return content_html, bibliography_html


def _convert_content_section(markdown: str) -> str:
    """Convert main content sections to HTML"""
    html = markdown

    # Remove title and front matter (first ## heading is handled separately)
    lines = html.split('\n')
    processed_lines = []
    skip_until_first_section = True

    for line in lines:
        # Skip everything until we hit "## Executive Summary" or first major section
        if skip_until_first_section:
            if line.startswith('## ') and not line.startswith('### '):
                skip_until_first_section = False
                processed_lines.append(line)
            continue
        processed_lines.append(line)

    html = '\n'.join(processed_lines)

    # Convert headers
    # ## Section Title → <div class="section"><h2 class="section-title">Section Title</h2></div>
    html = re.sub(
        r'^## (.+)$',
        r'<div class="section"><h2 class="section-title">\1</h2>',
        html,
        flags=re.MULTILINE
    )

    # ### Subsection → <h3 class="subsection-title">Subsection</h3>
    html = re.sub(
        r'^### (.+)$',
        r'<h3 class="subsection-title">\1</h3>',
        html,
        flags=re.MULTILINE
    )

    # #### Subsubsection → <h4 class="subsubsection-title">Title</h4>
    html = re.sub(
        r'^#### (.+)$',
        r'<h4 class="subsubsection-title">\1</h4>',
        html,
        flags=re.MULTILINE
    )

    # Convert **bold** text
    html = re.sub(r'\*\*(.+?)\*\*', r'<strong>\1</strong>', html)

    # Convert *italic* text
    html = re.sub(r'\*(.+?)\*', r'<em>\1</em>', html)

    # Convert inline code `code`
    html = re.sub(r'`(.+?)`', r'<code>\1</code>', html)

    # Convert unordered lists
    html = _convert_lists(html)

    # Convert tables
    html = _convert_tables(html)

    # Convert paragraphs (wrap non-HTML lines in <p> tags)
    html = _convert_paragraphs(html)

    # Close all open sections
    html = _close_sections(html)

    # Wrap executive summary if present
    html = html.replace(
        '<h2 class="section-title">Executive Summary</h2>',
        '<div class="executive-summary"><h2 class="section-title">Executive Summary</h2>'
    )
    if '<div class="executive-summary">' in html:
        # Close executive summary at the next section
        html = html.replace(
            '</h2>\n<div class="section">',
            '</h2></div>\n<div class="section">',
            1
        )

    return html


def _convert_bibliography_section(markdown: str) -> str:
    """Convert bibliography section to HTML"""
    if not markdown.strip():
        return ""

    html = markdown

    # Convert each [N] citation to a proper bibliography entry
    # Look for patterns like [1] Title - URL
    html = re.sub(
        r'\[(\d+)\]\s*(.+?)\s*-\s*(https?://[^\s\)]+)',
        r'<div class="bib-entry"><span class="bib-number">[\1]</span> <a href="\3" target="_blank">\2</a></div>',
        html
    )

    # Convert any remaining **bold** sections
    html = re.sub(r'\*\*(.+?)\*\*', r'<strong>\1</strong>', html)

    # Wrap in bibliography content div
    html = f'<div class="bibliography-content">{html}</div>'

    return html


def _convert_lists(html: str) -> str:
    """Convert markdown lists to HTML lists"""
    lines = html.split('\n')
    result = []
    in_list = False
    list_level = 0

    for i, line in enumerate(lines):
        stripped = line.strip()

        # Check for unordered list item
        if stripped.startswith('- ') or stripped.startswith('* '):
            if not in_list:
                result.append('<ul>')
                in_list = True
                list_level = len(line) - len(line.lstrip())

            # Get the content after the marker
            content = stripped[2:]
            result.append(f'<li>{content}</li>')

        # Check for ordered list item
        elif re.match(r'^\d+\.\s', stripped):
            if not in_list:
                result.append('<ol>')
                in_list = True
                list_level = len(line) - len(line.lstrip())

            # Get the content after the number and period
            content = re.sub(r'^\d+\.\s', '', stripped)
            result.append(f'<li>{content}</li>')

        else:
            # Not a list item
            if in_list:
                # Check if we're still in the list (indented continuation)
                current_level = len(line) - len(line.lstrip())
                if current_level > list_level and stripped:
                    # Continuation of previous list item
                    if result[-1].endswith('</li>'):
                        result[-1] = result[-1][:-5] + ' ' + stripped + '</li>'
                    continue
                else:
                    # End of list
                    result.append('</ul>' if '<ul>' in '\n'.join(result[-10:]) else '</ol>')
                    in_list = False
                    list_level = 0

            result.append(line)

    # Close any remaining open list
    if in_list:
        result.append('</ul>' if '<ul>' in '\n'.join(result[-10:]) else '</ol>')

    return '\n'.join(result)


def _convert_tables(html: str) -> str:
    """Convert markdown tables to HTML tables"""
    lines = html.split('\n')
    result = []
    in_table = False

    for i, line in enumerate(lines):
        if '|' in line and line.strip().startswith('|'):
            if not in_table:
                result.append('<table>')
                in_table = True
                # This is the header row
                cells = [cell.strip() for cell in line.split('|')[1:-1]]
                result.append('<thead><tr>')
                for cell in cells:
                    result.append(f'<th>{cell}</th>')
                result.append('</tr></thead>')
                result.append('<tbody>')
            elif '---' in line:
                # Skip separator row
                continue
            else:
                # Data row
                cells = [cell.strip() for cell in line.split('|')[1:-1]]
                result.append('<tr>')
                for cell in cells:
                    result.append(f'<td>{cell}</td>')
                result.append('</tr>')
        else:
            if in_table:
                result.append('</tbody></table>')
                in_table = False
            result.append(line)

    if in_table:
        result.append('</tbody></table>')

    return '\n'.join(result)


def _convert_paragraphs(html: str) -> str:
    """Wrap non-HTML lines in paragraph tags"""
    lines = html.split('\n')
    result = []
    in_paragraph = False

    for line in lines:
        stripped = line.strip()

        # Skip empty lines
        if not stripped:
            if in_paragraph:
                result.append('</p>')
                in_paragraph = False
            result.append(line)
            continue

        # Skip lines that are already HTML tags
        if (stripped.startswith('<') and stripped.endswith('>')) or \
           stripped.startswith('</') or \
           '<h' in stripped or '<div' in stripped or '<ul' in stripped or \
           '<ol' in stripped or '<li' in stripped or '<table' in stripped or \
           '</div>' in stripped or '</ul>' in stripped or '</ol>' in stripped:
            if in_paragraph:
                result.append('</p>')
                in_paragraph = False
            result.append(line)
            continue

        # Regular text line - wrap in paragraph
        if not in_paragraph:
            result.append('<p>' + line)
            in_paragraph = True
        else:
            result.append(line)

    if in_paragraph:
        result.append('</p>')

    return '\n'.join(result)


def _close_sections(html: str) -> str:
    """Close all open section divs"""
    # Count open and closed divs
    open_divs = html.count('<div class="section">')
    closed_divs = html.count('</div>')

    # Add closing divs for sections
    # Each section should be closed before the next section starts
    lines = html.split('\n')
    result = []
    section_open = False

    for i, line in enumerate(lines):
        if '<div class="section">' in line:
            if section_open:
                result.append('</div>')  # Close previous section
            section_open = True
        result.append(line)

    # Close final section if still open
    if section_open:
        result.append('</div>')

    return '\n'.join(result)


def main():
    """Test the converter with a sample markdown file"""
    import sys

    if len(sys.argv) < 2:
        print("Usage: python md_to_html.py <markdown_file>")
        sys.exit(1)

    md_file = Path(sys.argv[1])
    if not md_file.exists():
        print(f"Error: File {md_file} not found")
        sys.exit(1)

    markdown_text = md_file.read_text()
    content_html, bib_html = convert_markdown_to_html(markdown_text)

    print("=== CONTENT HTML ===")
    print(content_html[:1000])
    print("\n=== BIBLIOGRAPHY HTML ===")
    print(bib_html[:500])


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Deep Research Engine — STATE SCAFFOLD (not a runtime orchestrator)

This file provides phase instruction templates and research state persistence.
It does NOT drive Claude Code — Claude is the orchestrator; this file provides
data structures and CLI utilities for state management.

For the actual research workflow, see reference/methodology.md.
For the evidence substrate, see scripts/citation_manager.py and scripts/evidence_store.py.
"""

import argparse
import json
import sys
import time
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, asdict
from enum import Enum


class ResearchPhase(Enum):
    """Research pipeline phases"""
    SCOPE = "scope"
    PLAN = "plan"
    RETRIEVE = "retrieve"
    TRIANGULATE = "triangulate"
    SYNTHESIZE = "synthesize"
    CRITIQUE = "critique"
    REFINE = "refine"
    PACKAGE = "package"


class ResearchMode(Enum):
    """Research depth modes"""
    QUICK = "quick"  # 3 phases: scope, retrieve, package
    STANDARD = "standard"  # 6 phases: skip refine and critique
    DEEP = "deep"  # Full 8 phases
    ULTRADEEP = "ultradeep"  # 8 phases + extended iterations


@dataclass
class Source:
    """Represents a research source"""
    url: str
    title: str
    snippet: str
    retrieved_at: str
    credibility_score: float = 0.0
    source_type: str = "web"  # web, academic, documentation, code
    verification_status: str = "unverified"  # unverified, verified, conflicted

    def to_citation(self, index: int) -> str:
        """Generate citation string"""
        return f"[{index}] {self.title} - {self.url} (Retrieved: {self.retrieved_at})"


@dataclass
class ResearchState:
    """Maintains research state across phases"""
    query: str
    mode: ResearchMode
    phase: ResearchPhase
    scope: Dict[str, Any]
    plan: Dict[str, Any]
    sources: List[Source]
    findings: List[Dict[str, Any]]
    synthesis: Dict[str, Any]
    critique: Dict[str, Any]
    report: str
    metadata: Dict[str, Any]

    def save(self, filepath: Path):
        """Save research state to file with retry logic"""
        max_retries = 3
        for attempt in range(max_retries):
            try:
                with open(filepath, 'w') as f:
                    json.dump(self._serialize(), f, indent=2)
                return  # Success
            except (IOError, OSError) as e:
                if attempt == max_retries - 1:
                    # Final attempt failed
                    raise IOError(f"Failed to save state after {max_retries} attempts: {e}")
                # Wait with exponential backoff before retry
                wait_time = (attempt + 1) * 0.5  # 0.5s, 1s, 1.5s
                time.sleep(wait_time)

    def _serialize(self) -> dict:
        """Convert to serializable dict"""
        return {
            'query': self.query,
            'mode': self.mode.value,
            'phase': self.phase.value,
            'scope': self.scope,
            'plan': self.plan,
            'sources': [asdict(s) for s in self.sources],
            'findings': self.findings,
            'synthesis': self.synthesis,
            'critique': self.critique,
            'report': self.report,
            'metadata': self.metadata
        }

    @classmethod
    def load(cls, filepath: Path) -> 'ResearchState':
        """Load research state from file"""
        with open(filepath, 'r') as f:
            data = json.load(f)

        return cls(
            query=data['query'],
            mode=ResearchMode(data['mode']),
            phase=ResearchPhase(data['phase']),
            scope=data['scope'],
            plan=data['plan'],
            sources=[Source(**s) for s in data['sources']],
            findings=data['findings'],
            synthesis=data['synthesis'],
            critique=data['critique'],
            report=data['report'],
            metadata=data['metadata']
        )


class ResearchEngine:
    """Main research orchestration engine"""

    def __init__(self, mode: ResearchMode = ResearchMode.STANDARD):
        self.mode = mode
        self.state: Optional[ResearchState] = None
        self.output_dir = Path.home() / ".claude" / "research_output"
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def initialize_research(self, query: str) -> ResearchState:
        """Initialize new research session"""
        self.state = ResearchState(
            query=query,
            mode=self.mode,
            phase=ResearchPhase.SCOPE,
            scope={},
            plan={},
            sources=[],
            findings=[],
            synthesis={},
            critique={},
            report="",
            metadata={
                'started_at': datetime.now().isoformat(),
                'version': '1.0'
            }
        )
        return self.state

    def get_phase_instructions(self, phase: ResearchPhase) -> str:
        """Get instructions for current phase"""
        instructions = {
            ResearchPhase.SCOPE: """
# Phase 1: SCOPE

Your task: Define research boundaries and success criteria

## Execute:
1. Decompose the question into 3-5 core components
2. Identify 2-4 key stakeholder perspectives
3. Define what's IN scope and what's OUT of scope
4. List 3-5 success criteria for this research
5. Document 3-5 assumptions that need validation

## Output Format:
```json
{
  "core_components": ["component1", "component2", ...],
  "stakeholder_perspectives": ["perspective1", "perspective2", ...],
  "in_scope": ["item1", "item2", ...],
  "out_of_scope": ["item1", "item2", ...],
  "success_criteria": ["criteria1", "criteria2", ...],
  "assumptions": ["assumption1", "assumption2", ...]
}
```

Use extended reasoning to explore multiple framings before finalizing scope.
""",
            ResearchPhase.PLAN: """
# Phase 2: PLAN

Your task: Create intelligent research roadmap

## Execute:
1. Identify 5-10 primary sources to investigate
2. List 5-10 secondary/backup sources
3. Map knowledge dependencies (what must be understood first)
4. Create 10-15 search query variations
5. Plan triangulation approach (how to verify claims)
6. Define 3-5 quality gates

## Output Format:
```json
{
  "primary_sources": ["source_type1", "source_type2", ...],
  "secondary_sources": ["source_type1", "source_type2", ...],
  "knowledge_dependencies": {"concept1": ["prerequisite1", "prerequisite2"], ...},
  "search_queries": ["query1", "query2", ...],
  "triangulation_strategy": "description of verification approach",
  "quality_gates": ["gate1", "gate2", ...]
}
```

Use Graph-of-Thoughts: branch into 3-4 potential research paths, evaluate, then converge on optimal strategy.
""",
            ResearchPhase.RETRIEVE: """
# Phase 3: RETRIEVE

Your task: Systematically collect information from multiple sources

## Execute:
1. Use WebSearch with iterative query refinement (minimum 10 searches)
2. Use WebFetch to deep-dive into 5-10 most promising sources
3. Extract key passages with metadata
4. Track information gaps
5. Follow 2-3 promising tangents
6. Ensure source diversity (different domains, perspectives)

## Tools to Use:
- WebSearch: For current information and broad coverage
- WebFetch: For detailed extraction from specific URLs
- Grep/Read: For local documentation if relevant
- Task: Spawn 2-3 parallel retrieval agents for efficiency

## Output:
Store all sources with metadata. Each source should include:
- URL/location
- Title
- Key excerpts
- Relevance score
- Source type
- Retrieved timestamp

Aim for 15-30 distinct sources minimum.
""",
            ResearchPhase.TRIANGULATE: """
# Phase 4: TRIANGULATE

Your task: Validate information across multiple independent sources

## Execute:
1. List all major claims from retrieved information
2. For each claim, find 3+ independent confirmatory sources
3. Flag any contradictions or uncertainties
4. Assess source credibility (domain expertise, recency, bias)
5. Document consensus areas vs. debate areas
6. Mark verification status for each claim

## Quality Standards:
- Core claims MUST have 3+ independent sources
- Flag any single-source claims as "unverified"
- Note information recency
- Identify potential biases

## Output Format:
```json
{
  "verified_claims": [
    {
      "claim": "statement",
      "sources": ["source1", "source2", "source3"],
      "confidence": "high|medium|low"
    }
  ],
  "unverified_claims": [...],
  "contradictions": [
    {
      "topic": "what's contradicted",
      "viewpoint1": {"claim": "...", "sources": [...]},
      "viewpoint2": {"claim": "...", "sources": [...]}
    }
  ]
}
```
""",
            ResearchPhase.SYNTHESIZE: """
# Phase 5: SYNTHESIZE

Your task: Connect insights and generate novel understanding

## Execute:
1. Identify 5-10 key patterns across sources
2. Map relationships between concepts
3. Generate 3-5 insights that go beyond source material
4. Create conceptual frameworks or mental models
5. Build argument structures
6. Develop evidence hierarchies

## Use Extended Reasoning:
- Explore non-obvious connections
- Consider second-order implications
- Think about what sources might be missing
- Generate novel hypotheses

## Output Format:
```json
{
  "patterns": ["pattern1", "pattern2", ...],
  "concept_relationships": {"concept1": ["related_to1", "related_to2"], ...},
  "novel_insights": ["insight1", "insight2", ...],
  "frameworks": ["framework_description1", ...],
  "key_arguments": [
    {
      "argument": "main claim",
      "supporting_evidence": ["evidence1", "evidence2"],
      "strength": "strong|moderate|weak"
    }
  ]
}
```
""",
            ResearchPhase.CRITIQUE: """
# Phase 6: CRITIQUE

Your task: Rigorously evaluate research quality

## Execute Red Team Analysis:
1. Check logical consistency
2. Verify citation completeness
3. Identify gaps or weaknesses
4. Assess balance and objectivity
5. Test alternative interpretations
6. Challenge assumptions

## Red Team Questions:
- What's missing from this research?
- What could be wrong?
- What alternative explanations exist?
- What biases might be present?
- What counterfactuals should be considered?
- What would a skeptic say?

## Output Format:
```json
{
  "strengths": ["strength1", "strength2", ...],
  "weaknesses": ["weakness1", "weakness2", ...],
  "gaps": ["gap1", "gap2", ...],
  "biases": ["bias1", "bias2", ...],
  "improvements_needed": [
    {
      "issue": "description",
      "recommendation": "how to fix",
      "priority": "high|medium|low"
    }
  ]
}
```
""",
            ResearchPhase.REFINE: """
# Phase 7: REFINE

Your task: Address gaps and strengthen weak areas

## Execute:
1. Conduct additional research for identified gaps
2. Strengthen weak arguments with more evidence
3. Add missing perspectives
4. Resolve contradictions where possible
5. Enhance clarity and structure
6. Verify all revised content

## Focus On:
- High priority improvements from critique
- Missing stakeholder perspectives
- Weak evidence chains
- Unclear explanations

## Output:
Updated findings, sources, and synthesis with improvements documented.
""",
            ResearchPhase.PACKAGE: """
# Phase 8: PACKAGE

Your task: Deliver professional, actionable research report

## Generate Complete Report:

```markdown
# Research Report: [Topic]

## Executive Summary
[3-5 key findings bullets]
[Primary recommendation]
[Confidence level: High/Medium/Low]

## Introduction
### Research Question
[Original question]

### Scope & Methodology
[What was investigated and how]

### Key Assumptions
[Important assumptions made]

## Main Analysis

### Finding 1: [Title]
[Detailed explanation with evidence]
[Citations: [1], [2], [3]]

### Finding 2: [Title]
[Detailed explanation with evidence]
[Citations: [4], [5], [6]]

[Continue for all findings...]

## Synthesis & Insights
[Patterns and connections]
[Novel insights]
[Implications]

## Limitations & Caveats
[Known gaps]
[Assumptions]
[Areas of uncertainty]

## Recommendations
[Action items]
[Next steps]
[Further research needs]

## Bibliography
[1] Source 1 full citation
[2] Source 2 full citation
...

## Appendix: Methodology
[Research process]
[Sources consulted]
[Verification approach]
```

Save report to file with timestamp.
"""
        }

        return instructions.get(phase, "No instructions available for this phase")

    def execute_phase(self, phase: ResearchPhase) -> Dict[str, Any]:
        """Execute a research phase"""
        print(f"\n{'='*80}")
        print(f"PHASE {phase.value.upper()}: Starting...")
        print(f"{'='*80}\n")

        instructions = self.get_phase_instructions(phase)
        print(instructions)

        # In real usage, Claude will execute these instructions
        # This returns a structured result that Claude should populate
        result = {
            'phase': phase.value,
            'status': 'instructions_displayed',
            'timestamp': datetime.now().isoformat()
        }

        return result

    def run_pipeline(self, query: str) -> str:
        """Run complete research pipeline"""
        print(f"\n{'#'*80}")
        print(f"# DEEP RESEARCH ENGINE")
        print(f"# Query: {query}")
        print(f"# Mode: {self.mode.value}")
        print(f"{'#'*80}\n")

        # Initialize research
        self.initialize_research(query)

        # Determine phases based on mode
        phases = self._get_phases_for_mode()

        # Execute each phase
        for phase in phases:
            self.state.phase = phase
            result = self.execute_phase(phase)

            # Save state after each phase
            state_file = self.output_dir / f"research_state_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
            self.state.save(state_file)
            print(f"\n✓ Phase {phase.value} complete. State saved to: {state_file}\n")

        # Generate report path
        report_file = self.output_dir / f"research_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.md"

        print(f"\n{'='*80}")
        print(f"RESEARCH PIPELINE COMPLETE")
        print(f"Report will be saved to: {report_file}")
        print(f"{'='*80}\n")

        return str(report_file)

    def _get_phases_for_mode(self) -> List[ResearchPhase]:
        """Get phases based on research mode"""
        if self.mode == ResearchMode.QUICK:
            return [
                ResearchPhase.SCOPE,
                ResearchPhase.RETRIEVE,
                ResearchPhase.PACKAGE
            ]
        elif self.mode == ResearchMode.STANDARD:
            return [
                ResearchPhase.SCOPE,
                ResearchPhase.PLAN,
                ResearchPhase.RETRIEVE,
                ResearchPhase.TRIANGULATE,
                ResearchPhase.SYNTHESIZE,
                ResearchPhase.PACKAGE
            ]
        elif self.mode == ResearchMode.DEEP:
            return list(ResearchPhase)
        elif self.mode == ResearchMode.ULTRADEEP:
            # In ultradeep, we might iterate some phases
            return list(ResearchPhase)

        return list(ResearchPhase)


def main():
    """CLI entry point"""
    parser = argparse.ArgumentParser(
        description="Deep Research Engine for Claude Code",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python research_engine.py --query "state of quantum computing 2025" --mode deep
  python research_engine.py --query "PostgreSQL vs Supabase comparison" --mode standard
  python research_engine.py -q "longevity biotech funding trends" -m ultradeep
        """
    )

    parser.add_argument(
        '--query', '-q',
        type=str,
        required=True,
        help='Research question or topic'
    )

    parser.add_argument(
        '--mode', '-m',
        type=str,
        choices=['quick', 'standard', 'deep', 'ultradeep'],
        default='standard',
        help='Research depth mode (default: standard)'
    )

    parser.add_argument(
        '--resume',
        type=str,
        help='Resume from saved state file'
    )

    args = parser.parse_args()

    # Initialize engine
    mode = ResearchMode(args.mode)
    engine = ResearchEngine(mode=mode)

    if args.resume:
        # Load previous state
        state_file = Path(args.resume)
        if not state_file.exists():
            print(f"Error: State file not found: {state_file}", file=sys.stderr)
            sys.exit(1)
        engine.state = ResearchState.load(state_file)
        print(f"Resumed research from: {state_file}")

    # Run pipeline
    report_path = engine.run_pipeline(args.query)

    print(f"\nResearch complete! Report path: {report_path}")
    print(f"\nNow Claude should execute each phase using the displayed instructions.")


if __name__ == '__main__':
    main()