Agent Research Aggregator

#!/usr/bin/env python3
"""
discover_logs.py — Phase 1 of agent-research-aggregator.

Scans known AI agent cache directories (.claude, .cursor, .antigravity,
.openclaw) plus general project files for experimentation logs.

Outputs a JSON manifest that downstream scripts and LLM calls use to decide
which files to read.

Usage:
    python discover_logs.py \\
        --search-roots . ~ \\
        --agents claude,cursor,antigravity,openclaw \\
        --depth 4 \\
        --since 2025-01-01 \\
        --out workspace/ara/discovered_logs.json
"""

import argparse
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path

# ---------------------------------------------------------------------------
# Config: per-agent directory names + file glob patterns
# ---------------------------------------------------------------------------

AGENT_SPECS = {
    "claude": {
        "cache_dirs": [".claude"],
        "global_dirs": [os.path.expanduser("~/.claude")],
        "patterns": [
            "memory/**/*.md",
            "projects/*/memory/**/*.md",
            "task-outputs/**/*.md",
            "task-outputs/**/*.txt",
            "todos/**/*.json",
        ],
        "root_files": ["CLAUDE.md"],
        "priority_dirs": ["memory"],
    },
    "cursor": {
        "cache_dirs": [".cursor"],
        "global_dirs": [
            os.path.expanduser("~/.cursor/User/globalStorage"),
        ],
        "patterns": [
            "chat/**/*.json",
            "chat/**/*.chat",
            "rules/**/*.md",
            "notes/**/*.md",
        ],
        "root_files": [".cursorrules"],
        "priority_dirs": ["chat"],
    },
    "antigravity": {
        "cache_dirs": [".antigravity"],
        "global_dirs": [],
        "patterns": [
            "workers/**/output.md",
            "workers/**/task.json",
            "workers/**/log.jsonl",
            "tasks/**/*.json",
            "task-registry.json",
        ],
        "root_files": [],
        "priority_dirs": ["workers"],
    },
    "openclaw": {
        "cache_dirs": [".openclaw"],
        "global_dirs": [],
        "patterns": [
            "sessions/**/conversation.md",
            "sessions/**/artifacts/**/*.json",
            "memory/**/*.md",
            "runs/**/stdout.log",
            "runs/**/metrics.json",
        ],
        "root_files": [],
        "priority_dirs": ["sessions", "memory"],
    },
}

# General project file patterns (agent-agnostic)
GENERAL_PATTERNS = [
    ("results*.json", "HIGH"),
    ("results*.csv", "HIGH"),
    ("results*.tsv", "HIGH"),
    ("experiments*.json", "HIGH"),
    ("experiments*.yaml", "HIGH"),
    ("metrics.json", "HIGH"),
    ("eval.json", "HIGH"),
    ("ablation*.md", "HIGH"),
    ("ablation*.json", "HIGH"),
    ("*.ipynb", "HIGH"),
    ("run_*.log", "HIGH"),
    ("train_*.log", "HIGH"),
    ("README.md", "MEDIUM"),
    ("notes*.md", "MEDIUM"),
    ("NOTES.md", "MEDIUM"),
    ("config*.yaml", "MEDIUM"),
    ("config*.json", "MEDIUM"),
    ("config*.toml", "MEDIUM"),
    ("*.log", "LOW"),
]

SKIP_DIRS = {
    "node_modules", "__pycache__", ".git", ".tox", ".venv", "venv",
    "env", ".mypy_cache", ".pytest_cache", "dist", "build", "target",
    "site-packages", ".cargo",
}

SKIP_EXTENSIONS = {
    ".pyc", ".pyo", ".so", ".dylib", ".dll", ".exe", ".bin",
    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".svg",
    ".pdf", ".zip", ".tar", ".gz", ".bz2", ".xz", ".7z",
    ".pem", ".key", ".p12", ".pfx", ".crt", ".cer",
    ".db", ".sqlite",  # SQLite noted separately; too risky to include wholesale
}

SKIP_NAMES = {
    ".env", ".env.local", ".env.production", "credentials.json",
    "secrets.json", "token.json",
}

MAX_FILE_BYTES = 200 * 1024  # 200 KB


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def is_binary(path: Path) -> bool:
    try:
        with path.open("rb") as f:
            chunk = f.read(512)
        return b"\x00" in chunk
    except OSError:
        return True


def modified_after(path: Path, since: datetime | None) -> bool:
    if since is None:
        return True
    mtime = datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc)
    return mtime >= since


def file_entry(path: Path, agent: str, priority: str, since: datetime | None) -> dict | None:
    """Build a manifest entry for a single file, or return None to skip."""
    if path.name in SKIP_NAMES:
        return None
    if path.suffix.lower() in SKIP_EXTENSIONS:
        return None
    if not path.is_file():
        return None
    if is_binary(path):
        return None
    try:
        size = path.stat().st_size
        mtime = path.stat().st_mtime
    except OSError:
        return None
    if not modified_after(path, since):
        return None
    return {
        "path": str(path),
        "agent": agent,
        "priority": priority,
        "size_bytes": size,
        "modified_iso": datetime.fromtimestamp(mtime, tz=timezone.utc).isoformat(),
        "truncated": size > MAX_FILE_BYTES,
    }


def scan_dir_glob(base: Path, pattern: str, agent: str,
                  priority: str, depth: int, since: datetime | None) -> list[dict]:
    """Glob-expand a pattern relative to base, respecting depth and skip rules."""
    results = []
    try:
        for path in base.glob(pattern):
            # Depth check: count path components relative to base
            rel = path.relative_to(base)
            if len(rel.parts) > depth:
                continue
            # Skip known junk directories anywhere in the path
            if any(part in SKIP_DIRS for part in rel.parts):
                continue
            entry = file_entry(path, agent, priority, since)
            if entry:
                results.append(entry)
    except (PermissionError, OSError):
        pass
    return results


def scan_root_files(base: Path, names: list[str], agent: str,
                    since: datetime | None) -> list[dict]:
    results = []
    for name in names:
        path = base / name
        entry = file_entry(path, agent, "HIGH", since)
        if entry:
            results.append(entry)
    return results


def scan_general(base: Path, depth: int, since: datetime | None) -> list[dict]:
    results = []
    for pattern, priority in GENERAL_PATTERNS:
        try:
            for path in base.glob(pattern):
                rel = path.relative_to(base)
                if len(rel.parts) > depth:
                    continue
                if any(part in SKIP_DIRS for part in rel.parts):
                    continue
                entry = file_entry(path, "general", priority, since)
                if entry:
                    results.append(entry)
        except (PermissionError, OSError):
            pass
    return results


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def decode_claude_project_path(dir_name: str) -> str | None:
    """
    Claude Code encodes project paths as directory names by replacing '/' with '-'.
    e.g. '-home-user-projects-myproject' → '/home/user/projects/myproject'
    Returns None if it doesn't look like an encoded path.
    """
    if not dir_name.startswith("-"):
        return None
    # Replace leading '-' then swap remaining '-' that correspond to path separators.
    # The heuristic: a segment starting with '-' followed by a lowercase letter or
    # digit is likely an encoded absolute path.
    decoded = dir_name.replace("-", "/")
    if decoded.startswith("/") and len(decoded) > 2:
        return decoded
    return None


def infer_project(path: Path, search_root: Path, agent: str) -> str:
    """
    Infer a human-readable project label for a file.

    For Claude Code logs stored under ~/.claude/projects/<encoded-path>/,
    decode the project directory name back to the original path.
    For all other files, use the search root name or a top-level subdirectory.
    """
    try:
        rel = path.relative_to(Path.home() / ".claude" / "projects")
        top = rel.parts[0] if rel.parts else ""
        decoded = decode_claude_project_path(top)
        if decoded:
            return decoded
    except ValueError:
        pass

    # For files inside a search root, use the root itself as the project label
    try:
        path.relative_to(search_root)
        return str(search_root)
    except ValueError:
        pass

    return str(path.parent)


def main():
    parser = argparse.ArgumentParser(description="Discover AI agent experiment logs")
    parser.add_argument("--search-roots", default=".", help="Comma-separated root dirs to scan")
    parser.add_argument("--agents", default="claude,cursor,antigravity,openclaw",
                        help="Comma-separated agent types to scan")
    parser.add_argument("--depth", type=int, default=4, help="Max directory depth")
    parser.add_argument("--since", default=None,
                        help="ISO 8601 date; only include files modified after this")
    parser.add_argument("--out", required=True, help="Output JSON path")
    parser.add_argument("--project", default=None,
                        help="Filter to files belonging to this project label only "
                             "(must match a label from a prior discovery run)")
    args = parser.parse_args()

    search_roots = [Path(r.strip()).expanduser().resolve()
                    for r in args.search_roots.split(",")]
    enabled_agents = [a.strip() for a in args.agents.split(",")]
    since_dt: datetime | None = None
    if args.since:
        since_dt = datetime.fromisoformat(args.since).replace(tzinfo=timezone.utc)

    all_entries: list[dict] = []
    agent_counts: dict[str, int] = {}

    for root in search_roots:
        if not root.exists():
            print(f"[WARN] search root does not exist: {root}", file=sys.stderr)
            continue

        # --- Agent-specific scans ---
        for agent, spec in AGENT_SPECS.items():
            if agent not in enabled_agents:
                continue
            agent_counts.setdefault(agent, 0)

            dirs_to_scan: list[Path] = []
            for cache_dir in spec["cache_dirs"]:
                candidate = root / cache_dir
                if candidate.exists():
                    dirs_to_scan.append(candidate)
            for global_dir in spec["global_dirs"]:
                candidate = Path(global_dir)
                if candidate.exists():
                    dirs_to_scan.append(candidate)

            for base in dirs_to_scan:
                for pattern in spec["patterns"]:
                    priority = "HIGH" if any(
                        p in pattern for p in spec.get("priority_dirs", [])
                    ) else "MEDIUM"
                    entries = scan_dir_glob(base, pattern, agent, priority,
                                            args.depth, since_dt)
                    for e in entries:
                        e["project"] = infer_project(Path(e["path"]), root, agent)
                    all_entries.extend(entries)
                    agent_counts[agent] += len(entries)

            # Root-level files (e.g. CLAUDE.md, .cursorrules)
            entries = scan_root_files(root, spec["root_files"], agent, since_dt)
            for e in entries:
                e["project"] = infer_project(Path(e["path"]), root, agent)
            all_entries.extend(entries)
            agent_counts[agent] += len(entries)

        # --- General project file scan ---
        entries = scan_general(root, args.depth, since_dt)
        for e in entries:
            e["project"] = infer_project(Path(e["path"]), root, "general")
        all_entries.extend(entries)
        agent_counts["general"] = agent_counts.get("general", 0) + len(entries)

    # Deduplicate by path (a file might match multiple patterns)
    seen_paths: set[str] = set()
    deduped: list[dict] = []
    for e in all_entries:
        if e["path"] not in seen_paths:
            seen_paths.add(e["path"])
            deduped.append(e)

    # Sort: HIGH > MEDIUM > LOW, then by modified desc
    priority_order = {"HIGH": 0, "MEDIUM": 1, "LOW": 2}
    deduped.sort(key=lambda e: (priority_order.get(e["priority"], 9),
                                 -Path(e["path"]).stat().st_mtime
                                 if Path(e["path"]).exists() else 0))

    # Build project → file list index (before optional filtering)
    by_project: dict[str, list[str]] = {}
    for e in deduped:
        proj = e.get("project", "unknown")
        by_project.setdefault(proj, []).append(e["path"])

    # Apply --project filter if requested
    if args.project:
        filtered = [e for e in deduped if e.get("project") == args.project]
        if not filtered:
            print(f"[ERROR] No files matched project '{args.project}'.", file=sys.stderr)
            print("Available projects:", file=sys.stderr)
            for proj in sorted(by_project):
                print(f"  {proj}", file=sys.stderr)
            sys.exit(1)
        deduped = filtered

    # Build output manifest
    manifest = {
        "generated_at": datetime.now(tz=timezone.utc).isoformat(),
        "search_roots": [str(r) for r in search_roots],
        "agents_scanned": enabled_agents,
        "since": args.since,
        "depth": args.depth,
        "selected_project": args.project,
        "total_files": len(deduped),
        "total_size_bytes": sum(e["size_bytes"] for e in deduped),
        "by_agent": agent_counts,
        "by_project": {p: len(paths) for p, paths in by_project.items()},
        "files": deduped,
    }

    out_path = Path(args.out)
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with out_path.open("w", encoding="utf-8") as f:
        json.dump(manifest, f, indent=2)

    # Human-readable summary to stdout
    print(f"\n=== Agent Log Discovery Summary ===")
    print(f"Search roots : {', '.join(str(r) for r in search_roots)}")
    print(f"Agents       : {', '.join(enabled_agents)}")
    print(f"Depth        : {args.depth}")
    print(f"Since        : {args.since or 'all time'}")
    print()

    # Always show the project list — this is the key output for project selection
    print("Projects found:")
    for i, (proj, paths) in enumerate(sorted(by_project.items()), 1):
        marker = " ◀ selected" if args.project and proj == args.project else ""
        print(f"  [{i}] {proj}  ({len(paths)} files){marker}")
    print()

    if args.project:
        print(f"Filtered to project: {args.project}")
    else:
        print("[ACTION REQUIRED] Select a project before proceeding to Phase 2.")
        print("Re-run with --project <label> to filter to one project.")
    print()

    print(f"Total files  : {len(deduped)}")
    print(f"Total size   : {sum(e['size_bytes'] for e in deduped) / 1024:.1f} KB")
    print()
    print("By agent:")
    for agent, count in sorted(agent_counts.items()):
        print(f"  {agent:20s} {count:4d} files")
    print()
    print("Priority breakdown:")
    for prio in ("HIGH", "MEDIUM", "LOW"):
        n = sum(1 for e in deduped if e["priority"] == prio)
        print(f"  {prio:8s} {n:4d} files")
    truncated = [e for e in deduped if e.get("truncated")]
    if truncated:
        print(f"\n[WARN] {len(truncated)} file(s) exceed 200 KB and will be truncated:")
        for e in truncated[:5]:
            print(f"  {e['path']} ({e['size_bytes']//1024} KB)")
        if len(truncated) > 5:
            print(f"  ... and {len(truncated)-5} more")
    print(f"\nManifest written to: {args.out}")
    if not args.project:
        # Exit with code 2 to signal to the caller that project selection is needed
        sys.exit(2)
    if len(deduped) > 50:
        print(f"\n[ACTION REQUIRED] {len(deduped)} files found — review the manifest")
        print("and confirm with the user before proceeding to Phase 2.")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
extract_experiments.py — Phase 2 helper for agent-research-aggregator.

This script has two modes:

1. BATCH MODE (called by the host agent to prepare batches):
   Reads discovered_logs.json, groups files into batches under a size budget,
   and prints the list of file paths per batch so the host LLM knows what to
   read and extract.

   python extract_experiments.py \\
       --discovered workspace/ara/discovered_logs.json \\
       --list-batches \\
       --batch-bytes 40000

2. VALIDATE MODE (called after host has done all LLM extraction calls):
   Reads raw_experiments.json produced by the host and validates it meets
   the minimum schema before Phase 3.

   python extract_experiments.py \\
       --discovered workspace/ara/discovered_logs.json \\
       --out workspace/ara/raw_experiments.json \\
       --validate-only

Usage for validate mode:
    python extract_experiments.py \\
        --out workspace/ara/raw_experiments.json \\
        --validate-only
"""

import argparse
import json
import sys
from pathlib import Path


REQUIRED_TOP_KEYS = {"experiments"}
EXPERIMENT_REQUIRED = {"experiment_id", "confidence"}
EXPERIMENT_ONE_OF = {"hypothesis", "method", "results", "research_question"}

VALID_CONFIDENCE = {"high", "medium", "low"}


# ---------------------------------------------------------------------------
# Batch listing (for host agent to know what to read)
# ---------------------------------------------------------------------------

def list_batches(discovered_path: str, batch_bytes: int):
    with open(discovered_path, encoding="utf-8") as f:
        manifest = json.load(f)

    files = manifest.get("files", [])
    if not files:
        print("No files in manifest.", file=sys.stderr)
        sys.exit(1)

    batches: list[list[dict]] = []
    current_batch: list[dict] = []
    current_size = 0

    for entry in files:
        size = min(entry["size_bytes"], 200 * 1024)  # cap at truncation limit
        if current_batch and current_size + size > batch_bytes:
            batches.append(current_batch)
            current_batch = []
            current_size = 0
        current_batch.append(entry)
        current_size += size

    if current_batch:
        batches.append(current_batch)

    print(f"Total batches: {len(batches)}")
    print(f"Total files  : {len(files)}")
    print()
    for i, batch in enumerate(batches, 1):
        total = sum(min(e["size_bytes"], 200*1024) for e in batch)
        print(f"--- Batch {i} ({len(batch)} files, ~{total//1024} KB) ---")
        for entry in batch:
            trunc = " [TRUNCATED]" if entry.get("truncated") else ""
            print(f"  [{entry['priority']:6}] [{entry['agent']:12}] {entry['path']}{trunc}")
        print()


# ---------------------------------------------------------------------------
# Validation
# ---------------------------------------------------------------------------

def validate_experiments(out_path: str) -> bool:
    path = Path(out_path)
    if not path.exists():
        print(f"[ERROR] File not found: {out_path}", file=sys.stderr)
        return False

    try:
        with path.open(encoding="utf-8") as f:
            data = json.load(f)
    except json.JSONDecodeError as e:
        print(f"[ERROR] Invalid JSON: {e}", file=sys.stderr)
        return False

    # Top-level structure
    missing_top = REQUIRED_TOP_KEYS - set(data.keys())
    if missing_top:
        print(f"[ERROR] Missing top-level keys: {missing_top}", file=sys.stderr)
        return False

    experiments = data["experiments"]
    if not isinstance(experiments, list):
        print("[ERROR] 'experiments' must be a list", file=sys.stderr)
        return False

    if len(experiments) == 0:
        print("[WARN] 'experiments' array is empty — no extractable data found.")
        print("This may be correct if logs contained no experiment data.")
        print("Proceeding to Phase 3 with empty input is allowed but will")
        print("produce a synthesis with no results — confirm with user.")

    errors = []
    warnings = []

    for i, exp in enumerate(experiments):
        label = exp.get("experiment_id", f"[index {i}]")

        # Required keys
        for key in EXPERIMENT_REQUIRED:
            if key not in exp:
                errors.append(f"{label}: missing required key '{key}'")

        # At least one of these must be present
        if not any(k in exp for k in EXPERIMENT_ONE_OF):
            errors.append(f"{label}: must have at least one of {EXPERIMENT_ONE_OF}")

        # Confidence value
        conf = exp.get("confidence", "")
        if conf not in VALID_CONFIDENCE:
            errors.append(f"{label}: 'confidence' must be one of {VALID_CONFIDENCE}, got '{conf}'")

        # Results tables shape
        results = exp.get("results", {})
        if isinstance(results, dict):
            for j, table in enumerate(results.get("tables", [])):
                if not isinstance(table.get("headers"), list):
                    errors.append(f"{label}: results.tables[{j}].headers must be a list")
                if not isinstance(table.get("rows"), list):
                    errors.append(f"{label}: results.tables[{j}].rows must be a list")

        # Warn about low-confidence experiments with no numeric data
        if conf == "low":
            key_nums = results.get("key_numbers", []) if isinstance(results, dict) else []
            tables = results.get("tables", []) if isinstance(results, dict) else []
            if not key_nums and not tables:
                warnings.append(f"{label}: low confidence + no numeric data")

    for w in warnings:
        print(f"[WARN] {w}")

    if errors:
        for e in errors:
            print(f"[ERROR] {e}", file=sys.stderr)
        print(f"\nValidation FAILED: {len(errors)} error(s), {len(warnings)} warning(s)")
        return False

    print(f"Validation PASSED: {len(experiments)} experiment(s), {len(warnings)} warning(s)")
    high = sum(1 for e in experiments if e.get("confidence") == "high")
    med  = sum(1 for e in experiments if e.get("confidence") == "medium")
    low  = sum(1 for e in experiments if e.get("confidence") == "low")
    print(f"  Confidence: {high} high / {med} medium / {low} low")
    tables_total = sum(
        len(e.get("results", {}).get("tables", []))
        for e in experiments if isinstance(e.get("results"), dict)
    )
    print(f"  Result tables found: {tables_total}")
    return True


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    parser = argparse.ArgumentParser(description="Phase 2 helper: batch listing and validation")
    parser.add_argument("--discovered", default=None,
                        help="Path to discovered_logs.json (required for --list-batches)")
    parser.add_argument("--out", default=None,
                        help="Path to raw_experiments.json (required for --validate-only)")
    parser.add_argument("--list-batches", action="store_true",
                        help="Print batches of files for the host agent to process")
    parser.add_argument("--batch-bytes", type=int, default=40000,
                        help="Soft byte budget per LLM extraction batch (default: 40000)")
    parser.add_argument("--validate-only", action="store_true",
                        help="Validate raw_experiments.json (requires --out)")
    args = parser.parse_args()

    if args.list_batches:
        if not args.discovered:
            print("[ERROR] --list-batches requires --discovered", file=sys.stderr)
            sys.exit(1)
        list_batches(args.discovered, args.batch_bytes)
        sys.exit(0)

    if args.validate_only:
        if not args.out:
            print("[ERROR] --validate-only requires --out", file=sys.stderr)
            sys.exit(1)
        ok = validate_experiments(args.out)
        sys.exit(0 if ok else 1)

    parser.print_help()
    sys.exit(1)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
format_po_inputs.py — Phase 4 of agent-research-aggregator.

Converts synthesis.json (Phase 3 output) into PaperOrchestra-compatible input
files: idea.md (Sparse variant) and experimental_log.md.

Optionally writes an aggregation_report.md audit trail.

Usage:
    python format_po_inputs.py \\
        --synthesis workspace/ara/synthesis.json \\
        --out workspace/inputs/ \\
        --report workspace/ara/aggregation_report.md

    # Dry-run (print to stdout, don't write files):
    python format_po_inputs.py \\
        --synthesis workspace/ara/synthesis.json \\
        --out workspace/inputs/ \\
        --dry-run
"""

import argparse
import json
import sys
from datetime import datetime, timezone
from pathlib import Path


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def load_synthesis(path: str) -> dict:
    p = Path(path)
    if not p.exists():
        print(f"[ERROR] synthesis.json not found: {path}", file=sys.stderr)
        sys.exit(1)
    with p.open(encoding="utf-8") as f:
        data = json.load(f)
    return data


def rows_to_markdown_table(headers: list, rows: list) -> str:
    """Convert headers + rows to a GitHub-Flavored Markdown table."""
    col_widths = [len(str(h)) for h in headers]
    for row in rows:
        for i, cell in enumerate(row):
            if i < len(col_widths):
                col_widths[i] = max(col_widths[i], len(str(cell)))

    def fmt_row(cells):
        parts = []
        for i, cell in enumerate(cells):
            w = col_widths[i] if i < len(col_widths) else len(str(cell))
            parts.append(str(cell).ljust(w))
        return "| " + " | ".join(parts) + " |"

    separator = "| " + " | ".join("-" * w for w in col_widths) + " |"
    lines = [fmt_row(headers), separator]
    for row in rows:
        lines.append(fmt_row(row))
    return "\n".join(lines)


def pluralise(n: int, singular: str, plural: str | None = None) -> str:
    if plural is None:
        plural = singular + "s"
    return f"{n} {singular if n == 1 else plural}"


# ---------------------------------------------------------------------------
# idea.md (PaperOrchestra Sparse variant)
# ---------------------------------------------------------------------------

def build_idea_md(s: dict) -> str:
    lines = []

    # Derive a title from research_question (first sentence, capitalised)
    rq = s.get("research_question", "").strip()
    title = rq.split(".")[0].strip() if rq else "Synthesized Research"
    lines.append(f"# {title}")
    lines.append("")

    # Problem section
    lines.append("## Problem")
    lines.append("")
    lines.append(rq if rq else "_[Research question not detected in logs]_")
    lines.append("")

    # Hypothesis
    hyp = s.get("hypothesis", "").strip()
    lines.append("## Hypothesis")
    lines.append("")
    lines.append(hyp if hyp else "_[Hypothesis not detected in logs]_")
    lines.append("")

    # Method
    method = s.get("method_summary", "").strip()
    lines.append("## Method")
    lines.append("")
    lines.append(method if method else "_[Method not detected in logs]_")
    lines.append("")

    # Key contributions
    contributions = s.get("key_contributions", [])
    lines.append("## Key Contributions")
    lines.append("")
    if contributions:
        for c in contributions:
            lines.append(f"- {c}")
    else:
        lines.append("_[Contributions not detected in logs]_")
    lines.append("")

    # Open questions (optional)
    open_qs = s.get("open_questions", [])
    if open_qs:
        lines.append("## Open Questions")
        lines.append("")
        for q in open_qs:
            lines.append(f"- {q}")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# experimental_log.md (PaperOrchestra Experimental Log format)
# ---------------------------------------------------------------------------

def build_experimental_log_md(s: dict) -> str:
    lines = []
    lines.append("# Experimental Log")
    lines.append("")
    lines.append(
        "_Generated by agent-research-aggregator from AI agent cache logs._"
    )
    lines.append("")

    # Section 1: Experimental Setup
    lines.append("## 1. Experimental Setup")
    lines.append("")
    setup = s.get("experimental_setup", {})

    datasets = setup.get("datasets", [])
    if datasets:
        lines.append("**Datasets:**")
        for d in datasets:
            lines.append(f"- {d}")
        lines.append("")

    baselines = setup.get("baselines", [])
    if baselines:
        lines.append("**Baselines:**")
        for b in baselines:
            lines.append(f"- {b}")
        lines.append("")

    metrics = setup.get("metrics", [])
    if metrics:
        lines.append("**Metrics:**")
        for m in metrics:
            lines.append(f"- {m}")
        lines.append("")

    impl = setup.get("implementation", "").strip()
    if impl:
        lines.append("**Implementation:**")
        lines.append("")
        lines.append(impl)
        lines.append("")

    notes = setup.get("notes", "").strip()
    if notes:
        lines.append("**Notes:**")
        lines.append("")
        lines.append(notes)
        lines.append("")

    if not any([datasets, baselines, metrics, impl, notes]):
        lines.append("_[Experimental setup not detected in logs]_")
        lines.append("")

    # Section 2: Raw Numeric Data
    lines.append("## 2. Raw Numeric Data")
    lines.append("")

    tables = s.get("results_tables", [])
    if tables:
        for table in tables:
            title = table.get("title", "Results")
            headers = table.get("headers", [])
            rows = table.get("rows", [])
            confidence = table.get("confidence", "")

            lines.append(f"### {title}")
            if confidence in ("low", "medium"):
                lines.append(f"_[Data confidence: {confidence}]_")
            lines.append("")

            if headers and rows:
                lines.append(rows_to_markdown_table(headers, rows))
            elif headers:
                lines.append(rows_to_markdown_table(headers, []))
                lines.append("_[No rows found in logs]_")
            else:
                lines.append("_[Table structure not recovered]_")
            lines.append("")
    else:
        lines.append(
            "_[No numeric result tables detected in logs. "
            "PaperOrchestra's section-writing agent needs numeric data — "
            "consider adding result tables manually before running paper-orchestra.]_"
        )
        lines.append("")

    # Section 3: Qualitative Observations
    lines.append("## 3. Qualitative Observations")
    lines.append("")

    qual = s.get("qualitative_observations", "").strip()
    if qual:
        lines.append(qual)
        lines.append("")
    else:
        lines.append("_[Qualitative observations not detected in logs]_")
        lines.append("")

    # Iteration History subsection (if present)
    iterations = s.get("iteration_history", [])
    if iterations:
        lines.append("### Iteration History")
        lines.append("")
        for it in iterations:
            iter_id = it.get("iteration_id", "")
            desc = it.get("description", "")
            outcome = it.get("outcome", "")
            lines.append(f"**{iter_id}:** {desc}")
            if outcome:
                lines.append(f"  - Outcome: {outcome}")
        lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Aggregation report
# ---------------------------------------------------------------------------

def build_report_md(s: dict, idea_path: Path, log_path: Path,
                    discovered_path: Path | None) -> str:
    now = datetime.now(tz=timezone.utc).isoformat()
    lines = []
    lines.append("# Aggregation Report")
    lines.append("")
    lines.append(f"_Generated at: {now}_")
    lines.append("")

    # Files written
    lines.append("## Files Written")
    lines.append("")
    lines.append(f"- `{idea_path}` ({idea_path.stat().st_size if idea_path.exists() else '?'} bytes)")
    lines.append(f"- `{log_path}` ({log_path.stat().st_size if log_path.exists() else '?'} bytes)")
    lines.append("")

    # Synthesis summary
    lines.append("## Synthesis Summary")
    lines.append("")
    tables = s.get("results_tables", [])
    iters = s.get("iteration_history", [])
    open_qs = s.get("open_questions", [])
    warnings = s.get("data_quality_warnings", [])

    lines.append(f"- Research question detected: {'yes' if s.get('research_question') else 'no'}")
    lines.append(f"- Hypothesis detected: {'yes' if s.get('hypothesis') else 'no'}")
    lines.append(f"- Result tables: {len(tables)}")
    lines.append(f"- Iterations detected: {len(iters)}")
    lines.append(f"- Open questions: {len(open_qs)}")
    lines.append(f"- Conflicting research questions: {s.get('research_question_count', 1)}")
    lines.append("")

    # Data quality warnings
    if warnings:
        lines.append("## Data Quality Warnings")
        lines.append("")
        for w in warnings:
            lines.append(f"- {w}")
        lines.append("")
        lines.append(
            "_Review these warnings before running paper-orchestra. "
            "Tables with `[UNVERIFIED]` data may cause the section-writing "
            "agent to produce inaccurate numeric claims._"
        )
        lines.append("")

    # Table confidence summary
    if tables:
        lines.append("## Result Table Confidence")
        lines.append("")
        for t in tables:
            conf = t.get("confidence", "unknown")
            src = ", ".join(t.get("source_experiment_ids", []))
            lines.append(f"- **{t.get('title', 'Untitled')}**: {conf} (sources: {src or 'unknown'})")
        lines.append("")

    # Missing required PaperOrchestra inputs
    lines.append("## Next Steps for PaperOrchestra")
    lines.append("")
    lines.append("The following files are still required before running `paper-orchestra`:")
    lines.append("")
    lines.append("| File | Status |")
    lines.append("|---|---|")
    lines.append(f"| `workspace/inputs/idea.md` | ✓ generated (review recommended) |")
    lines.append(f"| `workspace/inputs/experimental_log.md` | ✓ generated (review recommended) |")
    lines.append(f"| `workspace/inputs/template.tex` | **MISSING — ask user to provide** |")
    lines.append(f"| `workspace/inputs/conference_guidelines.md` | **MISSING — ask user to provide** |")
    lines.append("")

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    parser = argparse.ArgumentParser(description="Format PaperOrchestra inputs from synthesis.json")
    parser.add_argument("--synthesis", required=True, help="Path to synthesis.json")
    parser.add_argument("--out", required=True, help="Output directory for idea.md + experimental_log.md")
    parser.add_argument("--report", default=None, help="Optional path for aggregation_report.md")
    parser.add_argument("--dry-run", action="store_true",
                        help="Print generated files to stdout instead of writing")
    parser.add_argument("--discovered", default=None,
                        help="Optional path to discovered_logs.json for richer report")
    args = parser.parse_args()

    s = load_synthesis(args.synthesis)

    # Warn if multiple research questions
    rq_count = s.get("research_question_count", 1)
    if rq_count > 1:
        print(
            f"[WARN] synthesis.json indicates {rq_count} distinct research questions.",
            file=sys.stderr,
        )
        print(
            "[WARN] The generated idea.md will contain all of them. "
            "Ask the user which to target before running paper-orchestra.",
            file=sys.stderr,
        )

    idea_content = build_idea_md(s)
    log_content = build_experimental_log_md(s)

    out_dir = Path(args.out)

    if args.dry_run:
        print("=== workspace/inputs/idea.md ===")
        print(idea_content)
        print("\n=== workspace/inputs/experimental_log.md ===")
        print(log_content)
        return

    out_dir.mkdir(parents=True, exist_ok=True)

    idea_path = out_dir / "idea.md"
    log_path = out_dir / "experimental_log.md"

    with idea_path.open("w", encoding="utf-8") as f:
        f.write(idea_content)
    print(f"Written: {idea_path}")

    with log_path.open("w", encoding="utf-8") as f:
        f.write(log_content)
    print(f"Written: {log_path}")

    # Report
    if args.report:
        discovered_path = Path(args.discovered) if args.discovered else None
        report_content = build_report_md(s, idea_path, log_path, discovered_path)
        report_path = Path(args.report)
        report_path.parent.mkdir(parents=True, exist_ok=True)
        with report_path.open("w", encoding="utf-8") as f:
            f.write(report_content)
        print(f"Written: {report_path}")

    # Final check: warn if no results tables
    if not s.get("results_tables"):
        print(
            "\n[WARN] No result tables were synthesized. "
            "PaperOrchestra's section-writing agent requires numeric data — "
            "consider adding result tables to experimental_log.md manually.",
            file=sys.stderr,
        )

    print("\nDone. Review idea.md and experimental_log.md with the user before")
    print("running paper-orchestra.")


if __name__ == "__main__":
    main()