Token Saver Context Compression

#!/usr/bin/env node
'use strict';

const fs = require('node:fs');
const path = require('node:path');
const { PROJECT_ROOT } = require('../../../lib/utils/project-root.cjs');

const REPORT_PATH = path.join(
  PROJECT_ROOT,
  '.claude',
  'context',
  'runtime',
  'token-saver-context-compression-last.json'
);

function parseInput() {
  try {
    return JSON.parse(process.argv[2] || '{}');
  } catch {
    return {};
  }
}

function main() {
  const payload = parseInput();
  fs.mkdirSync(path.dirname(REPORT_PATH), { recursive: true });
  fs.writeFileSync(
    REPORT_PATH,
    JSON.stringify(
      {
        timestamp: new Date().toISOString(),
        payload,
      },
      null,
      2
    ) + '\n',
    'utf8'
  );
  process.exit(0);
}

if (require.main === module) {
  main();
}

module.exports = { main };

#!/usr/bin/env node
'use strict';

function parseInput() {
  try {
    return JSON.parse(process.argv[2] || '{}');
  } catch {
    return {};
  }
}

function main() {
  const input = parseInput();
  const query = String(input.query || input.prompt || '').trim();
  if (!query) {
    console.error('token-saver-context-compression pre-execute: missing query');
    process.exit(1);
  }
  process.exit(0);
}

if (require.main === module) {
  main();
}

module.exports = { main };

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "token-saver-context-compression Input Schema",
  "type": "object",
  "required": ["query"],
  "properties": {
    "query": {
      "type": "string",
      "minLength": 3
    },
    "mode": {
      "type": "string",
      "enum": ["baseline", "query_guided", "evidence_aware"]
    },
    "limit": {
      "type": "integer",
      "minimum": 1,
      "maximum": 100
    },
    "failOnInsufficientEvidence": {
      "type": "boolean"
    },
    "persistFiles": {
      "type": "boolean",
      "description": "Local test utility path. Production persistence should use MemoryRecord."
    }
  },
  "additionalProperties": true
}

{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "token-saver-context-compression Output Schema",
  "type": "object",
  "required": ["ok"],
  "properties": {
    "ok": {
      "type": "boolean"
    },
    "search": {
      "type": "object",
      "properties": {
        "query": { "type": "string" },
        "hits": { "type": "integer" },
        "limit": { "type": "integer" }
      },
      "additionalProperties": true
    },
    "evidence": {
      "type": "object",
      "properties": {
        "sufficient": { "type": "boolean" }
      },
      "additionalProperties": true
    },
    "memoryRecords": {
      "type": "object",
      "properties": {
        "patterns": { "type": "array" },
        "gotchas": { "type": "array" },
        "issues": { "type": "array" },
        "decisions": { "type": "array" }
      },
      "additionalProperties": false
    },
    "error": {
      "type": "string"
    },
    "stage": {
      "type": "string"
    }
  },
  "additionalProperties": true
}

#!/usr/bin/env python3
"""Self-contained text compression and evidence scoring engine."""

from __future__ import annotations

import math
import re
from dataclasses import dataclass
from typing import Any, Dict, List

from _token_utils import count_tokens


SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+|\n+")
TOKEN_SPLIT = re.compile(r"[A-Za-z0-9_]+")


def _tokenize(text: str) -> List[str]:
    return [t.lower() for t in TOKEN_SPLIT.findall(text)]


def _jaccard(a: str, b: str) -> float:
    sa = set(_tokenize(a))
    sb = set(_tokenize(b))
    if not sa or not sb:
        return 0.0
    return len(sa & sb) / len(sa | sb)


def _split_segments(text: str) -> List[str]:
    segments = [s.strip() for s in SENTENCE_SPLIT.split(text) if s.strip()]
    return segments if segments else [text.strip()]


@dataclass
class CompressionResult:
    mode: str
    original_text: str
    compressed_text: str
    segments: List[Dict[str, Any]]
    original_tokens: int
    compressed_tokens: int
    compression_ratio: float
    token_savings_pct: float


def _score_segment(segment: str, query: str | None, idx: int) -> float:
    # Stable deterministic blend: relevance + light position prior.
    relevance = _jaccard(segment, query or "")
    position = 1.0 / (1.0 + math.log2(idx + 2))
    return 0.75 * relevance + 0.25 * position


def compress_text(
    text: str,
    mode: str = "baseline",
    query: str = "",
    skeleton_ratio: float = 0.2,
    top_k: int = 5,
) -> CompressionResult:
    segments = _split_segments(text)
    target_keep = max(1, int(round(len(segments) * max(0.05, min(0.95, skeleton_ratio)))))

    segment_rows: List[Dict[str, Any]] = []
    for idx, seg in enumerate(segments):
        score = _score_segment(seg, query if mode != "baseline" else "", idx)
        segment_rows.append(
            {
                "segment_id": idx,
                "score": round(score, 4),
                "tokens": count_tokens(seg),
                "text": seg,
            }
        )

    ranked = sorted(segment_rows, key=lambda r: (r["score"], -r["segment_id"]), reverse=True)
    if mode == "baseline":
        chosen = ranked[:target_keep]
    elif mode == "query_guided":
        chosen = ranked[: max(target_keep, min(top_k, len(ranked)))]
    else:  # evidence_aware
        chosen = ranked[: max(target_keep, top_k)]

    chosen_ids = {row["segment_id"] for row in chosen}
    ordered = [row for row in segment_rows if row["segment_id"] in chosen_ids]
    compressed_text = "\n".join(row["text"] for row in ordered)

    original_tokens = count_tokens(text)
    compressed_tokens = count_tokens(compressed_text)
    ratio = round(original_tokens / max(compressed_tokens, 1), 3)
    savings = round((1 - compressed_tokens / max(original_tokens, 1)) * 100, 2)

    for row in segment_rows:
        row["selected"] = row["segment_id"] in chosen_ids

    return CompressionResult(
        mode=mode,
        original_text=text,
        compressed_text=compressed_text,
        segments=segment_rows,
        original_tokens=original_tokens,
        compressed_tokens=compressed_tokens,
        compression_ratio=ratio,
        token_savings_pct=savings,
    )


def evaluate_evidence(
    compressed: CompressionResult,
    query: str,
    min_similarity: float = 0.35,
    top_k: int = 5,
) -> Dict[str, Any]:
    scored = []
    for row in compressed.segments:
        if not row["selected"]:
            continue
        sim = _jaccard(row["text"], query)
        scored.append(
            {
                "segment_id": row["segment_id"],
                "similarity": round(sim, 4),
                "text": row["text"],
            }
        )
    scored.sort(key=lambda r: r["similarity"], reverse=True)
    top = scored[:top_k]
    best = top[0]["similarity"] if top else 0.0
    sufficient = best >= min_similarity
    return {
        "query": query,
        "sufficient": sufficient,
        "best_score": round(best, 4),
        "threshold": min_similarity,
        "top_matches": top,
        "used_expanded_search": False,
        "message": "Evidence sufficient." if sufficient else "Evidence below threshold.",
    }

#!/usr/bin/env python3
"""Output format routing for json/toon/auto with safe fallback behavior."""

from __future__ import annotations

from typing import Any, Dict, Tuple

from _token_utils import compact_json, count_tokens
from _toon_codec import encode_toon, is_uniform_object_array


def _find_uniform_candidate(data: Any, min_rows: int) -> bool:
    if is_uniform_object_array(data, min_rows=min_rows):
        return True
    if isinstance(data, dict):
        for value in data.values():
            if _find_uniform_candidate(value, min_rows=min_rows):
                return True
    if isinstance(data, list):
        for value in data:
            if _find_uniform_candidate(value, min_rows=min_rows):
                return True
    return False


def render_output(
    payload: Dict[str, Any],
    output_format: str,
    auto_min_rows: int = 8,
) -> Tuple[str, str, Dict[str, Any]]:
    """
    Render payload as JSON or TOON.

    Returns tuple: (serialized_text, resolved_format, format_meta)
    """
    requested = output_format.lower()
    if requested not in {"json", "toon", "auto"}:
        requested = "json"

    if requested == "json":
        text = compact_json(payload)
        return text, "json", {"json_tokens": count_tokens(text)}

    if requested == "auto":
        resolved = "toon" if _find_uniform_candidate(payload, min_rows=auto_min_rows) else "json"
    else:
        resolved = "toon"

    if resolved == "toon":
        toon_text = encode_toon(payload)
        json_text = compact_json(payload)
        toon_tokens = count_tokens(toon_text)
        json_tokens = count_tokens(json_text)
        if toon_tokens <= json_tokens:
            savings = round((1 - toon_tokens / max(json_tokens, 1)) * 100, 2)
            return (
                toon_text,
                "toon",
                {
                    "toon_tokens": toon_tokens,
                    "json_tokens": json_tokens,
                    "token_savings_pct_vs_json": savings,
                },
            )
        # Safe fallback: TOON is not better for this payload.
        return (
            json_text,
            "json",
            {
                "fallback_reason": "toon_not_better_than_json",
                "toon_tokens": toon_tokens,
                "json_tokens": json_tokens,
                "token_savings_pct_vs_json": round(
                    (1 - toon_tokens / max(json_tokens, 1)) * 100, 2
                ),
            },
        )

    json_text = compact_json(payload)
    return json_text, "json", {"json_tokens": count_tokens(json_text)}

#!/usr/bin/env python3
"""Shared runtime helpers for portable, self-contained skill scripts."""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path
from typing import Any


def read_text_input(args: argparse.Namespace) -> str:
    """Read text from --text, --file, or stdin."""
    if getattr(args, "text", ""):
        return args.text
    if getattr(args, "file", None):
        return Path(args.file).read_text(encoding="utf-8")
    return sys.stdin.read()


def read_json_input(args: argparse.Namespace) -> Any:
    """Read JSON from --json, --json-file, or stdin."""
    if getattr(args, "json", ""):
        return json.loads(args.json)
    if getattr(args, "json_file", None):
        return json.loads(Path(args.json_file).read_text(encoding="utf-8"))
    raw = sys.stdin.read().strip()
    if not raw:
        raise ValueError("No JSON input provided.")
    return json.loads(raw)

#!/usr/bin/env python3
"""Token counting helpers with graceful fallback when tiktoken is unavailable."""

from __future__ import annotations

import re
from typing import Any


_WORD_OR_PUNCT = re.compile(r"\w+|[^\w\s]", re.UNICODE)


def count_tokens(text: str, encoding_name: str = "cl100k_base") -> int:
    """
    Count tokens using tiktoken when available.

    Falls back to a stable approximation for out-of-the-box portability.
    """
    try:
        import tiktoken  # type: ignore

        enc = tiktoken.get_encoding(encoding_name)
        return len(enc.encode(text))
    except Exception:
        # Conservative fallback approximation.
        return len(_WORD_OR_PUNCT.findall(text))


def compact_json(data: Any) -> str:
    """Compact JSON for token-efficient transmission."""
    import json

    return json.dumps(data, ensure_ascii=False, separators=(",", ":"))

#!/usr/bin/env python3
"""TOON encoding/decoding helpers for self-contained skill usage."""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any, Dict, List


Primitive = (str, int, float, bool, type(None))


def _is_primitive(value: Any) -> bool:
    return isinstance(value, Primitive)


def _escape_cell(value: Any) -> str:
    raw = "" if value is None else str(value)
    return raw.replace("\\", "\\\\").replace(",", "\\,").replace("\n", "\\n")


def _unescape_cell(value: str) -> str:
    out: List[str] = []
    i = 0
    while i < len(value):
        ch = value[i]
        if ch == "\\" and i + 1 < len(value):
            nxt = value[i + 1]
            if nxt == "n":
                out.append("\n")
            else:
                out.append(nxt)
            i += 2
        else:
            out.append(ch)
            i += 1
    return "".join(out)


def _split_escaped_csv(line: str) -> List[str]:
    cells: List[str] = []
    buf: List[str] = []
    escape = False
    for ch in line:
        if escape:
            if ch == "n":
                buf.append("\n")
            else:
                buf.append(ch)
            escape = False
            continue
        if ch == "\\":
            escape = True
            continue
        if ch == ",":
            cells.append("".join(buf))
            buf = []
            continue
        buf.append(ch)
    cells.append("".join(buf))
    return cells


def is_uniform_object_array(value: Any, min_rows: int = 3) -> bool:
    if not isinstance(value, list) or len(value) < min_rows:
        return False
    if not all(isinstance(item, dict) for item in value):
        return False

    first_keys = list(value[0].keys())
    if not first_keys:
        return False
    for item in value:
        if list(item.keys()) != first_keys:
            return False
        if not all(_is_primitive(v) for v in item.values()):
            return False
    return True


def _maybe_official_encode(data: Any) -> str | None:
    """Use official toon-format package when present."""
    try:
        from toon_format import dumps as toon_dumps  # type: ignore

        return toon_dumps(data)
    except Exception:
        return None


def _encode_scalar(value: Any) -> str:
    if isinstance(value, bool):
        return "true" if value else "false"
    if value is None:
        return "null"
    return str(value)


def encode_table(name: str, rows: List[Dict[str, Any]], indent: int = 0) -> str:
    if not rows:
        return f"{' ' * indent}{name}[0]{{}}:"
    fields = list(rows[0].keys())
    lines = [f"{' ' * indent}{name}[{len(rows)}]{{{','.join(fields)}}}:"]
    row_prefix = " " * (indent + 2)
    for row in rows:
        lines.append(row_prefix + ",".join(_escape_cell(row.get(field, "")) for field in fields))
    return "\n".join(lines)


def encode_toon(data: Any, root_name: str = "data", indent: int = 0) -> str:
    """Encode Python data to TOON-like text (lossless for supported structures)."""
    official = _maybe_official_encode(data)
    if official is not None:
        return official

    pad = " " * indent
    if isinstance(data, dict):
        lines: List[str] = []
        for key, value in data.items():
            if isinstance(value, dict):
                lines.append(f"{pad}{key}:")
                lines.append(encode_toon(value, root_name=key, indent=indent + 2))
            elif is_uniform_object_array(value, min_rows=1):
                lines.append(encode_table(key, value, indent=indent))
            elif isinstance(value, list) and all(_is_primitive(v) for v in value):
                lines.append(
                    f"{pad}{key}[{len(value)}]: " + ",".join(_escape_cell(v) for v in value)
                )
            elif isinstance(value, list):
                lines.append(f"{pad}{key}[{len(value)}]:")
                for item in value:
                    if isinstance(item, dict):
                        lines.append(f"{pad}  -")
                        lines.append(encode_toon(item, root_name=key, indent=indent + 4))
                    else:
                        lines.append(f"{pad}  - {_escape_cell(item)}")
            else:
                lines.append(f"{pad}{key}: {_encode_scalar(value)}")
        return "\n".join(lines)

    if is_uniform_object_array(data, min_rows=1):
        return encode_table(root_name, data, indent=indent)

    if isinstance(data, list) and all(_is_primitive(v) for v in data):
        return f"{pad}{root_name}[{len(data)}]: " + ",".join(_escape_cell(v) for v in data)

    return f"{pad}{root_name}: {_encode_scalar(data)}"


@dataclass
class DecodeResult:
    rows: List[Dict[str, str]]
    row_count: int
    fields: List[str]


def decode_table(table_toon: str) -> DecodeResult:
    """
    Decode a TOON table in the form: name[N]{field1,field2}:<rows>.

    This powers round-trip checks and retrieval benchmarks for uniform arrays.
    """
    lines = [ln.rstrip() for ln in table_toon.splitlines() if ln.strip()]
    if not lines:
        return DecodeResult(rows=[], row_count=0, fields=[])

    header = lines[0].strip()
    lb = header.find("[")
    rb = header.find("]")
    lcb = header.find("{")
    rcb = header.find("}")
    if min(lb, rb, lcb, rcb) == -1 or not header.endswith(":"):
        raise ValueError("Invalid TOON table header.")

    row_count = int(header[lb + 1 : rb])
    fields = header[lcb + 1 : rcb].split(",") if rcb > lcb + 1 else []

    rows: List[Dict[str, str]] = []
    for raw_line in lines[1:]:
        raw_line = raw_line.strip()
        if not raw_line:
            continue
        cells = _split_escaped_csv(raw_line)
        row = {
            field: _unescape_cell(cells[i]) if i < len(cells) else ""
            for i, field in enumerate(fields)
        }
        rows.append(row)

    return DecodeResult(rows=rows, row_count=row_count, fields=fields)

#!/usr/bin/env python3
"""Benchmark TOON vs JSON for token delta, round-trip, and retrieval accuracy."""

from __future__ import annotations

import argparse
import json
from dataclasses import dataclass
from typing import Any, Dict, List, Tuple

from _token_utils import compact_json, count_tokens
from _toon_codec import decode_table, encode_table, is_uniform_object_array
from _output_format import render_output


def _build_uniform_rows(n: int = 200) -> List[Dict[str, Any]]:
    return [
        {
            "id": i,
            "name": f"user_{i}",
            "role": "admin" if i % 7 == 0 else "user",
            "tier": "pro" if i % 5 == 0 else "free",
        }
        for i in range(1, n + 1)
    ]


def _build_mixed_payload() -> Dict[str, Any]:
    return {
        "meta": {"page": 1, "next": None, "ok": True},
        "items": [
            {"id": 1, "tags": ["a", "b"], "attrs": {"k": "v"}},
            {"id": 2, "tags": ["x"], "attrs": {"k2": "v2"}},
        ],
    }


def _qa_set(rows: List[Dict[str, Any]], k: int = 40) -> List[Tuple[int, str]]:
    out: List[Tuple[int, str]] = []
    for i in range(min(k, len(rows))):
        row = rows[i]
        out.append((int(row["id"]), str(row["role"])))
    return out


def _answer_from_rows(rows: List[Dict[str, Any]], row_id: int) -> str | None:
    target = str(row_id)
    for row in rows:
        if str(row.get("id")) == target:
            val = row.get("role")
            return None if val is None else str(val)
    return None


@dataclass
class BenchmarkSummary:
    dataset: str
    json_tokens: int
    toon_tokens: int | None
    token_savings_pct: float | None
    roundtrip_ok: bool | None
    retrieval_accuracy_json: float
    retrieval_accuracy_toon: float | None
    auto_selected_format: str

    def to_dict(self) -> Dict[str, Any]:
        return {
            "dataset": self.dataset,
            "json_tokens": self.json_tokens,
            "toon_tokens": self.toon_tokens,
            "token_savings_pct": self.token_savings_pct,
            "roundtrip_ok": self.roundtrip_ok,
            "retrieval_accuracy_json": self.retrieval_accuracy_json,
            "retrieval_accuracy_toon": self.retrieval_accuracy_toon,
            "auto_selected_format": self.auto_selected_format,
        }


def _accuracy(rows: List[Dict[str, Any]], qa: List[Tuple[int, str]]) -> float:
    correct = 0
    for row_id, gold in qa:
        pred = _answer_from_rows(rows, row_id)
        if pred == gold:
            correct += 1
    return round(correct / max(len(qa), 1), 4)


def benchmark_uniform() -> BenchmarkSummary:
    rows = _build_uniform_rows()
    qa = _qa_set(rows)
    payload = {"rows": rows}

    json_text = compact_json(payload)
    json_tokens = count_tokens(json_text)
    json_acc = _accuracy(rows, qa)

    toon_text = encode_table("rows", rows)
    toon_tokens = count_tokens(toon_text)
    savings = round((1 - toon_tokens / max(json_tokens, 1)) * 100, 2)

    decoded = decode_table(toon_text)
    decoded_rows = decoded.rows
    roundtrip_ok = len(decoded_rows) == len(rows) and all(
        str(a["id"]) == str(b["id"]) and str(a["role"]) == str(b["role"])
        for a, b in zip(decoded_rows, rows)
    )
    toon_acc = _accuracy(decoded_rows, qa)
    _, auto_selected_format, _ = render_output(payload, "auto", auto_min_rows=8)

    return BenchmarkSummary(
        dataset="uniform_rows",
        json_tokens=json_tokens,
        toon_tokens=toon_tokens,
        token_savings_pct=savings,
        roundtrip_ok=roundtrip_ok,
        retrieval_accuracy_json=json_acc,
        retrieval_accuracy_toon=toon_acc,
        auto_selected_format=auto_selected_format,
    )


def benchmark_mixed() -> BenchmarkSummary:
    data = _build_mixed_payload()
    json_text = compact_json(data)
    json_tokens = count_tokens(json_text)
    _, auto_selected_format, _ = render_output(data, "auto", auto_min_rows=8)

    # Mixed data isn't a TOON sweet spot. We record this explicitly.
    toon_tokens = None
    if is_uniform_object_array(data):  # always false, retained for completeness
        toon_tokens = count_tokens(encode_table("items", data))  # type: ignore[arg-type]

    return BenchmarkSummary(
        dataset="mixed_nested",
        json_tokens=json_tokens,
        toon_tokens=toon_tokens,
        token_savings_pct=None,
        roundtrip_ok=None,
        retrieval_accuracy_json=1.0,
        retrieval_accuracy_toon=None,
        auto_selected_format=auto_selected_format,
    )


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Benchmark TOON vs JSON within portable skill package."
    )
    parser.add_argument("--indent", type=int, default=2, help="JSON indent for benchmark output.")
    args = parser.parse_args()

    results = [benchmark_uniform().to_dict(), benchmark_mixed().to_dict()]
    summary = {
        "benchmarks": results,
        "guard": {
            "uniform_min_token_savings_pct": 20.0,
            "uniform_roundtrip_required": True,
            "uniform_min_retrieval_accuracy": 0.95,
            "uniform_auto_should_select": "toon",
            "mixed_auto_should_select": "json",
        },
    }

    # Apply simple guard checks.
    uniform = results[0]
    mixed = results[1]
    passes = (
        (uniform["token_savings_pct"] or 0) >= 20.0
        and uniform["roundtrip_ok"] is True
        and (uniform["retrieval_accuracy_toon"] or 0) >= 0.95
        and uniform["auto_selected_format"] == summary["guard"]["uniform_auto_should_select"]
        and mixed["auto_selected_format"] == summary["guard"]["mixed_auto_should_select"]
    )
    summary["guard"]["pass"] = passes

    print(json.dumps(summary, indent=args.indent))
    return 0 if passes else 1


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Compress context with baseline/query-guided/evidence-aware modes."""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

from _compression_engine import compress_text, evaluate_evidence
from _output_format import render_output
from _runtime import read_text_input


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Generate compressed context for Claude Skill workflows."
    )
    parser.add_argument("--text", type=str, default="", help="Inline text input.")
    parser.add_argument("--file", type=Path, help="Path to UTF-8 text file.")
    parser.add_argument(
        "--file-id", type=str, default="skill_context_doc", help="Document identifier."
    )
    parser.add_argument(
        "--mode",
        choices=["baseline", "query_guided", "evidence_aware"],
        default="baseline",
        help="Compression selection mode.",
    )
    parser.add_argument(
        "--query", type=str, default="", help="Query for guided/evidence-aware modes."
    )
    parser.add_argument("--top-k", type=int, default=5, help="Evidence top-k segment count.")
    parser.add_argument(
        "--min-similarity", type=float, default=0.35, help="Evidence sufficiency threshold."
    )
    parser.add_argument("--skeleton-ratio", type=float, default=0.2, help="Compression keep ratio.")
    parser.add_argument(
        "--output-format",
        choices=["json", "toon", "auto"],
        default="auto",
        help="Output format for the emitted payload.",
    )
    parser.add_argument(
        "--auto-min-rows",
        type=int,
        default=8,
        help="Minimum uniform rows before auto format selects TOON.",
    )
    return parser


def main() -> int:
    parser = build_parser()
    args = parser.parse_args()
    text = read_text_input(args)
    if not text.strip():
        print("No input text provided.", file=sys.stderr)
        return 2

    if args.mode != "baseline" and not args.query.strip():
        print("--query is required for query_guided and evidence_aware modes.", file=sys.stderr)
        return 2

    compressed = compress_text(
        text=text,
        mode=args.mode,
        query=args.query,
        skeleton_ratio=args.skeleton_ratio,
        top_k=args.top_k,
    )

    selected_segments = [row for row in compressed.segments if row["selected"]]
    payload = {
        "file_id": args.file_id,
        "mode": args.mode,
        "query": args.query or None,
        "original_tokens": compressed.original_tokens,
        "compressed_tokens": compressed.compressed_tokens,
        "compression_ratio": compressed.compression_ratio,
        "token_savings_pct": compressed.token_savings_pct,
        "compressed_text": compressed.compressed_text,
        "segments": selected_segments,
    }

    if args.mode == "evidence_aware":
        payload["evidence"] = evaluate_evidence(
            compressed=compressed,
            query=args.query,
            min_similarity=args.min_similarity,
            top_k=args.top_k,
        )

    rendered, resolved, meta = render_output(
        payload, args.output_format, auto_min_rows=args.auto_min_rows
    )
    if args.output_format == "toon" and resolved != "toon":
        payload_meta = {
            "requested_output_format": args.output_format,
            "resolved_output_format": resolved,
            **meta,
        }
        rendered, _, _ = render_output({**payload, "_format_meta": payload_meta}, "json")

    print(rendered)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env node
'use strict';

const fs = require('node:fs');
const path = require('node:path');
const { spawnSync } = require('node:child_process');

const { PROJECT_ROOT } = require('../../../lib/utils/project-root.cjs');

const RUNTIME_DIR = path.join(
  PROJECT_ROOT,
  '.claude',
  'context',
  'runtime',
  'token-saver-context-compression'
);

function parseArgs(argv) {
  const options = {};
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    if (!arg.startsWith('--')) continue;
    const key = arg.slice(2);
    const next = argv[i + 1];
    const hasValue = next && !next.startsWith('--');
    options[key] = hasValue ? argv[++i] : true;
  }
  return options;
}

function runCommand(cmd, args, cwd = PROJECT_ROOT) {
  return spawnSync(cmd, args, {
    cwd,
    encoding: 'utf8',
    windowsHide: true,
    shell: false,
    env: {
      ...process.env,
      PYTHONIOENCODING: 'utf-8', // Force UTF-8 on Windows (cp1252 breaks on unicode)
      PYTHONUTF8: '1', // Python 3.15+ UTF-8 mode
    },
  });
}

function runSearchQuery(run, query) {
  const pnpmResult = run('pnpm', ['search:code', '--', query], PROJECT_ROOT);
  if (pnpmResult.status === 0) return pnpmResult;

  const fallback = run(
    process.execPath,
    [path.join(PROJECT_ROOT, '.claude', 'tools', 'cli', 'hybrid-search.cjs'), query],
    PROJECT_ROOT
  );
  if (fallback.status === 0) return fallback;

  return {
    status: 1,
    stdout: fallback.stdout || pnpmResult.stdout || '',
    stderr: fallback.stderr || pnpmResult.stderr || '',
  };
}

function stripAnsi(str) {
  // Strip ANSI escape codes and common emoji byte sequences
  return str
    .replace(/\x1b\[[0-9;]*m/g, '') // ANSI color codes
    .replace(/[\u{1F300}-\u{1FAFF}]/gu, '') // Emoji unicode
    .replace(/[^\x20-\x7E\t]/g, '') // Non-printable
    .trim();
}

function normalizeSearchResults(rawText, limit) {
  const lines = String(rawText || '')
    .split(/\r?\n/)
    .map(line => stripAnsi(line))
    .filter(Boolean);

  const filePattern = /^\d+\.\s+(.+?)\s+\(\d+(\.\d+)?%\)$/;
  const hits = [];
  let current = null;
  for (const line of lines) {
    const match = line.match(filePattern);
    if (match) {
      current = { file: match[1], snippets: [] };
      hits.push(current);
      continue;
    }
    if (current && !line.startsWith('Search completed')) {
      current.snippets.push(line.replace(/^[-•]\s*/, '').trim());
    }
    if (hits.length >= limit) break;
  }
  return hits;
}

function flattenEvidenceStrings(value, bucket = []) {
  if (value == null) return bucket;
  if (typeof value === 'string') {
    const clean = value.trim();
    if (clean.length > 0) bucket.push(clean);
    return bucket;
  }
  if (Array.isArray(value)) {
    for (const item of value) flattenEvidenceStrings(item, bucket);
    return bucket;
  }
  if (typeof value === 'object') {
    // Python workflow output: extract compressed segments as evidence
    // Structure: { profile, compressed: { compressed_text, segments: [{ text }] }, evidence_validation }
    if (value.compressed && Array.isArray(value.compressed.segments)) {
      for (const seg of value.compressed.segments) {
        if (seg.text && seg.text.length > 20 && seg.selected !== false) {
          bucket.push(seg.text.trim());
        }
      }
      if (bucket.length > 0) return bucket;
    }

    const preferredKeys = [
      'compressed_text',
      'text',
      'content',
      'summary',
      'snippet',
      'note',
      'claim',
      'decision',
      'finding',
      'evidence',
    ];
    for (const key of preferredKeys) {
      if (key in value) flattenEvidenceStrings(value[key], bucket);
    }
    if (bucket.length === 0) {
      for (const key of Object.keys(value)) {
        flattenEvidenceStrings(value[key], bucket);
      }
    }
  }
  return bucket;
}

function classifyMemoryTarget(text) {
  const normalized = String(text || '').toLowerCase();
  if (/(gotcha|pitfall|anti-pattern|risk|warning|failure)/.test(normalized)) return 'gotchas';
  if (/(issue|bug|error|incident|defect|gap)/.test(normalized)) return 'issues';
  if (/(decision|tradeoff|choose|selected|rationale)/.test(normalized)) return 'decisions';
  return 'patterns';
}

function mapCompressionToMemoryRecords(compressionOutput, metadata = {}) {
  const rawTexts = flattenEvidenceStrings(compressionOutput, []);
  const unique = Array.from(new Set(rawTexts)).slice(0, 24);
  const timestamp = new Date().toISOString();
  const sourceQuery = String(metadata.query || '').trim();

  const records = {
    patterns: [],
    gotchas: [],
    issues: [],
    decisions: [],
  };

  for (const text of unique) {
    const target = classifyMemoryTarget(text);
    if (target === 'patterns' || target === 'gotchas') {
      records[target].push({
        text,
        timestamp,
        source: sourceQuery || 'token-saver-context-compression',
      });
      continue;
    }
    records[target].push({
      text,
      timestamp,
      source: sourceQuery || 'token-saver-context-compression',
      section: 'token-saver-context-compression',
    });
  }

  return records;
}

function mergeUniqueJsonEntries(filePath, incoming) {
  const existing = fs.existsSync(filePath)
    ? JSON.parse(fs.readFileSync(filePath, 'utf8') || '[]')
    : [];
  const map = new Map();
  for (const item of existing) {
    const key = typeof item === 'string' ? item : item?.text;
    if (!key) continue;
    map.set(key, item);
  }
  for (const item of incoming) {
    map.set(item.text, item);
  }
  fs.writeFileSync(filePath, JSON.stringify(Array.from(map.values()), null, 2) + '\n', 'utf8');
}

function appendMarkdownEntries(filePath, heading, entries) {
  if (!entries.length) return;
  const prior = fs.existsSync(filePath) ? fs.readFileSync(filePath, 'utf8') : '';
  const block = [
    '',
    `## ${heading} (${new Date().toISOString().slice(0, 10)})`,
    ...entries.map(entry => `- ${entry.text}`),
    '',
  ].join('\n');
  fs.writeFileSync(filePath, prior + block, 'utf8');
}

function loadExistingTextsFromMemory(memoryDir) {
  const existingTexts = new Set();
  for (const file of ['patterns.json', 'gotchas.json']) {
    const filePath = path.join(memoryDir, file);
    try {
      if (!fs.existsSync(filePath)) continue;
      const entries = JSON.parse(fs.readFileSync(filePath, 'utf8'));
      if (!Array.isArray(entries)) continue;
      for (const entry of entries) {
        const text = typeof entry === 'string' ? entry : entry?.text;
        if (text) existingTexts.add(text.toLowerCase().trim());
      }
    } catch (_e) {
      // Corrupt JSON or read error — skip this file, keep all incoming records for this category
    }
  }
  return existingTexts;
}

function deduplicateAgainstMemory(records, memoryDir) {
  const existingTexts = loadExistingTextsFromMemory(memoryDir);

  let total = 0;
  let filtered = 0;

  const dedupedRecords = {};
  for (const category of Object.keys(records)) {
    dedupedRecords[category] = [];
    for (const record of records[category]) {
      total++;
      const key = (record.text || '').toLowerCase().trim();
      if (key && existingTexts.has(key)) {
        filtered++;
      } else {
        dedupedRecords[category].push(record);
      }
    }
  }

  return {
    dedupedRecords,
    stats: { total, kept: total - filtered, filtered },
  };
}

function computeAdaptiveRatio(corpusTokens) {
  if (corpusTokens < 8000) return 0.8;
  if (corpusTokens < 32000) return 0.5;
  if (corpusTokens < 100000) return 0.2;
  return 0.1;
}

function applyMemoryRecordsToFiles(records, memoryDir) {
  fs.mkdirSync(memoryDir, { recursive: true });
  mergeUniqueJsonEntries(path.join(memoryDir, 'patterns.json'), records.patterns);
  mergeUniqueJsonEntries(path.join(memoryDir, 'gotchas.json'), records.gotchas);
  appendMarkdownEntries(path.join(memoryDir, 'issues.md'), 'Token Saver Issues', records.issues);
  appendMarkdownEntries(
    path.join(memoryDir, 'decisions.md'),
    'Token Saver Decisions',
    records.decisions
  );
}

function runTokenSaverWorkflow({
  corpusFile,
  query,
  mode,
  failOnInsufficientEvidence,
  skeletonRatio,
}) {
  const scriptPath = path.join(__dirname, 'run_skill_workflow.py');
  const args = [
    scriptPath,
    '--file',
    corpusFile,
    '--mode',
    mode,
    '--query',
    query,
    '--output-format',
    'json',
  ];
  if (failOnInsufficientEvidence) args.push('--fail-on-insufficient-evidence');
  if (skeletonRatio != null) args.push('--skeleton-ratio', String(skeletonRatio));
  const proc = runCommand('python', args);

  // Try to parse JSON output even on non-zero exit codes.
  // Python returns exit 1 for insufficient evidence (valid result, not a crash)
  // and exit 2 for actual errors (missing input, bad args).
  const stdout = (proc.stdout || '').trim();
  if (stdout.startsWith('{')) {
    try {
      return { ok: true, data: JSON.parse(stdout) };
    } catch (_parseErr) {
      // Fall through to error handling
    }
  }

  if (proc.status !== 0) {
    return {
      ok: false,
      status: proc.status || 1,
      stdout: proc.stdout || '',
      stderr: proc.stderr || '',
    };
  }

  try {
    return { ok: true, data: JSON.parse(stdout || '{}') };
  } catch (error) {
    return {
      ok: false,
      status: 1,
      stdout: proc.stdout || '',
      stderr: `Failed to parse workflow JSON: ${error.message}`,
    };
  }
}

function inferEvidenceSufficiency(workflowResult) {
  if (!workflowResult || typeof workflowResult !== 'object') return false;
  if ('evidence_sufficient' in workflowResult) return Boolean(workflowResult.evidence_sufficient);
  if ('sufficient' in workflowResult) return Boolean(workflowResult.sufficient);
  const validation = workflowResult.validation || workflowResult.evidence || null;
  if (validation && typeof validation === 'object') {
    if ('sufficient' in validation) return Boolean(validation.sufficient);
    if ('is_sufficient' in validation) return Boolean(validation.is_sufficient);
  }
  return true;
}

// eslint-disable-next-line complexity
function main(input = {}, deps = {}) {
  const run = deps.runCommand || runCommand;
  const runWorkflow = deps.runTokenSaverWorkflow || runTokenSaverWorkflow;

  const query = String(input.query || '').trim();
  if (!query) {
    return { ok: false, error: 'query is required' };
  }

  const mode = ['baseline', 'query_guided', 'evidence_aware'].includes(input.mode)
    ? input.mode
    : 'evidence_aware';
  const limit = Number.isFinite(Number(input.limit)) ? Math.max(1, Number(input.limit)) : 20;
  const failOnInsufficientEvidence = input.failOnInsufficientEvidence !== false;
  const persistFiles = input.persistFiles === true;

  const searchCmd = runSearchQuery(run, query);
  if (searchCmd.status !== 0) {
    return {
      ok: false,
      stage: 'search',
      error: 'search command failed',
      details: searchCmd.stderr || searchCmd.stdout || '',
    };
  }

  const hits = normalizeSearchResults(searchCmd.stdout, limit);
  fs.mkdirSync(RUNTIME_DIR, { recursive: true });
  const corpusFile = path.join(
    RUNTIME_DIR,
    `corpus-${Date.now()}-${Math.random().toString(16).slice(2)}.txt`
  );

  // Build corpus: for each hit, include file content (not just path)
  // Semantic search results often have no inline snippets — read the files directly
  const MAX_FILE_CHARS = 8000; // ~2K tokens per file to keep corpus bounded
  const corpusParts = [];
  for (const hit of hits) {
    let content = hit.snippets.join('\n').trim();
    if (!content && hit.file) {
      // Read actual file content for files with no inline snippets
      try {
        const filePath = path.isAbsolute(hit.file) ? hit.file : path.join(PROJECT_ROOT, hit.file);
        const raw = fs.readFileSync(filePath, 'utf8');
        content = raw.slice(0, MAX_FILE_CHARS);
        if (raw.length > MAX_FILE_CHARS) content += '\n[... truncated]';
      } catch (_e) {
        content = '(file not readable)';
      }
    }
    corpusParts.push(`FILE: ${hit.file}\n${content}`);
  }
  const corpus = corpusParts.join('\n\n---\n\n');
  fs.writeFileSync(corpusFile, corpus || String(searchCmd.stdout || ''), 'utf8');

  // Compute adaptive skeleton ratio from corpus size unless user explicitly provided one
  const corpusTokens = Math.ceil(corpus.length / 4);
  const skeletonRatio =
    input.skeletonRatio != null ? Number(input.skeletonRatio) : computeAdaptiveRatio(corpusTokens);

  const workflow = runWorkflow({
    corpusFile,
    query,
    mode,
    failOnInsufficientEvidence,
    skeletonRatio,
  });

  if (!workflow.ok) {
    return {
      ok: false,
      stage: 'compression',
      error: workflow.stderr || 'token-saver workflow failed',
      details: workflow.stdout || '',
    };
  }

  const sufficient = inferEvidenceSufficiency(workflow.data);
  if (failOnInsufficientEvidence && !sufficient) {
    return {
      ok: false,
      stage: 'evidence_gate',
      error: 'insufficient evidence',
      evidenceSufficient: false,
    };
  }

  const rawMemoryRecords = mapCompressionToMemoryRecords(workflow.data, { query });
  const memoryDir = path.join(PROJECT_ROOT, '.claude', 'context', 'memory');
  const { dedupedRecords: memoryRecords, stats: dedupStats } = deduplicateAgainstMemory(
    rawMemoryRecords,
    memoryDir
  );
  if (persistFiles) {
    applyMemoryRecordsToFiles(memoryRecords, memoryDir);
  }

  // --- Token & Cost Savings Telemetry ---
  const outputTokens = Math.ceil(JSON.stringify(workflow.data).length / 4);
  const savedTokens = Math.max(0, corpusTokens - outputTokens);

  let activeModelStr = 'claude-sonnet-4.6';
  try {
    const { getState } = require('../../../lib/routing/router-state.cjs');
    const { resolveAgentModel } = require('../../../lib/utils/agent-config-reader.cjs');

    const state = getState();
    const agentName = state.mode === 'agent' ? state.taskDescription : 'router';
    const resolved = resolveAgentModel(agentName);

    if (resolved && resolved.model) {
      activeModelStr = resolved.model;
    }
  } catch (_e) {
    // Graceful fallback
  }

  const model = String(input.model || activeModelStr).toLowerCase();
  let costPerMillion = 3.0; // Default: Sonnet 4.6
  if (model.includes('opus')) {
    costPerMillion = 5.0;
  } else if (model.includes('haiku')) {
    costPerMillion = 1.0;
  }

  const costSavingsUsd = (savedTokens / 1_000_000) * costPerMillion;

  const telemetryData = {
    timestamp: new Date().toISOString(),
    query,
    model,
    originalTokens: corpusTokens,
    compressedTokens: outputTokens,
    savedTokens,
    estimatedSavingsUsd: costSavingsUsd,
  };

  try {
    const statsFile = path.join(RUNTIME_DIR, 'token-saver-telemetry.jsonl');
    fs.appendFileSync(statsFile, JSON.stringify(telemetryData) + '\n', 'utf8');
  } catch (_e) {
    // Non-blocking telemetry
  }

  return {
    ok: true,
    search: { query, hits: hits.length, limit },
    evidence: { sufficient },
    compression: {
      mode,
      corpusFile,
      skeletonRatio,
    },
    telemetry: telemetryData,
    memoryRecords,
    dedupStats,
    persistMode: persistFiles ? 'files' : 'memoryrecord_payload_only',
    memoryRecordHint:
      'Use MemoryRecord to persist these payloads so sync-memory-index hook updates the search index.',
  };
}

if (require.main === module) {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    console.log(`
token-saver-context-compression wrapper

Usage:
  node main.cjs --query "<question>" [--mode evidence_aware|query_guided|baseline] [--limit 20]
                [--no-fail-on-insufficient-evidence] [--persist-files] [--skeleton-ratio 0.5]
                [--model claude-sonnet-4.6]
`);
    process.exit(0);
  }

  const result = main({
    query: options.query,
    mode: options.mode,
    limit: options.limit ? Number(options.limit) : undefined,
    failOnInsufficientEvidence: !(
      options['no-fail-on-insufficient-evidence'] === true ||
      String(options['fail-on-insufficient-evidence']).toLowerCase() === 'false'
    ),
    persistFiles: options['persist-files'] === true,
    skeletonRatio: options['skeleton-ratio'] ? Number(options['skeleton-ratio']) : undefined,
    model: options.model,
  });

  if (!result.ok) {
    console.error(JSON.stringify(result, null, 2));
    process.exit(1);
  }
  console.log(JSON.stringify(result, null, 2));
}

module.exports = {
  parseArgs,
  normalizeSearchResults,
  flattenEvidenceStrings,
  classifyMemoryTarget,
  mapCompressionToMemoryRecords,
  deduplicateAgainstMemory,
  computeAdaptiveRatio,
  applyMemoryRecordsToFiles,
  inferEvidenceSufficiency,
  runSearchQuery,
  main,
};

#!/usr/bin/env python3
"""Profile token counts for raw vs compressed context (self-contained)."""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

from _compression_engine import compress_text
from _output_format import render_output
from _runtime import read_text_input


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="Profile token usage before/after compression.")
    parser.add_argument("--text", type=str, default="", help="Inline text input.")
    parser.add_argument("--file", type=Path, help="Path to UTF-8 text file.")
    parser.add_argument(
        "--file-id", type=str, default="skill_profile_doc", help="Document identifier."
    )
    parser.add_argument("--skeleton-ratio", type=float, default=0.2, help="Compression keep ratio.")
    parser.add_argument(
        "--output-format",
        choices=["json", "toon", "auto"],
        default="json",
        help="Output format for the emitted payload.",
    )
    parser.add_argument(
        "--auto-min-rows",
        type=int,
        default=8,
        help="Minimum uniform rows before auto format selects TOON.",
    )
    return parser


def main() -> int:
    parser = build_parser()
    args = parser.parse_args()
    text = read_text_input(args)
    if not text.strip():
        print("No input text provided.", file=sys.stderr)
        return 2

    compressed = compress_text(
        text=text,
        mode="baseline",
        query="",
        skeleton_ratio=args.skeleton_ratio,
    )

    payload = {
        "file_id": args.file_id,
        "mode": "baseline",
        "original_tokens": compressed.original_tokens,
        "compressed_tokens": compressed.compressed_tokens,
        "compression_ratio": compressed.compression_ratio,
        "token_savings_pct": compressed.token_savings_pct,
        "selected_segments": sum(1 for s in compressed.segments if s["selected"]),
        "total_segments": len(compressed.segments),
    }

    rendered, resolved, meta = render_output(
        payload, args.output_format, auto_min_rows=args.auto_min_rows
    )
    if args.output_format == "toon" and resolved != "toon":
        payload_meta = {
            "requested_output_format": args.output_format,
            "resolved_output_format": resolved,
            **meta,
        }
        rendered, _, _ = render_output({**payload, "_format_meta": payload_meta}, "json")

    print(rendered)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Run profile + compression + evidence validation workflow."""

from __future__ import annotations

import argparse
import sys
from pathlib import Path

from _compression_engine import compress_text, evaluate_evidence
from _output_format import render_output
from _runtime import read_text_input


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description="Run profile + compression + evidence validation workflow."
    )
    parser.add_argument("--text", type=str, default="", help="Inline text input.")
    parser.add_argument("--file", type=Path, help="Path to UTF-8 text file.")
    parser.add_argument(
        "--file-id", type=str, default="skill_workflow_doc", help="Document identifier."
    )
    parser.add_argument(
        "--mode",
        choices=["baseline", "query_guided", "evidence_aware"],
        default="baseline",
        help="Compression selection mode.",
    )
    parser.add_argument(
        "--query", type=str, default="", help="Query for guided/evidence-aware modes."
    )
    parser.add_argument("--top-k", type=int, default=5, help="Evidence top-k segment count.")
    parser.add_argument("--min-similarity", type=float, default=0.35, help="Evidence threshold.")
    parser.add_argument("--skeleton-ratio", type=float, default=0.2, help="Compression keep ratio.")
    parser.add_argument(
        "--output-format",
        choices=["json", "toon", "auto"],
        default="auto",
        help="Output format for the emitted payload.",
    )
    parser.add_argument(
        "--auto-min-rows",
        type=int,
        default=8,
        help="Minimum uniform rows before auto format selects TOON.",
    )
    parser.add_argument(
        "--fail-on-insufficient-evidence",
        action="store_true",
        help="Exit non-zero if evidence validation is insufficient.",
    )
    return parser


def main() -> int:
    parser = build_parser()
    args = parser.parse_args()
    text = read_text_input(args)
    if not text.strip():
        print("No input text provided.", file=sys.stderr)
        return 2
    if args.mode != "baseline" and not args.query.strip():
        print("--query is required for query_guided and evidence_aware modes.", file=sys.stderr)
        return 2

    compressed = compress_text(
        text=text,
        mode=args.mode,
        query=args.query,
        skeleton_ratio=args.skeleton_ratio,
        top_k=args.top_k,
    )

    profile = {
        "file_id": args.file_id,
        "original_tokens": compressed.original_tokens,
        "compressed_tokens": compressed.compressed_tokens,
        "compression_ratio": compressed.compression_ratio,
        "token_savings_pct": compressed.token_savings_pct,
    }

    selected_segments = [row for row in compressed.segments if row["selected"]]
    compressed_payload = {
        "mode": args.mode,
        "query": args.query or None,
        "compressed_text": compressed.compressed_text,
        "segments": selected_segments,
    }

    evidence_validation = None
    if args.query.strip():
        evidence_validation = evaluate_evidence(
            compressed=compressed,
            query=args.query,
            min_similarity=args.min_similarity,
            top_k=args.top_k,
        )

    payload = {
        "profile": profile,
        "compressed": compressed_payload,
        "evidence_validation": evidence_validation,
    }

    rendered, resolved, meta = render_output(
        payload, args.output_format, auto_min_rows=args.auto_min_rows
    )
    if args.output_format == "toon" and resolved != "toon":
        payload_meta = {
            "requested_output_format": args.output_format,
            "resolved_output_format": resolved,
            **meta,
        }
        rendered, _, _ = render_output({**payload, "_format_meta": payload_meta}, "json")
    print(rendered)

    if (
        args.fail_on_insufficient_evidence
        and evidence_validation
        and not evidence_validation["sufficient"]
    ):
        return 1
    return 0


if __name__ == "__main__":
    raise SystemExit(main())