Caveman

#!/usr/bin/env python3
"""caveman_compressor.py — Apply Matt Pocock's caveman compression rules to text.

Stdlib-only. Deterministic regex-based compression matching the rules in
Matt Pocock's caveman skill SKILL.md:

  1. Drop articles (a/an/the)
  2. Drop filler (just/really/basically/actually/simply)
  3. Drop pleasantries (sure/certainly/of course/happy to)
  4. Drop hedging (might/maybe/perhaps/likely/possibly)
  5. Abbreviate common technical terms (database -> DB, configuration -> config, etc.)
  6. Strip conjunctions where safe (and/but at sentence start)
  7. Use arrows for "leads to" / "causes" phrases (-> )
  8. Strip "as you can see / it should be noted / it's worth mentioning"

PRESERVES:
- Code blocks (```...```) unchanged
- Inline code (`...`) unchanged
- Technical terms named verbatim
- Quoted strings unchanged

NO LLM CALLS. Stdlib only.

Usage:
    python caveman_compressor.py                          # uses embedded sample
    python caveman_compressor.py "your text here"
    python caveman_compressor.py --file path/to/input.txt
    python caveman_compressor.py "text" --output json
"""

import argparse
import json
import re
import sys
from typing import Any, Dict, List, Tuple


# Filler/pleasantry/hedging vocabularies (per Matt's rules)
ARTICLES = {"a", "an", "the"}
FILLER = {"just", "really", "basically", "actually", "simply", "obviously", "literally"}
PLEASANTRIES_PHRASES = [
    "sure!", "sure,", "certainly!", "certainly,",
    "of course!", "of course,",
    "happy to help", "i'd be happy to", "i would be happy to",
    "great question", "good question",
    "absolutely!", "absolutely,",
    "no problem!", "no problem,",
]
HEDGING = {"might", "maybe", "perhaps", "likely", "possibly", "probably"}
METATALK_PHRASES = [
    "as you can see",
    "it should be noted",
    "it's worth mentioning",
    "it is worth mentioning",
    "needless to say",
    "to be clear",
    "in other words",
    "that said",
    "having said that",
]

# Technical term abbreviations
ABBREVIATIONS = [
    (r"\bdatabase\b", "DB"),
    (r"\bdatabases\b", "DBs"),
    (r"\bauthentication\b", "auth"),
    (r"\bauthorization\b", "authz"),
    (r"\bconfiguration\b", "config"),
    (r"\bconfigurations\b", "configs"),
    (r"\brequest\b", "req"),
    (r"\brequests\b", "reqs"),
    (r"\bresponse\b", "res"),
    (r"\bresponses\b", "ress"),
    (r"\bfunction\b", "fn"),
    (r"\bfunctions\b", "fns"),
    (r"\bimplementation\b", "impl"),
    (r"\bimplementations\b", "impls"),
    (r"\benvironment\b", "env"),
    (r"\bdependencies\b", "deps"),
    (r"\bdependency\b", "dep"),
    (r"\brepository\b", "repo"),
    (r"\brepositories\b", "repos"),
    (r"\bdocumentation\b", "docs"),
    (r"\bapplication\b", "app"),
    (r"\bapplications\b", "apps"),
]

# Causality phrase -> arrow
CAUSALITY_PATTERNS = [
    (re.compile(r"\b(which\s+)?(leads?|causes?|results?\s+in|gives?\s+you|produces?)\s+", re.IGNORECASE), "-> "),
    (re.compile(r"\bbecause\s+of\b", re.IGNORECASE), "<- "),
]

# Embedded sample
SAMPLE_INPUT = (
    "Sure! I'd be happy to help you with that. The issue you're experiencing is "
    "likely caused by a misconfiguration in the authentication middleware, where "
    "the token expiry check is actually using a strict less-than comparison "
    "instead of less-than-or-equal. This basically means tokens at the exact "
    "expiry timestamp will get rejected. To fix this, you should simply update "
    "the configuration of the auth function to use `<=` instead of `<`."
)


def _protect_code(text: str) -> Tuple[str, List[str]]:
    """Replace code blocks + inline code with placeholders, return text + protected list."""
    protected: List[str] = []

    def replace_block(m: re.Match) -> str:
        protected.append(m.group(0))
        return f"\x00CODE{len(protected) - 1}\x00"

    text = re.sub(r"```.*?```", replace_block, text, flags=re.DOTALL)
    text = re.sub(r"`[^`]+`", replace_block, text)
    return text, protected


def _restore_code(text: str, protected: List[str]) -> str:
    for i, code in enumerate(protected):
        text = text.replace(f"\x00CODE{i}\x00", code)
    return text


def _drop_articles(text: str) -> str:
    pattern = re.compile(r"\b(" + "|".join(ARTICLES) + r")\s+", re.IGNORECASE)
    return pattern.sub("", text)


def _drop_word_set(text: str, words: set) -> str:
    pattern = re.compile(r"\b(" + "|".join(words) + r")\b\s*", re.IGNORECASE)
    return pattern.sub("", text)


def _drop_phrases(text: str, phrases: List[str]) -> str:
    for phrase in phrases:
        text = re.sub(re.escape(phrase) + r"\s*", "", text, flags=re.IGNORECASE)
        text = re.sub(re.escape(phrase.rstrip(",!")) + r"\s*", "", text, flags=re.IGNORECASE)
    return text


def _apply_abbreviations(text: str) -> str:
    for pattern, replacement in ABBREVIATIONS:
        text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
    return text


def _apply_causality_arrows(text: str) -> str:
    for pattern, replacement in CAUSALITY_PATTERNS:
        text = pattern.sub(replacement, text)
    return text


def _strip_leading_conjunctions(text: str) -> str:
    return re.sub(r"(^|\.\s+)(and|but|so)\s+", r"\1", text, flags=re.IGNORECASE)


def _collapse_whitespace(text: str) -> str:
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\s+([.,;:!?])", r"\1", text)
    return text.strip()


def compress(text: str) -> str:
    """Apply Matt Pocock's caveman rules. Returns compressed text."""
    text, protected = _protect_code(text)
    text = _drop_phrases(text, PLEASANTRIES_PHRASES)
    text = _drop_phrases(text, METATALK_PHRASES)
    text = _drop_word_set(text, FILLER)
    text = _drop_word_set(text, HEDGING)
    text = _drop_articles(text)
    text = _apply_abbreviations(text)
    text = _apply_causality_arrows(text)
    text = _strip_leading_conjunctions(text)
    text = _collapse_whitespace(text)
    text = _restore_code(text, protected)
    return text


def analyze(original: str, compressed: str) -> Dict[str, Any]:
    orig_words = len(original.split())
    new_words = len(compressed.split())
    saved = orig_words - new_words
    pct = round(100.0 * saved / max(orig_words, 1), 1)
    return {
        "original_chars": len(original),
        "compressed_chars": len(compressed),
        "original_words": orig_words,
        "compressed_words": new_words,
        "words_saved": saved,
        "percent_savings": pct,
        "compressed_text": compressed,
    }


def render_text(original: str, result: Dict[str, Any]) -> str:
    lines = []
    lines.append("=" * 72)
    lines.append("CAVEMAN COMPRESSOR")
    lines.append("=" * 72)
    lines.append("")
    lines.append("ORIGINAL:")
    lines.append(f"  {original}")
    lines.append("")
    lines.append("COMPRESSED:")
    lines.append(f"  {result['compressed_text']}")
    lines.append("")
    lines.append("-" * 72)
    lines.append(f"Chars: {result['original_chars']} -> {result['compressed_chars']}")
    lines.append(f"Words: {result['original_words']} -> {result['compressed_words']}")
    lines.append(f"Savings: {result['words_saved']} words ({result['percent_savings']}%)")
    return "\n".join(lines)


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Compress text per Matt Pocock's caveman rules.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )
    parser.add_argument("text", nargs="?", help="Input text (uses embedded sample if omitted)")
    parser.add_argument("--file", help="Read input from file")
    parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format")
    args = parser.parse_args()

    if args.file:
        try:
            with open(args.file, "r", encoding="utf-8") as f:
                original = f.read()
        except (IOError, OSError) as e:
            print(f"error: {e}", file=sys.stderr)
            return 1
    elif args.text:
        original = args.text
    else:
        original = SAMPLE_INPUT

    compressed = compress(original)
    result = analyze(original, compressed)

    if args.output == "json":
        print(json.dumps(result, indent=2))
    else:
        print(render_text(original, result))
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""caveman_lint.py — Lint a response for caveman-mode compliance.

Stdlib-only. Detects banned vocabulary in a response that's supposed to be in
caveman mode. Returns specific findings + verdict.

Banned categories per Matt Pocock's caveman rules:
  - Pleasantries (sure, certainly, of course, happy to)
  - Filler (just, really, basically, actually, simply)
  - Hedging (might, maybe, perhaps, likely)
  - Metatalk (as you can see, worth noting)
  - Verbose phrases ("the implementation of a solution for")

Whitelist (NOT banned even in caveman mode):
  - Words inside code blocks
  - Words inside inline code
  - Words inside quoted strings
  - Caveman exception zones (security warnings, destructive op confirmations)

Usage:
    python caveman_lint.py                          # uses embedded samples
    python caveman_lint.py "response text"
    python caveman_lint.py --file path/to/response.txt
    python caveman_lint.py "text" --output json
"""

import argparse
import json
import re
import sys
from typing import Any, Dict, List


BANNED_PHRASES = {
    "pleasantry": [
        "sure!", "sure,", "certainly", "of course", "happy to help",
        "i'd be happy", "i would be happy", "great question", "good question",
        "absolutely", "no problem!",
    ],
    "filler": ["just", "really", "basically", "actually", "simply", "obviously", "literally"],
    "hedging": ["might", "maybe", "perhaps", "likely", "possibly", "probably"],
    "metatalk": [
        "as you can see", "it should be noted", "worth mentioning",
        "needless to say", "to be clear", "in other words",
        "that said", "having said that",
    ],
    "verbose": [
        "implement a solution for", "the implementation of",
        "in order to", "for the purpose of", "with respect to",
        "due to the fact that",
    ],
}

# Patterns that DROP caveman temporarily (whitelisted zones)
EXCEPTION_MARKERS = [
    re.compile(r"\*\*warning:\*\*", re.IGNORECASE),
    re.compile(r"\bdestructive\b", re.IGNORECASE),
    re.compile(r"\birreversible\b", re.IGNORECASE),
    re.compile(r"\bcannot be undone\b", re.IGNORECASE),
]


SAMPLE_BAD = (
    "Sure! I'd be happy to help. The issue is actually quite simple — basically, "
    "you just need to update the configuration. It's worth mentioning that this might "
    "cause a slight performance hit, but probably not noticeable."
)
SAMPLE_GOOD = "Bug in auth middleware. Token expiry check use `<` not `<=`. Fix: change to `<=`."


def _protect_code(text: str) -> str:
    """Mask code blocks + inline code so banned-word matching skips them."""
    text = re.sub(r"```.*?```", lambda m: "\x00" * len(m.group(0)), text, flags=re.DOTALL)
    text = re.sub(r"`[^`]+`", lambda m: "\x00" * len(m.group(0)), text)
    return text


def _has_exception_context(text: str) -> bool:
    return any(p.search(text) for p in EXCEPTION_MARKERS)


def _count_phrase(phrase: str, masked: str) -> int:
    return len(re.findall(r"\b" + re.escape(phrase) + r"\b", masked, re.IGNORECASE))


def _violation_record(category: str, phrase: str, count: int) -> Dict[str, Any]:
    return {"category": category, "phrase": phrase, "count": count}


def find_violations(text: str) -> List[Dict[str, Any]]:
    """Find banned phrases. Returns list of {category, phrase, count}."""
    masked = _protect_code(text)
    violations: List[Dict[str, Any]] = []
    for category, phrases in BANNED_PHRASES.items():
        for phrase in phrases:
            count = _count_phrase(phrase, masked)
            if count > 0:
                violations.append(_violation_record(category, phrase, count))
    return violations


def analyze(text: str) -> Dict[str, Any]:
    violations = find_violations(text)
    total_violations = sum(v["count"] for v in violations)
    has_exception = _has_exception_context(text)

    # Verdict logic:
    #   0 violations + reasonable length     -> CLEAN
    #   <= 2 violations OR exception context -> WARN
    #   >  2 violations                      -> FAIL
    if total_violations == 0:
        verdict = "CLEAN"
    elif has_exception:
        verdict = "WARN"
        # When there's a security warning, some normal language is allowed
    elif total_violations <= 2:
        verdict = "WARN"
    else:
        verdict = "FAIL"

    return {
        "char_count": len(text),
        "word_count": len(text.split()),
        "violation_categories": sorted(set(v["category"] for v in violations)),
        "total_violations": total_violations,
        "has_exception_context": has_exception,
        "violations": violations,
        "verdict": verdict,
    }


def render_text(text: str, r: Dict[str, Any]) -> str:
    lines = []
    lines.append("=" * 72)
    lines.append("CAVEMAN LINT")
    lines.append("=" * 72)
    lines.append("")
    preview = text[:200] + ("..." if len(text) > 200 else "")
    lines.append(f"Text ({r['char_count']} chars, {r['word_count']} words):")
    lines.append(f"  {preview}")
    lines.append("")
    lines.append("-" * 72)
    lines.append(f"Violations: {r['total_violations']}")
    lines.append(f"Categories hit: {r['violation_categories']}")
    if r["has_exception_context"]:
        lines.append("Exception context detected (warning/destructive zone — some prose allowed)")
    lines.append("")
    if r["violations"]:
        for v in r["violations"]:
            lines.append(f"  [{v['category']:11s}] x{v['count']:2d}  '{v['phrase']}'")
    else:
        lines.append("  No banned phrases found.")
    lines.append("")
    lines.append("-" * 72)
    lines.append(f"Verdict: {r['verdict']}")
    return "\n".join(lines)


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Lint a response for caveman-mode compliance.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )
    parser.add_argument("text", nargs="?", help="Input text (uses embedded sample if omitted)")
    parser.add_argument("--file", help="Read input from file")
    parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format")
    args = parser.parse_args()

    if args.file:
        try:
            with open(args.file, "r", encoding="utf-8") as f:
                text = f.read()
        except (IOError, OSError) as e:
            print(f"error: {e}", file=sys.stderr)
            return 1
    elif args.text:
        text = args.text
    else:
        text = SAMPLE_BAD

    result = analyze(text)
    if args.output == "json":
        print(json.dumps({"text": text, **result}, indent=2))
    else:
        print(render_text(text, result))
    return 0 if result["verdict"] == "CLEAN" else 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""token_savings_estimator.py — Estimate token-cost savings from caveman compression.

Stdlib-only. Uses a chars-per-token heuristic (4 chars/token average for English
prose; 3.5 for technical text) to estimate output tokens before vs after caveman
compression.

Why heuristic and not real tokenizer:
- No external dependencies (stdlib only)
- Tokenizer accuracy varies by model (cl100k_base vs o200k_base vs others)
- Heuristic is within 10-15% of real tokenizer output for English prose
- Reports both heuristic + character count so user can apply their own multiplier

Usage:
    python token_savings_estimator.py                         # uses embedded sample
    python token_savings_estimator.py "your text"
    python token_savings_estimator.py --file path/to/input.txt
    python token_savings_estimator.py "text" --output json
    python token_savings_estimator.py "text" --price-per-mtok 3.00
"""

import argparse
import json
import sys
from typing import Any, Dict

# Import the compressor as a module
import os
_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
from caveman_compressor import compress, SAMPLE_INPUT  # noqa: E402


# Heuristic: average chars per token
CHARS_PER_TOKEN_PROSE = 4.0
CHARS_PER_TOKEN_TECHNICAL = 3.5
TECHNICAL_TOKEN_INDICATORS = ("```", "{", "}", "()", "->", "==", "//", "/*", "import ", "function ")


def _estimate_chars_per_token(text: str) -> float:
    """Heuristic: technical text has more tokens per char than prose."""
    hit_count = sum(1 for sig in TECHNICAL_TOKEN_INDICATORS if sig in text)
    if hit_count >= 3:
        return CHARS_PER_TOKEN_TECHNICAL
    return CHARS_PER_TOKEN_PROSE


def estimate_tokens(text: str) -> int:
    return int(round(len(text) / _estimate_chars_per_token(text)))


def analyze(original: str, price_per_mtok: float = 0.0) -> Dict[str, Any]:
    compressed = compress(original)
    orig_tokens = estimate_tokens(original)
    new_tokens = estimate_tokens(compressed)
    saved = orig_tokens - new_tokens
    pct = round(100.0 * saved / max(orig_tokens, 1), 1)

    out: Dict[str, Any] = {
        "original_chars": len(original),
        "compressed_chars": len(compressed),
        "chars_per_token_used": _estimate_chars_per_token(original),
        "estimated_original_tokens": orig_tokens,
        "estimated_compressed_tokens": new_tokens,
        "tokens_saved": saved,
        "percent_token_savings": pct,
        "compressed_preview": compressed[:200] + ("..." if len(compressed) > 200 else ""),
    }

    if price_per_mtok > 0:
        cost_per_token = price_per_mtok / 1_000_000.0
        out["price_per_million_tokens"] = price_per_mtok
        out["cost_saved_per_response_usd"] = round(saved * cost_per_token, 6)
        out["cost_saved_per_1k_responses_usd"] = round(saved * cost_per_token * 1000, 4)

    return out


def render_text(r: Dict[str, Any]) -> str:
    lines = []
    lines.append("=" * 72)
    lines.append("TOKEN SAVINGS ESTIMATOR (caveman compression)")
    lines.append("=" * 72)
    lines.append("")
    lines.append(f"Chars/token heuristic: {r['chars_per_token_used']:.1f} (prose=4.0; technical=3.5)")
    lines.append("")
    lines.append(f"Original:   {r['original_chars']} chars  ~ {r['estimated_original_tokens']} tokens")
    lines.append(f"Compressed: {r['compressed_chars']} chars  ~ {r['estimated_compressed_tokens']} tokens")
    lines.append("")
    lines.append(f"Savings: {r['tokens_saved']} tokens ({r['percent_token_savings']}%)")
    if "price_per_million_tokens" in r:
        lines.append("")
        lines.append(f"At ${r['price_per_million_tokens']}/Mtok:")
        lines.append(f"  Cost saved per response:   ${r['cost_saved_per_response_usd']:.6f}")
        lines.append(f"  Cost saved per 1k responses: ${r['cost_saved_per_1k_responses_usd']:.4f}")
    lines.append("")
    lines.append("-" * 72)
    lines.append("Compressed preview:")
    lines.append(f"  {r['compressed_preview']}")
    return "\n".join(lines)


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Estimate token + cost savings from caveman compression.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )
    price_help = "Per-million-token price (USD) to estimate cost savings"
    parser.add_argument("text", nargs="?", help="Input text (uses embedded sample if omitted)")
    parser.add_argument("--file", help="Read input from file")
    parser.add_argument("--output", choices=("text", "json"), default="text", help="Output format")
    parser.add_argument("--price-per-mtok", type=float, default=0.0, help=price_help)
    args = parser.parse_args()

    if args.file:
        try:
            with open(args.file, "r", encoding="utf-8") as f:
                original = f.read()
        except (IOError, OSError) as e:
            print(f"error: {e}", file=sys.stderr)
            return 1
    elif args.text:
        original = args.text
    else:
        original = SAMPLE_INPUT

    result = analyze(original, args.price_per_mtok)
    if args.output == "json":
        print(json.dumps(result, indent=2))
    else:
        print(render_text(result))
    return 0


if __name__ == "__main__":
    sys.exit(main())