Skill Logger

Skill Logger

Track, measure, and improve skill quality through systematic logging and scoring.

When to Use This Skill

Use for:

• Setting up skill usage logging
• Defining quality metrics for skill outputs
• Analyzing skill performance over time
• Identifying skills that need improvement
• Building feedback loops for skill enhancement
• A/B testing skill variations

NOT for:

• Creating new skills → use agent-creator
• Skill documentation → use skill-coach
• Runtime debugging → use appropriate debugger skills
• General logging/monitoring → use devops-automator

Core Logging Architecture

┌────────────────────────────────────────────────────────────────┐
│                    SKILL LOGGING PIPELINE                       │
├────────────────────────────────────────────────────────────────┤
│                                                                 │
│  1. CAPTURE          2. ANALYZE           3. SCORE              │
│  ├─ Invocation       ├─ Output parse      ├─ Quality metrics    │
│  ├─ Input context    ├─ Token usage       ├─ User satisfaction  │
│  ├─ Output           ├─ Tool calls        ├─ Goal completion    │
│  └─ Timing           └─ Error patterns    └─ Efficiency         │
│                                                                 │
│  4. AGGREGATE        5. ALERT             6. IMPROVE            │
│  ├─ Per-skill stats  ├─ Quality drops     ├─ Identify patterns  │
│  ├─ Trend analysis   ├─ Error spikes      ├─ Suggest changes    │
│  └─ Comparisons      └─ Underuse          └─ Track experiments  │
│                                                                 │
└────────────────────────────────────────────────────────────────┘

What to Log

Invocation Data

{
  "invocation_id": "uuid",
  "timestamp": "ISO8601",
  "skill_name": "wedding-immortalist",
  "skill_version": "1.2.0",

  "input": {
    "user_query": "Create a 3D model from my wedding photos",
    "context_tokens": 1500,
    "files_referenced": ["photos/", "config.json"]
  },

  "execution": {
    "duration_ms": 45000,
    "tool_calls": [
      {"tool": "Bash", "count": 5},
      {"tool": "Write", "count": 3}
    ],
    "tokens_used": {
      "input": 8500,
      "output": 3200
    },
    "errors": []
  },

  "output": {
    "type": "code_generation",
    "artifacts_created": ["pipeline.py", "config.yaml"],
    "response_length": 3200
  }
}

Quality Signals

QUALITY_SIGNALS = {
    # Implicit signals (automated)
    'completion': 'Did the skill complete without errors?',
    'token_efficiency': 'Output quality per token used',
    'tool_success_rate': 'Tool calls that succeeded',
    'retry_count': 'How many retries needed?',

    # Explicit signals (user feedback)
    'user_edit_ratio': 'How much did user modify output?',
    'user_accepted': 'Did user accept/use the output?',
    'follow_up_needed': 'Did user need to ask for fixes?',
    'explicit_rating': 'Thumbs up/down if available',

    # Outcome signals (delayed)
    'code_ran_successfully': 'Did generated code work?',
    'tests_passed': 'Did it pass tests?',
    'reverted': 'Was the output later reverted?',
}

Scoring Framework

Multi-Dimensional Quality Score

def calculate_skill_score(invocation_log):
    """Score a skill invocation 0-100."""

    scores = {
        # Completion (25%)
        'completion': (
            25 if invocation_log['errors'] == [] else
            15 if invocation_log['recovered'] else
            0
        ),

        # Efficiency (20%)
        'efficiency': min(20, 20 * (
            BASELINE_TOKENS / invocation_log['tokens_used']
        )),

        # Output Quality (30%)
        'quality': (
            30 if invocation_log['user_accepted'] else
            20 if invocation_log['user_edit_ratio'] < 0.2 else
            10 if invocation_log['user_edit_ratio'] < 0.5 else
            0
        ),

        # User Satisfaction (25%)
        'satisfaction': (
            25 if invocation_log['explicit_rating'] == 'positive' else
            15 if invocation_log['no_follow_up'] else
            5 if invocation_log['follow_up_resolved'] else
            0
        ),
    }

    return sum(scores.values())

Score Interpretation

Log Storage Schema

SQLite Schema (Local)

CREATE TABLE skill_invocations (
    id TEXT PRIMARY KEY,
    skill_name TEXT NOT NULL,
    skill_version TEXT,
    timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,

    -- Input
    user_query TEXT,
    context_tokens INTEGER,

    -- Execution
    duration_ms INTEGER,
    tokens_input INTEGER,
    tokens_output INTEGER,
    tool_calls_json TEXT,
    errors_json TEXT,

    -- Output
    output_type TEXT,
    artifacts_json TEXT,
    response_length INTEGER,

    -- Quality signals
    user_accepted BOOLEAN,
    user_edit_ratio REAL,
    follow_up_needed BOOLEAN,
    explicit_rating TEXT,

    -- Computed
    quality_score REAL,

    INDEX idx_skill_name (skill_name),
    INDEX idx_timestamp (timestamp),
    INDEX idx_quality (quality_score)
);

CREATE TABLE skill_aggregates (
    skill_name TEXT,
    period TEXT,  -- 'daily', 'weekly', 'monthly'
    period_start DATE,

    invocation_count INTEGER,
    avg_quality_score REAL,
    error_rate REAL,
    avg_tokens_used INTEGER,
    avg_duration_ms INTEGER,

    PRIMARY KEY (skill_name, period, period_start)
);

JSON Log Format (Portable)

{
  "logs_version": "1.0",
  "skill_name": "wedding-immortalist",
  "entries": [
    {
      "id": "uuid",
      "timestamp": "2025-01-15T14:30:00Z",
      "input": {...},
      "execution": {...},
      "output": {...},
      "quality": {
        "signals": {...},
        "score": 85,
        "computed_at": "2025-01-15T14:35:00Z"
      }
    }
  ]
}

Analytics Queries

Skill Performance Dashboard

-- Overall skill rankings
SELECT
    skill_name,
    COUNT(*) as uses,
    AVG(quality_score) as avg_quality,
    AVG(tokens_output) as avg_tokens,
    SUM(CASE WHEN errors_json != '[]' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as error_rate
FROM skill_invocations
WHERE timestamp > datetime('now', '-30 days')
GROUP BY skill_name
ORDER BY avg_quality DESC;

-- Quality trend (weekly)
SELECT
    skill_name,
    strftime('%Y-%W', timestamp) as week,
    AVG(quality_score) as avg_quality,
    COUNT(*) as uses
FROM skill_invocations
GROUP BY skill_name, week
ORDER BY skill_name, week;

-- Problem detection
SELECT skill_name, COUNT(*) as failures
FROM skill_invocations
WHERE quality_score < 50
  AND timestamp > datetime('now', '-7 days')
GROUP BY skill_name
HAVING failures >= 3
ORDER BY failures DESC;

Improvement Opportunities

def identify_improvement_opportunities(skill_name, logs):
    """Analyze logs to suggest skill improvements."""

    opportunities = []

    # Pattern 1: Common follow-up questions
    follow_ups = extract_follow_up_patterns(logs)
    if follow_ups:
        opportunities.append({
            'type': 'missing_capability',
            'description': f'Users frequently ask: {follow_ups[0]}',
            'suggestion': 'Add guidance for this common need'
        })

    # Pattern 2: High edit ratio in specific output types
    edit_patterns = analyze_edit_patterns(logs)
    if edit_patterns['code'] > 0.4:
        opportunities.append({
            'type': 'code_quality',
            'description': 'Users frequently edit generated code',
            'suggestion': 'Review code examples and templates'
        })

    # Pattern 3: Repeated errors
    error_patterns = cluster_errors(logs)
    for error_type, count in error_patterns:
        if count >= 3:
            opportunities.append({
                'type': 'recurring_error',
                'description': f'{error_type} occurred {count} times',
                'suggestion': 'Add error handling or documentation'
            })

    return opportunities

Implementation Guide

Basic Logger Hook

# hooks/skill_logger.py
import json
import sqlite3
from datetime import datetime
from pathlib import Path

LOG_DB = Path.home() / '.claude' / 'skill_logs.db'

def log_skill_invocation(
    skill_name: str,
    user_query: str,
    output: str,
    tool_calls: list,
    duration_ms: int,
    tokens: dict,
    errors: list = None
):
    """Log a skill invocation to the database."""

    conn = sqlite3.connect(LOG_DB)
    cursor = conn.cursor()

    cursor.execute('''
        INSERT INTO skill_invocations
        (id, skill_name, timestamp, user_query, duration_ms,
         tokens_input, tokens_output, tool_calls_json, errors_json,
         response_length)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    ''', (
        str(uuid.uuid4()),
        skill_name,
        datetime.utcnow().isoformat(),
        user_query,
        duration_ms,
        tokens.get('input', 0),
        tokens.get('output', 0),
        json.dumps(tool_calls),
        json.dumps(errors or []),
        len(output)
    ))

    conn.commit()
    conn.close()

Quality Signal Collection

def collect_quality_signals(invocation_id: str, signals: dict):
    """Update an invocation with quality signals."""

    conn = sqlite3.connect(LOG_DB)
    cursor = conn.cursor()

    # Update with user feedback
    cursor.execute('''
        UPDATE skill_invocations
        SET user_accepted = ?,
            user_edit_ratio = ?,
            follow_up_needed = ?,
            explicit_rating = ?,
            quality_score = ?
        WHERE id = ?
    ''', (
        signals.get('accepted'),
        signals.get('edit_ratio'),
        signals.get('follow_up'),
        signals.get('rating'),
        calculate_score(signals),
        invocation_id
    ))

    conn.commit()
    conn.close()

Alerting & Notifications

Alert Conditions

ALERT_CONDITIONS = {
    'quality_drop': {
        'condition': 'avg_quality_7d < avg_quality_30d * 0.8',
        'message': 'Skill {skill} quality dropped 20%+ in past week',
        'severity': 'warning'
    },
    'error_spike': {
        'condition': 'error_rate_24h > error_rate_7d * 2',
        'message': 'Skill {skill} error rate doubled in past 24h',
        'severity': 'critical'
    },
    'underused': {
        'condition': 'uses_7d < uses_30d_avg * 0.5',
        'message': 'Skill {skill} usage down 50%+ this week',
        'severity': 'info'
    },
    'high_performer': {
        'condition': 'avg_quality_7d > 90 AND uses_7d > 10',
        'message': 'Skill {skill} performing excellently',
        'severity': 'positive'
    }
}

Anti-Patterns

"Log Everything"

Wrong: Logging complete input/output for every invocation. Why: Privacy concerns, storage explosion, noise. Right: Log metadata, summaries, and opt-in detailed logging.

"Score Once, Forget"

Wrong: Calculating quality score immediately after completion. Why: Misses delayed signals (did code work? was it reverted?). Right: Collect signals over time, recalculate periodically.

"Averages Only"

Wrong: Only tracking average quality scores. Why: Hides distribution, misses failure modes. Right: Track percentiles, failure rates, and patterns.

"No Baseline"

Wrong: Measuring quality without establishing baselines. Why: Can't detect improvement or regression. Right: Establish baselines per skill, compare trends.

Output Reports

Weekly Skill Health Report

# Skill Health Report - Week of 2025-01-13

## Overview
- Total invocations: 247
- Average quality: 78.3 (up 2.1 from last week)
- Error rate: 4.2% (down 1.8%)

## Top Performers
1. **wedding-immortalist** - 92.1 avg quality, 18 uses
2. **skill-coach** - 89.4 avg quality, 34 uses
3. **api-architect** - 87.2 avg quality, 22 uses

## Needs Attention
1. **legacy-code-converter** - 52.3 avg quality (down 15%)
   - Common issue: Missing dependency detection
   - Suggested fix: Add dependency scanning step

## Improvement Opportunities
- `partner-text-coach`: Users frequently ask for tone adjustment
- `yard-landscaper`: High edit ratio on plant recommendations

Integration Points

• skill-coach: Feed quality data for skill improvements
• agent-creator: Use metrics when designing new skills
• automatic-stateful-prompt-improver: Quality signals for prompt optimization

---

Core Philosophy: What gets measured gets improved. Skill logging transforms intuition about skill quality into actionable data, enabling continuous improvement of the entire skill ecosystem.

Skill Scoring Rubric

Overview

This rubric defines how skill invocations are scored for quality, enabling data-driven improvement of the skill ecosystem.

Multi-Dimensional Scoring Model

Score Components

┌─────────────────────────────────────────────────────────────────┐
│                    SKILL QUALITY SCORE (0-100)                   │
├─────────────────────────────────────────────────────────────────┤
│                                                                  │
│  COMPLETION (25%)        EFFICIENCY (20%)                        │
│  ├─ Task completed?      ├─ Token economy                       │
│  ├─ No errors?           ├─ Tool call efficiency                │
│  └─ Graceful recovery?   └─ Response time                       │
│                                                                  │
│  OUTPUT QUALITY (30%)    USER SATISFACTION (25%)                 │
│  ├─ Accuracy             ├─ Accepted without edits?             │
│  ├─ Completeness         ├─ Follow-up needed?                   │
│  └─ Code quality (if applicable) └─ Explicit feedback           │
│                                                                  │
└─────────────────────────────────────────────────────────────────┘

Component Breakdown

1. Completion Score (25 points max)

def score_completion(invocation: dict) -> int:
    """Score task completion (0-25 points)."""

    errors = invocation.get('errors', [])
    recovered = invocation.get('recovered', False)
    partial = invocation.get('partial_completion', False)

    if not errors:
        return 25  # Perfect completion

    if recovered:
        return 20  # Recovered from error

    if partial:
        return 15  # Partial completion

    if invocation.get('workaround_used'):
        return 10  # Alternative path

    if invocation.get('failure_explained'):
        return 5   # At least explained the issue

    return 0  # Hard failure

2. Efficiency Score (20 points max)

# Baseline values by skill category
EFFICIENCY_BASELINES = {
    'code_generation': {
        'tokens_per_loc': 50,      # Tokens per line of code generated
        'calls_per_file': 3,       # Tool calls per file modified
        'time_per_task': 30_000,   # Milliseconds per typical task
    },
    'analysis': {
        'tokens_per_insight': 200,
        'calls_per_analysis': 5,
        'time_per_task': 20_000,
    },
    'design': {
        'tokens_per_component': 150,
        'calls_per_design': 4,
        'time_per_task': 45_000,
    },
    'research': {
        'tokens_per_finding': 100,
        'calls_per_search': 8,
        'time_per_task': 60_000,
    },
}

def score_efficiency(invocation: dict, skill_category: str) -> int:
    """Score efficiency (0-20 points)."""

    baselines = EFFICIENCY_BASELINES.get(skill_category, EFFICIENCY_BASELINES['analysis'])

    # Token efficiency (0-8 points)
    actual_tokens = invocation['tokens_used']
    output_size = invocation.get('output_size', 1)  # LOC, components, etc.
    expected_tokens = baselines['tokens_per_loc'] * output_size
    token_score = 8 * min(1.0, expected_tokens / max(actual_tokens, 1))

    # Tool call efficiency (0-6 points)
    actual_calls = len(invocation.get('tool_calls', []))
    expected_calls = baselines['calls_per_file'] * invocation.get('files_changed', 1)
    call_score = 6 * min(1.0, expected_calls / max(actual_calls, 1))

    # Response time (0-6 points)
    actual_time = invocation['duration_ms']
    expected_time = baselines['time_per_task']
    time_score = 6 * min(1.0, expected_time / max(actual_time, 1))

    return int(token_score + call_score + time_score)

3. Output Quality Score (30 points max)

def score_output_quality(invocation: dict, feedback: dict = None) -> int:
    """Score output quality (0-30 points)."""

    total = 0

    # Accuracy (0-12 points)
    if feedback:
        # Direct user feedback
        if feedback.get('accurate') == True:
            total += 12
        elif feedback.get('mostly_accurate'):
            total += 9
        elif feedback.get('partially_accurate'):
            total += 6
    else:
        # Heuristic scoring
        if invocation.get('output_validated'):
            total += 12
        elif not invocation.get('errors'):
            total += 8  # Assume reasonable accuracy if no errors

    # Completeness (0-10 points)
    requested_items = invocation.get('requested_items', 1)
    delivered_items = invocation.get('delivered_items', 1)
    completeness_ratio = delivered_items / max(requested_items, 1)
    total += int(10 * min(1.0, completeness_ratio))

    # Code quality (0-8 points) - if applicable
    if invocation.get('output_type') == 'code':
        quality_signals = invocation.get('code_quality', {})

        # Linter passed
        if quality_signals.get('linter_passed', True):
            total += 3

        # Type safe
        if quality_signals.get('types_valid', True):
            total += 2

        # Tests pass (if tests were run)
        if quality_signals.get('tests_passed'):
            total += 3
    else:
        total += 8  # Full points for non-code output

    return total

4. User Satisfaction Score (25 points max)

def score_user_satisfaction(invocation: dict, follow_ups: list = None) -> int:
    """Score user satisfaction (0-25 points)."""

    total = 0

    # Accepted without changes (0-10 points)
    if invocation.get('user_accepted'):
        if invocation.get('user_edit_ratio', 0) < 0.05:
            total += 10  # Accepted as-is
        else:
            total += 7   # Accepted with minor edits
    elif invocation.get('user_used_output'):
        total += 5       # Used but modified

    # Edit ratio (0-8 points)
    edit_ratio = invocation.get('user_edit_ratio', 0.5)
    total += int(8 * (1 - min(edit_ratio, 1.0)))

    # No follow-up needed (0-7 points)
    if follow_ups is None or len(follow_ups) == 0:
        total += 7
    elif len(follow_ups) == 1:
        total += 4  # One clarifying question is okay
    elif all(f.get('resolved') for f in follow_ups):
        total += 2  # Follow-ups were resolved

    return total

Score Interpretation

Quality Tiers

Trend Analysis

def analyze_skill_trends(skill_name: str, days: int = 30) -> dict:
    """Analyze quality trends for a skill."""

    # Get recent invocations
    invocations = get_invocations(skill_name, days=days)

    if len(invocations) < 10:
        return {'status': 'insufficient_data'}

    # Calculate rolling averages
    scores = [inv['quality_score'] for inv in invocations]
    recent_avg = np.mean(scores[-7:])    # Last week
    previous_avg = np.mean(scores[:-7])  # Before that

    # Trend detection
    trend = 'stable'
    change_pct = (recent_avg - previous_avg) / max(previous_avg, 1) * 100

    if change_pct > 10:
        trend = 'improving'
    elif change_pct < -10:
        trend = 'declining'

    # Identify weak components
    component_avgs = {
        'completion': np.mean([inv['scores']['completion'] for inv in invocations]),
        'efficiency': np.mean([inv['scores']['efficiency'] for inv in invocations]),
        'quality': np.mean([inv['scores']['quality'] for inv in invocations]),
        'satisfaction': np.mean([inv['scores']['satisfaction'] for inv in invocations]),
    }

    weak_component = min(component_avgs, key=lambda k: component_avgs[k] / COMPONENT_MAX[k])

    return {
        'current_avg': recent_avg,
        'previous_avg': previous_avg,
        'trend': trend,
        'change_percent': change_pct,
        'weak_component': weak_component,
        'component_scores': component_avgs,
        'recommendation': get_improvement_recommendation(weak_component, component_avgs[weak_component])
    }

def get_improvement_recommendation(component: str, score: float) -> str:
    """Get specific improvement recommendation based on weak component."""

    recommendations = {
        'completion': {
            'low': 'Add more error handling and recovery patterns to SKILL.md',
            'medium': 'Review common failure cases and add guidance',
        },
        'efficiency': {
            'low': 'Reduce context size, consider progressive disclosure',
            'medium': 'Optimize tool call patterns, reduce unnecessary reads',
        },
        'quality': {
            'low': 'Add more examples and anti-patterns to skill',
            'medium': 'Include validation steps in skill workflow',
        },
        'satisfaction': {
            'low': 'Gather user feedback, analyze edit patterns',
            'medium': 'Add clarifying questions to skill workflow',
        },
    }

    level = 'low' if score < 50 else 'medium'
    return recommendations.get(component, {}).get(level, 'Review skill performance')

Automated Quality Gates

QUALITY_GATES = {
    'publish': {
        'min_score': 70,
        'min_invocations': 10,
        'max_error_rate': 0.1,
        'description': 'Minimum requirements to publish a skill'
    },
    'feature': {
        'min_score': 85,
        'min_invocations': 50,
        'max_error_rate': 0.05,
        'description': 'Requirements to be featured in showcase'
    },
    'deprecation': {
        'max_score': 40,
        'min_invocations': 20,
        'max_age_without_improvement': 90,
        'description': 'Triggers deprecation review'
    }
}

def check_quality_gate(skill_name: str, gate: str) -> dict:
    """Check if skill passes a quality gate."""

    gate_config = QUALITY_GATES[gate]
    stats = get_skill_stats(skill_name)

    passed = True
    failures = []

    if stats['avg_score'] < gate_config.get('min_score', 0):
        passed = False
        failures.append(f"Score {stats['avg_score']:.1f} < {gate_config['min_score']}")

    if stats['invocation_count'] < gate_config.get('min_invocations', 0):
        passed = False
        failures.append(f"Invocations {stats['invocation_count']} < {gate_config['min_invocations']}")

    if stats['error_rate'] > gate_config.get('max_error_rate', 1.0):
        passed = False
        failures.append(f"Error rate {stats['error_rate']:.2%} > {gate_config['max_error_rate']:.2%}")

    return {
        'gate': gate,
        'passed': passed,
        'failures': failures,
        'stats': stats
    }

Dashboard Queries

-- Skill leaderboard
SELECT
    skill_name,
    COUNT(*) as uses,
    AVG(quality_score) as avg_score,
    AVG(completion_score) as avg_completion,
    AVG(efficiency_score) as avg_efficiency,
    AVG(quality_score_component) as avg_quality,
    AVG(satisfaction_score) as avg_satisfaction
FROM skill_invocations
WHERE timestamp > datetime('now', '-30 days')
GROUP BY skill_name
HAVING uses >= 5
ORDER BY avg_score DESC;

-- Quality distribution
SELECT
    skill_name,
    CASE
        WHEN quality_score >= 90 THEN 'Excellent'
        WHEN quality_score >= 75 THEN 'Good'
        WHEN quality_score >= 60 THEN 'Acceptable'
        WHEN quality_score >= 40 THEN 'Below Average'
        ELSE 'Poor'
    END as tier,
    COUNT(*) as count
FROM skill_invocations
WHERE timestamp > datetime('now', '-30 days')
GROUP BY skill_name, tier;

-- Improvement opportunities
SELECT
    skill_name,
    'completion' as weak_area,
    AVG(completion_score) / 25.0 as normalized_score
FROM skill_invocations
WHERE timestamp > datetime('now', '-30 days')
GROUP BY skill_name
HAVING normalized_score < 0.7

UNION ALL

SELECT
    skill_name,
    'efficiency' as weak_area,
    AVG(efficiency_score) / 20.0 as normalized_score
FROM skill_invocations
WHERE timestamp > datetime('now', '-30 days')
GROUP BY skill_name
HAVING normalized_score < 0.7

ORDER BY normalized_score ASC;