Vision Bench

{
  "name": "vision-bench",
  "description": "Score and compare images using vision LLMs as judges. YAML-defined criteria presets for 11 use cases (text-to-image, pho",
  "author": {
    "name": "Gleb Kalinin"
  },
  "repository": "https://github.com/glebis/claude-skills",
  "license": "MIT"
}

__pycache__/
.enzyme/
.enzyme-embeddings/
secrets.enc.yaml

creation_rules:
  - path_regex: \.yaml$
    age: age1g3q8la6ekarm8ynw0mq7chfpprfcrykn7atrefdm7s6ukg0v5arq60kmpt

#!/usr/bin/env python3
"""
vision-bench: Evaluate and compare images using vision LLMs as judges.

Usage:
  python bench.py img1.png img2.png --criteria text_to_image --prompt "a cat in space"
  python bench.py img1.png --criteria document_ocr --judge gemini-2.0-flash
  python bench.py img1.png img2.png --judges gpt-4o gemini-2.0-flash claude-opus-4-5-20251022
"""
import argparse
import sys
from pathlib import Path

import yaml

from judge import score_images
from report import generate_report

CRITERIA_DIR = Path(__file__).parent / "criteria"


def list_presets() -> list[str]:
    return [p.stem for p in sorted(CRITERIA_DIR.glob("*.yaml"))]


def load_criteria(name: str) -> dict:
    path = Path(name) if (name.endswith(".yaml") or name.endswith(".yml")) else CRITERIA_DIR / f"{name}.yaml"
    if not path.exists():
        print(f"Criteria not found: {name}")
        print(f"Available presets: {', '.join(list_presets())}")
        sys.exit(1)
    with open(path) as f:
        return yaml.safe_load(f)


def main():
    parser = argparse.ArgumentParser(
        description="Score images with vision LLM judges",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=f"Available presets: {', '.join(list_presets())}"
    )
    parser.add_argument("images", nargs="*", help="Image paths to evaluate")
    parser.add_argument("--criteria", "-c", default="text_to_image",
                        help="Preset name or path to .yaml file (default: text_to_image)")
    parser.add_argument("--judge", "-j", default="gemini-2.5-flash",
                        help="Single judge model")
    parser.add_argument("--judges", nargs="+",
                        help="Multiple judge models for consensus scoring (overrides --judge)")
    parser.add_argument("--prompt", "-p", help="Original generation prompt (used for prompt_adherence)")
    parser.add_argument("--output", "-o", choices=["markdown", "json", "table"], default="markdown")
    parser.add_argument("--save", "-s", help="Save report to file")
    parser.add_argument("--list-presets", action="store_true", help="List available presets and exit")
    args = parser.parse_args()

    if args.list_presets:
        for p in list_presets():
            criteria = load_criteria(p)
            print(f"  {p:<25} {criteria.get('description', '')}")
        return

    criteria = load_criteria(args.criteria)
    judges = args.judges or [args.judge]

    print(f"Evaluating {len(args.images)} image(s) with {len(judges)} judge(s)...", file=sys.stderr)

    results = score_images(args.images, criteria, judges, args.prompt)
    report = generate_report(results, criteria, args.output)

    if args.save:
        Path(args.save).write_text(report)
        print(f"Report saved to {args.save}", file=sys.stderr)
    else:
        print(report)


if __name__ == "__main__":
    main()

name: Accessibility Alt-Text Generation
description: Evaluate quality of image descriptions generated for screen readers and accessibility
mode: absolute

criteria:
  accuracy_and_completeness:
    weight: 0.40
    description: Description correctly captures all essential visual content
    rubric:
      5: "All essential content described accurately, nothing important omitted"
      4: "Main content accurate, minor secondary elements missed"
      3: "Core subject described, context or background elements missing"
      2: "Significant content omitted or incorrectly described"
      1: "Description is inaccurate or missing essential information"

  appropriate_detail_level:
    weight: 0.25
    description: Description is neither too sparse nor too verbose for screen reader use
    rubric:
      5: "Perfect balance — essential detail without overwhelming length"
      4: "Good length with minor verbosity or slight sparseness"
      3: "Acceptable but noticeably too long or too brief"
      2: "Excessively long (>150 words) or too terse (<5 words)"
      1: "Completely wrong length for accessibility use"

  context_and_purpose:
    weight: 0.20
    description: Conveys why the image matters in its context, not just what it depicts
    rubric:
      5: "Captures semantic meaning and purpose of image, not just literal content"
      4: "Good contextual understanding with minor gaps"
      3: "Describes content but misses the communicative purpose"
      2: "Purely literal description with no contextual awareness"
      1: "No contextual understanding"

  decorative_classification:
    weight: 0.15
    description: Correctly identifies decorative images that should have empty alt text
    rubric:
      5: "Always correctly distinguishes informational vs decorative images"
      4: "Usually correct, rare misclassification"
      3: "Sometimes treats decorative as informational or vice versa"
      2: "Frequently misclassifies image purpose"
      1: "Cannot distinguish decorative from informational"

name: Artistic Style Evaluation
description: Assess creative/artistic images for style consistency, composition, and craft
mode: absolute

criteria:
  style_consistency:
    weight: 0.35
    description: Coherence of the chosen artistic style throughout the image
    rubric:
      5: "Perfect stylistic unity; every element belongs to the same aesthetic"
      4: "Consistent style with minor mixing of incompatible elements"
      3: "General style recognizable but several inconsistencies"
      2: "Style incoherent, multiple conflicting aesthetics"
      1: "No discernible style or complete stylistic chaos"

  composition:
    weight: 0.30
    description: Balance, focal point, rule of thirds, visual flow
    rubric:
      5: "Masterful composition; clear focal point, strong visual flow"
      4: "Good composition with intentional choices"
      3: "Acceptable composition, somewhat static or unbalanced"
      2: "Poor balance, no clear focal point, distracting clutter"
      1: "No compositional intent"

  color_harmony:
    weight: 0.20
    description: Color palette cohesion, emotional resonance, contrast
    rubric:
      5: "Colors create strong mood, palette is cohesive and purposeful"
      4: "Pleasing palette with good contrast and mood"
      3: "Colors acceptable but palette feels accidental"
      2: "Clashing or poorly chosen colors that undermine the image"
      1: "Colors actively unpleasant or completely incoherent"

  originality:
    weight: 0.15
    description: Distinctive artistic voice, avoids clichés
    rubric:
      5: "Strikingly original; clear artistic vision not seen elsewhere"
      4: "Memorable with distinctive creative choices"
      3: "Competent work but derivative of common styles"
      2: "Generic, stock-image aesthetic, no distinctive voice"
      1: "Complete pastiche with no original contribution"

name: Chart & Diagram Analysis
description: Evaluate understanding of charts, graphs, and data visualizations
mode: absolute

criteria:
  data_accuracy:
    weight: 0.40
    description: Correctly reads data values, scales, and numerical information
    rubric:
      5: "All data points and values read precisely"
      4: "Values correct with minor rounding errors (<5%)"
      3: "Approximate values correct, some errors on fine-grained data"
      2: "Several significant data reading errors"
      1: "Cannot accurately read data values"

  trend_identification:
    weight: 0.30
    description: Correctly identifies patterns, trends, peaks, and relationships
    rubric:
      5: "All trends and patterns correctly identified and described"
      4: "Main trends correct, minor patterns missed"
      3: "Obvious trends identified, subtle patterns missed"
      2: "Trends misidentified or oversimplified"
      1: "Cannot identify meaningful trends"

  structural_comprehension:
    weight: 0.20
    description: Understands chart type, axes, legends, units, and labels
    rubric:
      5: "All axes, labels, units, and legend elements correctly interpreted"
      4: "Structure mostly correct, minor label confusion"
      3: "Chart type and main structure understood, some label errors"
      2: "Significant structural misinterpretation"
      1: "Cannot understand chart structure"

  insight_quality:
    weight: 0.10
    description: Draws meaningful, non-obvious conclusions from the data
    rubric:
      5: "Insightful, non-obvious conclusions correctly derived"
      4: "Good analysis beyond surface-level description"
      3: "Describes data but limited analytical depth"
      2: "Mostly restates visible data without analysis"
      1: "No meaningful insights"

name: Document & OCR Understanding
description: Evaluate how well a vision model reads and understands documents, forms, invoices
mode: absolute

criteria:
  text_extraction_accuracy:
    weight: 0.45
    description: Correct reading of all text including numbers, dates, special chars
    rubric:
      5: "All text extracted correctly, including handwriting and degraded fonts"
      4: "99%+ accuracy, 1-2 minor character errors"
      3: "Most text correct, several errors in difficult regions"
      2: "Significant text errors affecting meaning"
      1: "Cannot reliably extract text"

  layout_preservation:
    weight: 0.25
    description: Tables, columns, headers, and spatial structure correctly interpreted
    rubric:
      5: "All tables, columns, and structure perfectly understood"
      4: "Structure mostly correct, minor alignment issues"
      3: "General structure captured, some table/column confusion"
      2: "Layout mostly lost, content out of sequence"
      1: "No structural understanding"

  semantic_understanding:
    weight: 0.20
    description: Correctly identifies document type, key fields, and their meaning
    rubric:
      5: "Correctly identifies all key fields and their semantic role"
      4: "Most fields identified correctly"
      3: "Document type understood, some field misclassification"
      2: "Minimal semantic understanding"
      1: "No semantic interpretation"

  completeness:
    weight: 0.10
    description: Nothing important is missed or skipped
    rubric:
      5: "All content accounted for, nothing omitted"
      4: "Minor omissions in peripheral areas"
      3: "Some content missed but main body captured"
      2: "Significant gaps in coverage"
      1: "Majority of content missing"

name: Invoice & Financial Document Processing
description: Extract and verify financial fields from invoices, receipts, and purchase orders
mode: absolute

criteria:
  field_extraction_accuracy:
    weight: 0.45
    description: Correct extraction of key fields — total, line items, dates, vendor, tax
    rubric:
      5: "All financial fields extracted with 100% accuracy including all line items"
      4: "All key fields correct, 1-2 minor secondary field errors"
      3: "Totals and dates correct, some line items missing or wrong"
      2: "Key amount or vendor errors that would cause reconciliation failures"
      1: "Cannot reliably extract financial fields"

  table_interpretation:
    weight: 0.30
    description: Line item tables parsed with correct quantities, prices, and descriptions
    rubric:
      5: "All rows and columns correctly mapped, merged cells handled"
      4: "Table structure correct, 1-2 cell content errors"
      3: "Table structure understood, some cell-level errors"
      2: "Table rows mixed up or column alignment wrong"
      1: "Cannot parse tabular line items"

  currency_and_format:
    weight: 0.15
    description: Correct handling of currencies, number formats, and date formats
    rubric:
      5: "All currencies, decimals, and date formats correctly identified"
      4: "Minor format inconsistency but values correct"
      3: "Currency identified, some format normalization errors"
      2: "Format errors causing ambiguous values"
      1: "Cannot correctly parse numerical formats"

  multilingual_robustness:
    weight: 0.10
    description: Handles non-English text, non-Latin scripts, and international formats
    rubric:
      5: "Correctly processes any language and regional format"
      4: "Most languages handled, minor issues with rare scripts"
      3: "Common European languages handled, others partially"
      2: "Significantly degrades for non-English documents"
      1: "English-only extraction"

name: Photorealism Evaluation
description: How convincingly an image looks like a real photograph
mode: absolute

criteria:
  realism:
    weight: 0.35
    description: Overall believability — would a viewer mistake this for a photograph?
    rubric:
      5: "Indistinguishable from a real photograph at first glance"
      4: "Very realistic with only subtle tells under close inspection"
      3: "Mostly realistic but clearly AI-generated on closer look"
      2: "Several unrealistic elements visible immediately"
      1: "Obviously synthetic, not attempting realism"

  lighting_accuracy:
    weight: 0.25
    description: Natural, consistent light sources, shadows, and reflections
    rubric:
      5: "Perfect light consistency, shadows match source, realistic reflections"
      4: "Mostly accurate lighting with minor inconsistencies"
      3: "Light direction inconsistent in some areas"
      2: "Multiple lighting errors, shadows mismatch source"
      1: "Flat or impossible lighting"

  texture_and_detail:
    weight: 0.25
    description: Fine-grained surface textures — skin, fabric, materials, backgrounds
    rubric:
      5: "Rich, accurate textures throughout; microscopic detail preserved"
      4: "Good textures with minor smoothing or repetition artefacts"
      3: "Textures present but noticeably AI-smooth or repetitive"
      2: "Textures largely missing or clearly synthetic"
      1: "No meaningful texture, plastic/waxy appearance"

  anatomical_accuracy:
    weight: 0.15
    description: Correct proportions, hands, eyes, faces, and physical structures
    rubric:
      5: "All anatomy correct, hands/faces perfect, proportions natural"
      4: "Minor anatomical issues (e.g. slight hand oddity)"
      3: "Noticeable but not extreme anatomy errors"
      2: "Significant errors in hands, faces, or proportions"
      1: "Major deformities or impossible anatomy"

name: Portrait Evaluation
description: Evaluate AI-generated or edited portraits for quality and realism
mode: absolute

criteria:
  facial_accuracy:
    weight: 0.35
    description: Correct facial proportions, symmetry, and natural features
    rubric:
      5: "Facial features perfectly proportioned, natural symmetry, no uncanny valley"
      4: "Natural face with very minor proportion issues"
      3: "Acceptable face but some proportion oddities or subtle strangeness"
      2: "Clear facial anomalies or uncanny valley effect"
      1: "Severely distorted or unnatural face"

  skin_and_texture:
    weight: 0.25
    description: Realistic skin texture, pores, hair, and surface detail
    rubric:
      5: "Photographic skin quality with natural texture variation"
      4: "Good skin texture, minor smoothing or repetition"
      3: "Skin present but clearly AI-smooth or plastic-like"
      2: "Texture largely missing, waxy or fake appearance"
      1: "No realistic skin texture"

  expression_and_emotion:
    weight: 0.25
    description: Natural expression, coherent eye gaze, believable emotion
    rubric:
      5: "Authentic emotion with natural expression and correct eye focus"
      4: "Clear expression, minor stiffness or glassy eyes"
      3: "Expression recognizable but lacks emotional depth"
      2: "Expression feels staged or eyes appear unfocused/dead"
      1: "No discernible expression or deeply unsettling"

  hair_and_details:
    weight: 0.15
    description: Natural hair strands, eyebrows, eyelashes, and fine details
    rubric:
      5: "Individual hair strands, natural eyelashes, every fine detail correct"
      4: "Good hair quality with minor clumping or artifacts"
      3: "Hair recognizable but visibly synthetic or overly smooth"
      2: "Hair significantly artificial or missing fine details"
      1: "Hair completely unrealistic or missing"

name: Product Photography Evaluation
description: Evaluate product images for e-commerce, marketing, and catalog use
mode: absolute

criteria:
  product_clarity:
    weight: 0.35
    description: Product is sharp, fully visible, and the hero of the image
    rubric:
      5: "Product perfectly sharp, no occlusion, all details visible"
      4: "Product clear with minor softness in edges"
      3: "Product mostly clear but some details obscured"
      2: "Product partially hidden or significantly soft"
      1: "Product unclear or not the visual focus"

  background_and_staging:
    weight: 0.25
    description: Background is clean, non-distracting, appropriate to context
    rubric:
      5: "Professional background perfectly suited to product and brand"
      4: "Clean background with minor distractions"
      3: "Background acceptable but not optimal"
      2: "Background distracts from product"
      1: "Background dominates or clashes with product"

  lighting_and_color:
    weight: 0.25
    description: Even lighting that shows product accurately, true-to-life colors
    rubric:
      5: "Studio-quality lighting, accurate colors, no harsh shadows"
      4: "Good lighting with minor hot spots or shadows"
      3: "Adequate lighting but some shadows or color cast"
      2: "Poor lighting obscuring texture or creating false color"
      1: "Lighting severely misrepresents the product"

  brand_consistency:
    weight: 0.15
    description: Image feels professional and consistent with commercial standards
    rubric:
      5: "Professional quality suitable for luxury/flagship catalog"
      4: "Commercial quality with minor polish needed"
      3: "Usable but requires post-processing"
      2: "Below commercial standard, significant retouching needed"
      1: "Not suitable for commercial use"

name: Scientific & Medical Image Evaluation
description: Evaluate scientific images — microscopy, medical scans, diagrams, data figures
mode: absolute

criteria:
  technical_accuracy:
    weight: 0.40
    description: Scientifically correct structures, labels, and representations
    rubric:
      5: "All structures/labels scientifically accurate, no errors"
      4: "Mostly accurate with very minor technical issues"
      3: "General accuracy, some simplification or minor errors"
      2: "Several scientific inaccuracies that could mislead"
      1: "Scientifically incorrect or misleading"

  detail_preservation:
    weight: 0.30
    description: Fine-grained structures, boundaries, and subtle features are visible
    rubric:
      5: "All fine details preserved, subtle structures clearly distinguishable"
      4: "Good detail with minor loss in complex regions"
      3: "Main structures clear, fine details partly lost"
      2: "Significant detail loss obscuring important features"
      1: "Detail insufficient for scientific use"

  annotation_quality:
    weight: 0.20
    description: Labels, arrows, scale bars, and annotations are correct and clear
    rubric:
      5: "All annotations accurate, well-placed, and clearly readable"
      4: "Annotations correct, minor placement or legibility issues"
      3: "Main annotations correct, some missing or misplaced"
      2: "Several annotation errors or missing critical labels"
      1: "Annotations absent, wrong, or misleading"

  reproducibility_clarity:
    weight: 0.10
    description: Image contains enough information to understand the experimental context
    rubric:
      5: "All experimental context inferable from image alone"
      4: "Most context clear"
      3: "Partial context visible"
      2: "Limited contextual information"
      1: "Image cannot stand alone without extensive text explanation"

name: Text-to-Image Generation Evaluation
description: Compare AI image generators (Midjourney, DALL-E, Flux, Stable Diffusion)
mode: absolute

criteria:
  prompt_adherence:
    weight: 0.40
    description: How accurately all elements from the prompt appear in the image
    rubric:
      5: "All key elements present, correct style, accurate details and relationships"
      4: "Most elements present, minor deviations from prompt"
      3: "Core concept captured but missing several specified elements"
      2: "Loosely related to prompt, significant elements missing or wrong"
      1: "Does not match prompt or fundamental misinterpretation"

  visual_quality:
    weight: 0.25
    description: Technical quality — sharpness, coherence, no artifacts
    rubric:
      5: "Crisp, coherent, no artifacts, proper proportions throughout"
      4: "High quality with minor artifacts or slight inconsistencies"
      3: "Acceptable quality, some noticeable artifacts or blurring"
      2: "Significant artifacts, blurriness, or anatomical/structural errors"
      1: "Poor quality, major incoherence, unusable result"

  aesthetic_appeal:
    weight: 0.20
    description: Composition, color harmony, visual balance
    rubric:
      5: "Outstanding composition, balanced colors, highly compelling image"
      4: "Good composition and palette with minor issues"
      3: "Acceptable aesthetics, average composition"
      2: "Poor composition or clashing colors that distract"
      1: "Visually jarring, unbalanced, unpleasant"

  creativity:
    weight: 0.15
    description: Originality and interesting interpretation of the prompt
    rubric:
      5: "Highly creative, unexpected yet apt interpretation"
      4: "Creative with distinctive elements beyond the literal prompt"
      3: "Competent but conventional execution"
      2: "Generic, clichéd, predictable interpretation"
      1: "No creative merit, robotic or template-like result"

name: UI / Screenshot Understanding
description: Evaluate how well a vision model understands app screenshots and web interfaces
mode: absolute

criteria:
  element_identification:
    weight: 0.35
    description: Correctly identifies UI elements — buttons, inputs, navigation, modals
    rubric:
      5: "All UI elements correctly identified and labeled"
      4: "Most elements identified, 1-2 minor misclassifications"
      3: "Common elements identified, some complex components missed"
      2: "Only obvious elements identified, most missed"
      1: "Cannot identify UI elements"

  layout_understanding:
    weight: 0.30
    description: Understands spatial hierarchy, grouping, and information architecture
    rubric:
      5: "Perfect understanding of layout hierarchy and information flow"
      4: "Correct layout understanding with minor spatial errors"
      3: "General layout captured, some hierarchy confusion"
      2: "Layout mostly misunderstood or described sequentially"
      1: "No spatial or hierarchical understanding"

  functional_inference:
    weight: 0.25
    description: Correctly infers what each element does and the screen's purpose
    rubric:
      5: "Correctly infers all element functions and overall screen purpose"
      4: "Most functions correctly inferred"
      3: "Screen purpose clear, some element functions misunderstood"
      2: "Limited functional inference"
      1: "Cannot infer purpose or functions"

  state_recognition:
    weight: 0.10
    description: Identifies UI state — active/inactive, selected, error, loading
    rubric:
      5: "All states correctly identified (hover, disabled, selected, error)"
      4: "Most states identified"
      3: "Obvious states (active, disabled) identified"
      2: "Only most prominent state identified"
      1: "Cannot identify UI states"

import base64
import json
import re
from pathlib import Path
import vault  # local secrets.py — renamed to avoid stdlib collision

SUPPORTED_FORMATS = {".jpg": "image/jpeg", ".jpeg": "image/jpeg",
                     ".png": "image/png", ".gif": "image/gif", ".webp": "image/webp"}


def load_image_b64(path: str) -> tuple[str, str]:
    p = Path(path)
    suffix = p.suffix.lower()
    if suffix not in SUPPORTED_FORMATS:
        raise ValueError(f"Unsupported image format '{suffix}'. Supported: {', '.join(SUPPORTED_FORMATS)}")
    with open(p, "rb") as f:
        return base64.standard_b64encode(f.read()).decode(), SUPPORTED_FORMATS[suffix]


def _extract_json(text: str) -> dict:
    """Extract outermost JSON object robustly, handling model preamble."""
    start = text.find("{")
    if start == -1:
        raise ValueError("No JSON object found in response")
    depth, end = 0, -1
    for i, ch in enumerate(text[start:], start):
        if ch == "{":
            depth += 1
        elif ch == "}":
            depth -= 1
            if depth == 0:
                end = i
                break
    if end == -1:
        raise ValueError("Unclosed JSON object in response")
    return json.loads(text[start:end + 1])


def build_criteria_details(criteria: dict) -> str:
    lines = []
    for name, cfg in criteria["criteria"].items():
        lines.append(f"\n**{name}** (weight: {cfg['weight']})")
        lines.append(f"  {cfg['description']}")
        if "rubric" in cfg:
            for score, desc in sorted(cfg["rubric"].items(), reverse=True):
                lines.append(f"  {score}: {desc}")
    return "\n".join(lines)


def build_judge_prompt(criteria: dict, original_prompt: str | None) -> str:
    prompt_context = f' generated from the prompt: "{original_prompt}"' if original_prompt else ""
    criteria_details = build_criteria_details(criteria)
    template = criteria.get("judge_prompt_template", DEFAULT_JUDGE_PROMPT)
    return template.format(prompt_context=prompt_context, criteria_details=criteria_details)


def call_openai(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    from openai import OpenAI
    client = OpenAI(api_key=vault.require("OPENAI_API_KEY"))
    resp = client.chat.completions.create(
        model=model,
        response_format={"type": "json_object"},
        messages=[{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{image_b64}"}}
        ]}]
    )
    return json.loads(resp.choices[0].message.content)


def call_anthropic(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    import anthropic
    client = anthropic.Anthropic(api_key=vault.require("ANTHROPIC_API_KEY"))
    resp = client.messages.create(
        model=model,
        max_tokens=1024,
        messages=[{"role": "user", "content": [
            {"type": "image", "source": {"type": "base64", "media_type": media_type, "data": image_b64}},
            {"type": "text", "text": prompt}
        ]}]
    )
    return _extract_json(resp.content[0].text)


def call_gemini(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    from openai import OpenAI
    client = OpenAI(
        api_key=vault.require("GEMINI_API_KEY"),
        base_url="https://generativelanguage.googleapis.com/v1beta/openai/"
    )
    resp = client.chat.completions.create(
        model=model,
        response_format={"type": "json_object"},
        messages=[{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{image_b64}"}}
        ]}]
    )
    return json.loads(resp.choices[0].message.content)


def call_openrouter(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    from openai import OpenAI
    or_model = model[len("openrouter/"):] if model.lower().startswith("openrouter/") else model
    client = OpenAI(
        api_key=vault.require("OPENROUTER_API_KEY"),
        base_url="https://openrouter.ai/api/v1"
    )
    resp = client.chat.completions.create(
        model=or_model,
        response_format={"type": "json_object"},
        messages=[{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{image_b64}"}}
        ]}]
    )
    return json.loads(resp.choices[0].message.content)


def call_mistral(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    from mistralai import Mistral
    client = Mistral()
    resp = client.chat.complete(
        model=model,
        response_format={"type": "json_object"},
        messages=[{"role": "user", "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": f"data:{media_type};base64,{image_b64}"}
        ]}]
    )
    return json.loads(resp.choices[0].message.content)


# Ordered so longer/more-specific prefixes match before shorter ones
PROVIDERS = [
    ("claude",      call_anthropic),
    ("gemini",      call_gemini),
    ("openrouter/", call_openrouter),
    ("pixtral",     call_mistral),
    ("ministral",   call_mistral),
    ("mistral",     call_mistral),
    ("gpt",         call_openai),
    ("o1",          call_openai),
    ("o3",          call_openai),
    ("o4",          call_openai),
]


def call_judge(model: str, image_b64: str, media_type: str, prompt: str) -> dict:
    lower = model.lower()
    for prefix, fn in PROVIDERS:
        if lower.startswith(prefix):
            return fn(model, image_b64, media_type, prompt)
    return call_openai(model, image_b64, media_type, prompt)


def compute_weighted_score(scores: dict, criteria: dict) -> float:
    total = 0.0
    for name, cfg in criteria["criteria"].items():
        if name in scores:
            total += scores[name]["score"] * cfg["weight"]
    return round(total, 2)


def score_images(image_paths: list[str], criteria: dict, judge_models: list[str],
                 original_prompt: str | None = None) -> list[dict]:
    prompt = build_judge_prompt(criteria, original_prompt)
    results = []

    for img_path in image_paths:
        model_scores = {}

        try:
            b64, media_type = load_image_b64(img_path)
        except Exception as e:
            results.append({"image": img_path, "error": str(e), "judges": {}})
            continue

        for model in judge_models:
            try:
                raw = call_judge(model, b64, media_type, prompt)
                scores = raw.get("scores", {})
                model_scores[model] = {
                    "scores": scores,
                    "weighted_total": compute_weighted_score(scores, criteria),
                    "overall_impression": raw.get("overall_impression", "")
                }
            except Exception as e:
                model_scores[model] = {"error": str(e)}

        results.append({"image": img_path, "judges": model_scores})

    return results


DEFAULT_JUDGE_PROMPT = """You are evaluating an image{prompt_context}.

Score this image on each criterion using the rubric below.
Return ONLY a JSON object with this exact structure:
{{
  "scores": {{
    "<criterion_name>": {{
      "score": <integer 1-5>,
      "reasoning": "<one sentence>"
    }}
  }},
  "overall_impression": "<one sentence summary>"
}}

{criteria_details}"""

from pathlib import Path


def generate_report(results: list[dict], criteria: dict, fmt: str = "markdown") -> str:
    if fmt == "json":
        import json
        return json.dumps(results, indent=2)
    if fmt == "table":
        return _table(results)
    return _markdown(results, criteria)


def _markdown(results: list[dict], criteria: dict) -> str:
    lines = [f"# Vision Benchmark: {criteria.get('name', 'Results')}\n"]
    criterion_names = list(criteria["criteria"].keys())

    for r in results:
        lines.append(f"## {Path(r['image']).name}\n")
        for model, data in r["judges"].items():
            lines.append(f"### Judge: `{model}`")
            if "error" in data:
                lines.append(f"> Error: {data['error']}\n")
                continue
            lines.append(f"**Weighted total: {data['weighted_total']:.2f} / 5.00**")
            lines.append(f"> {data['overall_impression']}\n")
            lines.append("| Criterion | Score | Reasoning |")
            lines.append("|---|---|---|")
            for name in criterion_names:
                if name in data["scores"]:
                    s = data["scores"][name]
                    lines.append(f"| {name} | {s['score']}/5 | {s['reasoning']} |")
            lines.append("")

    if len(results) > 1:
        lines.append("## Ranking\n")
        lines.append(_ranking_table(results))

    return "\n".join(lines)


def _ranking_table(results: list[dict]) -> str:
    rows = []
    for r in results:
        name = Path(r["image"]).name
        totals = [d["weighted_total"] for d in r["judges"].values() if "weighted_total" in d]
        avg = sum(totals) / len(totals) if totals else 0
        rows.append((name, avg, totals))

    rows.sort(key=lambda x: x[1], reverse=True)
    lines = ["| Rank | Image | Avg Score |", "|---|---|---|"]
    for i, (name, avg, _) in enumerate(rows, 1):
        lines.append(f"| {i} | {name} | {avg:.2f} |")
    return "\n".join(lines)


def _table(results: list[dict]) -> str:
    rows = []
    for r in results:
        name = Path(r["image"]).name
        for model, data in r["judges"].items():
            total = data.get("weighted_total", "ERR")
            rows.append(f"{name:<40} {model:<35} {total}")
    header = f"{'Image':<40} {'Judge':<35} {'Score'}"
    sep = "-" * len(header)
    return "\n".join([header, sep] + sorted(rows, key=lambda x: x.split()[-1], reverse=True))

pyyaml>=6.0
openai>=1.0
anthropic>=0.40
mistralai>=1.0

"""Auto-decrypt secrets.enc.yaml via SOPS + age, fall back to environment variables."""
import os
import subprocess
from pathlib import Path

SECRETS_FILE = Path(__file__).parent / "secrets.enc.yaml"
_cache: dict = {}


def _load_sops() -> dict:
    if not SECRETS_FILE.exists():
        return {}
    age_key = Path.home() / ".config/sops/age/keys.txt"
    env = os.environ.copy()
    if age_key.exists():
        env["SOPS_AGE_KEY_FILE"] = str(age_key)
    try:
        result = subprocess.run(
            ["sops", "--decrypt", "--output-type", "yaml", str(SECRETS_FILE)],
            capture_output=True, text=True, env=env, timeout=10
        )
        if result.returncode == 0:
            import yaml
            return yaml.safe_load(result.stdout) or {}
    except (FileNotFoundError, subprocess.TimeoutExpired):
        pass
    return {}


def get(key: str) -> str | None:
    """Return secret: SOPS file first, then environment variable."""
    global _cache
    if not _cache:
        _cache = _load_sops()
    return _cache.get(key) or os.environ.get(key)


def require(key: str) -> str:
    val = get(key)
    if not val:
        raise RuntimeError(
            f"Missing secret '{key}'. Add it to secrets.enc.yaml or set ${key} in your environment."
        )
    return val