Vision Language Models

Vision Language Models ()

Integrate vision capabilities from leading multimodal models for image understanding, document analysis, and visual reasoning.

Overview

• Image captioning and description generation
• Visual question answering (VQA)
• Document/chart/diagram analysis with OCR
• Multi-image comparison and reasoning
• Bounding box detection and region analysis
• Video frame analysis

Model Comparison (January )

Image Input Methods

Base64 Encoding (All Providers)

import base64
import mimetypes

def encode_image_base64(image_path: str) -> tuple[str, str]:
    """Encode local image to base64 with MIME type."""
    mime_type, _ = mimetypes.guess_type(image_path)
    mime_type = mime_type or "image/png"

    with open(image_path, "rb") as f:
        base64_data = base64.standard_b64encode(f.read()).decode("utf-8")

    return base64_data, mime_type

OpenAI GPT-5/4o Vision

from openai import OpenAI

client = OpenAI()

def analyze_image_openai(image_path: str, prompt: str) -> str:
    """Analyze image using GPT-5 or GPT-4o."""
    base64_data, mime_type = encode_image_base64(image_path)

    response = client.chat.completions.create(
        model="gpt-5.2",  # or "gpt-4.1" for cost optimization
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": prompt},
                {"type": "image_url", "image_url": {
                    "url": f"data:{mime_type};base64,{base64_data}",
                    "detail": "high"  # low, high, or auto
                }}
            ]
        }],
        max_tokens=4096  # Required for vision
    )
    return response.choices[0].message.content

Claude 4.5 Vision (Anthropic)

import anthropic

client = anthropic.Anthropic()

def analyze_image_claude(image_path: str, prompt: str) -> str:
    """Analyze image using Claude Opus 4.6 or Sonnet 4.5."""
    base64_data, media_type = encode_image_base64(image_path)

    response = client.messages.create(
        model="claude-opus-4-6",  # or claude-sonnet-4-5
        max_tokens=4096,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": base64_data
                    }
                },
                {"type": "text", "text": prompt}
            ]
        }]
    )
    return response.content[0].text

Gemini 2.5/3 Vision (Google)

import google.generativeai as genai
from PIL import Image

genai.configure(api_key="YOUR_API_KEY")

def analyze_image_gemini(image_path: str, prompt: str) -> str:
    """Analyze image using Gemini 2.5 Pro or Gemini 3."""
    model = genai.GenerativeModel("gemini-2.5-pro")  # or gemini-3-pro

    image = Image.open(image_path)

    response = model.generate_content([prompt, image])
    return response.text

# For video analysis (Gemini excels here)
def analyze_video_gemini(video_path: str, prompt: str) -> str:
    """Analyze video using Gemini's native video support."""
    model = genai.GenerativeModel("gemini-2.5-pro")

    video_file = genai.upload_file(video_path)

    response = model.generate_content([prompt, video_file])
    return response.text

Grok 4 Vision (xAI)

from openai import OpenAI  # Grok uses OpenAI-compatible API

client = OpenAI(
    api_key="YOUR_XAI_API_KEY",
    base_url="https://api.x.ai/v1"
)

def analyze_image_grok(image_path: str, prompt: str) -> str:
    """Analyze image using Grok 4 with real-time capabilities."""
    base64_data, mime_type = encode_image_base64(image_path)

    response = client.chat.completions.create(
        model="grok-4",  # or grok-2-vision-1212
        messages=[{
            "role": "user",
            "content": [
                {"type": "text", "text": prompt},
                {"type": "image_url", "image_url": {
                    "url": f"data:{mime_type};base64,{base64_data}"
                }}
            ]
        }]
    )
    return response.choices[0].message.content

Multi-Image Analysis

async def compare_images(images: list[str], prompt: str) -> str:
    """Compare multiple images (Claude supports up to 100)."""
    content = []

    for img_path in images:
        base64_data, media_type = encode_image_base64(img_path)
        content.append({
            "type": "image",
            "source": {
                "type": "base64",
                "media_type": media_type,
                "data": base64_data
            }
        })

    content.append({"type": "text", "text": prompt})

    response = client.messages.create(
        model="claude-opus-4-6",
        max_tokens=8192,
        messages=[{"role": "user", "content": content}]
    )
    return response.content[0].text

Object Detection (Gemini 2.5+)

def detect_objects_gemini(image_path: str) -> list[dict]:
    """Detect objects with bounding boxes using Gemini 2.5+."""
    model = genai.GenerativeModel("gemini-2.5-pro")
    image = Image.open(image_path)

    response = model.generate_content([
        "Detect all objects in this image. Return bounding boxes "
        "as JSON with format: {objects: [{label, box: [x1,y1,x2,y2]}]}",
        image
    ])

    import json
    return json.loads(response.text)

Token Cost Optimization

# Cost-optimized simple classification
response = client.chat.completions.create(
    model="gpt-5.2-mini",  # Cheaper for simple tasks
    messages=[{
        "role": "user",
        "content": [
            {"type": "text", "text": "Is there a person? Reply: yes/no"},
            {"type": "image_url", "image_url": {
                "url": image_url,
                "detail": "low"  # Minimal tokens
            }}
        ]
    }]
)

Image Size Limits ()

Key Decisions

Common Mistakes

• Not setting max_tokens (responses truncated)
• Sending oversized images (resize to 2048px max)
• Using high detail for yes/no questions
• Not validating image format before encoding
• Ignoring rate limits on vision endpoints
• Using deprecated models (GPT-4V retired)

Limitations

• Cannot identify specific people (privacy restriction)
• May hallucinate on low-quality/rotated images (<200px)
• GPT-5: may struggle with precise spatial reasoning on edge cases
• No real-time video (use frame extraction except Gemini)

Related Skills

• audio-language-models - Audio/speech processing
• multimodal-rag - Image + text retrieval
• llm-streaming - Streaming vision responses

Capability Details

image-captioning

Keywords: caption, describe, image description, alt text, accessibility Solves:

• Generate descriptive captions for images
• Create accessibility alt text
• Extract visual content summary

visual-qa

Keywords: VQA, visual question, image question, analyze image Solves:

• Answer questions about image content
• Extract specific information from visuals
• Reason about image elements

document-vision

Keywords: document, PDF, chart, diagram, OCR, extract, table Solves:

• Extract text from documents and charts
• Analyze diagrams and flowcharts
• Process forms and tables with structure

Claude Code PDF Handling (CC 2.1.30+)

Read Tool Pages Parameter

For large PDFs (>10 pages), use the pages parameter to read specific ranges:

# Read first 5 pages of a large PDF
Read(file_path="/path/to/document.pdf", pages="1-5")

# Read specific page
Read(file_path="/path/to/document.pdf", pages="10")

# Read range in middle
Read(file_path="/path/to/document.pdf", pages="15-25")

Large PDF Strategy

For documents >100 pages, process incrementally:

# 1. Initial scan - read first pages for structure
Read(file_path=pdf_path, pages="1-5")

# 2. Identify key sections from TOC/headers
# 3. Read relevant sections
Read(file_path=pdf_path, pages="45-55")  # e.g., "Implementation" section

# 4. Process remaining sections as needed
Read(file_path=pdf_path, pages="80-90")  # e.g., "Appendix" section

Limits

multi-image-analysis

Keywords: compare images, multiple images, image comparison, batch Solves:

• Compare visual elements across images
• Track changes between versions
• Analyze image sequences

object-detection

Keywords: bounding box, detect objects, locate, segmentation Solves:

• Detect and locate objects in images
• Generate bounding box coordinates
• Segment image regions (Gemini 2.5+)

Vision Language Models Checklist

Image Input

• [ ] Support base64 encoding
• [ ] Support URL-based images
• [ ] Validate image format (PNG, JPEG, WebP)
• [ ] Resize images to max 2048px
• [ ] Set appropriate detail level (low/high/auto)

Provider Integration

• [ ] OpenAI GPT-5/4o integration
• [ ] Claude 4.5 integration
• [ ] Gemini 2.5/3 integration
• [ ] Grok 4 integration (if needed)
• [ ] Provider fallback chain

Multi-Image

• [ ] Handle up to 10 images (OpenAI)
• [ ] Handle up to 100 images (Claude)
• [ ] Handle batch image analysis
• [ ] Implement image comparison

Document Analysis

• [ ] PDF to image conversion
• [ ] Chart/graph data extraction
• [ ] Table extraction
• [ ] OCR for scanned documents

Cost Optimization

• [ ] Use low detail for classification
• [ ] Use high detail for OCR/charts
• [ ] Batch requests when possible
• [ ] Cache analysis results
• [ ] Use mini models for simple tasks

Error Handling

• [ ] Handle oversized images
• [ ] Handle unsupported formats
• [ ] Validate API responses
• [ ] Set max_tokens for vision
• [ ] Handle rate limits

Vision API Cost Optimization

Strategies to minimize costs while maintaining quality for vision workloads.

Token Cost by Provider (January 2026)

Detail Level Strategy

def select_detail_level(task: str, image_size: tuple) -> str:
    """Select optimal detail level for cost/quality balance."""
    width, height = image_size

    # Low detail tasks (65 tokens)
    low_detail_tasks = [
        "classification",
        "presence_detection",
        "yes_no_question",
        "simple_count",
        "color_identification"
    ]

    if task in low_detail_tasks:
        return "low"

    # High detail required (129+ tokens/tile)
    high_detail_tasks = [
        "ocr",
        "document_analysis",
        "chart_reading",
        "fine_detail",
        "small_text"
    ]

    if task in high_detail_tasks:
        return "high"

    # Auto for everything else
    return "auto"

Image Preprocessing

from PIL import Image

def optimize_image_for_api(
    image_path: str,
    max_dimension: int = 2048,
    quality: int = 85
) -> str:
    """Resize and compress image to minimize tokens."""
    img = Image.open(image_path)

    # Resize if larger than max dimension
    if max(img.size) > max_dimension:
        ratio = max_dimension / max(img.size)
        new_size = (int(img.size[0] * ratio), int(img.size[1] * ratio))
        img = img.resize(new_size, Image.LANCZOS)

    # Convert RGBA to RGB if needed
    if img.mode == "RGBA":
        background = Image.new("RGB", img.size, (255, 255, 255))
        background.paste(img, mask=img.split()[3])
        img = background

    # Save optimized
    output_path = "/tmp/optimized.jpg"
    img.save(output_path, "JPEG", quality=quality, optimize=True)

    return output_path

Batch Processing

async def batch_analyze_images(
    images: list[str],
    prompt: str,
    batch_size: int = 5
) -> list[str]:
    """Batch images to reduce API calls and costs."""
    results = []

    for i in range(0, len(images), batch_size):
        batch = images[i:i + batch_size]

        # Send multiple images in one request
        content = []
        for img_path in batch:
            base64_data, media_type = encode_image_base64(img_path)
            content.append({
                "type": "image",
                "source": {
                    "type": "base64",
                    "media_type": media_type,
                    "data": base64_data
                }
            })

        content.append({
            "type": "text",
            "text": f"Analyze each image and provide: {prompt}\n"
                    f"Format: Image 1: ..., Image 2: ..."
        })

        response = client.messages.create(
            model="claude-sonnet-4-5",  # Cheaper than Opus
            max_tokens=4096,
            messages=[{"role": "user", "content": content}]
        )

        results.append(response.content[0].text)

    return results

Model Tiering

def select_model_for_task(
    task_complexity: str,
    budget: str = "normal"
) -> str:
    """Select cost-appropriate model for task."""
    models = {
        "simple": {
            "budget": "gpt-5.2-mini",
            "normal": "gemini-2.5-flash",
            "quality": "gpt-5.2"
        },
        "moderate": {
            "budget": "gemini-2.5-flash",
            "normal": "claude-sonnet-4-5",
            "quality": "gpt-5"
        },
        "complex": {
            "budget": "claude-sonnet-4-5",
            "normal": "claude-opus-4-6",
            "quality": "claude-opus-4-6"
        }
    }

    return models.get(task_complexity, models["moderate"])[budget]

Cost Comparison Example

Best Practices

1. Start with Flash/Mini: Use cheapest model first, upgrade if quality insufficient 2. Resize images: Never send 4K images for simple tasks 3. Use low detail: For classification, presence detection 4. Batch requests: Multiple images per API call when possible 5. Cache results: Store analysis results, don't re-analyze 6. Gemini for volume: $0.15/M tokens for high-volume workloads

Document Vision Analysis

Patterns for analyzing PDFs, charts, diagrams, and scanned documents using vision models.

Best Model Selection

PDF Processing Pipeline

from pdf2image import convert_from_path
import anthropic

client = anthropic.Anthropic()

async def analyze_pdf(
    pdf_path: str,
    questions: list[str],
    max_pages: int = 20
) -> dict:
    """Analyze PDF document with vision model."""
    # Convert PDF to images (150 DPI for balance)
    pages = convert_from_path(pdf_path, dpi=150)

    results = {}

    for i, page in enumerate(pages[:max_pages]):
        # Save as PNG (better quality than JPEG for text)
        temp_path = f"/tmp/page_{i}.png"
        page.save(temp_path, "PNG")

        # Encode for API
        base64_data, media_type = encode_image_base64(temp_path)

        # Analyze with Claude
        response = client.messages.create(
            model="claude-opus-4-6",
            max_tokens=4096,
            messages=[{
                "role": "user",
                "content": [
                    {
                        "type": "image",
                        "source": {
                            "type": "base64",
                            "media_type": media_type,
                            "data": base64_data
                        }
                    },
                    {
                        "type": "text",
                        "text": f"Analyze this document page and answer:\n" +
                               "\n".join(f"- {q}" for q in questions)
                    }
                ]
            }]
        )

        results[f"page_{i}"] = response.content[0].text

    return results

Chart Analysis

async def analyze_chart(image_path: str) -> dict:
    """Extract data and insights from charts/graphs."""
    prompt = """Analyze this chart and provide:
1. Chart type (bar, line, pie, scatter, etc.)
2. Title and axis labels
3. All data points with values
4. Key trends or insights
5. Any anomalies or notable patterns

Format as JSON:
{
    "chart_type": "...",
    "title": "...",
    "x_axis": "...",
    "y_axis": "...",
    "data_points": [...],
    "insights": [...]
}"""

    response = await analyze_image_claude(image_path, prompt)
    return json.loads(response)

Table Extraction

async def extract_table(image_path: str) -> list[dict]:
    """Extract structured table data from image."""
    prompt = """Extract all data from this table.
Return as JSON array where each row is an object.
Use column headers as keys.
Handle merged cells appropriately.

Example output:
[
    {"Column1": "value1", "Column2": "value2"},
    {"Column1": "value3", "Column2": "value4"}
]"""

    response = await analyze_image_claude(image_path, prompt)
    return json.loads(response)

Multi-Page Document Context

async def analyze_multipage_document(
    images: list[str],
    question: str
) -> str:
    """Analyze document across multiple pages with context."""
    # Use Gemini for long context
    import google.generativeai as genai

    model = genai.GenerativeModel("gemini-2.5-pro")

    content = [Image.open(img) for img in images]
    content.append(
        f"This is a multi-page document. "
        f"Analyze all pages together and answer: {question}"
    )

    response = model.generate_content(content)
    return response.text

Quality Tips

1. DPI Selection: Use 150 DPI for documents, 300 for detailed diagrams 2. Page ordering: Place images before questions in the prompt 3. Chunking: For >20 pages, summarize sections first 4. Validation: Cross-reference extracted numbers with source 5. Fallback: Use OCR libraries (Tesseract) for pure text extraction

Image Captioning Patterns

Best practices for generating high-quality image descriptions and captions using vision models.

Model Selection for Captioning

Basic Captioning

import anthropic

client = anthropic.Anthropic()

def generate_caption(
    image_path: str,
    style: str = "descriptive"
) -> str:
    """Generate image caption with style control."""
    prompts = {
        "descriptive": "Describe this image in detail. Include objects, actions, setting, and mood.",
        "concise": "Write a one-sentence caption for this image.",
        "alt_text": "Write an alt text description for accessibility. Be concise but include key visual information.",
        "creative": "Write a creative, engaging caption for this image suitable for social media.",
        "technical": "Describe the technical aspects of this image: composition, lighting, colors, and style."
    }

    base64_data, media_type = encode_image_base64(image_path)

    response = client.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=500,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": base64_data
                    }
                },
                {"type": "text", "text": prompts.get(style, prompts["descriptive"])}
            ]
        }]
    )

    return response.content[0].text

Structured Caption Output

from pydantic import BaseModel
from typing import Optional

class ImageCaption(BaseModel):
    short_caption: str
    detailed_description: str
    objects: list[str]
    scene_type: str
    mood: Optional[str]
    colors: list[str]
    alt_text: str

def generate_structured_caption(image_path: str) -> ImageCaption:
    """Generate structured caption with multiple formats."""
    prompt = """Analyze this image and provide:
1. short_caption: One sentence summary
2. detailed_description: 2-3 sentences with full details
3. objects: List of main objects/subjects
4. scene_type: indoor, outdoor, portrait, product, etc.
5. mood: emotional tone if applicable
6. colors: dominant colors
7. alt_text: accessibility description

Return as JSON matching the schema."""

    base64_data, media_type = encode_image_base64(image_path)

    response = client.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=1000,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": base64_data
                    }
                },
                {"type": "text", "text": prompt}
            ]
        }]
    )

    import json
    data = json.loads(response.content[0].text)
    return ImageCaption(**data)

Batch Captioning

async def batch_caption_images(
    image_paths: list[str],
    batch_size: int = 5
) -> list[str]:
    """Efficiently caption multiple images."""
    captions = []

    for i in range(0, len(image_paths), batch_size):
        batch = image_paths[i:i + batch_size]

        content = []
        for j, path in enumerate(batch):
            base64_data, media_type = encode_image_base64(path)
            content.append({
                "type": "image",
                "source": {
                    "type": "base64",
                    "media_type": media_type,
                    "data": base64_data
                }
            })

        content.append({
            "type": "text",
            "text": f"Caption each image (1-2 sentences each).\n"
                    f"Format:\nImage 1: [caption]\nImage 2: [caption]\n..."
        })

        response = client.messages.create(
            model="claude-sonnet-4-5",
            max_tokens=2000,
            messages=[{"role": "user", "content": content}]
        )

        # Parse individual captions
        text = response.content[0].text
        for line in text.split("\n"):
            if line.startswith("Image"):
                caption = line.split(":", 1)[1].strip()
                captions.append(caption)

    return captions

Accessibility Alt Text

def generate_alt_text(
    image_path: str,
    context: str = None
) -> str:
    """Generate WCAG-compliant alt text."""
    prompt = """Write alt text for this image following accessibility best practices:
- Be concise (under 125 characters ideally)
- Describe key visual content
- Skip "image of" or "picture of"
- Include text visible in the image
- Convey the purpose/meaning, not just appearance"""

    if context:
        prompt += f"\n\nContext where image appears: {context}"

    base64_data, media_type = encode_image_base64(image_path)

    response = client.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=200,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": base64_data
                    }
                },
                {"type": "text", "text": prompt}
            ]
        }]
    )

    return response.content[0].text.strip()

Caption for Search Indexing

def generate_search_caption(image_path: str) -> str:
    """Generate caption optimized for search/retrieval."""
    prompt = """Describe this image for search indexing. Include:
- All visible objects and subjects
- Actions taking place
- Text visible in the image
- Colors, brands, or identifiable items
- Setting and context
- Any notable details

Be thorough but factual. Use keywords that someone might search for."""

    base64_data, media_type = encode_image_base64(image_path)

    response = client.messages.create(
        model="claude-sonnet-4-5",
        max_tokens=500,
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "image",
                    "source": {
                        "type": "base64",
                        "media_type": media_type,
                        "data": base64_data
                    }
                },
                {"type": "text", "text": prompt}
            ]
        }]
    )

    return response.content[0].text

Quality Guidelines

1. Be specific: "Golden retriever running on beach" > "Dog outside" 2. Include context: Mention setting, time of day, mood 3. Avoid assumptions: Describe what's visible, not interpretations 4. For alt text: Focus on function, not just appearance 5. For search: Include synonyms and related terms 6. For social: Match brand voice and platform norms