Ai Understanding Images

AI Understanding Images

Use DSPy's dspy.Image type to pass images into signatures alongside text. Vision LLMs return structured data from photos, screenshots, documents, and charts.

---

Step 1 - Understand the image task

Before writing code, ask:

• What images will you process? (URLs, local files, base64, cloud storage?)
• What do you need to extract? (text, categories, attributes, descriptions?)
• Does the output need to be structured? (typed fields vs. free text?)
• Are you processing images in batch or one at a time?
• Does the task require reasoning about the image, or just direct extraction?

---

Step 2 - Build a basic image analyzer

import dspy

lm = dspy.LM("openai/gpt-4o-mini")  # or "anthropic/claude-sonnet-4-5-20250929", etc.
dspy.configure(lm=lm)

class AnalyzeImage(dspy.Signature):
    """Analyze the image and answer the question."""
    image: dspy.Image = dspy.InputField(desc="The image to analyze")
    question: str = dspy.InputField(desc="What to extract or analyze")
    answer: str = dspy.OutputField(desc="Analysis result")

analyzer = dspy.Predict(AnalyzeImage)

# From a URL
result = analyzer(
    image=dspy.Image(url="https://example.com/photo.jpg"),
    question="What product is shown?"
)

# From a local file
result = analyzer(
    image=dspy.Image(url="photo.jpg"),
    question="What product is shown?"
)

print(result.answer)

---

Step 3 - Vision model selection

All models listed here support image inputs. Always verify vision support before deploying.

---

Step 4 - Combine image with text context

For richer analysis, pass supplemental text alongside the image:

from typing import Literal
from pydantic import BaseModel

class ProductAttributes(BaseModel):
    category: str
    color: str
    condition: Literal["new", "used", "damaged"]
    description: str

class CategorizeProduct(dspy.Signature):
    """Categorize a product from its photo and any provided context."""
    image: dspy.Image = dspy.InputField(desc="Product photo")
    context: str = dspy.InputField(desc="Additional context such as listing title or seller notes")
    attributes: ProductAttributes = dspy.OutputField(desc="Extracted product attributes")

categorizer = dspy.Predict(CategorizeProduct)
result = categorizer(
    image=dspy.Image(url="https://example.com/item.jpg"),
    context="Listed as: Vintage leather jacket, size M"
)
print(result.attributes.category, result.attributes.condition)

---

Step 5 - Common patterns

Alt text generation

class GenerateAltText(dspy.Signature):
    """Generate concise, accurate alt text for accessibility."""
    image: dspy.Image = dspy.InputField(desc="Image to describe")
    context: str = dspy.InputField(desc="Page or article context where the image appears")
    alt_text: str = dspy.OutputField(desc="Alt text under 125 characters, describing the image content")

alt_gen = dspy.Predict(GenerateAltText)
result = alt_gen(
    image=dspy.Image(url="https://example.com/team-photo.jpg"),
    context="About page of a SaaS startup"
)

Receipt and invoice OCR

from typing import List

class LineItem(BaseModel):
    description: str
    quantity: int
    unit_price: float
    total: float

class ExtractReceipt(dspy.Signature):
    """Extract all line items and totals from a receipt or invoice photo."""
    image: dspy.Image = dspy.InputField(desc="Photo of receipt or invoice")
    line_items: List[LineItem] = dspy.OutputField(desc="All line items found")
    subtotal: float = dspy.OutputField(desc="Subtotal before tax")
    tax: float = dspy.OutputField(desc="Tax amount")
    total: float = dspy.OutputField(desc="Total amount due")

extractor = dspy.Predict(ExtractReceipt)
result = extractor(image=dspy.Image(url="receipt.jpg"))

Chart and graph data extraction

class ExtractChart(dspy.Signature):
    """Extract the data series and labels from a chart or graph image."""
    image: dspy.Image = dspy.InputField(desc="Chart or graph image")
    chart_type: str = dspy.OutputField(desc="Type of chart - bar, line, pie, etc.")
    title: str = dspy.OutputField(desc="Chart title if present")
    data_summary: str = dspy.OutputField(desc="Summary of the data shown, including key values")

chart_reader = dspy.Predict(ExtractChart)

UI screenshot analysis

class AnalyzeUI(dspy.Signature):
    """Analyze a UI screenshot and identify components and issues."""
    image: dspy.Image = dspy.InputField(desc="UI screenshot")
    focus: str = dspy.InputField(desc="What aspect to analyze - layout, accessibility, bugs, etc.")
    findings: str = dspy.OutputField(desc="Detailed findings about the UI")
    suggestions: List[str] = dspy.OutputField(desc="Actionable improvement suggestions")

ui_analyzer = dspy.ChainOfThought(AnalyzeUI)
result = ui_analyzer(
    image=dspy.Image(url="screenshot.png"),
    focus="accessibility issues"
)

---

Step 6 - OCR vs vision model tradeoff

---

Step 7 - Image preprocessing

Vision models have token budgets per image. Large images consume more tokens and slow responses.

from PIL import Image as PILImage
import io, base64

def resize_for_vision(image_path: str, max_side: int = 1024) -> dspy.Image:
    """Resize image so the longest side is at most max_side pixels."""
    img = PILImage.open(image_path)
    ratio = min(max_side / img.width, max_side / img.height, 1.0)
    if ratio < 1.0:
        new_size = (int(img.width * ratio), int(img.height * ratio))
        img = img.resize(new_size, PILImage.LANCZOS)
    buf = io.BytesIO()
    img.save(buf, format="JPEG", quality=85)
    b64 = base64.b64encode(buf.getvalue()).decode()
    return dspy.Image(url=f"data:image/jpeg;base64,{b64}")

image = resize_for_vision("large_photo.jpg")
result = analyzer(image=image, question="What is shown?")

Recommended limits:

• Max 1024px on the longest side for most tasks
• JPEG quality 80-85 for photos; PNG for screenshots with text
• Avoid sending multiple large images in a single call

---

Step 8 - Evaluate visual tasks

For description quality, use an AI judge:

class ImageDescriptionJudge(dspy.Signature):
    """Judge whether an image description is accurate and complete."""
    image: dspy.Image = dspy.InputField(desc="The original image")
    description: str = dspy.InputField(desc="Description to evaluate")
    score: int = dspy.OutputField(desc="Score from 1 to 5, where 5 is fully accurate and complete")
    reasoning: str = dspy.OutputField(desc="Explanation of the score")

judge = dspy.Predict(ImageDescriptionJudge)

For structured extraction (OCR, receipts), use exact match or field-level comparison:

def eval_receipt_extraction(prediction, ground_truth):
    correct_items = sum(
        1 for item in prediction.line_items
        if item.description in [g.description for g in ground_truth.line_items]
    )
    recall = correct_items / max(len(ground_truth.line_items), 1)
    total_match = abs(prediction.total - ground_truth.total) < 0.01
    return {"item_recall": recall, "total_correct": total_match}

---

When NOT to use vision LLMs

• Object detection at scale - use YOLO, Detectron2, or a dedicated CV API
• Simple OCR on clean printed text - Tesseract or cloud OCR is faster and cheaper
• Pixel-level segmentation - use Segment Anything or dedicated segmentation models
• Real-time video processing - vision LLMs have too much latency
• Sub-100ms latency - vision LLMs typically take 1-5 seconds per image
• High-volume identical-format documents - train a specialized model or use template OCR

---

Key patterns

# Pattern 1 - Direct extraction with typed output
class ExtractFields(dspy.Signature):
    """Extract structured fields from the image."""
    image: dspy.Image = dspy.InputField()
    fields: MyDataModel = dspy.OutputField()

extractor = dspy.Predict(ExtractFields)

# Pattern 2 - Reasoning about image content
class ReasonAboutImage(dspy.Signature):
    """Reason step by step about what the image shows."""
    image: dspy.Image = dspy.InputField()
    question: str = dspy.InputField()
    answer: str = dspy.OutputField()

reasoner = dspy.ChainOfThought(ReasonAboutImage)

# Pattern 3 - Batch processing
images = [dspy.Image(url=p) for p in image_paths]
results = [extractor(image=img) for img in images]

# Pattern 4 - Iterative refinement when quality is low
refiner = dspy.Refine(dspy.Predict(AnalyzeImage), N=3, reward_fn=my_reward)

---

Gotchas

• Wrap image inputs in `dspy.Image` - Claude writes raw URL strings as image inputs instead of wrapping them. Always wrap the URL or local path directly: dspy.Image(url="https://...") or dspy.Image(url="local/path.jpg"). Raw strings are treated as text, not images. (The old dspy.Image(url=) / from_file() classmethods are deprecated in favor of dspy.Image(url=...).)

• Verify the model supports vision - Claude picks a model that does not support image inputs. Not all LLMs handle images. Confirm vision support for your chosen model before deploying (GPT-4o, Claude 3.5+, Gemini 2.x all work).

• Use `dspy.Refine` not `dspy.Assert` - Claude uses dspy.Assert/dspy.Suggest to validate image outputs. Use dspy.Refine with a reward function for iterative improvement instead.

• Resize before sending - Claude sends full-resolution images without resizing. Large images (4K, RAW photos) consume excessive tokens and can hit context limits. Resize to max 1024px on the longest side before processing.

• Match module to task - Claude applies dspy.ChainOfThought to all image tasks. Use dspy.Predict for direct extraction (OCR, field parsing). Reserve dspy.ChainOfThought for tasks that genuinely benefit from image reasoning, like diagnosing a bug from a screenshot.

---

Cross-references

Install any skill: npx skills add lebsral/DSPy-Programming-not-prompting-LMs-skills --skill <name>

• /ai-parsing-data - parse structured data from text; complement with image extraction for mixed inputs
• /ai-stopping-hallucinations - reduce made-up field values in vision extraction pipelines
• /ai-checking-outputs - validate extracted fields after vision model output
• /dspy-refine - iterative refinement when initial image analysis quality is low
• /dspy-modules - understand dspy.Predict vs dspy.ChainOfThought for image tasks
• Install `/ai-do` if you do not have it — it routes any AI problem to the right skill and is the fastest way to work: npx skills add lebsral/DSPy-Programming-not-prompting-LMs-skills --skill ai-do

Additional resources

See examples.md for worked examples:

• Product photo categorizer
• Alt text generator
• Receipt/invoice OCR pipeline

Examples - AI Understanding Images

---

Example 1 - Product photo categorizer

Classify e-commerce product photos and extract structured attributes.

import dspy
from pydantic import BaseModel
from typing import List, Literal

lm = dspy.LM("openai/gpt-4o-mini")  # or "anthropic/claude-sonnet-4-5-20250929", etc.
dspy.configure(lm=lm)

class ProductAttributes(BaseModel):
    category: str
    subcategory: str
    primary_color: str
    secondary_colors: List[str]
    condition: Literal["new", "like_new", "used", "damaged"]
    material: str
    description: str

class CategorizeProduct(dspy.Signature):
    """
    Examine the product photo and extract structured attributes for catalog indexing.
    Be specific about category and subcategory. Use 'unknown' only if truly unclear.
    """
    image: dspy.Image = dspy.InputField(desc="Product photo")
    listing_title: str = dspy.InputField(desc="Seller-provided title, may be incomplete or misleading")
    attributes: ProductAttributes = dspy.OutputField(desc="Extracted product attributes")

categorizer = dspy.Predict(CategorizeProduct)

# Process a single product
result = categorizer(
    image=dspy.Image(url="https://example.com/jacket.jpg"),
    listing_title="vintage jacket size M great condition"
)
print(result.attributes.category)       # Clothing
print(result.attributes.subcategory)    # Jackets & Coats
print(result.attributes.primary_color)  # Brown
print(result.attributes.condition)      # like_new

# Batch process a catalog
import json

products = [
    {"url": "https://example.com/item1.jpg", "title": "old lamp"},
    {"url": "https://example.com/item2.jpg", "title": "ceramic bowl set"},
    {"url": "https://example.com/item3.jpg", "title": "running shoes"},
]

results = []
for product in products:
    res = categorizer(
        image=dspy.Image(url=product["url"]),
        listing_title=product["title"]
    )
    results.append({
        "title": product["title"],
        "category": res.attributes.category,
        "subcategory": res.attributes.subcategory,
        "condition": res.attributes.condition,
    })

print(json.dumps(results, indent=2))

Reward function for optimization

VALID_CONDITIONS = {"new", "like_new", "used", "damaged"}
REQUIRED_FIELDS = ["category", "subcategory", "primary_color", "condition", "description"]

def product_categorization_reward(example, prediction, trace=None):
    attrs = prediction.attributes
    # Check all required fields are populated
    for field in REQUIRED_FIELDS:
        if not getattr(attrs, field, None):
            return 0.0
    # Check condition is valid
    if attrs.condition not in VALID_CONDITIONS:
        return 0.0
    # Check description is meaningful (not just "unknown")
    if len(attrs.description) < 20:
        return 0.5
    return 1.0

# Optimize with a labeled dataset
optimizer = dspy.MIPROv2(metric=product_categorization_reward)
optimized = optimizer.compile(categorizer, trainset=labeled_examples)

---

Example 2 - Alt text generator

Generate accessible alt text for images on web pages and in content management systems.

import dspy

lm = dspy.LM("openai/gpt-4o-mini")  # or "anthropic/claude-sonnet-4-5-20250929", etc.
dspy.configure(lm=lm)

class AltTextOutput(dspy.Signature):
    """
    Generate concise, accurate alt text for web accessibility (WCAG 2.1 AA).
    Alt text should describe the image content and function, not appearance.
    Decorative images should receive empty alt text.
    Keep alt text under 125 characters.
    """
    image: dspy.Image = dspy.InputField(desc="Image to describe")
    page_context: str = dspy.InputField(
        desc="The surrounding page content or article topic where the image appears"
    )
    image_role: str = dspy.InputField(
        desc="Role of the image - informative, decorative, functional, or complex"
    )
    alt_text: str = dspy.OutputField(
        desc="Alt text under 125 characters. Empty string if decorative."
    )
    long_description: str = dspy.OutputField(
        desc="Extended description for complex images like charts. Empty if not needed."
    )
    is_decorative: bool = dspy.OutputField(
        desc="True if the image is purely decorative and alt should be empty"
    )

alt_gen = dspy.Predict(AltTextOutput)

# Single image
result = alt_gen(
    image=dspy.Image(url="https://example.com/team.jpg"),
    page_context="About page of a B2B SaaS company, section titled 'Our Team'",
    image_role="informative"
)
print(result.alt_text)          # "Five team members smiling in a modern office"
print(result.is_decorative)     # False

# CMS batch processing
cms_images = [
    {
        "url": "https://example.com/hero-bg.jpg",
        "context": "Homepage hero section background",
        "role": "decorative"
    },
    {
        "url": "https://example.com/quarterly-chart.png",
        "context": "Q4 investor report, revenue growth section",
        "role": "complex"
    },
    {
        "url": "https://example.com/cta-button.png",
        "context": "Sign up call to action",
        "role": "functional"
    },
]

for item in cms_images:
    res = alt_gen(
        image=dspy.Image(url=item["url"]),
        page_context=item["context"],
        image_role=item["role"]
    )
    if res.is_decorative:
        print(f'{item["url"]} -> alt=""')
    else:
        print(f'{item["url"]} -> alt="{res.alt_text}"')
        if res.long_description:
            print(f'  longdesc: {res.long_description[:80]}...')

Evaluation

def alt_text_reward(example, prediction, trace=None):
    alt = prediction.alt_text or ""
    # Decorative images should have empty alt
    if example.is_decorative and alt == "":
        return 1.0
    if example.is_decorative and alt != "":
        return 0.0
    # Non-decorative images must have meaningful alt text
    if len(alt) == 0:
        return 0.0
    if len(alt) > 125:
        return 0.5  # Too long for WCAG compliance
    # Penalize generic filler phrases
    filler = ["image of", "photo of", "picture of"]
    if any(alt.lower().startswith(f) for f in filler):
        return 0.7
    return 1.0

---

Example 3 - Receipt and invoice OCR pipeline

Extract structured line items and totals from photos of receipts and invoices.

import dspy
from pydantic import BaseModel
from typing import List, Optional

lm = dspy.LM("openai/gpt-4o")  # Use a higher-quality model for OCR accuracy
dspy.configure(lm=lm)

class LineItem(BaseModel):
    description: str
    quantity: float
    unit_price: float
    total: float

class ReceiptData(dspy.Signature):
    """
    Extract all line items, taxes, and totals from a receipt or invoice photo.
    Normalize prices to float. If quantity is not shown, assume 1.
    Use 0.0 for any monetary field that is not present or legible.
    """
    image: dspy.Image = dspy.InputField(desc="Photo of receipt or invoice")
    currency_hint: str = dspy.InputField(
        desc="Expected currency code if known, e.g. USD, EUR. Use 'unknown' if not sure."
    )
    merchant_name: str = dspy.OutputField(desc="Name of the merchant or vendor")
    date: str = dspy.OutputField(desc="Transaction date in ISO 8601 format if readable, else empty string")
    line_items: List[LineItem] = dspy.OutputField(desc="All line items on the receipt")
    subtotal: float = dspy.OutputField(desc="Subtotal before tax and tip")
    tax: float = dspy.OutputField(desc="Tax amount")
    tip: float = dspy.OutputField(desc="Tip amount, 0.0 if not present")
    total: float = dspy.OutputField(desc="Final total paid")
    currency: str = dspy.OutputField(desc="Currency code detected")

extractor = dspy.Predict(ReceiptData)

# Process a single receipt
result = extractor(
    image=dspy.Image(url="receipt.jpg"),
    currency_hint="USD"
)

print(f"Merchant - {result.merchant_name}")
print(f"Date - {result.date}")
print(f"Items -")
for item in result.line_items:
    print(f"  {item.description} x{item.quantity} @ ${item.unit_price:.2f} = ${item.total:.2f}")
print(f"Subtotal - ${result.subtotal:.2f}")
print(f"Tax - ${result.tax:.2f}")
print(f"Tip - ${result.tip:.2f}")
print(f"Total - ${result.total:.2f}")

# Validate totals (simple sanity check)
computed = sum(i.total for i in result.line_items) + result.tax + result.tip
if abs(computed - result.total) > 0.05:
    print(f"WARNING - Computed total ${computed:.2f} does not match extracted total ${result.total:.2f}")

Pipeline with preprocessing and retry

from PIL import Image as PILImage
import io, base64

def preprocess_receipt(image_path: str) -> dspy.Image:
    """Resize and enhance contrast for better OCR accuracy."""
    img = PILImage.open(image_path).convert("RGB")
    # Resize so longest side is 2048px (receipts benefit from higher resolution)
    ratio = min(2048 / img.width, 2048 / img.height, 1.0)
    if ratio < 1.0:
        img = img.resize(
            (int(img.width * ratio), int(img.height * ratio)),
            PILImage.LANCZOS
        )
    buf = io.BytesIO()
    img.save(buf, format="JPEG", quality=92)
    b64 = base64.b64encode(buf.getvalue()).decode()
    return dspy.Image(url=f"data:image/jpeg;base64,{b64}")

def receipt_reward(example, prediction, trace=None):
    """Reward function for receipt extraction quality."""
    if not prediction.line_items:
        return 0.0
    # Check total reconciliation
    computed = sum(i.total for i in prediction.line_items) + prediction.tax + prediction.tip
    total_ok = abs(computed - prediction.total) < 0.10
    # Check required fields
    has_merchant = bool(prediction.merchant_name)
    has_items = len(prediction.line_items) > 0
    score = (0.5 * int(total_ok)) + (0.3 * int(has_merchant)) + (0.2 * int(has_items))
    return score

# Use dspy.Refine for retry on low-confidence extractions
refining_extractor = dspy.Refine(dspy.Predict(ReceiptData), N=2, reward_fn=receipt_reward)

image = preprocess_receipt("crumpled_receipt.jpg")
result = refining_extractor(image=image, currency_hint="USD")
print(f"Total - ${result.total:.2f} ({len(result.line_items)} items)")