Ai Scoring

Build an AI Scorer

Guide the user through building AI that scores, grades, or evaluates work against defined criteria. The pattern: define a rubric, score each criterion independently, calibrate with examples, and validate scorer quality.

Step 1: Define the rubric

Ask the user: 1. What are you scoring? (essays, code, support responses, applications, etc.) 2. What criteria matter? (clarity, accuracy, completeness, tone, security, etc.) 3. What scale? (1-5, 1-10, pass/fail, letter grade) 4. Are criteria weighted equally? (e.g., accuracy 50%, clarity 30%, formatting 20%)

A good rubric has:

• 3-7 criteria — more than that and scorers lose focus
• Clear scale anchors — what does a "2" vs a "4" look like?
• Observable evidence — criteria should reference things you can point to, not vibes

Step 2: Build the scoring signature

import dspy
from pydantic import BaseModel, Field

class CriterionScore(BaseModel):
    criterion: str = Field(description="Name of the criterion being scored")
    score: int = Field(ge=1, le=5, description="Score from 1 (poor) to 5 (excellent)")
    justification: str = Field(description="Evidence from the input that supports this score")

class ScoringResult(BaseModel):
    criterion_scores: list[CriterionScore] = Field(description="Score for each criterion")
    overall_score: float = Field(ge=1.0, le=5.0, description="Weighted overall score")
    summary: str = Field(description="Brief overall assessment")

Define what's being scored and the criteria:

CRITERIA = [
    "clarity: Is the writing clear and easy to follow? (1=confusing, 5=crystal clear)",
    "argument: Is the argument well-structured and logical? (1=no structure, 5=compelling)",
    "evidence: Does the writing cite relevant evidence? (1=no evidence, 5=strong support)",
]

class ScoreCriterion(dspy.Signature):
    """Score the submission on a single criterion. Be specific — cite evidence from the text."""
    submission: str = dspy.InputField(desc="The work being evaluated")
    criterion: str = dspy.InputField(desc="The criterion to score, including scale description")
    score: int = dspy.OutputField(desc="Score from 1 to 5")
    justification: str = dspy.OutputField(desc="Specific evidence from the submission supporting this score")

Step 3: Score per criterion independently

Scoring all criteria at once causes "halo effect" — a strong first impression biases all scores. Instead, score each criterion in its own call:

class RubricScorer(dspy.Module):
    def __init__(self, criteria: list[str], weights: list[float] = None):
        self.criteria = criteria
        self.weights = weights or [1.0 / len(criteria)] * len(criteria)
        self.score_criterion = dspy.ChainOfThought(ScoreCriterion)

    def forward(self, submission: str):
        criterion_scores = []

        for criterion in self.criteria:
            result = self.score_criterion(
                submission=submission,
                criterion=criterion,
            )

            criterion_scores.append(CriterionScore(
                criterion=criterion.split(":")[0],
                score=result.score,
                justification=result.justification,
            ))

        overall = sum(
            cs.score * w for cs, w in zip(criterion_scores, self.weights)
        )

        return dspy.Prediction(
            criterion_scores=criterion_scores,
            overall_score=round(overall, 2),
        )

Using ChainOfThought here is important — reasoning through the evidence before assigning a score produces more calibrated results than jumping straight to a number.

Step 4: Calibrate with anchor examples

Without anchors, the scorer doesn't know what a "2" vs a "4" looks like. Provide reference examples at each level:

ANCHORS = """
Score 2 example for clarity: "The thing with the data is that it does stuff and the results are what they are."
→ Vague language, no specific referents, reader can't follow what's being described.

Score 4 example for clarity: "The customer churn model reduced false positives by 30% compared to the rule-based approach, though it still struggles with seasonal patterns."
→ Specific claims with numbers, clear comparison, one caveat noted.
"""

class ScoreCriterionCalibrated(dspy.Signature):
    """Score the submission on a single criterion. Use the anchor examples to calibrate your scoring."""
    submission: str = dspy.InputField(desc="The work being evaluated")
    criterion: str = dspy.InputField(desc="The criterion to score, including scale description")
    anchors: str = dspy.InputField(desc="Reference examples showing what different score levels look like")
    score: int = dspy.OutputField(desc="Score from 1 to 5")
    justification: str = dspy.OutputField(desc="Specific evidence from the submission supporting this score")

Then pass anchors per criterion:

class CalibratedScorer(dspy.Module):
    def __init__(self, criteria: list[str], anchors: dict[str, str], weights: list[float] = None):
        self.criteria = criteria
        self.anchors = anchors
        self.weights = weights or [1.0 / len(criteria)] * len(criteria)
        self.score_criterion = dspy.ChainOfThought(ScoreCriterionCalibrated)

    def forward(self, submission: str):
        criterion_scores = []

        for criterion in self.criteria:
            criterion_name = criterion.split(":")[0]
            result = self.score_criterion(
                submission=submission,
                criterion=criterion,
                anchors=self.anchors.get(criterion_name, "No anchors provided."),
            )

            criterion_scores.append(CriterionScore(
                criterion=criterion_name,
                score=result.score,
                justification=result.justification,
            ))

        overall = sum(cs.score * w for cs, w in zip(criterion_scores, self.weights))
        return dspy.Prediction(
            criterion_scores=criterion_scores,
            overall_score=round(overall, 2),
        )

Writing good anchors takes effort, but it's the single biggest lever for scoring quality. Start with 2-3 anchors per criterion at the low, mid, and high ends of the scale.

Step 5: Handle edge cases

Validate score consistency

The overall score should be consistent with per-criterion scores:

def validate_scores(criterion_scores, weights, overall_score):
    expected = sum(cs.score * w for cs, w in zip(criterion_scores, weights))
    if abs(expected - overall_score) >= 0.1:
        raise ValueError(
            f"Overall score {overall_score} doesn't match weighted criteria ({expected:.2f})"
        )

Handle "not applicable" criteria

Some criteria don't apply to every submission:

class CriterionScoreOptional(BaseModel):
    criterion: str
    score: int = Field(ge=0, le=5, description="Score 1-5, or 0 if not applicable")
    justification: str
    applicable: bool = Field(description="Whether this criterion applies to this submission")

Score ranges for pass/fail decisions

def pass_fail(overall_score: float, threshold: float = 3.0) -> str:
    if overall_score >= threshold:
        return "pass"
    return "fail"

# Or with a "needs review" band
def tiered_decision(overall_score: float) -> str:
    if overall_score >= 4.0:
        return "pass"
    elif overall_score >= 2.5:
        return "needs_review"
    return "fail"

Step 6: Multi-rater ensemble

For high-stakes scoring, run multiple independent scorers and flag disagreements:

class EnsembleScorer(dspy.Module):
    def __init__(self, criteria, anchors, num_raters=3, weights=None):
        self.raters = [
            CalibratedScorer(criteria, anchors, weights)
            for _ in range(num_raters)
        ]

    def forward(self, submission: str):
        all_results = [rater(submission=submission) for rater in self.raters]

        # Check for disagreement per criterion
        flagged = []
        for i, criterion in enumerate(self.raters[0].criteria):
            criterion_name = criterion.split(":")[0]
            scores = [r.criterion_scores[i].score for r in all_results]
            spread = max(scores) - min(scores)
            if spread > 1:
                flagged.append({
                    "criterion": criterion_name,
                    "scores": scores,
                    "spread": spread,
                })

        # Average the overall scores
        avg_overall = sum(r.overall_score for r in all_results) / len(all_results)

        return dspy.Prediction(
            overall_score=round(avg_overall, 2),
            all_results=all_results,
            flagged_disagreements=flagged,
            needs_human_review=len(flagged) > 0,
        )

When raters disagree by more than 1 point on any criterion, flag it for human review. This catches the submissions that are genuinely ambiguous — exactly where human judgment matters most.

Step 7: Evaluate scorer quality

Prepare gold-standard scores

You need human-scored examples to evaluate your AI scorer:

scored_examples = [
    dspy.Example(
        submission="...",
        gold_scores={"clarity": 4, "argument": 3, "evidence": 5},
        gold_overall=4.0,
    ).with_inputs("submission"),
    # 20-50+ scored examples
]

Mean absolute error metric

def scoring_metric(example, prediction, trace=None):
    """Measures how close AI scores are to human gold scores."""
    errors = []
    for cs in prediction.criterion_scores:
        gold = example.gold_scores.get(cs.criterion)
        if gold is not None:
            errors.append(abs(cs.score - gold))

    if not errors:
        return 0.0

    mae = sum(errors) / len(errors)
    # Convert to 0-1 scale (0 error = 1.0, 4 error = 0.0)
    return max(0.0, 1.0 - mae / 4.0)

Agreement rate metric

def agreement_metric(example, prediction, trace=None):
    """Score is 1.0 if all criteria are within 1 point of gold."""
    for cs in prediction.criterion_scores:
        gold = example.gold_scores.get(cs.criterion)
        if gold is not None and abs(cs.score - gold) > 1:
            return 0.0
    return 1.0

Optimize the scorer

from dspy.evaluate import Evaluate

evaluator = Evaluate(devset=scored_examples, metric=scoring_metric, num_threads=4)
baseline = evaluator(scorer)

optimizer = dspy.MIPROv2(metric=scoring_metric, auto="medium")
optimized_scorer = optimizer.compile(scorer, trainset=trainset)

optimized_score = evaluator(optimized_scorer)
print(f"Baseline agreement: {baseline:.1f}%")
print(f"Optimized agreement: {optimized_score:.1f}%")
# Typical improvement: 50-65% agreement -> 75-90% after MIPROv2 with 30+ gold examples

When NOT to use scoring

• You need discrete categories, not numbers — if the output is "spam / not spam" or "bug / feature / question", use /ai-sorting instead. Scoring adds complexity when you just need buckets.
• There is no rubric and you cannot define one — scoring without criteria produces arbitrary numbers. If you cannot articulate what a "3" vs a "5" means, start with qualitative analysis first.
• You need deterministic, auditable grading — if regulatory or legal requirements demand exact reproducibility, rule-based scoring (point deductions for specific violations) is more defensible than LM-based judgment.
• The evaluation is purely objective — if "correct" has one answer (e.g., math problems, factual lookups), use exact-match or programmatic checks. AI scoring is for subjective or multi-dimensional evaluation.

Scoring approach comparison

Key patterns

• Score per criterion independently — prevents halo effect where one strong dimension inflates all scores
• Use anchor examples — the single biggest lever for calibration quality
• ChainOfThought for scoring — reasoning before scoring produces better-calibrated results
• Require justifications — forces the scorer to cite evidence, catches lazy scoring
• Multi-rater for high stakes — flag disagreements for human review
• Validate consistency — overall score should match weighted criterion scores
• Pydantic for structure — Field(ge=1, le=5) enforces valid score ranges automatically

Gotchas

• Claude defaults to generous scoring (central tendency bias). Without anchors, Claude clusters scores around 3-4 on a 1-5 scale and rarely gives 1s or 5s. Provide anchor examples at the extremes to calibrate the full range — explicitly show what a "1" and a "5" look like.
• Scoring all criteria in one call causes halo effect. Claude lets a strong first impression bleed across all criteria. Always score each criterion in a separate ChainOfThought call, even though it costs more.
• Claude invents justifications that sound plausible but cite nothing specific. Use Pydantic Field(min_length=20) on the justification field, or wrap the scorer with dspy.Refine and a reward function that penalizes vague justifications.
• `result.score` can be a string instead of int depending on adapter. Always use int(result.score) or Pydantic Field(ge=1, le=5) to enforce the type. Comparing string "3" > int 2 silently passes in some contexts.
• Ensemble scorers with identical prompts produce correlated scores, not independent judgments. For true multi-rater benefit, vary the temperature or use different prompt configurations for each rater. Three identical calls add cost without adding much signal.

Cross-references

Install any skill: npx skills add lebsral/DSPy-Programming-not-prompting-LMs-skills --skill <name>

• Need discrete categories instead of scores? Use /ai-sorting
• Need to validate AI output (not score human work)? Use /ai-checking-outputs
• Measure and improve scorer accuracy — see /ai-improving-accuracy
• ChainOfThought for reasoning before scoring — see /dspy-chain-of-thought
• Refine for enforcing score ranges and justification quality — see /dspy-refine
• Install `/ai-do` if you do not have it — it routes any AI problem to the right skill and is the fastest way to work: npx skills add lebsral/DSPy-Programming-not-prompting-LMs-skills --skill ai-do

Additional resources

• For worked examples (essay grading, code review, support QA), see examples.md

Scoring Examples

Essay Grading

import dspy
from pydantic import BaseModel, Field

lm = dspy.LM("openai/gpt-4o-mini")  # or "anthropic/claude-sonnet-4-5-20250929", etc.
dspy.configure(lm=lm)

# Define the rubric
ESSAY_CRITERIA = [
    "clarity: Is the writing clear and easy to follow? (1=confusing, 5=crystal clear)",
    "argument: Is the argument well-structured with a clear thesis? (1=no structure, 5=compelling)",
    "evidence: Does the essay cite relevant evidence and examples? (1=none, 5=strong support)",
]

ESSAY_WEIGHTS = [0.3, 0.4, 0.3]  # Argument weighted highest

ESSAY_ANCHORS = {
    "clarity": """Score 2: "The thing about climate change is that it affects things and there are many reasons for it happening."
→ Vague referents, no specifics, reads like filler.
Score 4: "Rising sea levels threaten coastal cities like Miami, where 2.7 million residents face increased flooding risk by 2050."
→ Specific claims, concrete examples, easy to follow.""",

    "argument": """Score 2: "Climate change is bad. Also, polar bears are dying. In conclusion, we should do something."
→ No logical progression, disconnected claims, no thesis.
Score 4: "While renewable energy adoption is accelerating, three structural barriers — grid infrastructure, storage costs, and regulatory fragmentation — prevent the transition speed needed to meet 2030 targets."
→ Clear thesis, enumerated supporting points, acknowledges complexity.""",

    "evidence": """Score 2: "Many scientists agree that this is a problem."
→ Vague appeal to authority, no specific sources.
Score 4: "According to the IPCC's 2023 Synthesis Report, global temperatures have risen 1.1°C since pre-industrial levels, with the rate of increase accelerating since 1970."
→ Specific source, concrete data points, verifiable claim.""",
}


class CriterionScore(BaseModel):
    criterion: str
    score: int = Field(ge=1, le=5)
    justification: str


class ScoreCriterion(dspy.Signature):
    """Score the essay on a single criterion. Use the anchor examples to calibrate."""
    submission: str = dspy.InputField(desc="The essay being graded")
    criterion: str = dspy.InputField(desc="The criterion to score")
    anchors: str = dspy.InputField(desc="Reference examples for calibration")
    score: int = dspy.OutputField(desc="Score from 1 to 5")
    justification: str = dspy.OutputField(desc="Evidence from the essay supporting this score")


class EssayGrader(dspy.Module):
    def __init__(self):
        self.score_criterion = dspy.ChainOfThought(ScoreCriterion)

    def forward(self, submission: str):
        scores = []
        for criterion, weight in zip(ESSAY_CRITERIA, ESSAY_WEIGHTS):
            name = criterion.split(":")[0]
            result = self.score_criterion(
                submission=submission,
                criterion=criterion,
                anchors=ESSAY_ANCHORS.get(name, ""),
            )
            scores.append(CriterionScore(
                criterion=name, score=result.score, justification=result.justification
            ))

        overall = sum(cs.score * w for cs, w in zip(scores, ESSAY_WEIGHTS))
        return dspy.Prediction(criterion_scores=scores, overall_score=round(overall, 2))


grader = EssayGrader()
result = grader(submission="Climate change poses an existential threat to coastal communities...")

for cs in result.criterion_scores:
    print(f"  {cs.criterion}: {cs.score}/5 — {cs.justification[:80]}...")
print(f"Overall: {result.overall_score}/5")

Code Review Scoring

CODE_CRITERIA = [
    "correctness: Does the code produce correct results for all cases? (1=broken, 5=handles all edge cases)",
    "readability: Is the code easy to understand and well-named? (1=cryptic, 5=self-documenting)",
    "security: Is the code free from vulnerabilities? (1=critical issues, 5=follows security best practices)",
]

CODE_WEIGHTS = [0.5, 0.25, 0.25]  # Correctness weighted highest

CODE_ANCHORS = {
    "correctness": """Score 2: Function returns wrong result for empty list input, no error handling for None values.
→ Fails basic edge cases that are easy to predict.
Score 4: Handles empty input, validates types, but doesn't account for concurrent access.
→ Covers common cases, misses only advanced scenarios.""",

    "readability": """Score 2: Variables named `x`, `tmp`, `d2`. No comments. 50-line function doing 4 things.
→ Reader has to reverse-engineer intent from implementation.
Score 4: Descriptive names like `user_email`, `retry_count`. Functions under 20 lines. One comment explaining a non-obvious business rule.
→ Intent is clear from reading, minimal mental overhead.""",

    "security": """Score 2: SQL query built with string concatenation. User input passed directly to shell command.
→ Classic injection vulnerabilities.
Score 4: Parameterized queries, input validation, no hardcoded secrets. Uses established auth library.
→ Follows OWASP basics, uses safe defaults.""",
}


class ScoreCodeCriterion(dspy.Signature):
    """Score the code submission on a single review criterion."""
    code: str = dspy.InputField(desc="The code being reviewed")
    criterion: str = dspy.InputField(desc="The review criterion")
    anchors: str = dspy.InputField(desc="Reference examples for calibration")
    score: int = dspy.OutputField(desc="Score from 1 to 5")
    justification: str = dspy.OutputField(desc="Specific code elements supporting this score")


class CodeReviewScorer(dspy.Module):
    def __init__(self):
        self.score_criterion = dspy.ChainOfThought(ScoreCodeCriterion)

    def forward(self, code: str):
        scores = []
        for criterion, weight in zip(CODE_CRITERIA, CODE_WEIGHTS):
            name = criterion.split(":")[0]
            result = self.score_criterion(
                code=code,
                criterion=criterion,
                anchors=CODE_ANCHORS.get(name, ""),
            )
            scores.append(CriterionScore(
                criterion=name, score=result.score, justification=result.justification
            ))

        overall = sum(cs.score * w for cs, w in zip(scores, CODE_WEIGHTS))
        return dspy.Prediction(criterion_scores=scores, overall_score=round(overall, 2))


reviewer = CodeReviewScorer()
result = reviewer(code="""
def get_user(user_id):
    query = f"SELECT * FROM users WHERE id = {user_id}"
    return db.execute(query)
""")
for cs in result.criterion_scores:
    print(f"  {cs.criterion}: {cs.score}/5")

Customer Support Quality Audit

SUPPORT_CRITERIA = [
    "helpfulness: Did the agent resolve the customer's issue? (1=ignored the problem, 5=fully resolved)",
    "accuracy: Was the information provided correct? (1=wrong info, 5=completely accurate)",
    "tone: Was the agent professional and empathetic? (1=rude/dismissive, 5=warm and professional)",
]

SUPPORT_WEIGHTS = [0.4, 0.4, 0.2]


class ScoreSupportCriterion(dspy.Signature):
    """Score a customer support interaction on a single quality criterion."""
    conversation: str = dspy.InputField(desc="The support conversation transcript")
    criterion: str = dspy.InputField(desc="The quality criterion to score")
    score: int = dspy.OutputField(desc="Score from 1 to 5")
    justification: str = dspy.OutputField(desc="Evidence from the conversation")


class SupportAuditor(dspy.Module):
    def __init__(self):
        self.score_criterion = dspy.ChainOfThought(ScoreSupportCriterion)

    def forward(self, conversation: str):
        scores = []
        for criterion, weight in zip(SUPPORT_CRITERIA, SUPPORT_WEIGHTS):
            name = criterion.split(":")[0]
            result = self.score_criterion(
                conversation=conversation,
                criterion=criterion,
            )
            scores.append(CriterionScore(
                criterion=name, score=result.score, justification=result.justification
            ))

        overall = sum(cs.score * w for cs, w in zip(scores, SUPPORT_WEIGHTS))
        decision = "pass" if overall >= 3.5 else "needs_coaching"

        return dspy.Prediction(
            criterion_scores=scores,
            overall_score=round(overall, 2),
            decision=decision,
        )


auditor = SupportAuditor()
result = auditor(conversation="""
Customer: I've been waiting 3 weeks for my refund and nobody is helping me.
Agent: I understand your frustration. Let me look into this right now.
Agent: I can see your refund was stuck in processing. I've escalated it — you'll receive it within 2 business days. I'll email you the confirmation.
Customer: Thank you so much!
""")
print(f"Overall: {result.overall_score}/5 — {result.decision}")

Evaluating Scorer Quality

# Gold-standard scored examples (human-scored)
scored_trainset = [
    dspy.Example(
        submission="Climate change is a big problem...",
        gold_scores={"clarity": 3, "argument": 2, "evidence": 2},
    ).with_inputs("submission"),
    # Add 20-50+ examples
]

def scoring_metric(example, prediction, trace=None):
    errors = []
    for cs in prediction.criterion_scores:
        gold = example.gold_scores.get(cs.criterion)
        if gold is not None:
            errors.append(abs(cs.score - gold))
    if not errors:
        return 0.0
    mae = sum(errors) / len(errors)
    return max(0.0, 1.0 - mae / 4.0)

# Optimize
optimizer = dspy.MIPROv2(metric=scoring_metric, auto="medium")
optimized_grader = optimizer.compile(grader, trainset=scored_trainset)