Prompt di valutazione basati su rubriche

Che cos'è la valutazione basata su rubriche

Anatomia di una rubrica di valutazione

Un prompt con tre criteri di valutazione

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

RUBRIC_PROMPT = (
    'Score this response on a scale of 1-5 for each criterion:\n\n'
    'ACCURACY: Is the response factually correct?\n'
    '  1=Contains significant factual errors\n'
    '  3=Mostly correct with minor inaccuracies\n'
    '  5=Completely accurate with no errors\n\n'
    'COMPLETENESS: Did it answer everything asked?\n'
    '  1=Missed most of the question\n'
    '  3=Answered the main question but missed sub-parts\n'
    '  5=Addressed every part of the question\n\n'
    'CLARITY: Is it easy to understand?\n'
    '  1=Confusing, hard to follow\n'
    '  3=Understandable but could be clearer\n'
    '  5=Exceptionally clear and well-organized\n\n'
    'Question: {question}\n'
    'Response: {response}\n\n'
    'Return JSON: {{"accuracy": N, "completeness": N, "clarity": N, '
    '"overall": N, "notes": "one sentence"}}'
)

def rubric_judge(question, response):
    prompt = RUBRIC_PROMPT.format(question=question, response=response)
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': prompt}]
    )
    return json.loads(r.content[0].text)

result = rubric_judge(
    question='What is a REST API?',
    response='A REST API is a way for applications to communicate over HTTP.'
)
print(result)

Valutazione ponderata

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def weighted_rubric_judge(question, response, weights):
    """
    weights: dict of criterion -> weight (should sum to 1.0)
    Example: {'accuracy': 0.5, 'completeness': 0.3, 'clarity': 0.2}
    """
    RUBRIC = (
        'Score this response 1-5 on:\n'
        'Accuracy: Is it factually correct?\n'
        'Completeness: Does it cover the full question?\n'
        'Clarity: Is it easy to understand?\n\n'
        'Q: {q}\nA: {a}\n\n'
        'Return JSON: {{"accuracy":N,"completeness":N,"clarity":N}}'
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=100,
        messages=[{'role': 'user', 'content': RUBRIC.format(q=question, a=response)}]
    )
    scores = json.loads(r.content[0].text)

    # Calculate weighted average
    weighted_score = sum(
        scores[criterion] * weight
        for criterion, weight in weights.items()
        if criterion in scores
    )
    print(f'Individual scores: {scores}')
    print(f'Weighted score: {weighted_score:.2f}/5')
    return weighted_score

weighted_rubric_judge(
    'How do I reverse a string in Python?',
    'Use slicing: s[::-1]',
    weights={'accuracy': 0.5, 'completeness': 0.3, 'clarity': 0.2}
)

Criterio: accuratezza fattuale

ACCURACY_RUBRIC = (
    'Evaluate FACTUAL ACCURACY of the following response.\n\n'
    'Score 1-5:\n'
    '1 = Multiple factual errors that fundamentally mislead the reader\n'
    '2 = At least one significant factual error (wrong date, number, or core fact)\n'
    '3 = Factually correct but includes minor imprecisions or over-generalizations\n'
    '4 = Factually correct with appropriate hedging of uncertain claims\n'
    '5 = Factually precise, no errors, and correctly acknowledges uncertainty where present\n\n'
    'Check specifically for:\n'
    '- Wrong dates, statistics, or numerical values\n'
    '- Misattributed quotes or inventions\n'
    '- Outdated information presented as current\n'
    '- Claims stated with false confidence (should be hedged)\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Score and list any errors found:'
)
print(ACCURACY_RUBRIC[:400])

Criterio: completezza

COMPLETENESS_RUBRIC = (
    'Evaluate COMPLETENESS of this response.\n\n'
    'First, list every distinct question or requirement in the original query.\n'
    'Then, check whether the response addressed each one.\n\n'
    'Score 1-5:\n'
    '1 = Only addressed 0-20% of what was asked\n'
    '2 = Addressed 20-50% — missed major components\n'
    '3 = Addressed 50-80% — answered main question but missed sub-parts\n'
    '4 = Addressed 80-95% — minor omissions only\n'
    '5 = Addressed 100% — every requirement was met\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Requirements checklist and completeness score:'
)

# This rubric forces the judge to decompose the question first,
# which is much more reliable than asking 'was it complete?'
print(COMPLETENESS_RUBRIC[:400])

Criterio: chiarezza

CLARITY_RUBRIC = (
    'Evaluate CLARITY of this response for a {audience} audience.\n\n'
    'Score 1-5:\n'
    '1 = Incomprehensible — cannot extract meaning\n'
    '2 = Very hard to follow — excessive jargon, poor structure\n'
    '3 = Understandable with effort — some confusing parts\n'
    '4 = Clear and well-organized — easy to read\n'
    '5 = Exceptionally clear — ideal structure, appropriate vocabulary, '
    'no unnecessary complexity\n\n'
    'Consider:\n'
    '- Is the vocabulary appropriate for the audience?\n'
    '- Is the response logically organized?\n'
    '- Are sentences a readable length?\n'
    '- Is the main point stated early and clearly?\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Clarity score and key issues:'
)

# Parameterize the audience for context-aware clarity assessment
print(CLARITY_RUBRIC.format(
    audience='non-technical business stakeholder',
    question='What is an API?',
    response='REST APIs use HTTP to transfer data between client and server.'
)[:300])

Rubriche specifiche per il compito

# Customer support response rubric
SUPPORT_RUBRIC = (
    'Evaluate this customer support response:\n\n'
    'EMPATHY (1-5): Does it acknowledge the customer emotion?\n'
    'RESOLUTION (1-5): Does it provide a clear solution or next step?\n'
    'TONE (1-5): Is it professional, warm, and not condescending?\n'
    'EFFICIENCY (1-5): Does it avoid unnecessary words or boilerplate?\n\n'
    'Customer message: {customer_message}\n'
    'Support response: {support_response}\n\n'
    'Scores and notes (JSON):'
)

# Code review rubric
CODE_REVIEW_RUBRIC = (
    'Evaluate this code explanation:\n\n'
    'CORRECTNESS (1-5): Is the code technically correct?\n'
    'EDGE_CASES (1-5): Does it handle edge cases (empty input, errors)?\n'
    'EFFICIENCY (1-5): Is it reasonably efficient (no obvious O(n^2) where O(n) is easy)?\n'
    'READABILITY (1-5): Is the code easy to read and understand?\n\n'
    'Task: {task}\n'
    'Code: {code}\n\n'
    'Scores and notes (JSON):'
)
print('Specialized rubrics produce better signal for your domain')

Verifica della coerenza della rubrica

import anthropic
import json
import statistics

client = anthropic.Anthropic(api_key='sk-ant-...')

def test_rubric_consistency(rubric_prompt, question, response, n_trials=5):
    scores = []
    for i in range(n_trials):
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=100,
            messages=[{'role': 'user', 'content': rubric_prompt.format(
                question=question, response=response
            )}]
        )
        try:
            data = json.loads(r.content[0].text)
            overall = data.get('overall', sum(data.values()) / len(data))
            scores.append(overall)
        except Exception:
            scores.append(None)

    valid = [s for s in scores if s is not None]
    if valid:
        print(f'Scores: {valid}')
        print(f'Mean: {statistics.mean(valid):.2f}')
        print(f'Std dev: {statistics.stdev(valid):.2f}')
        if statistics.stdev(valid) > 0.5:
            print('WARNING: High variance — rubric may be underspecified')

    return valid

Restituire JSON dal giudice

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def structured_rubric_judge(question, response):
    prompt = (
        'Score this response 1-5 on three criteria and return JSON.\n\n'
        'Q: {q}\nA: {a}\n\n'
        'Return ONLY this JSON structure (no other text):\n'
        '{{\n'
        '  "accuracy": <1-5>,\n'
        '  "completeness": <1-5>,\n'
        '  "clarity": <1-5>,\n'
        '  "overall": <1-5>,\n'
        '  "primary_issue": "<what most needs improvement>",\n'
        '  "primary_strength": "<what the response does best>"\n'
        '}}'
    ).format(q=question, a=response)

    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': prompt}]
    )

    text = r.content[0].text.strip()
    # Strip markdown code fences if present
    if text.startswith('###'):
        text = text.split('###')[1]
        if text.startswith('json'):
            text = text[4:]
    return json.loads(text.strip())

result = structured_rubric_judge(
    'Explain big O notation.',
    'Big O describes algorithm time complexity.'
)
print(json.dumps(result, indent=2))

Iterare sulla progettazione della rubrica

Verifica delle conoscenze: ancoraggi dei punteggi

Riepilogo: prompt per la valutazione basata su rubriche