Prompty oceniania oparte na rubrykach

Czym jest ocenianie na podstawie rubryki?

Budowa rubryki oceniania

Prompt rubryki z trzema kryteriami

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

RUBRIC_PROMPT = (
    'Score this response on a scale of 1-5 for each criterion:\n\n'
    'ACCURACY: Is the response factually correct?\n'
    '  1=Contains significant factual errors\n'
    '  3=Mostly correct with minor inaccuracies\n'
    '  5=Completely accurate with no errors\n\n'
    'COMPLETENESS: Did it answer everything asked?\n'
    '  1=Missed most of the question\n'
    '  3=Answered the main question but missed sub-parts\n'
    '  5=Addressed every part of the question\n\n'
    'CLARITY: Is it easy to understand?\n'
    '  1=Confusing, hard to follow\n'
    '  3=Understandable but could be clearer\n'
    '  5=Exceptionally clear and well-organized\n\n'
    'Question: {question}\n'
    'Response: {response}\n\n'
    'Return JSON: {{"accuracy": N, "completeness": N, "clarity": N, '
    '"overall": N, "notes": "one sentence"}}'
)

def rubric_judge(question, response):
    prompt = RUBRIC_PROMPT.format(question=question, response=response)
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': prompt}]
    )
    return json.loads(r.content[0].text)

result = rubric_judge(
    question='What is a REST API?',
    response='A REST API is a way for applications to communicate over HTTP.'
)
print(result)

Ocenianie ważone

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def weighted_rubric_judge(question, response, weights):
    """
    weights: dict of criterion -> weight (should sum to 1.0)
    Example: {'accuracy': 0.5, 'completeness': 0.3, 'clarity': 0.2}
    """
    RUBRIC = (
        'Score this response 1-5 on:\n'
        'Accuracy: Is it factually correct?\n'
        'Completeness: Does it cover the full question?\n'
        'Clarity: Is it easy to understand?\n\n'
        'Q: {q}\nA: {a}\n\n'
        'Return JSON: {{"accuracy":N,"completeness":N,"clarity":N}}'
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=100,
        messages=[{'role': 'user', 'content': RUBRIC.format(q=question, a=response)}]
    )
    scores = json.loads(r.content[0].text)

    # Calculate weighted average
    weighted_score = sum(
        scores[criterion] * weight
        for criterion, weight in weights.items()
        if criterion in scores
    )
    print(f'Individual scores: {scores}')
    print(f'Weighted score: {weighted_score:.2f}/5')
    return weighted_score

weighted_rubric_judge(
    'How do I reverse a string in Python?',
    'Use slicing: s[::-1]',
    weights={'accuracy': 0.5, 'completeness': 0.3, 'clarity': 0.2}
)

Kryterium: trafność faktograficzna

ACCURACY_RUBRIC = (
    'Evaluate FACTUAL ACCURACY of the following response.\n\n'
    'Score 1-5:\n'
    '1 = Multiple factual errors that fundamentally mislead the reader\n'
    '2 = At least one significant factual error (wrong date, number, or core fact)\n'
    '3 = Factually correct but includes minor imprecisions or over-generalizations\n'
    '4 = Factually correct with appropriate hedging of uncertain claims\n'
    '5 = Factually precise, no errors, and correctly acknowledges uncertainty where present\n\n'
    'Check specifically for:\n'
    '- Wrong dates, statistics, or numerical values\n'
    '- Misattributed quotes or inventions\n'
    '- Outdated information presented as current\n'
    '- Claims stated with false confidence (should be hedged)\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Score and list any errors found:'
)
print(ACCURACY_RUBRIC[:400])

Kryterium: kompletność

COMPLETENESS_RUBRIC = (
    'Evaluate COMPLETENESS of this response.\n\n'
    'First, list every distinct question or requirement in the original query.\n'
    'Then, check whether the response addressed each one.\n\n'
    'Score 1-5:\n'
    '1 = Only addressed 0-20% of what was asked\n'
    '2 = Addressed 20-50% — missed major components\n'
    '3 = Addressed 50-80% — answered main question but missed sub-parts\n'
    '4 = Addressed 80-95% — minor omissions only\n'
    '5 = Addressed 100% — every requirement was met\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Requirements checklist and completeness score:'
)

# This rubric forces the judge to decompose the question first,
# which is much more reliable than asking 'was it complete?'
print(COMPLETENESS_RUBRIC[:400])

Kryterium: jasność

CLARITY_RUBRIC = (
    'Evaluate CLARITY of this response for a {audience} audience.\n\n'
    'Score 1-5:\n'
    '1 = Incomprehensible — cannot extract meaning\n'
    '2 = Very hard to follow — excessive jargon, poor structure\n'
    '3 = Understandable with effort — some confusing parts\n'
    '4 = Clear and well-organized — easy to read\n'
    '5 = Exceptionally clear — ideal structure, appropriate vocabulary, '
    'no unnecessary complexity\n\n'
    'Consider:\n'
    '- Is the vocabulary appropriate for the audience?\n'
    '- Is the response logically organized?\n'
    '- Are sentences a readable length?\n'
    '- Is the main point stated early and clearly?\n\n'
    'Q: {question}\nA: {response}\n\n'
    'Clarity score and key issues:'
)

# Parameterize the audience for context-aware clarity assessment
print(CLARITY_RUBRIC.format(
    audience='non-technical business stakeholder',
    question='What is an API?',
    response='REST APIs use HTTP to transfer data between client and server.'
)[:300])

Rubryki właściwe dla zadania

# Customer support response rubric
SUPPORT_RUBRIC = (
    'Evaluate this customer support response:\n\n'
    'EMPATHY (1-5): Does it acknowledge the customer emotion?\n'
    'RESOLUTION (1-5): Does it provide a clear solution or next step?\n'
    'TONE (1-5): Is it professional, warm, and not condescending?\n'
    'EFFICIENCY (1-5): Does it avoid unnecessary words or boilerplate?\n\n'
    'Customer message: {customer_message}\n'
    'Support response: {support_response}\n\n'
    'Scores and notes (JSON):'
)

# Code review rubric
CODE_REVIEW_RUBRIC = (
    'Evaluate this code explanation:\n\n'
    'CORRECTNESS (1-5): Is the code technically correct?\n'
    'EDGE_CASES (1-5): Does it handle edge cases (empty input, errors)?\n'
    'EFFICIENCY (1-5): Is it reasonably efficient (no obvious O(n^2) where O(n) is easy)?\n'
    'READABILITY (1-5): Is the code easy to read and understand?\n\n'
    'Task: {task}\n'
    'Code: {code}\n\n'
    'Scores and notes (JSON):'
)
print('Specialized rubrics produce better signal for your domain')

Testowanie spójności rubryki

import anthropic
import json
import statistics

client = anthropic.Anthropic(api_key='sk-ant-...')

def test_rubric_consistency(rubric_prompt, question, response, n_trials=5):
    scores = []
    for i in range(n_trials):
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=100,
            messages=[{'role': 'user', 'content': rubric_prompt.format(
                question=question, response=response
            )}]
        )
        try:
            data = json.loads(r.content[0].text)
            overall = data.get('overall', sum(data.values()) / len(data))
            scores.append(overall)
        except Exception:
            scores.append(None)

    valid = [s for s in scores if s is not None]
    if valid:
        print(f'Scores: {valid}')
        print(f'Mean: {statistics.mean(valid):.2f}')
        print(f'Std dev: {statistics.stdev(valid):.2f}')
        if statistics.stdev(valid) > 0.5:
            print('WARNING: High variance — rubric may be underspecified')

    return valid

Zwracanie JSON przez sędziego

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def structured_rubric_judge(question, response):
    prompt = (
        'Score this response 1-5 on three criteria and return JSON.\n\n'
        'Q: {q}\nA: {a}\n\n'
        'Return ONLY this JSON structure (no other text):\n'
        '{{\n'
        '  "accuracy": <1-5>,\n'
        '  "completeness": <1-5>,\n'
        '  "clarity": <1-5>,\n'
        '  "overall": <1-5>,\n'
        '  "primary_issue": "<what most needs improvement>",\n'
        '  "primary_strength": "<what the response does best>"\n'
        '}}'
    ).format(q=question, a=response)

    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': prompt}]
    )

    text = r.content[0].text.strip()
    # Strip markdown code fences if present
    if text.startswith('###'):
        text = text.split('###')[1]
        if text.startswith('json'):
            text = text[4:]
    return json.loads(text.strip())

result = structured_rubric_judge(
    'Explain big O notation.',
    'Big O describes algorithm time complexity.'
)
print(json.dumps(result, indent=2))

Iteracyjne projektowanie rubryki

Sprawdzenie wiedzy: zakotwiczenia ocen

Podsumowanie: prompty do oceniania na podstawie rubryki