Using LLM to Evaluate LLM Outputs

Why Use an LLM as a Judge?

Why LLM Judges Work

A Simple LLM Judge

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def simple_llm_judge(question, response, criterion):
    judge_prompt = (
        f'Rate the following response on {criterion} from 1 to 5.\n\n'
        f'Question: {question}\n'
        f'Response: {response}\n\n'
        f'Return JSON: {{"score": <1-5>, "reason": "<one sentence>"}}'
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=100,
        messages=[{'role': 'user', 'content': judge_prompt}]
    )
    try:
        result = json.loads(r.content[0].text)
        return result['score'], result['reason']
    except Exception:
        return None, r.content[0].text

score, reason = simple_llm_judge(
    question='What is recursion in programming?',
    response='Recursion is when a function calls itself.',
    criterion='clarity and completeness'
)
print(f'Score: {score}/5 — {reason}')

Where LLM Judges Fail: Position Bias

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

def demonstrate_position_bias(question, response_a, response_b):
    def ask_judge(first, second, order):
        prompt = (
            f'Question: {question}\n\n'
            f'Response 1: {first}\n\n'
            f'Response 2: {second}\n\n'
            f'Which response is better? Reply with 1 or 2.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=10,
            messages=[{'role': 'user', 'content': prompt}]
        )
        choice = r.content[0].text.strip()
        # Map back to original labels
        if order == 'AB':
            return 'A' if choice == '1' else 'B'
        else:  # BA
            return 'B' if choice == '1' else 'A'

    result_ab = ask_judge(response_a, response_b, 'AB')
    result_ba = ask_judge(response_b, response_a, 'BA')

    print(f'Order A-B: Judge picked {result_ab}')
    print(f'Order B-A: Judge picked {result_ba}')
    if result_ab != result_ba:
        print('Position bias detected: different results!')

    return result_ab, result_ba

Where LLM Judges Fail: Verbosity Bias

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

def length_aware_judge(question, response):
    judge_prompt = (
        f'Evaluate this response for quality. Be aware of verbosity bias: '
        f'do NOT score longer responses higher just because they are longer.\n\n'
        f'Question: {question}\n'
        f'Response: {response}\n\n'
        f'Evaluate on:\n'
        f'1. Accuracy (does it correctly answer the question?)\n'
        f'2. Conciseness (does it avoid unnecessary filler?)\n'
        f'3. Helpfulness (does it serve the user well?)\n\n'
        f'Penalize responses that add filler, repetition, or irrelevant information.\n'
        f'Score each 1-5 and provide an overall score. Return JSON.'
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': judge_prompt}]
    )
    print(r.content[0].text)

Where LLM Judges Fail: Self-Preference

import anthropic
import openai

anthropic_client = anthropic.Anthropic(api_key='sk-ant-...')
openai_client = openai.OpenAI(api_key='sk-...')

def multi_model_judge(question, response):
    judge_prompt = (
        f'Rate this response 1-10 for overall quality.\n'
        f'Q: {question}\nA: {response}\n'
        f'Reply with only a number.'
    )

    # Judge 1: Claude
    r_claude = anthropic_client.messages.create(
        model='claude-opus-4-5',
        max_tokens=10,
        messages=[{'role': 'user', 'content': judge_prompt}]
    )
    score_claude = float(r_claude.content[0].text.strip())

    # Judge 2: GPT-4o
    r_gpt = openai_client.chat.completions.create(
        model='gpt-4o',
        max_tokens=10,
        messages=[{'role': 'user', 'content': judge_prompt}]
    )
    score_gpt = float(r_gpt.choices[0].message.content.strip())

    avg = (score_claude + score_gpt) / 2
    print(f'Claude judge: {score_claude}, GPT judge: {score_gpt}, Average: {avg}')
    if abs(score_claude - score_gpt) > 2:
        print('WARNING: High disagreement — consider human review')
    return avg

Scoring Inflation

# Anti-inflation judge prompt with explicit score anchors
CALIBRATED_JUDGE_PROMPT = (
    'Rate this response 1-5 using these STRICT score definitions:\n'
    '1 = Completely wrong, harmful, or completely off-topic\n'
    '2 = Partially relevant but contains significant errors or omissions\n'
    '3 = Correct and addresses the question but lacks depth or precision\n'
    '4 = Correct, reasonably complete, and clearly expressed\n'
    '5 = Exceptional: correct, complete, insightful, and concise\n\n'
    'Only give 5 if the response is genuinely outstanding.\n'
    'Give 3 for any adequate-but-not-impressive response.\n\n'
    'Question: {question}\n'
    'Response: {response}\n\n'
    'Score (1-5) and one-sentence reason:'
)

# Compare to non-anchored prompt which tends to cluster at 4-5
print('Anchored rubrics force the judge to use the full scale')

When LLM Judges Work Best

When Human Evaluation Is Required

def triage_for_human_review(question, response, llm_score, confidence_threshold=0.7):
    """
    Route low-confidence or high-stakes evaluations to human review.
    """
    # Route to human if judge is uncertain
    if llm_score is None:
        return 'human_review', 'LLM judge failed to produce a score'

    # Route to human for borderline scores (near decision boundaries)
    if 2.5 <= llm_score <= 3.5:
        return 'human_review', f'Borderline score {llm_score} — needs human judgment'

    # Route to human for domain-specific high-risk content
    HIGH_RISK_KEYWORDS = ['medication', 'legal advice', 'financial advice', 'security']
    if any(kw in question.lower() for kw in HIGH_RISK_KEYWORDS):
        return 'human_review', 'High-risk domain — human verification required'

    # Else: LLM score is sufficient
    return 'auto_accept', f'Score {llm_score} — LLM judgment sufficient'

routing, reason = triage_for_human_review('What medication should I take?', 'Take aspirin.', 4.5)
print(f'{routing}: {reason}')

Building an Evaluation Pipeline

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def evaluate_batch(examples, product_under_test, criteria):
    results = []
    for ex in examples:
        response = product_under_test(ex['question'])
        score, reason = simple_llm_judge(ex['question'], response, criteria)

        results.append({
            'question': ex['question'],
            'response': response,
            'score': score,
            'reason': reason,
            'needs_review': score is None or 2.5 <= (score or 0) <= 3.5
        })

    # Summarize
    valid_scores = [r['score'] for r in results if r['score'] is not None]
    avg_score = sum(valid_scores) / len(valid_scores) if valid_scores else 0
    review_count = sum(1 for r in results if r['needs_review'])

    print(f'Average score: {avg_score:.2f}/5')
    print(f'Cases needing review: {review_count}/{len(results)}')
    return results, avg_score

# results, avg = evaluate_batch(test_cases, my_product, 'helpfulness')

Reference-Free vs Reference-Based Evaluation

Knowledge Check: Position Bias

Recap: LLM-as-Judge Evaluation