Comparative Judging: A vs B

What Is Pairwise (A vs B) Evaluation?

Basic Pairwise Judge Prompt

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

PAIRWISE_PROMPT = (
    'Given the same question, compare these two responses and decide which is better.\n\n'
    'Question: {question}\n\n'
    'Response A:\n{response_a}\n\n'
    'Response B:\n{response_b}\n\n'
    'Which response is better? You must pick A, B, or TIE (use TIE only if '
    'they are truly equal in all meaningful ways).\n\n'
    'Return JSON: {{"winner": "A" or "B" or "TIE", '
    '"reason": "<one sentence explaining why the winner is better>"}}'
)

def pairwise_judge(question, response_a, response_b):
    prompt = PAIRWISE_PROMPT.format(
        question=question,
        response_a=response_a,
        response_b=response_b
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=150,
        messages=[{'role': 'user', 'content': prompt}]
    )
    return json.loads(r.content[0].text)

result = pairwise_judge(
    'What is machine learning?',
    'Machine learning is a subset of AI.',
    'Machine learning is a method of data analysis that automates model building.'
)
print(result)

Randomizing Order to Reduce Position Bias

import anthropic
import json
import random

client = anthropic.Anthropic(api_key='sk-ant-...')

def debiased_pairwise_judge(question, response_a, response_b):
    def single_comparison(first, second, first_label, second_label):
        prompt = (
            f'Question: {question}\n\n'
            f'Response {first_label}:\n{first}\n\n'
            f'Response {second_label}:\n{second}\n\n'
            f'Which is better? Reply with {first_label}, {second_label}, or TIE.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=10,
            messages=[{'role': 'user', 'content': prompt}]
        )
        return r.content[0].text.strip()

    # Run A-first
    result_ab = single_comparison(response_a, response_b, 'A', 'B')
    # Run B-first
    result_ba = single_comparison(response_b, response_a, 'B', 'A')

    if result_ab == 'A' and result_ba == 'A':
        return 'A', 'Consistent: A wins in both orderings'
    elif result_ab == 'B' and result_ba == 'B':
        return 'B', 'Consistent: B wins in both orderings'
    else:
        return 'TIE', f'Inconsistent: {result_ab} then {result_ba} — position bias detected'

winner, reason = debiased_pairwise_judge(
    'Explain a hash table.',
    'A hash table maps keys to values.',
    'A hash table is a data structure using a hash function to store key-value pairs for O(1) lookup.'
)
print(f'Winner: {winner} — {reason}')

Pairwise with Multiple Criteria

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

DIMENSIONAL_PAIRWISE = (
    'Compare Response A and Response B on each dimension.\n\n'
    'Question: {question}\n\n'
    'Response A: {response_a}\n\n'
    'Response B: {response_b}\n\n'
    'For each dimension, say A, B, or TIE:\n'
    '1. ACCURACY: Which is more factually correct?\n'
    '2. COMPLETENESS: Which answers the question more fully?\n'
    '3. CLARITY: Which is easier to understand?\n'
    '4. CONCISENESS: Which avoids unnecessary length?\n\n'
    'Return JSON: {{"accuracy":"A/B/TIE", "completeness":"A/B/TIE", '
    '"clarity":"A/B/TIE", "conciseness":"A/B/TIE", "overall":"A/B/TIE"}}'
)

def dimensional_compare(question, a, b):
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=200,
        messages=[{'role': 'user', 'content': DIMENSIONAL_PAIRWISE.format(
            question=question, response_a=a, response_b=b
        )}]
    )
    return json.loads(r.content[0].text)

result = dimensional_compare(
    'How does HTTPS work?',
    'HTTPS encrypts web traffic using SSL/TLS.',
    'HTTPS secures HTTP using TLS. The browser and server perform a handshake, exchange certificates, and establish an encrypted channel for all data transfer.'
)
print(result)

Building a Tournament: Round Robin

from itertools import combinations
from collections import defaultdict

def round_robin_tournament(question, responses):
    """
    responses: list of (label, text) tuples
    Returns ranking by win count.
    """
    wins = defaultdict(int)
    ties = defaultdict(int)

    # Every pair compared once
    for (label_a, text_a), (label_b, text_b) in combinations(responses, 2):
        winner, reason = debiased_pairwise_judge(question, text_a, text_b)

        if winner == 'A':
            wins[label_a] += 1
        elif winner == 'B':
            wins[label_b] += 1
        else:  # TIE
            ties[label_a] += 1
            ties[label_b] += 1

        print(f'{label_a} vs {label_b}: {winner}')

    # Rank by wins, then ties
    ranking = sorted(
        responses,
        key=lambda x: (wins[x[0]], ties[x[0]]),
        reverse=True
    )
    print('\nFinal ranking:')
    for i, (label, _) in enumerate(ranking, 1):
        print(f'{i}. {label}: {wins[label]} wins, {ties[label]} ties')
    return ranking

Elo Rating for Continuous Ranking

import math

def elo_update(rating_a, rating_b, winner, k=32):
    """
    Update Elo ratings after a match.
    winner: 'A' (A won), 'B' (B won), 'TIE' (draw)
    Returns (new_rating_a, new_rating_b)
    """
    expected_a = 1 / (1 + 10 ** ((rating_b - rating_a) / 400))
    expected_b = 1 - expected_a

    if winner == 'A':
        score_a, score_b = 1, 0
    elif winner == 'B':
        score_a, score_b = 0, 1
    else:  # TIE
        score_a, score_b = 0.5, 0.5

    new_a = rating_a + k * (score_a - expected_a)
    new_b = rating_b + k * (score_b - expected_b)
    return new_a, new_b

# Simulate ratings for 4 model variants
ratings = {'ModelA': 1000, 'ModelB': 1000, 'ModelC': 1000, 'ModelD': 1000}

# After running many pairwise comparisons:
ratings['ModelA'], ratings['ModelB'] = elo_update(ratings['ModelA'], ratings['ModelB'], 'A')
ratings['ModelC'], ratings['ModelD'] = elo_update(ratings['ModelC'], ratings['ModelD'], 'TIE')

ranking = sorted(ratings.items(), key=lambda x: x[1], reverse=True)
for model, score in ranking:
    print(f'{model}: {score:.0f}')

When to Use Pairwise vs Absolute Scoring

Sampling-Based Pairwise at Scale

import random
from collections import defaultdict

def sampled_tournament(question, responses, k_opponents=5):
    """
    Compare each response against k random opponents.
    More efficient than full round-robin for large N.
    """
    wins = defaultdict(int)
    n = len(responses)

    for i, (label, text) in enumerate(responses):
        # Sample k random opponents (not self)
        opponent_indices = random.sample(
            [j for j in range(n) if j != i],
            min(k_opponents, n - 1)
        )
        for j in opponent_indices:
            opp_label, opp_text = responses[j]
            winner, _ = debiased_pairwise_judge(question, text, opp_text)
            if winner == 'A':
                wins[label] += 1
            elif winner == 'B':
                wins[opp_label] += 1

    ranking = sorted(responses, key=lambda x: wins[x[0]], reverse=True)
    for i, (label, _) in enumerate(ranking, 1):
        print(f'{i}. {label}: {wins[label]} wins')
    return ranking

Interpreting Disagreements

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def deep_analyze_tie(question, response_a, response_b):
    """When A vs B is a genuine tie, ask the judge to explain strengths of each."""
    analysis_prompt = (
        f'These two responses to the same question are closely matched in quality.\n\n'
        f'Question: {question}\n\n'
        f'Response A: {response_a}\n\n'
        f'Response B: {response_b}\n\n'
        f'Analyze both:\n'
        f'1. What does A do better than B?\n'
        f'2. What does B do better than A?\n'
        f'3. For what audience or context would you prefer A?\n'
        f'4. For what audience or context would you prefer B?'
    )
    r = client.messages.create(
        model='claude-opus-4-5',
        max_tokens=400,
        messages=[{'role': 'user', 'content': analysis_prompt}]
    )
    return r.content[0].text

analysis = deep_analyze_tie(
    'Explain async/await in Python.',
    'Async/await allows non-blocking code execution.',
    'Async/await lets you write asynchronous code that looks synchronous, '
    'using the asyncio event loop to handle I/O without blocking.'
)
print(analysis[:300])

Logging Pairwise Results

import json
import datetime

def logged_pairwise(question, response_a, response_b,
                    label_a='A', label_b='B', log_file='pairwise_log.jsonl'):
    winner, reason = debiased_pairwise_judge(question, response_a, response_b)

    entry = {
        'timestamp': datetime.datetime.utcnow().isoformat(),
        'question': question[:100],  # Truncate for storage
        'label_a': label_a,
        'label_b': label_b,
        'winner': winner,
        'reason': reason,
        'response_a_length': len(response_a.split()),
        'response_b_length': len(response_b.split()),
    }

    with open(log_file, 'a') as f:
        f.write(json.dumps(entry) + '\n')

    print(f'{label_a} vs {label_b}: {winner} — {reason}')
    return winner

logged_pairwise(
    'What is SQL?',
    'SQL is a database query language.',
    'SQL (Structured Query Language) is used to query and manage relational databases.',
    label_a='ModelV1',
    label_b='ModelV2'
)

Pairwise Evaluation vs Absolute Scoring: Trade-offs

Knowledge Check: Debiasing Pairwise Evaluation

Recap: Comparative Judging A vs B