LLM 심사자의 보정과 편향

평가자 보정이 중요한 이유

위치 편향: 심층 분석

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

def measure_position_bias(question, n_pairs=20):
    """
    Measure position bias by comparing IDENTICAL responses.
    If both responses are the same, wins should be 50/50.
    Any deviation from 50/50 is pure position bias.
    """
    response = 'Machine learning is a subset of AI that learns from data.'
    first_wins = 0

    for _ in range(n_pairs):
        prompt = (
            f'Which response is better?\nQ: {question}\n'
            f'Response A: {response}\n'
            f'Response B: {response}\n'
            f'Reply with A or B.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=5,
            messages=[{'role': 'user', 'content': prompt}]
        )
        if 'A' in r.content[0].text:
            first_wins += 1

    bias = first_wins / n_pairs
    print(f'First-position win rate with IDENTICAL responses: {bias:.0%}')
    print(f'Expected (no bias): 50%')
    print(f'Measured bias: {(bias - 0.5) * 100:+.0f}%')
    return bias

장황함 편향: 심층 분석

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-...')

def measure_verbosity_bias(question, correct_answer):
    """
    Compare a concise correct answer against a verbose one with filler.
    A good judge should prefer the concise version or call it a tie.
    """
    concise = correct_answer
    verbose = (
        f'That is a great question! I am happy to help. '
        f'{correct_answer} '
        f'I hope this comprehensive explanation addresses all your needs. '
        f'Please feel free to ask if you need any further clarification!'
    )

    for label, resp in [('Concise', concise), ('Verbose', verbose)]:
        prompt = (
            f'Rate this response 1-5 for quality.\n'
            f'Q: {question}\nA: {resp}\n'
            f'Return only a number 1-5.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=5,
            messages=[{'role': 'user', 'content': prompt}]
        )
        score = r.content[0].text.strip()
        print(f'{label} response score: {score}')
        print(f'  ({len(resp.split())} words)')

자기 선호 편향: 심층 분석

import anthropic
import openai

anthropic_client = anthropic.Anthropic(api_key='sk-ant-...')
openai_client = openai.OpenAI(api_key='sk-...')

def test_self_preference(question):
    # Generate one response per model
    claude_answer = anthropic_client.messages.create(
        model='claude-opus-4-5', max_tokens=100,
        messages=[{'role': 'user', 'content': question}]
    ).content[0].text

    gpt_answer = openai_client.chat.completions.create(
        model='gpt-4o', max_tokens=100,
        messages=[{'role': 'user', 'content': question}]
    ).choices[0].message.content

    judge_prompt = (
        f'Which response is better?\nQ: {question}\n'
        f'Response A: {claude_answer}\n'
        f'Response B: {gpt_answer}\n'
        f'Reply: A or B'
    )

    # Claude judges the comparison
    claude_verdict = anthropic_client.messages.create(
        model='claude-opus-4-5', max_tokens=5,
        messages=[{'role': 'user', 'content': judge_prompt}]
    ).content[0].text.strip()

    print(f'Claude judge verdict: {claude_verdict}')
    print('(A=Claude response, B=GPT response)')
    print('Self-preference: did Claude prefer its own response?')

완화 방법 1: 순서 바꾸기

import anthropic
import json

client = anthropic.Anthropic(api_key='sk-ant-...')

def debiased_compare(question, response_a, response_b, label_a='A', label_b='B'):
    def single_pass(first, second, first_label, second_label):
        prompt = (
            f'Q: {question}\n\n'
            f'{first_label}: {first}\n\n'
            f'{second_label}: {second}\n\n'
            f'Which is better? Reply with {first_label}, {second_label}, or TIE.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=10,
            messages=[{'role': 'user', 'content': prompt}]
        )
        return r.content[0].text.strip()

    # Pass 1: A first
    p1 = single_pass(response_a, response_b, label_a, label_b)
    # Pass 2: B first — but labels stay the same (we just swap presentation order)
    p2 = single_pass(response_b, response_a, label_b, label_a)

    # Normalize p2: if judge said label_b in pass 2, that means they preferred first-shown
    # Need to map back: if p2 = label_b, the 'first' won; if p2 = label_a, the 'second' won
    if p1 == p2 and p1 != 'TIE':
        return p1, 'Consistent result'
    else:
        return 'TIE', f'Inconsistent: pass1={p1}, pass2={p2}'

winner, reason = debiased_compare(
    'What is Docker?',
    'Docker is a containerization platform.',
    'Docker allows you to package applications into containers for consistent deployment.'
)
print(f'{winner}: {reason}')

완화 방법 2: 다양성을 갖춘 여러 평가자

import anthropic
import openai

anthropic_client = anthropic.Anthropic(api_key='sk-ant-...')
openai_client = openai.OpenAI(api_key='sk-...')

def multi_model_pairwise(question, response_a, response_b):
    results = {}

    # Judge 1: Claude
    r1 = anthropic_client.messages.create(
        model='claude-opus-4-5', max_tokens=5,
        messages=[{'role': 'user', 'content':
            f'Q: {question}\nA: {response_a}\nB: {response_b}\n'
            f'Which is better? Reply A, B, or TIE.'
        }]
    )
    results['Claude'] = r1.content[0].text.strip()

    # Judge 2: GPT-4o
    r2 = openai_client.chat.completions.create(
        model='gpt-4o', max_tokens=5,
        messages=[{'role': 'user', 'content':
            f'Q: {question}\nA: {response_a}\nB: {response_b}\n'
            f'Which is better? Reply A, B, or TIE.'
        }]
    )
    results['GPT4o'] = r2.choices[0].message.content.strip()

    print(f'Claude judge: {results["Claude"]}')
    print(f'GPT-4o judge: {results["GPT4o"]}')

    # Aggregate: majority vote
    votes = list(results.values())
    if votes.count('A') >= 2: return 'A'
    if votes.count('B') >= 2: return 'B'
    return 'TIE'

final = multi_model_pairwise(
    'Explain recursion.',
    'A function that calls itself.',
    'Recursion is when a function solves a problem by solving a smaller version of the same problem.'
)
print(f'Final verdict: {final}')

완화 방법 3: 장황함 방지 지침

ANTI_VERBOSITY_JUDGE = (
    'Evaluate this response. Apply these corrections for known judge biases:\n\n'
    'VERBOSITY CORRECTION: Do NOT rate a response higher simply because it is longer. '
    'A concise, accurate 20-word answer is better than a 200-word answer that says the same thing. '
    'Actively penalize unnecessary padding, filler phrases like "Great question!", '
    'and repetition.\n\n'
    'LENGTH PENALTY: If the response contains introductory filler, closing remarks, '
    'or restates the question, subtract 1 point from your score.\n\n'
    'Question: {question}\n'
    'Response: {response}\n\n'
    'Score 1-5 (apply verbosity correction above):'
)

# This instruction significantly reduces verbosity inflation in practice
print('Anti-verbosity instructions reduce the length-quality conflation')

사람 평가자를 기준으로 보정하기

import anthropic
import json
from scipy import stats  # pip install scipy

client = anthropic.Anthropic(api_key='sk-ant-...')

def calibrate_judge(calibration_set):
    """
    calibration_set: list of dicts with:
    {'question': str, 'response': str, 'human_score': float}
    """
    llm_scores = []
    human_scores = []

    for item in calibration_set:
        prompt = (
            f'Rate this response 1-5.\n'
            f'Q: {item["question"]}\nA: {item["response"]}\n'
            f'Return only a number.'
        )
        r = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=5,
            messages=[{'role': 'user', 'content': prompt}]
        )
        try:
            llm_score = float(r.content[0].text.strip())
        except ValueError:
            llm_score = 3.0  # Default on parse error

        llm_scores.append(llm_score)
        human_scores.append(item['human_score'])

    correlation, p_value = stats.pearsonr(llm_scores, human_scores)
    print(f'LLM-Human correlation: {correlation:.3f} (p={p_value:.4f})')
    print(f'LLM mean score: {sum(llm_scores)/len(llm_scores):.2f}')
    print(f'Human mean score: {sum(human_scores)/len(human_scores):.2f}')
    return correlation

체계적 편향 패턴 탐지

import json
from collections import defaultdict

def analyze_judge_bias(log_file='pairwise_log.jsonl'):
    """
    Analyze pairwise logs to detect systematic bias.
    """
    wins_by_label = defaultdict(int)
    total_by_label = defaultdict(int)

    with open(log_file) as f:
        for line in f:
            entry = json.loads(line)
            winner = entry['winner']
            label_a = entry['label_a']
            label_b = entry['label_b']

            if winner == 'A':
                wins_by_label[label_a] += 1
            elif winner == 'B':
                wins_by_label[label_b] += 1

            total_by_label[label_a] += 1
            total_by_label[label_b] += 1

    print('Win rate by model:')
    for label in sorted(total_by_label):
        total = total_by_label[label]
        wins = wins_by_label[label]
        print(f'  {label}: {wins}/{total} = {wins/total:.0%}')

    # Flag if any model wins >60% — likely systematic bias
    for label in total_by_label:
        rate = wins_by_label[label] / total_by_label[label]
        if rate > 0.65 or rate < 0.35:
            print(f'WARNING: {label} win rate {rate:.0%} suggests systematic bias')

편향 완화 점검 목록

감사 추적과 설명 가능성

지식 확인: 자기 선호 완화

복습: LLM 평가자의 조정과 편향