모델 업데이트에 따른 회귀 테스트

모델 업데이트가 프롬프트를 망가뜨리는 이유

모델 업데이트 문제

MODEL_HISTORY = [
    {'model': 'gpt-4', 'deployed': '2023-03-14', 'pass_rate': 0.87},
    {'model': 'gpt-4-turbo', 'deployed': '2023-11-06', 'pass_rate': 0.91},
    {'model': 'gpt-4o', 'deployed': '2024-05-13', 'pass_rate': 0.93},
    {'model': 'gpt-4o-2024-11-20', 'deployed': '2024-11-20', 'pass_rate': None},  # to be measured
]

# Goal: measure pass_rate for the new model before deploying to production

새 모델에서 테스트 모음 실행하기

import openai
client = openai.OpenAI(api_key='sk-...')

def run_suite_on_model(test_cases, system_prompt, model):
    results = []
    for test in test_cases:
        resp = client.chat.completions.create(
            model=model,
            messages=[
                {'role': 'system', 'content': system_prompt},
                {'role': 'user', 'content': test['input']}
            ],
            temperature=0
        )
        output = resp.choices[0].message.content
        passed = test['evaluator'](output)
        results.append({'id': test['id'], 'passed': passed, 'output': output})
    pass_rate = sum(r['passed'] for r in results) / len(results)
    return results, pass_rate

old_results, old_rate = run_suite_on_model(TEST_CASES, PROMPT, 'gpt-4o')
new_results, new_rate = run_suite_on_model(TEST_CASES, PROMPT, 'gpt-4o-2024-11-20')
print(f'Old: {old_rate:.1%} | New: {new_rate:.1%} | Delta: {(new_rate-old_rate):+.1%}')

모델 버전 간 결과 비교

def diff_results(old_results, new_results):
    old_by_id = {r['id']: r for r in old_results}
    new_by_id = {r['id']: r for r in new_results}

    regressions = []
    improvements = []

    for test_id, new_r in new_by_id.items():
        old_r = old_by_id.get(test_id)
        if old_r is None:
            continue
        if old_r['passed'] and not new_r['passed']:
            regressions.append({'id': test_id, 'old_output': old_r['output'], 'new_output': new_r['output']})
        elif not old_r['passed'] and new_r['passed']:
            improvements.append({'id': test_id})

    print(f'Regressions: {len(regressions)}')
    print(f'Improvements: {len(improvements)}')
    for reg in regressions:
        print(f'  REGRESSED: {reg["id"]}')
        print(f'    Old: {reg["old_output"][:60]}')
        print(f'    New: {reg["new_output"][:60]}')
    return regressions, improvements

regressions, improvements = diff_results(old_results, new_results)

테스트 로그에서 모델 버전 추적하기

import openai, json
from datetime import datetime, timezone

client = openai.OpenAI(api_key='sk-...')

def run_test_with_version_tracking(test, system_prompt, model_alias):
    resp = client.chat.completions.create(
        model=model_alias,
        messages=[
            {'role': 'system', 'content': system_prompt},
            {'role': 'user', 'content': test['input']}
        ],
        temperature=0
    )
    output = resp.choices[0].message.content
    return {
        'test_id': test['id'],
        'model_alias': model_alias,
        'model_actual': resp.model,  # The exact versioned model ID returned by the API
        'timestamp': datetime.now(timezone.utc).isoformat(),
        'output': output,
        'passed': test['evaluator'](output)
    }

회귀 조사하기

def investigate_regression(test_id, old_output, new_output, prompt, user_input):
    # Step 1: check if old behavior was actually wrong
    judge_prompt = (
        f'Given this task prompt: {prompt}\n'
        f'And this user input: {user_input}\n\n'
        f'Which output is better?\n'
        f'A) {old_output}\n'
        f'B) {new_output}\n\n'
        'Reply with A or B and one sentence explanation.'
    )
    resp = client.chat.completions.create(
        model='gpt-4o',
        messages=[{'role': 'user', 'content': judge_prompt}],
        temperature=0
    )
    verdict = resp.choices[0].message.content
    print(f'Judge verdict for {test_id}: {verdict}')
    # If judge says B (new output) is better: update the test, not the prompt

프롬프트와 테스트 중 무엇을 업데이트할지 결정하기

REGRESSION_ACTIONS = {
    'prompt_updated': {
        'when': 'New model ignores instruction the old model followed. Same behavior needed.',
        'action': 'Add stronger/rephrased instruction. Re-run tests on new model.'
    },
    'test_updated': {
        'when': 'New model output is objectively better but fails old test criteria.',
        'action': 'Update expected output in test. Document the improvement.'
    },
    'model_pinned': {
        'when': 'New model cannot perform this task reliably despite prompt changes.',
        'action': 'Pin the model alias to the old versioned ID for this endpoint.'
    }
}

모델 버전 고정하기

# Bad practice: alias can point to different model versions over time
client.chat.completions.create(
    model='gpt-4o',  # could be any version at any time
    messages=[...]
)

# Good practice: pin to a specific version for production
client.chat.completions.create(
    model='gpt-4o-2024-11-20',  # guaranteed behavior
    messages=[...]
)

# Track in config
MODEL_CONFIG = {
    'sentiment_classifier': 'gpt-4o-2024-11-20',
    'code_generator': 'gpt-4o-2024-11-20',
    'summarizer': 'gpt-4o-mini-2024-07-18',
}

CI에서 자동화된 회귀 테스트

# .github/workflows/prompt_regression.yml (YAML, shown as comment)
# name: Prompt Regression Tests
# on:
#   push:
#     paths:
#       - 'config/models.json'  # triggers when model version changes
#       - 'prompts/*.txt'       # triggers when prompts change
# jobs:
#   test:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v4
#       - name: Install dependencies
#         run: pip install pytest openai jsonschema
#       - name: Run prompt test suite
#         run: pytest tests/prompts/ -v --junitxml=results.xml
#         env:
#           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

# In Python: read model version from config
import json
with open('config/models.json') as f:
    MODEL_CONFIG = json.load(f)
MODEL = MODEL_CONFIG['sentiment_classifier']

제공업체 간 회귀 처리하기

def cross_provider_test(test_cases, system_prompt, providers):
    all_results = {}
    for provider, config in providers.items():
        results, rate = run_suite_on_model(
            test_cases, system_prompt, model=config['model']
        )
        all_results[provider] = {'rate': rate, 'results': results}
        print(f'{provider}: {rate:.1%}')

    # Find cases that fail on one provider but not another
    for test in test_cases:
        outcomes = {
            p: next(r for r in all_results[p]['results'] if r['id'] == test['id'])['passed']
            for p in providers
        }
        if not all(outcomes.values()):
            failing = [p for p, passed in outcomes.items() if not passed]
            print(f'  {test["id"]}: FAILS on {failing}')

    return all_results

통과율 추세 모니터링하기

import json

def plot_pass_rate_trend(history_file='test_history.jsonl'):
    runs = []
    with open(history_file) as f:
        for line in f:
            runs.append(json.loads(line))

    runs.sort(key=lambda r: r['run_id'])

    print('Pass rate trend:')
    for run in runs[-10:]:  # last 10 runs
        bar = '#' * int(run['pass_rate'] * 40)
        print(f"{run['run_id'][:10]} {run['model']:30} {run['pass_rate']:.0%} |{bar}|")

    # Detect regression
    if len(runs) >= 2:
        delta = runs[-1]['pass_rate'] - runs[-2]['pass_rate']
        if delta < -0.05:
            print(f'ALERT: pass rate dropped {delta:.0%} since last run!')

plot_pass_rate_trend()

지식 확인

요약: 모델 업데이트 전반의 회귀 테스트