プロンプトの根本原因分析

# Failing prompt — no context
prompt_v1 = 'What is the current price of our Pro plan?'

# Context problem test: inject the information
prompt_v2 = '''
Our pricing (as of today):
- Free: $0/month
- Pro: $19/month
- Enterprise: $99/month

Question: What is the current price of our Pro plan?
'''
# If v2 succeeds and v1 fails -> root cause is context problem

# Ambiguous
prompt_ambiguous = 'Summarize the following article briefly.'

# Precise — ambiguity removed
prompt_precise = (
    'Summarize the following article in exactly 2 sentences. '
    'Do not exceed 50 words. Output only the summary, nothing else.'
)

# Test: if precise version succeeds, root cause was ambiguity
# Fix: replace vague qualifiers with exact specifications

# Format conflict example
system_prompt = 'You are a helpful assistant. Always respond in plain text without any formatting.'

user_message = 'List the top 5 benefits of exercise. Use markdown bullet points.'

# The model faces a conflict: plain text vs markdown.
# Detection: if output format is inconsistent across runs, look for conflicting instructions.

# Fix: ensure all format instructions agree. Move format to system prompt only.
system_prompt_fixed = (
    'You are a helpful assistant. '
    'Always respond using markdown bullet points for lists.'
)

# Capability limit test: try the same task on different models
models = ['gpt-4o-mini', 'gpt-4o', 'gpt-4o-2024-11-20']
results = {}

for model in models:
    resp = client.chat.completions.create(
        model=model,
        messages=[{'role': 'user', 'content': complex_reasoning_prompt}]
    )
    results[model] = evaluate(resp.choices[0].message.content)

# If accuracy improves with more capable models -> capability limit
for model, score in results.items():
    print(f'{model}: {score:.0%} accuracy')

def rca_test(base_prompt, test_input, expected_output):
    results = {}

    # Test 1: base (failing) prompt
    results['base'] = run_and_evaluate(base_prompt, test_input, expected_output)

    # Test 2: add context
    results['with_context'] = run_and_evaluate(
        base_prompt + '\nContext: ' + get_context(test_input),
        test_input, expected_output
    )

    # Test 3: clarify instructions
    results['clarified'] = run_and_evaluate(
        clarify(base_prompt), test_input, expected_output
    )

    for name, passed in results.items():
        print(f'{name}: {"PASS" if passed else "FAIL"}')

def confirm_root_cause(fix_fn, test_cases, threshold=0.9):
    '''fix_fn: a function that takes a prompt and returns a fixed prompt'''
    passed = 0
    for case in test_cases:
        fixed_prompt = fix_fn(case['prompt'])
        result = run_and_evaluate(fixed_prompt, case['input'], case['expected'])
        if result:
            passed += 1

    pass_rate = passed / len(test_cases)
    print(f'Fix pass rate: {pass_rate:.0%}')
    if pass_rate >= threshold:
        print('Root cause CONFIRMED — fix is reliable.')
    else:
        print('Root cause NOT confirmed — failure has multiple causes.')

rca_record = {
    'prompt_id': 'summarize_v3',
    'failure_type': 'wrong_format',
    'root_cause': 'format_conflict',
    'hypothesis': 'System prompt said plain text, user message asked for markdown',
    'evidence': 'Removing markdown instruction from user message resolved failure on 8/8 test cases',
    'fix_applied': 'Moved all format instructions to system prompt; removed format instructions from user template',
    'fix_date': '2024-11-15'
}

def measure_failure_rate(prompt, test_input, expected, runs=20):
    failures = 0
    for _ in range(runs):
        resp = client.chat.completions.create(
            model='gpt-4o',
            messages=[{'role': 'user', 'content': prompt + '\n' + test_input}],
            temperature=0.7
        )
        if not evaluate(resp.choices[0].message.content, expected):
            failures += 1
    print(f'Failure rate: {failures}/{runs} = {failures/runs:.0%}')