Systematic Debugging Approach

The Debugging Mindset

Step 1: Reproduce the Failure

import openai
client = openai.OpenAI(api_key='sk-...')

def run_prompt(prompt, user_input, temperature=0):
    resp = client.chat.completions.create(
        model='gpt-4o',
        messages=[
            {'role': 'system', 'content': prompt},
            {'role': 'user', 'content': user_input}
        ],
        temperature=temperature
    )
    return resp.choices[0].message.content

# Reproduce with temperature=0 to eliminate randomness
for i in range(5):
    output = run_prompt(failing_prompt, test_input, temperature=0)
    print(f'Run {i+1}:', output[:100])

Step 2: Create a Minimal Reproducible Prompt

def binary_search_prompt(prompt_lines, user_input, fail_fn):
    '''Binary search: find the minimal set of lines that causes the failure.'''
    if len(prompt_lines) == 1:
        return prompt_lines  # Minimal failing unit found

    mid = len(prompt_lines) // 2
    first_half = prompt_lines[:mid]
    second_half = prompt_lines[mid:]

    # Test first half
    if fail_fn('\n'.join(first_half), user_input):
        return binary_search_prompt(first_half, user_input, fail_fn)
    # Test second half
    elif fail_fn('\n'.join(second_half), user_input):
        return binary_search_prompt(second_half, user_input, fail_fn)
    else:
        # Both halves pass — interaction effect between halves
        return prompt_lines

The Removal Test

sections = {
    'role_instruction': 'You are a precise JSON API. Respond only with valid JSON.',
    'context': 'The user is asking about our product catalog.',
    'format_spec': 'Return a JSON object with keys: name, price, available.',
    'examples': 'Example: {"name": "Widget", "price": 9.99, "available": true}',
    'safety': 'Do not reveal internal pricing strategy.'
}

def test_without(section_to_remove, user_input):
    reduced = {k: v for k, v in sections.items() if k != section_to_remove}
    prompt = '\n'.join(reduced.values())
    output = run_prompt(prompt, user_input)
    print(f'Without {section_to_remove}: {evaluate(output)}')

for section in sections:
    test_without(section, 'What is the price of a Widget?')

A/B Testing Prompt Sections

# A/B test: vague vs precise format instruction
variant_A = 'Return a JSON object.'
variant_B = 'Return a valid JSON object. No markdown, no code fences, no prose. Only the raw JSON.'

test_inputs = [
    'What is the price of Widget A?',
    'List all available products.',
    'Is Widget B in stock?'
]

def run_ab_test(base_prompt, variant, inputs, n_runs=5):
    pass_count = 0
    for inp in inputs:
        for _ in range(n_runs):
            prompt = base_prompt.replace('{{FORMAT}}', variant)
            output = run_prompt(prompt, inp)
            if is_valid_json(output):
                pass_count += 1
    return pass_count / (len(inputs) * n_runs)

print('A pass rate:', run_ab_test(template, variant_A, test_inputs))
print('B pass rate:', run_ab_test(template, variant_B, test_inputs))

Differential Testing

import difflib

def show_prompt_diff(prompt_v1, prompt_v2):
    diff = difflib.unified_diff(
        prompt_v1.splitlines(),
        prompt_v2.splitlines(),
        fromfile='v1',
        tofile='v2',
        lineterm=''
    )
    for line in diff:
        print(line)

show_prompt_diff(working_prompt, failing_prompt)
# Output shows exactly which lines changed between versions
# Test reverting each changed section individually

Testing Inputs vs Testing Prompts

# Input complexity ladder
inputs_by_complexity = [
    'What is 2 + 2?',             # trivially simple
    'Summarize this sentence.',    # simple task
    'Analyze this 500-word essay.',  # moderate
    'Compare 10 documents and extract contradictions.'  # complex
]

# Find the complexity level where the prompt starts failing
for inp in inputs_by_complexity:
    output = run_prompt(failing_prompt, inp)
    result = 'PASS' if evaluate(output) else 'FAIL'
    print(f'{result}: {inp[:60]}')
# First FAIL indicates where the prompt breaks down

The Minimal Reproducible Prompt Pattern

# Start minimal
MINIMAL_PROMPT = (
    'You are a data extractor.\n'
    'Extract the product name and price from the text.\n'
    'Respond with JSON: {"name": "...", "price": ...}\n'
)

minimal_input = 'Widget Pro costs $49.'

# Test: if this works, the problem is in something added on top
output = run_prompt(MINIMAL_PROMPT, minimal_input)
print(output)
# Expected: {"name": "Widget Pro", "price": 49.0}

Tracking the Debugging Session

debug_log = [
    {
        'test': 'base_prompt_v5',
        'hypothesis': 'failing due to format conflict',
        'result': 'FAIL',
        'notes': 'JSON prefix still present'
    },
    {
        'test': 'base_prompt_v5_no_markdown_hint',
        'hypothesis': 'removing markdown hint from user message fixes conflict',
        'result': 'PASS',
        'notes': 'Output is clean JSON. Root cause confirmed: format conflict.'
    }
]

import json
with open('debug_session.json', 'w') as f:
    json.dump(debug_log, f, indent=2)

When to Stop Debugging and Change Strategy

Fixing vs Hardening

def safe_run_prompt(prompt, user_input):
    output = run_prompt(prompt, user_input)
    try:
        parsed = json.loads(output)
        return parsed
    except json.JSONDecodeError:
        # Fallback: ask the model to fix its own output
        fix_prompt = f'The following is not valid JSON. Rewrite it as valid JSON only:\n{output}'
        fixed = run_prompt('', fix_prompt)
        return json.loads(fixed)

Knowledge Check

Recap: Systematic Debugging