Why Testing Agents Is Different

Testing Software vs. Testing Agents

Non-Determinism: Same Input, Different Output

import openai

client = openai.OpenAI(api_key='YOUR_API_KEY')

# Same prompt, potentially different outputs each run
for i in range(3):
    response = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': 'Name a planet.'}],
        temperature=0.9  # High randomness
    )
    print(f'Run {i+1}: {response.choices[0].message.content}')
# Run 1: Mars
# Run 2: Jupiter
# Run 3: Saturn

The Cost Problem: Real LLM Calls Are Expensive

# A test that calls the real API costs tokens every run
# 100 tests x 500 tokens each x 10 CI runs/day = 500,000 tokens/day
# At $0.15/1M tokens (gpt-4o-mini): ~$0.075/day = ~$27/year for a tiny suite
# For gpt-4o: 15x more expensive = ~$400/year

# This is why mocking and recording API responses is essential
print('Real API calls in tests = expensive and slow')
print('Solution: Mock or record LLM responses in unit tests')
print('Reserve real calls for scheduled integration tests')

The Latency Problem

import time

# Simulating what a test suite looks like with real vs mocked calls
num_tests = 50

# Real API calls
real_time = num_tests * 5  # avg 5 seconds per call
print(f'With real API calls: {real_time}s = {real_time/60:.1f} minutes')

# Mocked calls
mock_time = num_tests * 0.001  # <1ms per mock
print(f'With mocked calls: {mock_time:.3f}s = nearly instant')

# Conclusion: mock in unit tests, use real calls in integration tests

External Dependencies in Agent Tests

# An agent might call multiple external services
# Each is a potential test failure point

def agent_pipeline(query: str) -> str:
    search_results = search_web(query)       # External: Tavily/Serper API
    documents = fetch_documents(search_results)  # External: HTTP calls
    answer = llm_summarize(documents)        # External: OpenAI API
    saved = database_store(answer)           # External: PostgreSQL
    return answer

# In unit tests: mock ALL of these
# In integration tests: use sandboxed versions of real services
print('Each external call is a test reliability risk')

What Standard Unit Tests Assume

# Standard unit test — works perfectly for deterministic code
def add(a, b):
    return a + b

def test_add():
    assert add(2, 3) == 5  # Always passes — deterministic

# Agent 'unit test' that calls a real LLM — problematic
# def test_agent_answers_question():
#     response = agent.run('What is 2+2?')
#     assert response == '4'  # Might return 'The answer is 4' or 'Four'

print('Exact string matching fails for LLM outputs')
print('Need structural or semantic assertions instead')

The Testing Pyramid for Agents

Structural vs. Semantic Assertions

# Fragile: exact string match
# assert response.content == 'The capital of France is Paris.'

# Better: structural assertion
# assert response.tool_calls[0]['function']['name'] == 'search_web'

# Better: semantic check
def test_capital_in_response(response_text: str) -> bool:
    key_words = ['paris', 'france', 'capital']
    lower = response_text.lower()
    return all(word in lower for word in key_words)

response = 'Paris is the capital city of France.'
print(test_capital_in_response(response))  # True

Evaluation Harnesses and LLM-as-Judge

import openai

client = openai.OpenAI(api_key='YOUR_API_KEY')

def llm_judge(question: str, answer: str) -> dict:
    prompt = f'Question: {question}\nAnswer: {answer}\nRate the answer 1-5 for accuracy. Reply with only a number.'
    response = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': prompt}]
    )
    score = int(response.choices[0].message.content.strip())
    return {'score': score, 'pass': score >= 4}

# result = llm_judge('What is the capital of France?', 'Paris')
# print(result)  # {'score': 5, 'pass': True}

Regression Testing for Agents

# golden_examples.py
GOLDEN_EXAMPLES = [
    {
        'input': 'Search for the weather in Paris',
        'expected_tool': 'get_weather',
        'expected_args': {'city': 'Paris'}
    },
    {
        'input': 'Calculate 15% tip on $45',
        'expected_tool': 'calculate',
        'expected_args': {'expression': '45 * 0.15'}
    }
]

def run_regressions(agent, examples: list) -> int:
    failures = 0
    for ex in examples:
        result = agent.plan(ex['input'])  # mocked LLM
        if result['tool'] != ex['expected_tool']:
            print(f'FAIL: expected {ex["expected_tool"]}, got {result["tool"]}')
            failures += 1
    return failures

# --- demo: a stub agent whose .plan() mimics an LLM's tool choice ---
class _StubAgent:
    def plan(self, text):
        if 'weather' in text.lower():
            return {'tool': 'get_weather'}
        if 'tip' in text.lower() or 'calculate' in text.lower():
            return {'tool': 'wrong_tool'}  # simulate a regression
        return {'tool': 'unknown'}

failures = run_regressions(_StubAgent(), GOLDEN_EXAMPLES)
print(f'{failures} of {len(GOLDEN_EXAMPLES)} golden examples failed')

Setting Up a Basic Agent Test File

# tests/test_agent.py
import pytest

# Fast unit tests — run on every commit
class TestAgentTools:
    def test_tool_returns_dict(self, mock_llm):
        result = my_tool(query='test')
        assert isinstance(result, dict)
        assert 'data' in result

    def test_agent_selects_correct_tool(self, mock_llm):
        response = agent.run('Search for Python tutorials')
        assert response['tool_used'] == 'web_search'

# Slow integration tests — run nightly or on release
@pytest.mark.integration
class TestAgentIntegration:
    def test_full_pipeline_with_real_api(self):
        # Uses real OpenAI + sandboxed services
        result = agent.run('Summarize the Python docs')
        assert len(result['answer']) > 50

Knowledge Check: Why Agent Testing Is Different

Recap: Why Testing Agents Is Different