Pourquoi tester les agents est différent

Vérifier les logiciels et vérifier les agents

Non-déterminisme : même entrée, sortie différente

import openai

client = openai.OpenAI(api_key='YOUR_API_KEY')

# Same prompt, potentially different outputs each run
for i in range(3):
    response = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': 'Name a planet.'}],
        temperature=0.9  # High randomness
    )
    print(f'Run {i+1}: {response.choices[0].message.content}')
# Run 1: Mars
# Run 2: Jupiter
# Run 3: Saturn

Le problème du coût : les appels réels aux LLM sont coûteux

# A test that calls the real API costs tokens every run
# 100 tests x 500 tokens each x 10 CI runs/day = 500,000 tokens/day
# At $0.15/1M tokens (gpt-4o-mini): ~$0.075/day = ~$27/year for a tiny suite
# For gpt-4o: 15x more expensive = ~$400/year

# This is why mocking and recording API responses is essential
print('Real API calls in tests = expensive and slow')
print('Solution: Mock or record LLM responses in unit tests')
print('Reserve real calls for scheduled integration tests')

Le problème de la latence

import time

# Simulating what a test suite looks like with real vs mocked calls
num_tests = 50

# Real API calls
real_time = num_tests * 5  # avg 5 seconds per call
print(f'With real API calls: {real_time}s = {real_time/60:.1f} minutes')

# Mocked calls
mock_time = num_tests * 0.001  # <1ms per mock
print(f'With mocked calls: {mock_time:.3f}s = nearly instant')

# Conclusion: mock in unit tests, use real calls in integration tests

Dépendances externes dans les vérifications d’agents

# An agent might call multiple external services
# Each is a potential test failure point

def agent_pipeline(query: str) -> str:
    search_results = search_web(query)       # External: Tavily/Serper API
    documents = fetch_documents(search_results)  # External: HTTP calls
    answer = llm_summarize(documents)        # External: OpenAI API
    saved = database_store(answer)           # External: PostgreSQL
    return answer

# In unit tests: mock ALL of these
# In integration tests: use sandboxed versions of real services
print('Each external call is a test reliability risk')

Ce que supposent les vérifications unitaires standard

# Standard unit test — works perfectly for deterministic code
def add(a, b):
    return a + b

def test_add():
    assert add(2, 3) == 5  # Always passes — deterministic

# Agent 'unit test' that calls a real LLM — problematic
# def test_agent_answers_question():
#     response = agent.run('What is 2+2?')
#     assert response == '4'  # Might return 'The answer is 4' or 'Four'

print('Exact string matching fails for LLM outputs')
print('Need structural or semantic assertions instead')

La pyramide de vérification des agents

Assertions structurelles ou sémantiques

# Fragile: exact string match
# assert response.content == 'The capital of France is Paris.'

# Better: structural assertion
# assert response.tool_calls[0]['function']['name'] == 'search_web'

# Better: semantic check
def test_capital_in_response(response_text: str) -> bool:
    key_words = ['paris', 'france', 'capital']
    lower = response_text.lower()
    return all(word in lower for word in key_words)

response = 'Paris is the capital city of France.'
print(test_capital_in_response(response))  # True

Harnais d’évaluation et évaluation par un LLM

import openai

client = openai.OpenAI(api_key='YOUR_API_KEY')

def llm_judge(question: str, answer: str) -> dict:
    prompt = f'Question: {question}\nAnswer: {answer}\nRate the answer 1-5 for accuracy. Reply with only a number.'
    response = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': prompt}]
    )
    score = int(response.choices[0].message.content.strip())
    return {'score': score, 'pass': score >= 4}

# result = llm_judge('What is the capital of France?', 'Paris')
# print(result)  # {'score': 5, 'pass': True}

Vérification des régressions pour les agents

# golden_examples.py
GOLDEN_EXAMPLES = [
    {
        'input': 'Search for the weather in Paris',
        'expected_tool': 'get_weather',
        'expected_args': {'city': 'Paris'}
    },
    {
        'input': 'Calculate 15% tip on $45',
        'expected_tool': 'calculate',
        'expected_args': {'expression': '45 * 0.15'}
    }
]

def run_regressions(agent, examples: list) -> int:
    failures = 0
    for ex in examples:
        result = agent.plan(ex['input'])  # mocked LLM
        if result['tool'] != ex['expected_tool']:
            print(f'FAIL: expected {ex["expected_tool"]}, got {result["tool"]}')
            failures += 1
    return failures

# --- demo: a stub agent whose .plan() mimics an LLM's tool choice ---
class _StubAgent:
    def plan(self, text):
        if 'weather' in text.lower():
            return {'tool': 'get_weather'}
        if 'tip' in text.lower() or 'calculate' in text.lower():
            return {'tool': 'wrong_tool'}  # simulate a regression
        return {'tool': 'unknown'}

failures = run_regressions(_StubAgent(), GOLDEN_EXAMPLES)
print(f'{failures} of {len(GOLDEN_EXAMPLES)} golden examples failed')

Configurer un fichier de vérification d’agent de base

# tests/test_agent.py
import pytest

# Fast unit tests — run on every commit
class TestAgentTools:
    def test_tool_returns_dict(self, mock_llm):
        result = my_tool(query='test')
        assert isinstance(result, dict)
        assert 'data' in result

    def test_agent_selects_correct_tool(self, mock_llm):
        response = agent.run('Search for Python tutorials')
        assert response['tool_used'] == 'web_search'

# Slow integration tests — run nightly or on release
@pytest.mark.integration
class TestAgentIntegration:
    def test_full_pipeline_with_real_api(self):
        # Uses real OpenAI + sandboxed services
        result = agent.run('Summarize the Python docs')
        assert len(result['answer']) > 50

Contrôle des connaissances : pourquoi la vérification des agents est différente

Récapitulatif : pourquoi les tests d'agents sont différents