Тестирование агентов на основе утверждений

Переход от точного сопоставления строк

Проверка выполнения вызовов инструментов

import json
from unittest.mock import patch, MagicMock

@patch('myagent.client.chat.completions.create')
def test_agent_calls_search_tool(mock_create):
    # Mock: agent decides to call search_web
    tool_call = MagicMock()
    tool_call.function.name = 'search_web'
    tool_call.function.arguments = json.dumps({'query': 'Python tutorials'})
    mock_create.return_value = MagicMock(
        choices=[MagicMock(message=MagicMock(tool_calls=[tool_call]))]
    )

    response = mock_create()  # simulating the agent call
    tc = response.choices[0].message.tool_calls

    assert tc is not None
    assert len(tc) > 0
    assert tc[0].function.name == 'search_web'

# --- demo: give unittest.mock.patch a real dotted path to patch ---
import sys
import types

_myagent = types.ModuleType('myagent')
_myagent.client = types.SimpleNamespace(
    chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=lambda *a, **k: None))
)
sys.modules['myagent'] = _myagent

test_agent_calls_search_tool()
print('test_agent_calls_search_tool: PASS')

Проверка правильного имени инструмента

import json
from unittest.mock import MagicMock

def extract_tool_calls(response) -> list:
    message = response.choices[0].message
    if not message.tool_calls:
        return []
    return [
        {
            'name': tc.function.name,
            'args': json.loads(tc.function.arguments)
        }
        for tc in message.tool_calls
    ]

# In a test:
# calls = extract_tool_calls(mock_response)
# assert calls[0]['name'] == 'get_weather'
# assert calls[0]['args']['city'] == 'Paris'
print('Tool name and argument assertions are the most reliable agent tests')

Проверка аргументов инструмента

import json
from unittest.mock import patch, MagicMock

@patch('myagent.client.chat.completions.create')
def test_weather_tool_gets_correct_city(mock_create):
    tool_call = MagicMock()
    tool_call.function.name = 'get_weather'
    tool_call.function.arguments = json.dumps({'city': 'Tokyo', 'unit': 'celsius'})
    mock_create.return_value = MagicMock(
        choices=[MagicMock(message=MagicMock(tool_calls=[tool_call]))]
    )

    response = mock_create()
    args = json.loads(response.choices[0].message.tool_calls[0].function.arguments)

    assert args['city'] == 'Tokyo'
    assert args.get('unit') in ['celsius', 'fahrenheit', None]  # flexible

# --- demo: give unittest.mock.patch a real dotted path to patch ---
import sys
import types

_myagent = types.ModuleType('myagent')
_myagent.client = types.SimpleNamespace(
    chat=types.SimpleNamespace(completions=types.SimpleNamespace(create=lambda *a, **k: None))
)
sys.modules['myagent'] = _myagent

test_weather_tool_gets_correct_city()
print('test_weather_tool_gets_correct_city: PASS')

Проверка выходных данных по JSON Schema

# pip install jsonschema
import jsonschema

AGENT_RESPONSE_SCHEMA = {
    'type': 'object',
    'required': ['answer', 'sources', 'confidence'],
    'properties': {
        'answer': {'type': 'string', 'minLength': 1},
        'sources': {
            'type': 'array',
            'items': {'type': 'string', 'format': 'uri'}
        },
        'confidence': {'type': 'number', 'minimum': 0, 'maximum': 1}
    }
}

def test_agent_output_schema(agent_output: dict):
    try:
        jsonschema.validate(instance=agent_output, schema=AGENT_RESPONSE_SCHEMA)
        print('Schema validation passed')
    except jsonschema.ValidationError as e:
        raise AssertionError(f'Invalid agent output: {e.message}')

Проверки наличия ключевых слов

def assert_keywords_present(text: str, keywords: list, require_all: bool = True):
    lower_text = text.lower()
    found = [kw.lower() in lower_text for kw in keywords]

    if require_all:
        missing = [kw for kw, f in zip(keywords, found) if not f]
        assert not missing, f'Missing keywords: {missing}'
    else:
        assert any(found), f'None of {keywords} found in: {text[:100]}'

# Tests
response = 'The capital city of France is Paris, located in western Europe.'
assert_keywords_present(response, ['paris', 'france', 'capital'])
print('All keywords present!')  # passes

assert_keywords_present(response, ['spain', 'france'], require_all=False)
print('At least one keyword present!')  # passes

Проверка формата ответа: проверка типов

def test_agent_returns_valid_structure(agent_result):
    # Type checks
    assert isinstance(agent_result, dict), 'Result must be a dict'
    assert isinstance(agent_result.get('answer'), str), 'answer must be a string'
    assert isinstance(agent_result.get('steps'), list), 'steps must be a list'

    # Non-empty checks
    assert len(agent_result['answer']) > 0, 'answer must not be empty'
    assert len(agent_result['steps']) >= 1, 'must have at least one step'

    # Range checks
    confidence = agent_result.get('confidence', 0)
    assert 0.0 <= confidence <= 1.0, 'confidence must be 0-1'

print('Structural assertions are fast and reliable')

Проверка причины завершения

from unittest.mock import MagicMock

def test_agent_stops_cleanly(mock_response):
    finish_reason = mock_response.choices[0].finish_reason
    assert finish_reason in ('stop', 'tool_calls'), \
        f'Unexpected finish_reason: {finish_reason}'

def test_no_truncation(mock_response):
    finish_reason = mock_response.choices[0].finish_reason
    assert finish_reason != 'length', \
        'Response was truncated — increase max_tokens'

# Example mock for a clean stop
mock = MagicMock()
mock.choices = [MagicMock(finish_reason='stop')]
test_agent_stops_cleanly(mock)
print('finish_reason: stop — clean termination')

Проверка числа шагов в цикле

def test_agent_completes_in_bounded_steps(mock_agent):
    result = mock_agent.run('Search for the weather in Paris')

    # Agent should complete within 5 steps
    assert result['steps_taken'] <= 5, \
        f'Agent took too many steps: {result["steps_taken"]}'

    # Agent should produce a final answer, not exit on timeout
    assert result['status'] == 'completed', \
        f'Agent did not complete: {result["status"]}'

    assert result['answer'] is not None

print('Bounding step count prevents runaway agents from passing tests')

Параметризация тестов для разных входных данных

import pytest
from unittest.mock import patch, MagicMock
import json

@pytest.mark.parametrize('query,expected_tool', [
    ('What is the weather in Tokyo?', 'get_weather'),
    ('Calculate 15% of 200', 'calculator'),
    ('Search for Python books', 'web_search'),
    ('What time is it in Berlin?', 'get_time'),
])
@patch('myagent.client.chat.completions.create')
def test_agent_tool_routing(mock_create, query, expected_tool):
    tool_call = MagicMock()
    tool_call.function.name = expected_tool
    tool_call.function.arguments = json.dumps({'input': query})
    mock_create.return_value = MagicMock(
        choices=[MagicMock(message=MagicMock(tool_calls=[tool_call]))]
    )
    response = mock_create()
    actual = response.choices[0].message.tool_calls[0].function.name
    assert actual == expected_tool

Написание вспомогательных функций для пользовательских проверок

import json

def assert_tool_called(response, tool_name: str, required_args: dict = None):
    message = response.choices[0].message
    assert message.tool_calls, 'Expected tool call but got plain text'
    names = [tc.function.name for tc in message.tool_calls]
    assert tool_name in names, f'Expected {tool_name}, got {names}'

    if required_args:
        for tc in message.tool_calls:
            if tc.function.name == tool_name:
                args = json.loads(tc.function.arguments)
                for key, val in required_args.items():
                    assert args.get(key) == val, \
                        f'Arg {key}: expected {val}, got {args.get(key)}'

# Clean test using the helper:
# assert_tool_called(response, 'get_weather', {'city': 'Paris'})

# --- demo ---
from unittest.mock import MagicMock

tool_call = MagicMock()
tool_call.function.name = 'get_weather'
tool_call.function.arguments = json.dumps({'city': 'Paris'})
response = MagicMock(choices=[MagicMock(message=MagicMock(tool_calls=[tool_call]))])

assert_tool_called(response, 'get_weather', {'city': 'Paris'})
print('assert_tool_called passed: agent called get_weather with city=Paris')

Проверка знаний: тестирование агентов на основе проверок

Итоги: тестирование агентов на основе проверок