アサーションベースのプロンプトテスト

LLM出力に対するアサーション

基本的なアサーション: contains と not_contains

import openai
client = openai.OpenAI(api_key='sk-...')

def call_prompt(system, user, temperature=0):
    resp = client.chat.completions.create(
        model='gpt-4o',
        messages=[
            {'role': 'system', 'content': system},
            {'role': 'user', 'content': user}
        ],
        temperature=temperature
    )
    return resp.choices[0].message.content

# Keyword presence assertion
def assert_contains(output, keyword, case_sensitive=False):
    text = output if case_sensitive else output.lower()
    kw = keyword if case_sensitive else keyword.lower()
    assert kw in text, f'Expected "{keyword}" in output, got: {output[:100]}'

# Keyword absence assertion
def assert_not_contains(output, forbidden, case_sensitive=False):
    text = output if case_sensitive else output.lower()
    kw = forbidden if case_sensitive else forbidden.lower()
    assert kw not in text, f'Forbidden "{forbidden}" found in output: {output[:100]}'

JSON Schemaの検証

import json
from jsonschema import validate, ValidationError

PRODUCT_SCHEMA = {
    'type': 'object',
    'properties': {
        'name': {'type': 'string'},
        'price': {'type': 'number', 'minimum': 0},
        'available': {'type': 'boolean'}
    },
    'required': ['name', 'price', 'available'],
    'additionalProperties': False
}

def assert_valid_json_schema(output, schema):
    try:
        data = json.loads(output.strip())
    except json.JSONDecodeError as e:
        raise AssertionError(f'Output is not valid JSON: {e}\nOutput: {output[:200]}')
    try:
        validate(instance=data, schema=schema)
    except ValidationError as e:
        raise AssertionError(f'JSON does not match schema: {e.message}\nOutput: {output[:200]}')
    return data

# Test
output = call_prompt(
    'Extract product info as JSON: {"name": ..., "price": ..., "available": ...}',
    'Widget Pro costs $49.99 and is in stock.'
)
product = assert_valid_json_schema(output, PRODUCT_SCHEMA)
print('Parsed product:', product)

正規表現マッチング

import re

def assert_matches_regex(output, pattern, flags=0):
    if not re.search(pattern, output, flags):
        raise AssertionError(
            f'Output does not match pattern /{pattern}/\nOutput: {output[:200]}'
        )

def assert_output_is_label(output, valid_labels):
    cleaned = output.strip().upper()
    assert cleaned in valid_labels, (
        f'Expected one of {valid_labels}, got: {repr(cleaned)}'
    )

# Examples
output = call_prompt('Classify sentiment as POSITIVE, NEGATIVE, or NEUTRAL:', 'Great product!')
assert_output_is_label(output, {'POSITIVE', 'NEGATIVE', 'NEUTRAL'})

date_output = call_prompt('Extract the date in YYYY-MM-DD format:', 'Meeting on November 15, 2024')
assert_matches_regex(date_output, r'^\d{4}-\d{2}-\d{2}$')

LLM-as-Judgeによるスコアリング

def llm_judge_score(original_prompt, output, criteria, max_score=10):
    judge_prompt = (
        f'Evaluate the following AI response on a scale of 1-{max_score}.\n'
        f'Evaluation criteria: {criteria}\n\n'
        f'Original prompt: {original_prompt}\n\n'
        f'AI response: {output}\n\n'
        f'Return only a number from 1 to {max_score}.'
    )
    resp = client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': judge_prompt}],
        temperature=0
    )
    score_text = resp.choices[0].message.content.strip()
    return int(score_text)

def assert_llm_score_above(original_prompt, output, criteria, min_score=7):
    score = llm_judge_score(original_prompt, output, criteria)
    assert score >= min_score, f'LLM judge score {score} < minimum {min_score}'

プロンプトテストでpytestを使う

# test_sentiment_prompt.py
import pytest
import openai

client = openai.OpenAI(api_key='sk-...')
SYSTEM_PROMPT = 'Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Return only the label.'

def classify(text):
    resp = client.chat.completions.create(
        model='gpt-4o',
        messages=[
            {'role': 'system', 'content': SYSTEM_PROMPT},
            {'role': 'user', 'content': text}
        ],
        temperature=0
    )
    return resp.choices[0].message.content.strip().upper()

# pytest automatically discovers functions starting with test_
def test_positive_sentiment():
    assert classify('I love this product!') == 'POSITIVE'

def test_negative_sentiment():
    assert classify('Terrible experience.') == 'NEGATIVE'

def test_neutral_sentiment():
    assert classify('It arrived on time.') == 'NEUTRAL'

# Run: pytest test_sentiment_prompt.py -v

pytestでのパラメータ化テスト

# test_sentiment_parametrized.py
import pytest

TEST_CASES = [
    ('I love this!', 'POSITIVE'),
    ('Worst purchase ever.', 'NEGATIVE'),
    ('It works.', 'NEUTRAL'),
    ('Amazing!', 'POSITIVE'),
    ('Terrible!', 'NEGATIVE'),
    ('OK I guess.', 'NEUTRAL'),
]

@pytest.mark.parametrize('text,expected', TEST_CASES)
def test_sentiment_classification(text, expected):
    result = classify(text)
    assert result == expected, f'For "{text}": expected {expected}, got {result}'

# pytest test_sentiment_parametrized.py -v
# Output shows each test case individually:
# PASSED test_sentiment_parametrized.py::test_sentiment_classification[I love this!-POSITIVE]
# PASSED test_sentiment_parametrized.py::test_sentiment_classification[Worst purchase ever.-NEGATIVE]

プロンプトの共有状態に対するフィクスチャ

# conftest.py — fixtures available to all test files in the directory
import pytest
import openai

@pytest.fixture(scope='session')
def llm_client():
    return openai.OpenAI(api_key='sk-...')

@pytest.fixture(scope='session')
def sentiment_prompt():
    with open('prompts/sentiment_v3.txt') as f:
        return f.read()

# test_sentiment.py
def test_positive_with_fixture(llm_client, sentiment_prompt):
    resp = llm_client.chat.completions.create(
        model='gpt-4o',
        messages=[
            {'role': 'system', 'content': sentiment_prompt},
            {'role': 'user', 'content': 'I love this!'}
        ],
        temperature=0
    )
    assert 'POSITIVE' in resp.choices[0].message.content.upper()

不安定なテストへの対処

import pytest

def run_with_retry(fn, n=3):
    '''Run fn up to n times, pass if any run succeeds.'''
    failures = []
    for _ in range(n):
        try:
            fn()
            return  # passed
        except AssertionError as e:
            failures.append(str(e))
    raise AssertionError(f'Failed all {n} attempts. Last: {failures[-1]}')

def test_positive_with_retry():
    def check():
        result = classify('I love this!')
        assert result == 'POSITIVE'
    run_with_retry(check, n=3)

# Or use pytest-retry plugin:
# @pytest.mark.flaky(reruns=3)
# def test_positive_sentiment():
#     assert classify('I love this!') == 'POSITIVE'

テストのパフォーマンスとコスト

import hashlib, json

RESPONSE_CACHE = {}

def cached_classify(text, use_cache=True):
    key = hashlib.md5(text.encode()).hexdigest()
    if use_cache and key in RESPONSE_CACHE:
        return RESPONSE_CACHE[key]
    result = classify(text)
    RESPONSE_CACHE[key] = result
    return result

# Persist cache to disk for CI
def load_cache(path='test_cache.json'):
    global RESPONSE_CACHE
    try:
        with open(path) as f:
            RESPONSE_CACHE = json.load(f)
    except FileNotFoundError:
        RESPONSE_CACHE = {}

def save_cache(path='test_cache.json'):
    with open(path, 'w') as f:
        json.dump(RESPONSE_CACHE, f, indent=2)

テスト結果レポート

# Run test suite and generate reports
# In terminal:
# pytest tests/prompt/ -v --tb=short --junitxml=test_results.xml

# In Python (for programmatic use):
import subprocess

def run_prompt_tests(test_dir='tests/prompt'):
    result = subprocess.run(
        ['pytest', test_dir, '-v', '--tb=short', '--junitxml=test_results.xml'],
        capture_output=True, text=True
    )
    print(result.stdout)
    if result.returncode != 0:
        print('TESTS FAILED')
        print(result.stderr)
    return result.returncode == 0

passed = run_prompt_tests()

理解度チェック

まとめ: アサーションベースのプロンプトテスト