Validating Tool Outputs (Pydantic)

Trust But Verify

Pydantic Models

from pydantic import BaseModel, HttpUrl

class SearchResult(BaseModel):
    title: str
    url: HttpUrl
    snippet: str
    score: float = 0.0

Validate at the Boundary

from pydantic import ValidationError

def search(query):
    raw = requests.post('https://api.tavily.com/search', json={'query': query}).json()
    results = []
    for r in raw.get('results', []):
        try:
            results.append(SearchResult.model_validate(r))
        except ValidationError as e:
            log.warning('Bad search result skipped: %s', e)
    return results

Lists with Model Validation

from pydantic import BaseModel
class SearchResponse(BaseModel):
    results: list[SearchResult]

response = SearchResponse.model_validate(raw)

Field Validators

from pydantic import field_validator

class SearchResult(BaseModel):
    score: float

    @field_validator('score')
    @classmethod
    def score_in_range(cls, v):
        if not 0 <= v <= 1:
            raise ValueError('score out of [0,1]')
        return v

Strict vs Lenient

from pydantic import StrictInt

class User(BaseModel):
    age: StrictInt   # rejects '25' (string)

Default Values for Missing Fields

class Tool(BaseModel):
    name: str
    description: str = 'No description provided'   # fallback

Validate LLM Structured Output

Validate Tool Arguments

class SearchArgs(BaseModel):
    query: str
    k: int = 5

args = SearchArgs.model_validate_json(tool_call.function.arguments)
results = search(args.query, args.k)

Validation Errors as Tool Results

try:
    args = SearchArgs.model_validate_json(...)
except ValidationError as e:
    return {'error': f'Invalid arguments: {e}'}

Schema Versioning

class SearchResultV2(BaseModel):
    title: str
    url: HttpUrl
    relevance: float       # was 'score' in v1

Generate JSON Schema From Pydantic

schema = SearchArgs.model_json_schema()
# Use directly as the 'parameters' for an OpenAI tool definition.

Validation Goal

Recap