不完全なデータと欠損データへの対処

不完全なドキュメントの実態

Pydanticでのオプションフィールド

from pydantic import BaseModel, Field
from typing import Optional

class JobPosting(BaseModel):
    title: str
    company: str
    salary_min: Optional[float] = Field(None, description='Minimum salary if stated')
    salary_max: Optional[float] = Field(None, description='Maximum salary if stated')
    remote: Optional[bool] = Field(None, description='True if remote, False if on-site, None if unspecified')

信頼度スコアの追加

from pydantic import BaseModel
from typing import Optional

class Confident(BaseModel):
    value: Optional[str]
    confidence: float  # 0.0 to 1.0

class ContractExtract(BaseModel):
    party_a: Confident
    party_b: Confident
    effective_date: Confident
    termination_clause: Confident

センチネル値とNoneの比較

from pydantic import BaseModel
from typing import Optional, Literal

class Discount(BaseModel):
    # 'none' = explicitly no discount; None = not mentioned
    discount_type: Optional[Literal['percentage', 'fixed', 'none']] = None
    discount_value: Optional[float] = None

フォールバック抽出戦略

def fill_missing(partial: JobPosting, raw_text: str) -> JobPosting:
    missing = [k for k, v in partial.model_dump().items() if v is None]
    if not missing:
        return partial
    prompt = f'From this text, extract ONLY these fields: {missing}.\n\n{raw_text}'
    supplement = client.chat.completions.create(
        model='gpt-4o-mini',
        response_model=JobPosting,
        messages=[{'role': 'user', 'content': prompt}]
    )
    merged = partial.model_dump()
    for field in missing:
        if getattr(supplement, field) is not None:
            merged[field] = getattr(supplement, field)
    return JobPosting(**merged)

デフォルト値とファクトリの利用

from pydantic import BaseModel, Field
from typing import List, Optional

class Article(BaseModel):
    title: str
    author: Optional[str] = None
    tags: List[str] = Field(default_factory=list)
    word_count: Optional[int] = None
    published_date: Optional[str] = None

信頼度の低い抽出結果の振り分け

CONFIDENCE_THRESHOLD = 0.75

def process_extraction(result: ContractExtract, doc_id: str):
    needs_review = any(
        field.confidence < CONFIDENCE_THRESHOLD
        for field in [result.party_a, result.party_b, result.effective_date]
    )
    if needs_review:
        queue_for_human_review(doc_id, result)
    else:
        store_in_production_table(doc_id, result)

曖昧なテキストスパンの処理

from pydantic import BaseModel
from typing import Optional

class DateField(BaseModel):
    raw_span: Optional[str] = None    # exact text from document
    iso_date: Optional[str] = None    # normalized YYYY-MM-DD
    confidence: float = 1.0

class Contract(BaseModel):
    effective_date: DateField
    expiration_date: DateField

欠損フィールドのパターンのログ記録

from collections import Counter

missing_counter = Counter()

def log_missing(result):
    for field, value in result.model_dump().items():
        if value is None:
            missing_counter[field] += 1

# After processing 1000 documents:
for field, count in missing_counter.most_common(5):
    print(f'{field}: {count} missing ({100*count//1000}%)')

複数パスにまたがる抽出結果の統合

def multi_pass_extract(pages: list) -> Invoice:
    # Pass 1: header info from first page
    header = extract_header(pages[0])
    # Pass 2: line items from middle pages
    items = []
    for page in pages[1:-1]:
        items.extend(extract_line_items(page))
    # Pass 3: totals from last page
    totals = extract_totals(pages[-1])
    return Invoice(
        vendor=header.vendor,
        invoice_number=header.invoice_number,
        line_items=items,
        total_amount=totals.total_amount
    )

スキーマ設計のベストプラクティス

from pydantic import BaseModel, Field

class Address(BaseModel):
    street: str = Field(description='Street number and name, e.g. 123 Main St')
    city: str = Field(description='City name only, no state')
    state: str = Field(description='Two-letter US state code, e.g. CA')
    zip_code: str = Field(description='5-digit ZIP code, e.g. 94105')
    country: str = Field(default='US', description='ISO 3166-1 alpha-2 country code')

理解度チェック

レッスンのまとめ