평가, 배포, 회고

마지막 단계: 출시 전 평가

전체 평가 도구 모음 실행하기

async def final_evaluation(system_url: str, test_set_path: str) -> dict:
    test_cases = load_test_set(test_set_path)
    results = []
    for case in test_cases:
        start = time.perf_counter()
        response = await query_system(system_url, case['question'])
        latency_ms = (time.perf_counter() - start) * 1000
        judge_score = await judge(case['question'], response['answer'], case.get('reference'))
        hit = any(case['relevant_doc'] in s for s in response.get('sources', []))
        results.append({'latency_ms': latency_ms, 'score': judge_score, 'hit': hit, 'cost': response.get('cost_usd', 0)})
    return compute_final_metrics(results)

운영 시스템 부하 테스트하기

# Locust load test (locustfile.py)
from locust import HttpUser, task, between
import random

QUESTIONS = [
    'What is the return policy?',
    'How do I cancel my subscription?',
    'Where are you located?',
]

class AIUser(HttpUser):
    wait_time = between(1, 3)  # realistic think time

    @task
    def query(self):
        self.client.post(
            '/query',
            json={'question': random.choice(QUESTIONS), 'tenant_id': 'load_test'},
            headers={'Authorization': 'Bearer test_token'}
        )

# Run: locust -f locustfile.py --headless -u 50 -r 5 --run-time 5m

운영 환경에 배포하기

# Deployment steps (pseudocode for AWS ECS or K8s):
# 1. Build and push new Docker image
#    docker build -t qa-assistant:v2.0 . && docker push ...
#
# 2. Deploy green (new) version alongside blue (current)
#    kubectl apply -f deploy/green.yaml
#
# 3. Run smoke tests against green
#    pytest tests/smoke/ --base-url https://green.internal
#
# 4. Canary traffic shift (ALB weighted routing)
#    5%  -> green (monitor 10min)
#    25% -> green (monitor 10min)
#    50% -> green (monitor 10min)
#    100% -> green
#
# 5. Decommission blue after 24h stability

배포 후 모니터링

# Post-deploy monitoring dashboard queries:
# (Assuming Grafana + Prometheus)

# Error rate (last 5 min):
# rate(http_requests_total{status=~'5..'}[5m]) / rate(http_requests_total[5m])

# p95 latency (last 5 min):
# histogram_quantile(0.95, rate(query_duration_seconds_bucket[5m]))

# Cache hit rate:
# rate(cache_hits_total[5m]) / rate(queries_total[5m])

# Average cost per query:
# rate(llm_cost_usd_total[5m]) / rate(queries_total[5m])

아키텍처 회고 작성하기

# RETROSPECTIVE.md structure:
#
# ## What Worked Well
# - Hybrid retrieval improved hit rate from 71% to 89%
# - Semantic cache reduced average cost by 31%
# - LangSmith tracing saved 2 days of debugging
#
# ## What Did Not Work
# - Semantic chunking was 4x slower than recursive chunking
#   with only 3% hit rate improvement -- not worth it
# - Cohere reranker had 400ms latency -- too slow for p95 target
#   Switched to BGE-reranker-v2 running locally
#
# ## What We'd Do Differently
# - Start with pgvector, not Pinecone (migration cost 3 days)
# - Add semantic cache BEFORE building the agent, not after

운영 대응 절차서 문서화하기

# runbooks/latency.md
# ## Alert: p95 Latency > 8000ms
#
# ### Likely Causes
# 1. OpenAI API degraded (check status.openai.com)
# 2. Cohere reranker slow (check Cohere status page)
# 3. pgvector query slow (check DB CPU in CloudWatch)
# 4. Redis cache full (check Redis memory usage)
#
# ### Diagnosis
# curl https://api.openai.com/v1/models -H 'Authorization: Bearer $KEY'
# Check LangSmith traces for which step is slow
# SELECT mean(duration) FROM traces GROUP BY step
#
# ### Fixes
# - If OpenAI slow: circuit breaker should auto-failover to Claude
# - If Cohere slow: disable reranking temporarily (env SKIP_RERANK=1)
# - If DB slow: increase pgvector ef_search from 40 to 20

비즈니스 영향 측정하기

# Business impact metrics (month 1):
# Technical:
# - 12,847 queries served, 11,439 (89%) answered without human
# - Avg query cost: $0.031 (within $0.05 budget)
# - System uptime: 99.94%
#
# Business:
# - Support tickets: 1,840/month -> 1,203/month (-35%)
# - Avg resolution time: 4.2h -> 23 seconds for AI-answered
# - User CSAT on AI answers: 4.1/5.0
# - Cost per resolved query: $12 (human) -> $0.031 (AI)
# - ROI: 157% in month 1 at current volumes

다음 개선 주기 계획

# Next iteration priorities (based on first month data):
NEXT_SPRINT = [
    # High impact / high confidence
    {
        'feature': 'Sentence-window retrieval',
        'expected_impact': 'Hit rate 89% -> 93%',
        'effort': 'medium',
        'evidence': '11% of failures due to answer split across chunks'
    },
    # High impact / medium confidence
    {
        'feature': 'Query decomposition for multi-hop questions',
        'expected_impact': 'Multi-hop correctness 61% -> 78%',
        'effort': 'high',
        'evidence': '23% of failures are multi-hop questions'
    },
    # Low effort quick win
    {
        'feature': 'Extend cache TTL from 1h to 24h',
        'expected_impact': 'Cache hit rate 31% -> 38%',
        'effort': 'trivial',
        'evidence': 'Same questions asked daily by different users'
    }
]

지식 공유 및 문서화

# INTERNALS.md — key non-obvious decisions:
#
# ## Chunk Size: 800 tokens with 100 token overlap
# We tested 400, 600, 800, 1200 tokens.
# 800 tokens maximizes hit rate (89%) while keeping context
# small enough for 5 chunks to fit comfortably in 4096 token prompt.
# Larger chunks improved recall but degraded precision.
#
# ## Cache Similarity Threshold: 0.92
# Tested 0.85, 0.90, 0.92, 0.95.
# 0.92 gives 31% hit rate with <2% incorrect cache hits.
# 0.85 gives 41% hit rate but 8% incorrect hits (too aggressive).
#
# ## Reranker Top-N: 3 (from initial 10)
# More than 3 chunks causes context stuffing without quality gain.

구축한 내용

# System capabilities summary:
SYSTEM_CAPABILITIES = {
    'retrieval': 'Hybrid BM25+dense with Cohere reranking, 89% hit rate',
    'generation': 'GPT-4o with Claude fallback, circuit breaker, streaming',
    'caching': 'Semantic cache (Redis+embeddings), 31% cache hit rate',
    'security': 'Injection filter (2-stage) + output scanning + tenant isolation',
    'observability': 'LangSmith traces + Prometheus metrics + PagerDuty alerts',
    'evaluation': 'Automated LLM-as-judge in CI, daily full suite, LangSmith evals',
    'reliability': '99.94% uptime, circuit breakers, graceful degradation ladder',
    'cost': '$0.031/query average, model routing saves 67% vs GPT-4o only',
}

빠른 확인

수업 요약