RAG에서 평가가 중요한 이유

# Diagnosis example: which stage is failing?

# Test 1: Is retrieval finding the right chunks?
retrieval_hit_rate = evaluate_retrieval(questions, ground_truth_chunks)
print(f'Retrieval hit rate@5: {retrieval_hit_rate:.1%}')
# If this is low (< 80%), fix chunking and embedding first

# Test 2: Given perfect context, does LLM generate correct answers?
generation_faithfulness = evaluate_generation(questions, perfect_context)
print(f'Generation faithfulness: {generation_faithfulness:.1%}')
# If retrieval is fine but this is low, fix your prompts

# Golden dataset format
golden_dataset = [
    {
        'question': 'How many vacation days do employees receive in their first year?',
        'answer': '15 days',
        'relevant_chunks': ['employee_handbook_p24', 'benefits_summary_p3'],
        'source_doc': 'employee_handbook_2025.pdf'
    },
    {
        'question': 'What is the parental leave duration for primary caregivers?',
        'answer': '16 weeks fully paid',
        'relevant_chunks': ['parental_leave_policy_p1'],
        'source_doc': 'parental_leave_policy.pdf'
    }
]

def generate_qa_pairs_for_chunk(chunk_text, llm_client):
    prompt = (
        'Given the following document excerpt, generate 3 diverse questions '
        'that can be answered using ONLY this text. '
        'For each question, provide the exact answer from the text.\n\n'
        f'Text:\n{chunk_text}\n\n'
        'Format each as JSON: {"question": ..., "answer": ...}'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o-mini',
        messages=[{'role': 'user', 'content': prompt}]
    )
    return response.choices[0].message.content

def compute_hit_rate(golden_dataset, retriever, top_k=5):
    hits = 0
    for item in golden_dataset:
        results = retriever.retrieve(item['question'], top_k=top_k)
        retrieved_ids = {r['id'] for r in results}
        relevant_ids = set(item['relevant_chunks'])
        if retrieved_ids & relevant_ids:  # intersection not empty
            hits += 1
    hit_rate = hits / len(golden_dataset)
    print(f'Hit rate@{top_k}: {hit_rate:.1%} ({hits}/{len(golden_dataset)})')
    return hit_rate

def evaluate_faithfulness(answer, context, llm_client):
    prompt = (
        'Given this context and answer, evaluate faithfulness.\n\n'
        f'Context: {context}\n\n'
        f'Answer: {answer}\n\n'
        'For each sentence in the answer, determine if it is '
        'supported by the context (FAITHFUL) or not (HALLUCINATED). '
        'Return a JSON: {"score": 0.0-1.0, "issues": ["sentence..."]}.'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o',
        messages=[{'role': 'user', 'content': prompt}]
    )
    return response.choices[0].message.content

def evaluate_answer_relevance(question, answer, llm_client):
    prompt = (
        f'Question: {question}\n\n'
        f'Answer: {answer}\n\n'
        'Rate how well this answer addresses the question on a 1-5 scale:\n'
        '5 = fully answers the question\n'
        '3 = partially answers but misses key aspects\n'
        '1 = does not address the question at all\n\n'
        'Return JSON: {"score": 1-5, "reason": "brief explanation"}'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o',
        messages=[{'role': 'user', 'content': prompt}]
    )
    return response.choices[0].message.content

import json
from datetime import datetime

def save_evaluation_results(metrics, config, output_file='eval_history.jsonl'):
    record = {
        'timestamp': datetime.utcnow().isoformat(),
        'config': config,
        'metrics': metrics
    }
    with open(output_file, 'a') as f:
        f.write(json.dumps(record) + '\n')
    print(f'Saved eval result: hit_rate={metrics["hit_rate"]:.1%}, '
          f'faithfulness={metrics["faithfulness"]:.1%}')

import json
from sklearn.model_selection import train_test_split

def split_golden_dataset(all_questions, test_ratio=0.3, seed=42):
    dev_set, test_set = train_test_split(
        all_questions,
        test_size=test_ratio,
        random_state=seed
    )
    print(f'Dev set: {len(dev_set)} questions')
    print(f'Test set: {len(test_set)} questions (FROZEN)')
    with open('eval/dev_set.json', 'w') as f:
        json.dump(dev_set, f, indent=2)
    with open('eval/test_set.json', 'w') as f:
        json.dump(test_set, f, indent=2)
    return dev_set, test_set