Oluşturma Ölçütleri: Sadakat ve Yanıt Uygunluğu

Measuring the Generation Stage

Faithfulness: Definition

# Faithfulness = supported_claims / total_claims

example_answer = (
    'Employees receive 15 vacation days per year. '
    'Remote work is allowed on Wednesdays and Fridays. '
    'The CEO is John Smith.'
)
example_context = (
    '15 vacation days are granted annually. '
    'Remote work is permitted on Wednesdays and Fridays.'
)

# Claim 1: 15 vacation days — SUPPORTED
# Claim 2: Remote work Wed+Fri — SUPPORTED
# Claim 3: CEO is John Smith — NOT IN CONTEXT (hallucinated)
# Faithfulness = 2/3 = 0.67

Implementing Faithfulness with LLM-as-Judge

import json

def evaluate_faithfulness(answer, context, llm_client):
    prompt = (
        'Task: Evaluate the faithfulness of an answer against a context.\n\n'
        f'Context:\n{context}\n\n'
        f'Answer:\n{answer}\n\n'
        'Steps:\n'
        '1. Break the answer into individual atomic claims.\n'
        '2. For each claim, check if it is supported by the context.\n'
        '3. Return JSON: {"claims": [{"text": "...", "supported": true/false}], "score": 0.0-1.0}'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o',
        response_format={'type': 'json_object'},
        messages=[{'role': 'user', 'content': prompt}]
    )
    return json.loads(response.choices[0].message.content)

Answer Relevance: Definition

Measuring Answer Relevance

def evaluate_answer_relevance(question, answer, llm_client, embed_fn):
    # Generate hypothetical questions for this answer
    prompt = (
        f'Given this answer: "{answer}"\n\n'
        'Generate 3 questions that this answer would be a good response to.\n'
        'Return as JSON: {"questions": ["...", "...", "..."]}'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o-mini',
        response_format={'type': 'json_object'},
        messages=[{'role': 'user', 'content': prompt}]
    )
    hyp_questions = json.loads(response.choices[0].message.content)['questions']

    # Measure similarity to original question
    orig_embedding = embed_fn(question)
    hyp_embeddings = [embed_fn(q) for q in hyp_questions]
    similarities = [cosine_similarity(orig_embedding, e) for e in hyp_embeddings]
    return sum(similarities) / len(similarities)

Context Recall: Did We Retrieve Enough?

def evaluate_context_recall(ground_truth_answer, retrieved_contexts, llm_client):
    prompt = (
        f'Ground truth answer:\n{ground_truth_answer}\n\n'
        f'Retrieved context:\n{",".join(retrieved_contexts)}\n\n'
        'For each sentence in the ground truth answer, determine if it '
        'can be attributed to the retrieved context.\n'
        'Return JSON: {"sentences": [{"text": "...", "in_context": true/false}], "recall": 0.0-1.0}'
    )
    response = llm_client.chat.completions.create(
        model='gpt-4o',
        response_format={'type': 'json_object'},
        messages=[{'role': 'user', 'content': prompt}]
    )
    return json.loads(response.choices[0].message.content)

Context Precision: Are Retrieved Chunks Relevant?

def evaluate_context_precision(question, answer, retrieved_contexts, llm_client):
    precision_scores = []
    for i, context in enumerate(retrieved_contexts, 1):
        prompt = (
            f'Question: {question}\n\n'
            f'Context chunk {i}: {context}\n\n'
            f'Answer: {answer}\n\n'
            'Was this context chunk useful in generating the answer? '
            'Return JSON: {"useful": true/false, "reason": "..."}'
        )
        response = llm_client.chat.completions.create(
            model='gpt-4o-mini',
            response_format={'type': 'json_object'},
            messages=[{'role': 'user', 'content': prompt}]
        )
        result = json.loads(response.choices[0].message.content)
        precision_scores.append(1.0 if result['useful'] else 0.0)
    return sum(precision_scores) / len(precision_scores)

Using RAGAS for All Metrics at Once

from ragas import evaluate
from ragas.metrics import (
    faithfulness, answer_relevancy,
    context_precision, context_recall
)
from datasets import Dataset

# Build evaluation dataset
eval_samples = [
    {
        'question': item['question'],
        'answer': item['generated_answer'],
        'contexts': item['retrieved_texts'],
        'ground_truth': item['expected_answer']
    }
    for item in test_results
]

eval_dataset = Dataset.from_list(eval_samples)
results = evaluate(eval_dataset,
    metrics=[faithfulness, answer_relevancy, context_precision, context_recall]
)
results.to_pandas().to_csv('eval_results.csv', index=False)

Diagnosing with Metric Combinations

# Diagnostic matrix
diagnostics = [
    {
        'condition': 'Low faithfulness + High context precision',
        'cause': 'LLM ignoring context, answering from parametric memory',
        'fix': 'Strengthen system prompt: "Answer ONLY from the provided context"'
    },
    {
        'condition': 'Low context recall + High faithfulness',
        'cause': 'Retriever missing relevant chunks',
        'fix': 'Improve chunking strategy, embedding model, or increase top-k'
    },
    {
        'condition': 'Low answer relevance + High context recall',
        'cause': 'Retrieved context is off-topic; LLM is answering wrong question',
        'fix': 'Add query rewriting or improve metadata filters'
    }
]

Human Evaluation for Calibration

from scipy.stats import spearmanr

def calibrate_judge_vs_humans(samples):
    '''samples: [{"llm_score": 0.9, "human_score": 4.2}, ...]'''
    llm_scores = [s['llm_score'] for s in samples]
    human_scores = [s['human_score'] / 5.0 for s in samples]  # normalize to 0-1

    correlation, pvalue = spearmanr(llm_scores, human_scores)
    print(f'LLM-Human Spearman correlation: {correlation:.3f} (p={pvalue:.4f})')

    mean_llm = sum(llm_scores) / len(llm_scores)
    mean_human = sum(human_scores) / len(human_scores)
    bias = mean_llm - mean_human
    print(f'LLM bias vs humans: {bias:+.3f}')
    return correlation, bias

Setting Metric Targets for Production

PRODUCTION_THRESHOLDS = {
    'faithfulness': 0.90,
    'answer_relevancy': 0.80,
    'context_precision': 0.60,
    'context_recall': 0.75
}

def check_production_readiness(metrics):
    ready = True
    print('Production readiness check:')
    for metric, threshold in PRODUCTION_THRESHOLDS.items():
        score = metrics.get(metric, 0)
        status = 'PASS' if score >= threshold else 'FAIL'
        print(f'  {metric}: {score:.2f} (>= {threshold}) -> {status}')
        if status == 'FAIL':
            ready = False
    print(f'Overall: {"READY" if ready else "NOT READY"}')
    return ready

Quick Check

Lesson Recap