Context Length and Relevance

The Context Window Budget

Context Window Sizes

import tiktoken

def estimate_tokens(text, model='gpt-4o'):
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))

# Quick token budget calculator
models = {
    'GPT-3.5 Turbo':  16385,
    'GPT-4o':        128000,
    'Claude Opus 4.5': 200000,
}

prompt = 'Explain the concept of technical debt in 500 words for a non-technical CEO.'
prompt_tokens = estimate_tokens(prompt)

for model_name, limit in models.items():
    reserved_for_output = 1024
    available = limit - prompt_tokens - reserved_for_output
    print(f'{model_name}: limit={limit:,} | prompt={prompt_tokens} | '
          f'context budget={available:,} tokens')

What to Include: Relevance Scoring

def score_context_element(element, task):
    '''
    Heuristic: does this context element directly constrain or shape the answer?
    Returns: HIGH / MEDIUM / LOW
    '''
    high_signals = ['stack', 'constraint', 'deadline', 'must', 'cannot', 'budget',
                    'audience', 'goal', 'version', 'scale', 'limit']
    low_signals  = ['founded', 'headquartered', 'fun fact', 'history', 'awards',
                    'team building', 'company culture', 'office location']

    el_lower = element.lower()
    if any(s in el_lower for s in high_signals):
        return 'HIGH'
    if any(s in el_lower for s in low_signals):
        return 'LOW'
    return 'MEDIUM'

context_elements = [
    'Our stack is Python FastAPI and PostgreSQL',
    'We cannot use any paid third-party APIs',
    'Our company was founded in Berlin in 2020',
    'We need the solution to handle 1000 requests/second',
    'We won a startup award last year',
]

for el in context_elements:
    score = score_context_element(el, task='optimize our API')
    print(f'[{score:6}] {el}')

The Lost-in-the-Middle Problem

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-your-key-here')

# Structure: critical constraint at the TOP, then the body, then the task
well_structured_prompt = (
    # Critical constraint FIRST
    'CRITICAL CONSTRAINT: Output must be under 50 words and contain no code.\n\n'
    # Background in the middle
    'Background: we are explaining our API rate limiting policy to non-technical support agents. '
    'They handle billing inquiries and need to explain errors to customers. '
    'Our rate limit is 100 requests per minute per API key.\n\n'
    # Task at the end
    'Task: Write the explanation.'
)

response = client.messages.create(
    model='claude-opus-4-5',
    max_tokens=128,
    messages=[{'role': 'user', 'content': well_structured_prompt}]
)
print(response.content[0].text)

Chunking Long Documents

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-your-key-here')

def chunk_and_summarize(long_text, chunk_size=2000):
    '''Split text into chunks, summarize each, combine summaries.'''
    words = long_text.split()
    chunks = []
    for i in range(0, len(words), chunk_size):
        chunk = ' '.join(words[i:i + chunk_size])
        chunks.append(chunk)

    summaries = []
    for idx, chunk in enumerate(chunks):
        response = client.messages.create(
            model='claude-opus-4-5',
            max_tokens=256,
            messages=[{
                'role': 'user',
                'content': f'Summarize this section in 3 bullet points:\n\n{chunk}'
            }]
        )
        summaries.append(f'Section {idx+1}:\n{response.content[0].text}')

    return '\n\n'.join(summaries)

# Example usage
long_doc = 'word ' * 5000  # placeholder for a real document
print('Chunks needed:', len(long_doc.split()) // 2000 + 1)

Relevance Filtering in Practice

import openai

client = openai.OpenAI(api_key='sk-your-key-here')

# Step 1: Filter first, then ask
full_schema = (
    'Table: users (id, name, email, created_at, role)\n'
    'Table: products (id, name, price, stock, category_id)\n'
    'Table: orders (id, user_id, total, status, created_at)\n'
    'Table: order_items (id, order_id, product_id, quantity, unit_price)\n'
    'Table: categories (id, name, parent_id)\n'
    'Table: reviews (id, product_id, user_id, rating, body)\n'
    'Table: sessions (id, user_id, token, expires_at)'
)

# Only include relevant tables for the specific question
relevant_context = (
    'Relevant tables for this query:\n'
    'Table: orders (id, user_id, total, status, created_at)\n'
    'Table: order_items (id, order_id, product_id, quantity, unit_price)\n'
)

response = client.chat.completions.create(
    model='gpt-4o',
    messages=[{
        'role': 'user',
        'content': f'{relevant_context}\nWrite SQL to find the top 5 orders by total value this month.'
    }]
)
print(response.choices[0].message.content)

Managing Conversation History

import openai

client = openai.OpenAI(api_key='sk-your-key-here')

def summarize_history(old_history):
    '''Compress old conversation turns into a brief summary.'''
    history_text = '\n'.join(
        f'{m["role"].upper()}: {m["content"]}' for m in old_history
    )
    response = client.chat.completions.create(
        model='gpt-4o',
        max_tokens=200,
        messages=[{
            'role': 'user',
            'content': (
                'Summarize this conversation history in 3 bullet points. '
                'Focus on decisions made and key information established.\n\n'
                + history_text
            )
        }]
    )
    return response.choices[0].message.content

# Example: compressing old history before continuing
old_turns = [
    {'role': 'user',      'content': 'We are building a Kanban app.'},
    {'role': 'assistant', 'content': 'Great, what is your stack?'},
    {'role': 'user',      'content': 'React + FastAPI + PostgreSQL.'},
    {'role': 'assistant', 'content': 'Good choice for a Kanban app.'}
]
summary = summarize_history(old_turns)
print('Summary of old history:', summary)

Context Compression Techniques

import tiktoken

def count_tokens(text):
    enc = tiktoken.encoding_for_model('gpt-4o')
    return len(enc.encode(text))

# Same information, different token counts
prose_context = (
    'Our company is a startup that was founded recently and we are building '
    'a data analytics platform. It is worth noting that we use Python for our backend. '
    'Additionally, we have chosen PostgreSQL as our primary database. '
    'Furthermore, we deploy on AWS using ECS containers.'
)

bullet_context = (
    'Company: data analytics startup\n'
    'Stack: Python backend, PostgreSQL, AWS ECS'
)

print('Prose context tokens: ', count_tokens(prose_context))
print('Bullet context tokens:', count_tokens(bullet_context))
print('Tokens saved:', count_tokens(prose_context) - count_tokens(bullet_context))
print('Same information? Yes — same facts, 60% fewer tokens')

Dynamic Context Selection

# Simplified RAG pattern: retrieve relevant chunks, inject into prompt
import openai

client = openai.OpenAI(api_key='sk-your-key-here')

# Simulated knowledge base (in production: vector database)
knowledge_base = [
    {'id': 1, 'topic': 'billing',   'text': 'Refunds are processed within 5-7 business days.'},
    {'id': 2, 'topic': 'shipping',  'text': 'Standard shipping takes 3-5 days.'},
    {'id': 3, 'topic': 'returns',   'text': 'Returns accepted within 30 days with receipt.'},
    {'id': 4, 'topic': 'warranty',  'text': 'All products come with a 1-year warranty.'},
]

def get_relevant_docs(user_query, kb, top_k=2):
    '''Simplified relevance: keyword match. Production uses embeddings.'''
    scored = [(doc, sum(w in user_query.lower() for w in doc['topic'].split())) for doc in kb]
    scored.sort(key=lambda x: x[1], reverse=True)
    return [doc['text'] for doc, _ in scored[:top_k]]

query = 'Can I return this and get my money back?'
relevant = get_relevant_docs(query, knowledge_base)
context = '\n'.join(relevant)
print('Injected context:', context)
response = client.chat.completions.create(
    model='gpt-4o', max_tokens=80,
    messages=[{'role': 'user', 'content': f'Context:\n{context}\n\nQuestion: {query}'}]
)
print('Answer:', response.choices[0].message.content.strip())

Context Budget Planning

# Context budget planner
MODEL_LIMIT = 128000  # GPT-4o

BUDGET = {
    'output_reserve':  int(MODEL_LIMIT * 0.25),  # 32,000 tokens
    'system_message':  int(MODEL_LIMIT * 0.05),  # 6,400 tokens
    'recent_history':  int(MODEL_LIMIT * 0.30),  # 38,400 tokens
    'dynamic_context': int(MODEL_LIMIT * 0.35),  # 44,800 tokens
    'task_prompt':     int(MODEL_LIMIT * 0.05),  # 6,400 tokens
}

total_input = sum(v for k, v in BUDGET.items() if k != 'output_reserve')
print('Context budget plan:')
for key, tokens in BUDGET.items():
    pct = round(tokens / MODEL_LIMIT * 100)
    print(f'  {key:<20}: {tokens:>7,} tokens ({pct}%)')
print(f'  {"total input":<20}: {total_input:>7,} tokens')
print(f'  {"+ output reserve":<20}: {BUDGET["output_reserve"]:>7,} tokens')
print(f'  {"= model limit":<20}: {MODEL_LIMIT:>7,} tokens')

When Relevance Beats Length

import anthropic

client = anthropic.Anthropic(api_key='sk-ant-your-key-here')

# Demonstrating: lean context produces sharper output
lean_context = (
    'Task: write a 50-word product tagline.\n'
    'Product: CLI tool that auto-generates Git commit messages from your diff.\n'
    'Audience: senior developers who hate writing commit messages.\n'
    'Tone: dry, witty, technical.'
)

response = client.messages.create(
    model='claude-opus-4-5',
    max_tokens=100,
    messages=[{'role': 'user', 'content': lean_context}]
)
print('Lean context output:')
print(response.content[0].text.strip())

Knowledge Check

Context Length and Relevance — Recap