컨텍스트 압축

def compress(query, chunks):
    # Goal: keep only spans that bear on the query,
    # dropping boilerplate, navigation, and off-topic sentences.
    return [extract_relevant(query, c) for c in chunks]

def extractive(query, chunk):
    sents = split_sentences(chunk.text)
    scored = [(s, relevance(query, s)) for s in sents]
    kept = [s for s, r in scored if r > TAU]
    return ' '.join(kept) or top1(scored)   # never return empty

def sentence_filter(query, chunk, keep_ratio=0.4):
    sents = split_sentences(chunk.text)
    scores = embed_sim_batch(query, sents)
    n_keep = max(1, int(len(sents) * keep_ratio))
    idx = sorted(range(len(sents)), key=lambda i: -scores[i])[:n_keep]
    return ' '.join(sents[i] for i in sorted(idx))  # keep original order

def llm_compress(query, chunk):
    prompt = (
        'Extract ONLY sentences from the passage relevant to the query. '
        'If none are relevant, output NONE. Do not paraphrase.\n'
        'Query: ' + query + '\nPassage: ' + chunk.text
    )
    out = llm(prompt, temperature=0).strip()
    return None if out == 'NONE' else out

def token_prune(context, target_ratio=0.3):
    # small LM estimates per-token information (perplexity-based);
    # drop the least informative tokens down to target_ratio length
    scores = small_lm_token_importance(context)
    return keep_top_fraction(context, scores, target_ratio)

# Offline: precompute a dense summary per chunk (query-agnostic)
chunk.summary = summarize(chunk.text)
# Online: query-aware trim over summaries (cheap) then verify on full
ctx = [sentence_filter(query, Chunk(c.summary)) for c in top_chunks]

def safe_compress(query, chunk):
    out = llm_compress(query, chunk)
    if out is None and chunk.rerank_score > 0.7:
        return chunk.text          # trust re-ranker over compressor
    return out or chunk.text

def compress_with_ids(query, chunks):
    out = []
    for c in chunks:
        span = safe_compress(query, c)
        out.append({'id': c.id, 'doc': c.meta['doc_id'], 'text': span})
    return out   # generator cites id; provenance preserved

def pack(spans, budget_tokens, tok):
    spans = sorted(spans, key=lambda s: -s['score'])
    used, packed = 0, []
    for s in spans:
        t = tok(s['text'])
        if used + t > budget_tokens:
            continue
        packed.append(s); used += t
    return packed

def eval_compression(eval_set, levels):
    return {
        lvl: {
            'ratio': mean_ratio(eval_set, lvl),
            'acc':   answer_accuracy(eval_set, lvl),
            'faith': faithfulness(eval_set, lvl),
        } for lvl in levels
    }

def pipeline(query):
    top = rerank(query, hybrid_retrieve(query, 50))[:12]
    spans = compress_with_ids(query, top)
    spans = [s for s in spans if s['text']]
    packed = pack(spans, budget_tokens=2000, tok=count_tokens)
    return generate_with_citations(query, packed)