Latenz und Kosten pro Schritt

Warum Metriken pro Schritt?

Latenz erfassen

start = time.time()
# ... do work ...
span.duration_ms = (time.time() - start) * 1000

Latenzkategorien

Kosten erfassen

PRICES = {
    'gpt-4o-mini':       {'in': 0.150 / 1e6, 'out': 0.600 / 1e6},
    'gpt-4o':            {'in': 2.50  / 1e6, 'out': 10.00 / 1e6},
    'claude-sonnet-4-5': {'in': 3.00  / 1e6, 'out': 15.00 / 1e6}
}

def compute_cost(model, tokens_in, tokens_out):
    p = PRICES[model]
    return tokens_in * p['in'] + tokens_out * p['out']

cost = compute_cost('gpt-4o-mini', 1000, 500)
print(f"Cost for 1000 in / 500 out tokens on gpt-4o-mini: ${cost:.6f}")

Pro Trace aggregieren

def trace_metrics(spans):
    return {
        'duration_ms': sum(s.duration_ms for s in spans),
        'cost_usd':    sum(s.cost_usd or 0 for s in spans),
        'tokens':      sum((s.tokens_in or 0) + (s.tokens_out or 0) for s in spans),
        'step_count':  len(spans)
    }

from types import SimpleNamespace
spans = [
    SimpleNamespace(duration_ms=120, cost_usd=0.002, tokens_in=100, tokens_out=50),
    SimpleNamespace(duration_ms=340, cost_usd=0.005, tokens_in=200, tokens_out=80),
]
print(trace_metrics(spans))

p50 / p95 / p99

import numpy as np
durations = [t.duration_ms for t in last_1000_traces]
print('p50:', np.percentile(durations, 50))
print('p95:', np.percentile(durations, 95))
print('p99:', np.percentile(durations, 99))

Kosten pro Benutzer

SELECT user_id, SUM(cost_usd) AS daily_cost
FROM traces
WHERE created_at::date = current_date
GROUP BY user_id
ORDER BY daily_cost DESC
LIMIT 100

Kosten pro Schrittnamen

SELECT name, SUM(cost_usd) AS total
FROM spans
WHERE trace_id IN (SELECT id FROM traces WHERE created_at::date = current_date)
GROUP BY name
ORDER BY total DESC

Latenz-Wasserfälle

Auf Ausreißer hinweisen

Zwei Modelle vergleichen

# Run 100 traces with gpt-4o-mini and 100 with haiku
# Compare p95 latency and average cost
# Pick the winner for the use case
print("Run 100 traces with gpt-4o-mini and 100 with haiku")
print("Compare p95 latency and average cost")
print("Pick the winner for the use case")

Tokenverteilung verfolgen

Warum Perzentile?

Zusammenfassung