Strukturiertes Logging

Was ist strukturiertes Logging?

import logging, json

def log_json(level, msg, **kw):
    record = {"level": level, "msg": msg, **kw}
    print(json.dumps(record))

log_json("INFO", "Request received", path="/api/users", method="GET")

Benutzerdefinierter JSON-Formatter

import logging, json

class JSONFormatter(logging.Formatter):
    def format(self, record):
        return json.dumps({
            "time": self.formatTime(record),
            "level": record.levelname,
            "name": record.name,
            "msg": record.getMessage(),
        })

handler = logging.StreamHandler()
handler.setFormatter(JSONFormatter())
logging.getLogger().addHandler(handler)

python-json-logger

# pip install python-json-logger
import logging
from pythonjsonlogger import jsonlogger

logger = logging.getLogger()
handler = logging.StreamHandler()
handler.setFormatter(jsonlogger.JsonFormatter())
logger.addHandler(handler)
logger.info("User created", extra={"user_id": 42})

structlog

# pip install structlog
import structlog

log = structlog.get_logger()
log = log.bind(request_id="abc123")
log.info("payment_processed", amount=49.99, currency="USD")

Kontext mit structlog binden

import structlog

log = structlog.get_logger().bind(
    service="auth",
    env="prod"
)
log.warning("login_failed", user="alice", reason="bad_password")
# {service:auth, env:prod, user:alice, reason:bad_password}

Zeitstempel und Aufruferinformationen hinzufügen

import structlog

structlog.configure(
    processors=[
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.CallsiteParameterAdder([structlog.processors.CallsiteParameter.FILENAME]),
        structlog.processors.JSONRenderer(),
    ]
)

Request-IDs in Web-Apps protokollieren

import structlog

def middleware(request, call_next):
    req_id = request.headers.get("X-Request-Id", generate_id())
    log = structlog.get_logger().bind(request_id=req_id)
    log.info("request_started", path=request.url.path)
    response = call_next(request)
    log.info("request_finished", status=response.status_code)
    return response

ECS-Log-Format

# pip install ecs-logging
import logging
import ecs_logging

logger = logging.getLogger("app")
handler = logging.StreamHandler()
handler.setFormatter(ecs_logging.StdlibFormatter())
logger.addHandler(handler)
logger.info("Event logged in ECS format")

Systeme zur Log-Aggregation

# Loki query example:
# {app="myservice"} | json | level="ERROR"

# CloudWatch Insights:
# fields @timestamp, level, msg
# | filter level = "ERROR"
# | sort @timestamp desc

Keine personenbezogenen Daten in Logs

import re

def mask_pii(_, __, event_dict):
    msg = event_dict.get("event", "")
    event_dict["event"] = re.sub(
        r"[\w.+-]+@[\w-]+\.[\w.]+", "***@***.***", msg
    )
    return event_dict

Vor- und Nachteile strukturierter und unstrukturierter Logs

import structlog, os

if os.getenv("ENV") == "production":
    renderer = structlog.processors.JSONRenderer()
else:
    renderer = structlog.dev.ConsoleRenderer()

structlog.configure(processors=[renderer])

Kurztest

Zusammenfassung