Вычисление показателей и гистограмм из потоков журналов

Зачем вычислять метрики по необработанным журналам?

Структура обычного журнала доступа

127.0.0.1 - frank [10/Oct/2024:13:55:36 -0700] "GET /api/users HTTP/1.1" 200 2326 0.042

Подсчёт частоты запросов в минуту

#!/usr/bin/env bash
# Requests per minute from an nginx/apache access log
# Usage: bash req_per_min.sh access.log

LOG="${1:-access.log}"

awk '{
    # Extract [day/Mon/year:HH:MM from field 4
    match($0, /\[([^:]+:[0-9]+:[0-9]+)/, arr)
    minute = arr[1]
    if (minute != "") count[minute]++
} END {
    for (m in count) print count[m], m
}' "$LOG" | sort -k2

Частота по скользящему окну в потоке реального времени

#!/usr/bin/env bash
# Live request rate — prints lines/10s from a tailed log
# Usage: bash live_rate.sh /var/log/nginx/access.log

LOG="${1:-/var/log/nginx/access.log}"
WINDOW=10

tail -f "$LOG" | awk -v win="$WINDOW" '
BEGIN { start = systime(); count = 0 }
{
    count++
    now = systime()
    if (now - start >= win) {
        printf "[%s] %d req/%ds (%.1f req/s)\n",
               strftime("%H:%M:%S", now), count, win, count/win
        count = 0
        start = now
    }
}'

Извлечение значений задержки для вычисления процентилей

#!/usr/bin/env bash
# Extract latency column from access log (field 11)
# Assumes last field on each line is response time in seconds

LOG="${1:-access.log}"
TMP=$(mktemp /tmp/latency_XXXXXX.txt)

awk '{ if ($NF ~ /^[0-9]+\.?[0-9]*$/) print $NF }' "$LOG" \
    | sort -n > "$TMP"

echo "Extracted $(wc -l < "$TMP") latency samples -> $TMP"
echo "$TMP"   # callers can read this file

Вычисление p50, p95 и p99 с помощью awk

#!/usr/bin/env bash
# Compute p50 / p95 / p99 from a sorted latency file
# Usage: bash percentiles.sh latency_sorted.txt

# Demo: generate 1000 random latencies if no file given
if [[ $# -eq 0 ]]; then
    SORTED=$(mktemp)
    for i in $(seq 1 1000); do
        awk 'BEGIN { srand(); printf "%.4f\n", 0.001 + rand()*0.999 }'
    done | sort -n > "$SORTED"
else
    SORTED="$1"
fi

awk '
{ values[NR] = $1 }
END {
    n = NR
    if (n == 0) { print "No data"; exit }
    p50  = values[int(n * 0.50 + 0.9999)]
    p95  = values[int(n * 0.95 + 0.9999)]
    p99  = values[int(n * 0.99 + 0.9999)]
    printf "p50  = %.4fs\np95  = %.4fs\np99  = %.4fs\nN    = %d\n",
           p50, p95, p99, n
}' "$SORTED"

Создание отчёта о конечных точках top-N

#!/usr/bin/env bash
# Top-10 most requested URL paths from access log
# Usage: bash top_endpoints.sh access.log [N]

LOG="${1:-access.log}"
N="${2:-10}"

echo "=== Top $N endpoints ==="
awk -F'"' '{ print $2 }' "$LOG" \
    | awk '{ print $2 }' \
    | sort \
    | uniq -c \
    | sort -rn \
    | head -n "$N" \
    | awk '{ printf "%6d  %s\n", $1, $2 }'

Top-N по коду состояния: быстрый поиск ошибок

#!/usr/bin/env bash
# Status code frequency breakdown
# Usage: bash status_breakdown.sh access.log

LOG="${1:-access.log}"

echo "=== HTTP Status Code Distribution ==="
awk '{ print $9 }' "$LOG" \
    | grep -E '^[0-9]{3}$' \
    | sort \
    | uniq -c \
    | sort -rn \
    | awk '{ printf "%6d  %s\n", $1, $2 }'

echo
echo "=== 5xx Error Spike Check ==="
awk '$9 ~ /^5/ { print $9, $7 }' "$LOG" \
    | sort | uniq -c | sort -rn | head -20

Гистограммы ASCII с помощью awk

#!/usr/bin/env bash
# ASCII latency histogram from a list of latency values in seconds
# Demo: generates synthetic data if no input file is provided

if [[ $# -eq 0 ]]; then
    # Generate 500 synthetic latencies (ms converted to s)
    python3 -c "
import random, math
for _ in range(500):
    # bimodal: fast cluster ~50ms, slow cluster ~300ms
    if random.random() < 0.75:
        v = max(1, random.gauss(50, 15))
    else:
        v = max(1, random.gauss(300, 80))
    print(f'{v/1000:.4f}')
" | sort -n | awk '
{ values[NR] = $1 * 1000 }  # convert to ms
END {
    buckets = 10; maxVal = 500
    step = maxVal / buckets
    for (i = 0; i < buckets; i++) hist[i] = 0
    for (i = 1; i <= NR; i++) {
        b = int(values[i] / step)
        if (b >= buckets) b = buckets - 1
        hist[b]++
    }
    maxCount = 0
    for (i = 0; i < buckets; i++) if (hist[i] > maxCount) maxCount = hist[i]
    print "Latency (ms)   Count   Distribution"
    print "---------------------------------------"
    for (i = 0; i < buckets; i++) {
        lo = i * step; hi = lo + step
        barLen = int(hist[i] / maxCount * 40)
        bar = ""
        for (j = 0; j < barLen; j++) bar = bar "#"
        printf "%4d-%4dms  %5d  %s\n", lo, hi, hist[i], bar
    }
}'
else
    echo "Usage: pipe a sorted latency file (in seconds) to this pattern"
fi

Объединение метрик: сводный отчёт за один запуск

#!/usr/bin/env bash
# One-shot log summary report
# Usage: bash log_summary.sh access.log

LOG="${1:-access.log}"
[[ -f "$LOG" ]] || { echo "File not found: $LOG"; exit 1; }

TOTAL=$(wc -l < "$LOG")
ERRORS=$(awk '$9 ~ /^[45]/' "$LOG" | wc -l)
ERR_RATE=$(awk "BEGIN { printf \"%.1f\", ($ERRORS / ($TOTAL || 1)) * 100 }")

echo "====================================="
echo " Log Summary: $LOG"
echo "====================================="
printf " Total requests : %d\n" "$TOTAL"
printf " 4xx/5xx errors : %d (%.1f%%)\n" "$ERRORS" "$ERR_RATE"

echo
echo "--- Top 5 Endpoints ---"
awk -F'"' '{ print $2 }' "$LOG" | awk '{ print $2 }' \
    | sort | uniq -c | sort -rn | head -5 \
    | awk '{ printf "  %6d  %s\n", $1, $2 }'

echo
echo "--- Latency Percentiles ---"
awk '{ if ($NF ~ /^[0-9]+\.?[0-9]*$/) print $NF * 1000 }' "$LOG" \
    | sort -n \
    | awk '
{ v[NR]=$1 }
END {
  if (NR==0) { print "  No latency data"; exit }
  printf "  p50 = %.1fms\n", v[int(NR*0.50+0.9999)]
  printf "  p95 = %.1fms\n", v[int(NR*0.95+0.9999)]
  printf "  p99 = %.1fms\n", v[int(NR*0.99+0.9999)]
}'

Потоковая передача метрик через именованные каналы и tee

#!/usr/bin/env bash
# Fan-out: write to archive AND count errors in real time
# Run: bash fanout_pipeline.sh /var/log/nginx/access.log

LOG="${1:-/var/log/nginx/access.log}"
ARCHIVE="/tmp/access_archive.log"
PIPE="/tmp/log_metrics_pipe"

# Clean up on exit
trap 'rm -f "$PIPE"' EXIT

mkfifo "$PIPE"

# Branch 1: count 5xx errors per minute from the pipe
awk '$9 ~ /^5/ {
    match($0, /\[([^:]+:[0-9]+:[0-9]+)/, arr)
    errors[arr[1]]++
} END {
    for (m in errors) printf "5xx errors at %s: %d\n", m, errors[m]
}' "$PIPE" &

# Branch 2: archive to disk + feed Branch 1 via pipe
tail -f "$LOG" | tee "$PIPE" >> "$ARCHIVE"

Какая техника awk правильно вычисляет 95-й процентиль из отсортированного массива из N значений задержки?

Итоги урока: метрики и гистограммы из потоков журналов