awk配列とグループ化による集計

awkの連想配列とは

グループごとの行数を数える

count[$1]++

#!/usr/bin/env bash
# Count how many log entries exist per HTTP status code
# Input format: <ip> <date> <method> <path> <status> <bytes>
printf '10.0.0.1 2024-01-01 GET /index 200 512
10.0.0.2 2024-01-01 POST /api 404 128
10.0.0.3 2024-01-01 GET /img 200 2048
10.0.0.4 2024-01-01 GET /api 500 64
10.0.0.5 2024-01-01 DELETE /api 404 32
' | awk '
{
    count[$5]++
}
END {
    for (status in count)
        print status, count[status]
}'

グループごとに数値フィールドを合計する

#!/usr/bin/env bash
# Sum bytes transferred per user from an access log
printf 'alice 1024
bob 512
alice 2048
charlie 256
bob 768
alice 128
' | awk '
{
    bytes[$1] += $2
}
END {
    for (user in bytes)
        printf "%-10s %d bytes\n", user, bytes[user]
}'

1回の走査で件数と合計を組み合わせる

#!/usr/bin/env bash
# Compute per-department headcount and average salary
printf 'Engineering Alice 95000
Engineering Bob 88000
Marketing Carol 72000
Marketing Dave 68000
Engineering Eve 102000
Marketing Frank 75000
' | awk '
{
    dept  = $1
    sal   = $3
    count[dept]++
    total[dept] += sal
}
END {
    printf "%-15s %5s %10s\n", "Department", "Count", "Avg Salary"
    for (d in count)
        printf "%-15s %5d %10.0f\n", d, count[d], total[d]/count[d]
}'

2次元グループ化のための複数フィールドキー

#!/usr/bin/env bash
# Count requests grouped by HTTP method AND status code
printf 'GET 200
POST 201
GET 404
GET 200
DELETE 204
POST 201
GET 404
POST 500
' | awk '
{
    key = $1 "|" $2
    count[key]++
}
END {
    printf "%-8s %-6s %s\n", "Method", "Status", "Count"
    for (k in count) {
        split(k, parts, "|")
        printf "%-8s %-6s %d\n", parts[1], parts[2], count[k]
    }
}'

グループごとの最小値と最大値を追跡する

#!/usr/bin/env bash
# Find min and max response time per endpoint
printf '/api/users 120
/api/orders 340
/api/users 98
/api/orders 512
/api/users 210
/api/orders 280
/health 5
/health 7
' | awk '
{
    ep  = $1
    rt  = $2
    if (!(ep in mn)) { mn[ep] = rt; mx[ep] = rt }
    if (rt < mn[ep]) mn[ep] = rt
    if (rt > mx[ep]) mx[ep] = rt
}
END {
    printf "%-15s %6s %6s\n", "Endpoint", "Min", "Max"
    for (ep in mn)
        printf "%-15s %6d %6d\n", ep, mn[ep], mx[ep]
}'

度数分布 — 上位Nグループ

#!/usr/bin/env bash
# Top 3 most active IP addresses from an access log
printf '192.168.1.10 GET /index
10.0.0.5 POST /api
192.168.1.10 GET /css
172.16.0.3 GET /index
10.0.0.5 GET /api
192.168.1.10 DELETE /api
172.16.0.3 POST /login
10.0.0.5 GET /index
10.0.0.5 POST /logout
' | awk '{count[$1]++} END {for (ip in count) print count[ip], ip}' \
  | sort -rn \
  | head -n 3

複数ファイルの集計にNRとFNRを使う

awk配列からソート済みの出力を表示する

#!/usr/bin/env bash
# Bytes per user, sorted alphabetically by username
printf 'zara 800
alice 1500
bob 300
alice 200
zara 400
bob 1100
' | awk '
{
    total[$1] += $2
}
END {
    for (u in total)
        print u, total[u]
}' | sort -k1,1

配列キーを削除して状態をクリアする

#!/usr/bin/env bash
# Sum values within each section delimited by "---"
printf 'alice 100
bob 200
---
alice 50
bob 75
charlie 300
---
' | awk '
/^---/ {
    print "-- Section totals --"
    for (u in total) printf "  %-10s %d\n", u, total[u]
    delete total
    next
}
{
    total[$1] += $2
}'

実践的なパイプライン：Nginxログの要約

#!/usr/bin/env bash
# Simulated Nginx combined log: host ip - - [date] "METHOD /path HTTP/1.1" status bytes
printf 'api.example.com 10.0.0.1 - - [01/Jan/2024] "GET /v1" 200 1024
www.example.com 10.0.0.2 - - [01/Jan/2024] "GET /" 200 4096
api.example.com 10.0.0.3 - - [01/Jan/2024] "POST /v1" 500 128
www.example.com 10.0.0.4 - - [01/Jan/2024] "GET /img" 404 64
api.example.com 10.0.0.5 - - [01/Jan/2024] "GET /v1" 200 2048
www.example.com 10.0.0.6 - - [01/Jan/2024] "GET /" 200 4096
api.example.com 10.0.0.7 - - [01/Jan/2024] "DELETE /v1" 403 32
' | awk '
{
    host = $1; status = $9; b = $10
    reqs[host]++
    bytes[host] += b
    if (status + 0 >= 400) errors[host]++
}
END {
    printf "%-20s %6s %10s %6s\n", "Host", "Reqs", "Bytes", "Errors"
    for (h in reqs)
        printf "%-20s %6d %10d %6d\n", h, reqs[h], bytes[h], errors[h]+0
}'

理解度チェック：最小値を正しく初期化する

レッスンのまとめ：awk配列による集計