实时跟踪日志与流式告警

实时跟踪日志为何重要

tail -f：跟踪实时日志

#!/usr/bin/env bash
# Simulate a live log and follow it
LOGFILE='/tmp/demo_app.log'

# Write a header
echo '[INFO] Application started' >> "$LOGFILE"

# In a real scenario you would run:
# tail -f "$LOGFILE"
# Below we demonstrate tail showing the last 3 lines then exit
tail -n 3 "$LOGFILE"

tail -F：应对日志轮换

# Follow nginx access log, surviving log rotation
tail -F /var/log/nginx/access.log

# Follow multiple files simultaneously
tail -F /var/log/nginx/access.log /var/log/nginx/error.log

使用 grep 筛选日志流

# Show only ERROR and WARN lines from a live application log
tail -F /var/log/myapp/app.log | grep --line-buffered -Ei 'error|warn|critical'

# Follow nginx and exclude health-check requests
tail -F /var/log/nginx/access.log | grep --line-buffered -v '/health'

使用 awk 添加时间戳和上下文

# Prepend a reception timestamp to every ERROR line
tail -F /var/log/myapp/app.log | \
  grep --line-buffered -i 'error' | \
  awk '{ print strftime("[%Y-%m-%d %H:%M:%S]"), $0; fflush() }'

# Extract HTTP status code (field 9) and URL (field 7) from nginx combined log
tail -F /var/log/nginx/access.log | \
  awk '{ print $9, $7; fflush() }' | \
  grep --line-buffered '^5'

使用 Slack Webhook 发送告警

#!/usr/bin/env bash
# Real-time alert: send every ERROR line to a Slack channel
LOGFILE='/var/log/myapp/app.log'
WEBHOOK_URL='https://hooks.slack.com/services/T000/B000/XXXX'

tail -F "$LOGFILE" | grep --line-buffered -i 'error' | while IFS= read -r line; do
  payload=$(printf '{"text":"*[ERROR ALERT]*\n%s"}' "$line")
  curl -s -o /dev/null -X POST -H 'Content-Type: application/json' \
    -d "$payload" "$WEBHOOK_URL"
done

限制告警频率，避免噪声

#!/usr/bin/env bash
# Alert on ERROR lines but no more than once every 60 seconds
LOGFILE='/var/log/myapp/app.log'
WEBHOOK_URL='https://hooks.slack.com/services/T000/B000/XXXX'
COOLDOWN=60
LAST_ALERT_FILE='/tmp/last_alert_ts'

tail -F "$LOGFILE" | grep --line-buffered -i 'error' | while IFS= read -r line; do
  now=$(date +%s)
  last=0
  [ -f "$LAST_ALERT_FILE" ] && last=$(cat "$LAST_ALERT_FILE")

  if (( now - last >= COOLDOWN )); then
    echo "$now" > "$LAST_ALERT_FILE"
    payload=$(printf '{"text":"*[ERROR ALERT]*\n%s"}' "$line")
    curl -s -o /dev/null -X POST -H 'Content-Type: application/json' \
      -d "$payload" "$WEBHOOK_URL"
    echo "[$(date)] Alert sent: $line"
  else
    echo "[$(date)] Suppressed (cooldown): $line"
  fi
done

使用滑动窗口统计错误突发

#!/usr/bin/env bash
# Alert when more than 10 errors occur within any 60-second window
LOGFILE='/var/log/myapp/app.log'
WINDOW=60
THRESHOLD=10
TS_FILE='/tmp/error_timestamps'
WEBHOOK_URL='https://hooks.slack.com/services/T000/B000/XXXX'

tail -F "$LOGFILE" | grep --line-buffered -i 'error' | while IFS= read -r line; do
  now=$(date +%s)
  echo "$now" >> "$TS_FILE"

  # Keep only timestamps within the window
  cutoff=$(( now - WINDOW ))
  tmp=$(mktemp)
  awk -v c="$cutoff" '$1 > c' "$TS_FILE" > "$tmp" && mv "$tmp" "$TS_FILE"

  count=$(wc -l < "$TS_FILE")
  if (( count > THRESHOLD )); then
    msg="*[BURST ALERT]* ${count} errors in ${WINDOW}s — last: ${line}"
    curl -s -o /dev/null -X POST -H 'Content-Type: application/json' \
      -d "{\"text\":\"$msg\"}" "$WEBHOOK_URL"
    # Clear to avoid re-alerting until next burst
    > "$TS_FILE"
  fi
done

journalctl -f：跟踪 systemd 日志

# Follow only error-and-above entries for nginx
journalctl -f -u nginx.service -p err

# Stream journal as JSON and extract MESSAGE field with jq
journalctl -f -u myapp.service -o json | \
  jq --unbuffered -r 'select(.PRIORITY <= "3") | .MESSAGE'

multitail 与彩色多源监控

# multitail: watch nginx access + error + app log in split panes
# (requires: apt install multitail  or  brew install multitail)
multitail /var/log/nginx/access.log /var/log/nginx/error.log /var/log/myapp/app.log

# Pure-Bash alternative — merge three streams with labeled prefixes
(
  tail -F /var/log/nginx/access.log | sed --unbuffered 's/^/[nginx-access] /' &
  tail -F /var/log/nginx/error.log  | sed --unbuffered 's/^/[nginx-error]  /' &
  tail -F /var/log/myapp/app.log    | sed --unbuffered 's/^/[myapp]        /' &
  wait
)

构建自包含的告警守护进程

#!/usr/bin/env bash
# log_alert_daemon.sh — tail a log and fire Slack alerts with cooldown
set -euo pipefail

LOGFILE=${1:-'/var/log/myapp/app.log'}
PATTERN=${2:-'error|critical|fatal'}
WEBHOOK_URL=${SLACK_WEBHOOK_URL:?'Set SLACK_WEBHOOK_URL env var'}
COOLDOWN=${ALERT_COOLDOWN:-120}
DAEMON_LOG='/var/log/log_alert_daemon.log'
LAST_SENT_FILE='/tmp/log_alert_last_sent'

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$DAEMON_LOG"; }

log "Starting alert daemon: watching $LOGFILE for pattern: $PATTERN"

tail -F "$LOGFILE" | grep --line-buffered -Ei "$PATTERN" | while IFS= read -r line; do
  now=$(date +%s)
  last=0
  [ -f "$LAST_SENT_FILE" ] && last=$(cat "$LAST_SENT_FILE")

  if (( now - last >= COOLDOWN )); then
    echo "$now" > "$LAST_SENT_FILE"
    msg=$(printf '{"text":"*[ALERT]* %s\n%s"}' "$(hostname)" "$line")
    if curl -s -o /dev/null -w '%{http_code}' -X POST \
         -H 'Content-Type: application/json' -d "$msg" "$WEBHOOK_URL" | grep -q '^200$'; then
      log "Alert sent: $line"
    else
      log "Alert FAILED to send: $line"
    fi
  else
    log "Suppressed (cooldown ${COOLDOWN}s): $line"
  fi
done

知识检查：流式日志流水线

课程回顾：实时日志跟踪与流式告警