スクリプトでjournalctlを使ったjournaldの検索

インシデントのトリアージにjournaldを使う理由

journalctlの基本的な呼び出し

#!/usr/bin/env bash
# Print the last 20 lines of the nginx service journal
journalctl --no-pager -u nginx.service -n 20

systemdユニットによるフィルタリング

#!/usr/bin/env bash
# Usage: ./unit_logs.sh nginx.service
UNIT="${1:?Usage: $0 <unit>}"

echo "=== Journal for ${UNIT} (last 50 lines) ==="
journalctl --no-pager -u "${UNIT}" -n 50

# Combine two related units
echo "=== Combined: api + worker ==="
journalctl --no-pager -u myapp-api.service -u myapp-worker.service -n 30

優先度レベルと -p フラグ

#!/usr/bin/env bash
# Extract only error-level and above entries for sshd
journalctl --no-pager \
  -u sshd.service \
  -p emerg..err \
  --since "1 hour ago"

# Exit non-zero if any errors were found (useful in CI health checks)
ERROR_COUNT=$(journalctl --no-pager -u sshd.service -p emerg..err \
  --since "1 hour ago" --output=cat | wc -l)

if [[ "${ERROR_COUNT}" -gt 0 ]]; then
  echo "[ALERT] ${ERROR_COUNT} error(s) detected in sshd" >&2
  exit 1
fi

--since と --until による時間範囲フィルタリング

#!/usr/bin/env bash
# Record deploy start time, then check logs afterwards
DEPLOY_START=$(date +"%Y-%m-%d %H:%M:%S")

echo "Deploying at ${DEPLOY_START}..."
# ... your deploy steps here ...
sleep 2  # simulate deploy

echo "=== Journal since deploy start ==="
journalctl --no-pager \
  -u myapp.service \
  --since "${DEPLOY_START}" \
  -p emerg..warning

JSON 形式による構造化出力

#!/usr/bin/env bash
# Extract error messages as a clean list for a Slack notification
MESSAGES=$(journalctl --no-pager \
  -u nginx.service \
  -p emerg..err \
  --since "30 minutes ago" \
  -o json \
  | jq -r '.MESSAGE' \
  | sort -u)

if [[ -n "${MESSAGES}" ]]; then
  echo "Errors detected:"
  echo "${MESSAGES}"
fi

ジャーナルのリアルタイム追跡

#!/usr/bin/env bash
# Cursor-based polling: read only new journal entries each run
CURSOR_FILE="/tmp/triage_cursor"

if [[ -f "${CURSOR_FILE}" ]]; then
  CURSOR=$(cat "${CURSOR_FILE}")
  NEW_ENTRIES=$(journalctl --no-pager \
    -u myapp.service \
    -p emerg..err \
    --after-cursor="${CURSOR}" \
    -o json)
else
  # First run: look back 5 minutes
  NEW_ENTRIES=$(journalctl --no-pager \
    -u myapp.service \
    -p emerg..err \
    --since "5 minutes ago" \
    -o json)
fi

# Save updated cursor for next poll
journalctl --no-pager -n 0 --show-cursor 2>&1 \
  | grep '^-- cursor:' \
  | sed 's/-- cursor: //' \
  > "${CURSOR_FILE}"

echo "${NEW_ENTRIES}" | jq -r '.MESSAGE // empty'

-b によるブート単位のクエリ

#!/usr/bin/env bash
# Post-mortem: collect critical logs from the previous boot
echo "=== Previous boot sessions ==="
journalctl --list-boots

echo ""
echo "=== Critical entries from previous boot ==="
journalctl --no-pager \
  -b -1 \
  -p emerg..crit \
  -o short-iso

journalctl 内での grep とネイティブマッチの比較

#!/usr/bin/env bash
# Native field match: errors from the postgres process only
journalctl --no-pager \
  _COMM=postgres \
  -p emerg..err \
  --since "1 hour ago"

# Full-text grep for a specific error string
journalctl --no-pager \
  -u postgresql.service \
  --since "1 hour ago" \
  -g "FATAL|PANIC" \
  --output=cat

再利用可能なトリアージ関数の作成

#!/usr/bin/env bash
# triage.sh — reusable journal triage function

triage_unit() {
  local unit="${1:?unit required}"
  local priority="${2:-emerg..err}"
  local since="${3:-1 hour ago}"
  local logfile="/tmp/triage_${unit//[^a-zA-Z0-9]/_}_$(date +%s).log"

  echo "[$(date -Iseconds)] Triaging ${unit} | prio=${priority} | since='${since}'" | tee "${logfile}"

  journalctl --no-pager \
    -u "${unit}" \
    -p "${priority}" \
    --since "${since}" \
    -o short-iso \
    | tee -a "${logfile}"

  local count
  count=$(wc -l < "${logfile}")
  # Subtract 1 for the header line
  (( count-- ))

  if [[ "${count}" -gt 0 ]]; then
    echo "[ALERT] ${count} line(s) logged to ${logfile}" >&2
    return 1
  fi
  return 0
}

# Example: triage nginx errors in the last 30 minutes
triage_unit nginx.service "emerg..err" "30 minutes ago"

インシデント自動トリアージスクリプト

#!/usr/bin/env bash
# incident_triage.sh — automated multi-service journal triage
set -euo pipefail

LOOKBACK="${1:-15 minutes ago}"
PRIORITY="emerg..err"
SERVICES=(nginx.service postgresql.service myapp-api.service myapp-worker.service)
REPORT="/tmp/incident_report_$(date +%Y%m%d_%H%M%S).txt"
FAILED=0

{
  echo "Incident Triage Report"
  echo "Generated : $(date -Iseconds)"
  echo "Lookback  : ${LOOKBACK}"
  echo "Priority  : ${PRIORITY}"
  echo "-----------------------------------"
} > "${REPORT}"

for svc in "${SERVICES[@]}"; do
  ENTRIES=$(journalctl --no-pager \
    -u "${svc}" \
    -p "${PRIORITY}" \
    --since "${LOOKBACK}" \
    --output=cat 2>/dev/null || true)

  COUNT=$(echo "${ENTRIES}" | grep -c . || true)

  if [[ "${COUNT}" -gt 0 ]]; then
    echo "[FAIL] ${svc}: ${COUNT} error(s)" | tee -a "${REPORT}"
    echo "${ENTRIES}" >> "${REPORT}"
    echo "-----------------------------------" >> "${REPORT}"
    FAILED=1
  else
    echo "[OK]   ${svc}"
  fi
done

echo ""
echo "Full report: ${REPORT}"
exit "${FAILED}"

理解度チェック: 優先度範囲のフィルタリング

レッスンのまとめ: スクリプトでの journalctl