Building System Health Check and Alert Scripts

Why System Health Checks Matter

Capturing Load Average

#!/usr/bin/env bash
# Read 1-minute load average from /proc/loadavg
LOAD_RAW=$(cut -d' ' -f1 /proc/loadavg)
echo "Raw load average: $LOAD_RAW"

# Number of CPU cores — used to normalise load
CPU_CORES=$(nproc)
echo "CPU cores: $CPU_CORES"

# Compute load percentage (load / cores * 100) using bc
LOAD_PCT=$(echo "scale=2; $LOAD_RAW / $CPU_CORES * 100" | bc)
echo "Load %: $LOAD_PCT"

Threshold Comparison with Floating-Point Values

#!/usr/bin/env bash
LOAD_RAW=$(cut -d' ' -f1 /proc/loadavg)
CPU_CORES=$(nproc)
THRESHOLD=80  # alert when load % exceeds 80%

LOAD_PCT=$(echo "scale=2; $LOAD_RAW / $CPU_CORES * 100" | bc)

# bc returns 1 if the expression is true
if [ "$(echo "$LOAD_PCT > $THRESHOLD" | bc)" -eq 1 ]; then
    echo "ALERT: Load is ${LOAD_PCT}% (threshold ${THRESHOLD}%)"
else
    echo "OK: Load is ${LOAD_PCT}%"
fi

Collecting Memory Metrics

#!/usr/bin/env bash
# Extract memory figures from /proc/meminfo (values in kB)
MEM_TOTAL=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
MEM_AVAIL=$(awk '/^MemAvailable:/ {print $2}' /proc/meminfo)

# Used memory percentage
MEM_USED_PCT=$(echo "scale=2; (1 - $MEM_AVAIL / $MEM_TOTAL) * 100" | bc)

echo "Total RAM : ${MEM_TOTAL} kB"
echo "Available : ${MEM_AVAIL} kB"
echo "Used      : ${MEM_USED_PCT}%"

Collecting Disk Usage Metrics

#!/usr/bin/env bash
DISK_THRESHOLD=85

# Skip header line with tail -n +2
# --output=pcent,target gives "85% /var" style lines
df --output=pcent,target | tail -n +2 | while read -r USED_PCT MOUNT; do
    # Remove the % sign for arithmetic
    USED_INT=${USED_PCT//%/}

    if [ "$USED_INT" -ge "$DISK_THRESHOLD" ]; then
        echo "ALERT: Disk $MOUNT is ${USED_PCT} full"
    else
        echo "OK   : Disk $MOUNT is ${USED_PCT} full"
    fi
done

Structured Alert Output with Timestamps

#!/usr/bin/env bash
LOG_FILE="/var/log/healthcheck.log"

alert() {
    local LEVEL="$1"   # OK | WARN | ALERT
    local CHECK="$2"
    local MSG="$3"
    local TS
    TS=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
    local LINE="[$TS] [$LEVEL] [$CHECK] $MSG"

    if [ "$LEVEL" = "ALERT" ]; then
        echo "$LINE" | tee -a "$LOG_FILE" >&2
    else
        echo "$LINE" | tee -a "$LOG_FILE"
    fi
}

# Usage examples
alert "OK"    "DISK"  "/ is 42% full"
alert "ALERT" "DISK"  "/var is 91% full"

Sending Email Alerts with mail and sendmail

#!/usr/bin/env bash
ALERT_EMAIL="ops@example.com"
LOCK_DIR="/tmp/healthcheck_locks"
mkdir -p "$LOCK_DIR"

send_alert() {
    local CHECK="$1"
    local MSG="$2"
    local LOCK="$LOCK_DIR/${CHECK}.lock"

    # Only send if no lock exists (prevents repeated emails within the hour)
    if [ ! -f "$LOCK" ]; then
        echo "$MSG" | mail -s "[ALERT] $CHECK on $(hostname)" "$ALERT_EMAIL"
        touch "$LOCK"
        # Lock expires after 1 hour via cron or find+delete
        echo "Alert sent for $CHECK"
    else
        echo "Alert suppressed for $CHECK (lock active)"
    fi
}

send_alert "HIGH_LOAD" "Load average exceeded 80% on $(hostname) at $(date)"

Composing the Full Health Check Script

#!/usr/bin/env bash
set -euo pipefail

# ── Thresholds ───────────────────────────────────────────
LOAD_THRESHOLD=80   # percent of CPU capacity
MEM_THRESHOLD=90    # percent used
DISK_THRESHOLD=85   # percent used
ALERT_EMAIL="ops@example.com"
LOG_FILE="/var/log/healthcheck.log"
ALERT_FIRED=0

# ── Helpers ──────────────────────────────────────────────
ts()    { date -u +"%Y-%m-%dT%H:%M:%SZ"; }
log()   { echo "[$(ts)] $*" | tee -a "$LOG_FILE"; }
alert() { log "ALERT: $*"; echo "$*" | mail -s "[ALERT] $(hostname)" "$ALERT_EMAIL" 2>/dev/null; ALERT_FIRED=1; }

# ── Checks ───────────────────────────────────────────────
check_load() {
    local raw cores pct
    raw=$(cut -d' ' -f1 /proc/loadavg)
    cores=$(nproc)
    pct=$(echo "scale=2; $raw / $cores * 100" | bc)
    if [ "$(echo "$pct > $LOAD_THRESHOLD" | bc)" -eq 1 ]; then
        alert "Load ${pct}% exceeds ${LOAD_THRESHOLD}%"
    else
        log "OK load=${pct}%"
    fi
}

check_memory() {
    local total avail pct
    total=$(awk '/^MemTotal:/    {print $2}' /proc/meminfo)
    avail=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
    pct=$(echo "scale=2; (1 - $avail / $total) * 100" | bc)
    if [ "$(echo "$pct > $MEM_THRESHOLD" | bc)" -eq 1 ]; then
        alert "Memory ${pct}% used (threshold ${MEM_THRESHOLD}%)"
    else
        log "OK memory=${pct}%"
    fi
}

check_disk() {
    df --output=pcent,target | tail -n +2 | while read -r used mnt; do
        local pct_int=${used//%/}
        if [ "$pct_int" -ge "$DISK_THRESHOLD" ]; then
            alert "Disk $mnt at ${used}"
        else
            log "OK disk $mnt=${used}"
        fi
    done
}

main() {
    log "=== Health check START ==="
    check_load
    check_memory
    check_disk
    log "=== Health check END (alerts=$ALERT_FIRED) ==="
    exit "$ALERT_FIRED"
}

main

Scheduling with Cron

# /etc/cron.d/healthcheck
# Run the health check every 5 minutes as root
MAILTO=""
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin

*/5 * * * * root /usr/local/sbin/healthcheck.sh >> /var/log/healthcheck.log 2>&1

Preventing Alert Storms with Cooldown Locks

#!/usr/bin/env bash
LOCK_DIR="/tmp/hc_locks"
COOLDOWN_MIN=60  # suppress repeat alerts for 60 minutes
mkdir -p "$LOCK_DIR"

should_alert() {
    local check="$1"
    local lock="$LOCK_DIR/${check}.lock"

    if [ ! -f "$lock" ]; then
        # No lock — allow alert and create lock
        touch "$lock"
        return 0  # true: send alert
    fi

    # Lock exists — check if it is older than the cooldown
    # find returns the filename only if it's OLDER than COOLDOWN_MIN
    local expired
    expired=$(find "$lock" -mmin +"$COOLDOWN_MIN" 2>/dev/null)

    if [ -n "$expired" ]; then
        touch "$lock"  # refresh lock timestamp
        return 0       # cooldown expired — allow alert
    fi

    return 1  # still within cooldown — suppress
}

# Usage
if should_alert "HIGH_LOAD"; then
    echo "Sending load alert..."
    # mail -s "..." ops@example.com <<< "Load too high"
else
    echo "Load alert suppressed (cooldown active)"
fi

Testing and Validating Your Health Check Script

#!/usr/bin/env bash
# Smoke-test the alert path without sending real email
MOCK_MAIL=true
ALERT_EMAIL="ops@example.com"

send_mail() {
    local subject="$1"
    local body="$2"
    if [ "$MOCK_MAIL" = true ]; then
        echo "[MOCK MAIL] To: $ALERT_EMAIL | Subject: $subject"
        echo "[MOCK MAIL] Body: $body"
    else
        echo "$body" | mail -s "$subject" "$ALERT_EMAIL"
    fi
}

# Override threshold to guarantee an alert fires
LOAD_THRESHOLD=0   # Any load will exceed 0%
LOAD_RAW=$(cut -d' ' -f1 /proc/loadavg)
CPU_CORES=$(nproc)
PCT=$(echo "scale=2; $LOAD_RAW / $CPU_CORES * 100" | bc)

if [ "$(echo "$PCT > $LOAD_THRESHOLD" | bc)" -eq 1 ]; then
    send_mail "[ALERT] Load on $(hostname)" "Load is ${PCT}%"
fi

Knowledge Check: Cooldown Strategy

Lesson Recap: System Health Check Scripts