290 lines
13 KiB
Bash
Executable File
290 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# server-health-check.sh — Argento health monitoring
|
|
# Checks: SMART disk health, ZFS pool status, disk space
|
|
# Outputs plain text to stdout by default, or sends an HTML email with --email.
|
|
# Use --error-only to suppress output when everything is healthy.
|
|
#
|
|
# Usage:
|
|
# ./server-health-check.sh # full report to stdout
|
|
# ./server-health-check.sh --error-only # only print errors (silent if healthy)
|
|
# ./server-health-check.sh --email # full report to email
|
|
# ./server-health-check.sh --error-only --email # email errors (no email if healthy)
|
|
#
|
|
# Install:
|
|
# chmod +x /opt/server-health-check.sh
|
|
# crontab -e:
|
|
# 0 6 * * * /opt/server-health-check.sh --error-only --email # daily alert
|
|
# 0 8 * * 1 /opt/server-health-check.sh --email # weekly report
|
|
#
|
|
# Dependencies: smartmontools, zfs, msmtp (as sendmail)
|
|
|
|
set -uo pipefail
|
|
|
|
MAILTO="timothykim@fastmail.fm"
|
|
HOSTNAME=$(hostname)
|
|
SPACE_THRESHOLD=85 # percent used — alert above this
|
|
|
|
ERROR_ONLY=false
|
|
USE_EMAIL=false
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--error-only) ERROR_ONLY=true ;;
|
|
--email) USE_EMAIL=true ;;
|
|
*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
PROBLEMS=""
|
|
STATUS=""
|
|
|
|
# HTML table rows for the email body, built alongside the plain-text STATUS.
|
|
# Styles are inlined on every element because most mail clients strip
|
|
# <style> blocks. nowrap keeps device paths and sizes from wrapping.
|
|
TH='style="padding:2px 10px;text-align:left;border-bottom:1px solid #ccc;white-space:nowrap"'
|
|
TD='style="padding:2px 10px;text-align:left;white-space:nowrap"'
|
|
TDR='style="padding:2px 10px;text-align:right;white-space:nowrap"'
|
|
TABLE='style="border-collapse:collapse;margin:4px 0 16px"'
|
|
HTML_SATA=""
|
|
HTML_NVME=""
|
|
HTML_ZFS=""
|
|
HTML_SPACE=""
|
|
|
|
# Map a usage percentage to a hex color: green at 0%, amber at 50%,
|
|
# red at 100%, linearly interpolated per RGB channel. Endpoints are the
|
|
# same dark shades used by state_html so they stay readable on white.
|
|
# Done in bash because email clients can't compute colors in CSS.
|
|
usage_color() {
|
|
local p=$1 r g b
|
|
[ "$p" -ge 0 ] 2>/dev/null || { printf '#222222'; return; }
|
|
[ "$p" -gt 100 ] && p=100
|
|
if [ "$p" -le 50 ]; then
|
|
# 1a7f37 (green) -> 9a6700 (amber)
|
|
r=$(( 26 + (154 - 26) * p / 50 ))
|
|
g=$(( 127 + (103 - 127) * p / 50 ))
|
|
b=$(( 55 + (0 - 55) * p / 50 ))
|
|
else
|
|
# 9a6700 (amber) -> cf222e (red)
|
|
r=$(( 154 + (207 - 154) * (p - 50) / 50 ))
|
|
g=$(( 103 + (34 - 103) * (p - 50) / 50 ))
|
|
b=$(( 0 + (46 - 0) * (p - 50) / 50 ))
|
|
fi
|
|
printf '#%02x%02x%02x' "$r" "$g" "$b"
|
|
}
|
|
|
|
# Green for healthy, red for anything else. UNKNOWN is amber because it
|
|
# means smartctl couldn't read the device, not that the device failed.
|
|
state_html() {
|
|
case "$1" in
|
|
PASSED|ONLINE) printf '<span style="color:#1a7f37;font-weight:bold">%s</span>' "$1" ;;
|
|
UNKNOWN) printf '<span style="color:#9a6700;font-weight:bold">%s</span>' "$1" ;;
|
|
*) printf '<span style="color:#cf222e;font-weight:bold">%s</span>' "$1" ;;
|
|
esac
|
|
}
|
|
|
|
# ─── SMART Disk Health (SATA/SAS) ───────────────────────────────
|
|
STATUS+="SMART (SATA):\n"
|
|
for disk in /dev/sd?; do
|
|
[ -b "$disk" ] || continue
|
|
|
|
# smartctl returns non-zero exit codes as a bitmask for disk
|
|
# conditions (standby, threshold exceeded, etc.), so don't
|
|
# treat non-zero as a script error — capture output and check it.
|
|
health=$(smartctl -H "$disk" 2>/dev/null | grep "test result" | awk -F': ' '{print $2}') || true
|
|
health="${health:-UNKNOWN}"
|
|
temp=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 194 || $1 == 190 {print $10; exit}') || true
|
|
temp="${temp:-?}"
|
|
hours=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 9 {print $10; exit}') || true
|
|
# Some Seagates report attribute 9 raw as "11170h+42m+51.807s" —
|
|
# keep only the whole hours.
|
|
hours=${hours%%[^0-9]*}
|
|
hours="${hours:-?}"
|
|
model=$(smartctl -i "$disk" 2>/dev/null | awk -F': +' '/Device Model|Model Number/{print $2; exit}') || true
|
|
model="${model:-unknown}"
|
|
|
|
STATUS+=" $disk ($model): $health, ${temp}C, ${hours}h\n"
|
|
HTML_SATA+="<tr><td $TD>$disk</td><td $TD>$model</td><td $TD>$(state_html "$health")</td><td $TDR>${temp}C</td><td $TDR>${hours}h</td></tr>"$'\n'
|
|
|
|
if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
|
|
PROBLEMS+="[SMART] $disk: overall health check FAILED ($health)\n"
|
|
fi
|
|
|
|
# Check critical SMART attributes
|
|
# 5 = Reallocated_Sector_Ct
|
|
# 187 = Reported_Uncorrect
|
|
# 197 = Current_Pending_Sector
|
|
# 198 = Offline_Uncorrectable
|
|
while IFS= read -r line; do
|
|
id=$(echo "$line" | awk '{print $1}')
|
|
name=$(echo "$line" | awk '{print $2}')
|
|
raw=$(echo "$line" | awk '{print $10}')
|
|
case "$id" in
|
|
5|187|197|198)
|
|
if [ "$raw" -gt 0 ] 2>/dev/null; then
|
|
PROBLEMS+="[SMART] $disk: $name = $raw (should be 0)\n"
|
|
fi
|
|
;;
|
|
esac
|
|
done < <(smartctl -A "$disk" 2>/dev/null | awk 'NR>7 && /^[[:space:]]*[0-9]/' || true)
|
|
done
|
|
|
|
# ─── SMART Disk Health (NVMe) ───────────────────────────────────
|
|
STATUS+="\nSMART (NVMe):\n"
|
|
for disk in /dev/nvme[0-9]*; do
|
|
# Match only controller devices (nvme0, nvme1), not namespaces (nvme0n1)
|
|
[[ "$disk" =~ ^/dev/nvme[0-9]+$ ]] || continue
|
|
[ -c "$disk" ] || continue
|
|
|
|
ns="${disk}n1" # namespace device for smartctl
|
|
[ -b "$ns" ] || continue
|
|
|
|
health=$(smartctl -H "$ns" 2>/dev/null | grep "SMART overall-health" | awk -F': ' '{print $2}') || true
|
|
health="${health:-UNKNOWN}"
|
|
model=$(smartctl -i "$ns" 2>/dev/null | awk -F': +' '/Model Number/{print $2; exit}') || true
|
|
model="${model:-unknown}"
|
|
# smartctl prints "38 Celsius" — keep just the number to match the SATA column
|
|
temp=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Temperature:/{print $2; exit}') || true
|
|
temp=${temp%% *}
|
|
temp="${temp:-?}"
|
|
# strip thousands separator ("9,852")
|
|
hours=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Power On Hours/{print $2; exit}' | tr -d ',') || true
|
|
hours="${hours:-?}"
|
|
# NVMe percentage used — 100% means full rated write endurance consumed
|
|
pct_used=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Percentage Used/{print $2; exit}' | tr -d '%') || true
|
|
|
|
STATUS+=" $ns ($model): $health, ${temp}C, ${hours}h\n"
|
|
HTML_NVME+="<tr><td $TD>$ns</td><td $TD>$model</td><td $TD>$(state_html "$health")</td><td $TDR>${temp}C</td><td $TDR>${hours}h</td></tr>"$'\n'
|
|
|
|
if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
|
|
PROBLEMS+="[SMART] $ns: overall health check FAILED ($health)\n"
|
|
fi
|
|
|
|
if [ -n "$pct_used" ] && [ "$pct_used" -gt 90 ] 2>/dev/null; then
|
|
PROBLEMS+="[SMART] $ns: NVMe endurance ${pct_used}% used (>90%)\n"
|
|
fi
|
|
done
|
|
|
|
# ─── ZFS Pool Health ─────────────────────────────────────────────
|
|
STATUS+="\nZFS Pools:\n"
|
|
while IFS= read -r pool; do
|
|
[ -n "$pool" ] || continue
|
|
state=$(zpool list -H -o health "$pool" 2>/dev/null) || true
|
|
state="${state:-UNKNOWN}"
|
|
pool_info=$(zpool list -H -o name,size,alloc,free,cap "$pool" 2>/dev/null) || true
|
|
STATUS+=" $pool_info $state\n"
|
|
|
|
# zpool -H output is tab-separated
|
|
IFS=$'\t' read -r p_name p_size p_alloc p_free p_cap <<< "$pool_info" || true
|
|
HTML_ZFS+="<tr><td $TD>${p_name:-$pool}</td><td $TDR>$p_size</td><td $TDR>$p_alloc</td><td $TDR>$p_free</td><td $TDR>$p_cap</td><td $TD>$(state_html "$state")</td></tr>"$'\n'
|
|
|
|
if [ "$state" != "ONLINE" ]; then
|
|
PROBLEMS+="[ZFS] Pool '$pool' state: $state\n"
|
|
fi
|
|
|
|
# Check for errors via zpool status -p (parseable).
|
|
# Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM.
|
|
# Flag any vdev that is not ONLINE, or has non-zero error counters.
|
|
while IFS= read -r vdev_line; do
|
|
vdev_state=$(echo "$vdev_line" | awk '{print $2}')
|
|
vdev_read=$(echo "$vdev_line" | awk '{print $3}')
|
|
vdev_write=$(echo "$vdev_line" | awk '{print $4}')
|
|
vdev_cksum=$(echo "$vdev_line" | awk '{print $5}')
|
|
vdev_name=$(echo "$vdev_line" | awk '{print $1}')
|
|
|
|
if [ "$vdev_state" != "ONLINE" ] && [ "$vdev_state" != "state" ]; then
|
|
PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' state: $vdev_state\n"
|
|
fi
|
|
for val in "$vdev_read" "$vdev_write" "$vdev_cksum"; do
|
|
if [ "$val" -gt 0 ] 2>/dev/null; then
|
|
PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' has errors (R:${vdev_read} W:${vdev_write} C:${vdev_cksum})\n"
|
|
break
|
|
fi
|
|
done
|
|
done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true)
|
|
done < <(zpool list -H -o name 2>/dev/null || true)
|
|
|
|
# ─── Disk Space ──────────────────────────────────────────────────
|
|
STATUS+="\nDisk Space:\n"
|
|
while IFS= read -r line; do
|
|
[ -n "$line" ] || continue
|
|
usage=$(echo "$line" | awk '{print $5}' | tr -d '%')
|
|
mount=$(echo "$line" | awk '{print $6}')
|
|
size=$(echo "$line" | awk '{print $2}')
|
|
used=$(echo "$line" | awk '{print $3}')
|
|
avail=$(echo "$line" | awk '{print $4}')
|
|
STATUS+=" $mount: ${usage}% (${used}/${size}, ${avail} free)\n"
|
|
weight=""
|
|
if [ "$usage" -gt "$SPACE_THRESHOLD" ] 2>/dev/null; then
|
|
PROBLEMS+="[SPACE] $mount is ${usage}% full\n"
|
|
weight=";font-weight:bold"
|
|
fi
|
|
usage_cell="<span style=\"color:$(usage_color "$usage")$weight\">${usage}%</span>"
|
|
HTML_SPACE+="<tr><td $TD>$mount</td><td $TDR>$usage_cell</td><td $TDR>$used</td><td $TDR>$size</td><td $TDR>$avail</td></tr>"$'\n'
|
|
done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true)
|
|
|
|
# ─── Output ──────────────────────────────────────────────────────
|
|
if [ "$ERROR_ONLY" = true ]; then
|
|
if [ -z "$PROBLEMS" ]; then
|
|
exit 0
|
|
fi
|
|
SUBJECT="[ALERT] $HOSTNAME health check failed"
|
|
BODY="Health check found issues on $HOSTNAME at $(date):\n\n$PROBLEMS\n---\n"
|
|
else
|
|
if [ -n "$PROBLEMS" ]; then
|
|
SUBJECT="[REPORT] $HOSTNAME health — ISSUES FOUND"
|
|
else
|
|
SUBJECT="[REPORT] $HOSTNAME health — all clear"
|
|
fi
|
|
BODY="Health report for $HOSTNAME at $(date):\n\n"
|
|
if [ -n "$PROBLEMS" ]; then
|
|
BODY+="*** ISSUES ***\n\n$PROBLEMS\n"
|
|
fi
|
|
BODY+="--- Status ---\n\n$STATUS\n---\n"
|
|
fi
|
|
|
|
if [ "$USE_EMAIL" = true ]; then
|
|
HTML_PROBLEMS=""
|
|
while IFS= read -r p; do
|
|
[ -n "$p" ] || continue
|
|
HTML_PROBLEMS+="<li style=\"color:#cf222e;font-weight:bold;margin:2px 0\">$p</li>"$'\n'
|
|
done < <(printf '%b' "$PROBLEMS")
|
|
|
|
# Built with a newline after each row/section — SMTP caps lines at
|
|
# ~1000 chars and one giant line can get mangled in transit.
|
|
HTML_BODY="<div style=\"font-family:Arial,Helvetica,sans-serif;font-size:14px;color:#222\">"$'\n'
|
|
if [ "$ERROR_ONLY" = true ]; then
|
|
HTML_BODY+="<p>Health check found issues on <b>$HOSTNAME</b> at $(date):</p>"$'\n'
|
|
HTML_BODY+="<ul style=\"padding-left:20px\">"$'\n'"$HTML_PROBLEMS</ul>"$'\n'
|
|
else
|
|
HTML_BODY+="<p>Health report for <b>$HOSTNAME</b> at $(date):</p>"$'\n'
|
|
if [ -n "$HTML_PROBLEMS" ]; then
|
|
HTML_BODY+="<h3 style=\"color:#cf222e\">Issues</h3>"$'\n'
|
|
HTML_BODY+="<ul style=\"padding-left:20px\">"$'\n'"$HTML_PROBLEMS</ul>"$'\n'
|
|
fi
|
|
HTML_BODY+="<h3>SMART (SATA)</h3><table $TABLE>"$'\n'
|
|
HTML_BODY+="<tr><th $TH>Disk</th><th $TH>Model</th><th $TH>Health</th><th $TH>Temp</th><th $TH>Hours</th></tr>"$'\n'
|
|
HTML_BODY+="$HTML_SATA</table>"$'\n'
|
|
HTML_BODY+="<h3>SMART (NVMe)</h3><table $TABLE>"$'\n'
|
|
HTML_BODY+="<tr><th $TH>Disk</th><th $TH>Model</th><th $TH>Health</th><th $TH>Temp</th><th $TH>Hours</th></tr>"$'\n'
|
|
HTML_BODY+="$HTML_NVME</table>"$'\n'
|
|
HTML_BODY+="<h3>ZFS Pools</h3><table $TABLE>"$'\n'
|
|
HTML_BODY+="<tr><th $TH>Pool</th><th $TH>Size</th><th $TH>Alloc</th><th $TH>Free</th><th $TH>Cap</th><th $TH>State</th></tr>"$'\n'
|
|
HTML_BODY+="$HTML_ZFS</table>"$'\n'
|
|
HTML_BODY+="<h3>Disk Space</h3><table $TABLE>"$'\n'
|
|
HTML_BODY+="<tr><th $TH>Mount</th><th $TH>Used %</th><th $TH>Used</th><th $TH>Size</th><th $TH>Free</th></tr>"$'\n'
|
|
HTML_BODY+="$HTML_SPACE</table>"$'\n'
|
|
fi
|
|
HTML_BODY+="</div>"
|
|
|
|
{
|
|
echo "Subject: $SUBJECT"
|
|
echo "From: $HOSTNAME <argento@fastmail.com>"
|
|
echo "To: $MAILTO"
|
|
echo "MIME-Version: 1.0"
|
|
echo "Content-Type: text/html; charset=utf-8"
|
|
echo ""
|
|
printf '%s\n' "$HTML_BODY"
|
|
} | sendmail "$MAILTO"
|
|
else
|
|
printf '%b' "$BODY"
|
|
fi
|