diff --git a/scripts/disk-health-check.sh b/scripts/disk-health-check.sh index cc8ec2b..af1d0ed 100755 --- a/scripts/disk-health-check.sh +++ b/scripts/disk-health-check.sh @@ -23,6 +23,8 @@ set -uo pipefail MAILTO="timothykim@fastmail.fm" HOSTNAME=$(hostname) SPACE_THRESHOLD=85 # percent used — alert above this +ZFS_CAP_THRESHOLD=80 # ZFS write performance degrades on nearly-full pools +QUOTA_THRESHOLD=85 # percent of dataset quota — alert above this ERROR_ONLY=false USE_EMAIL=false @@ -174,12 +176,19 @@ while IFS= read -r pool; do # zpool -H output is tab-separated IFS=$'\t' read -r p_name p_size p_alloc p_free p_cap <<< "$pool_info" || true - HTML_ZFS+="${p_name:-$pool}$p_size$p_alloc$p_free$p_cap$(state_html "$state")"$'\n' + cap_cell="$p_cap" + HTML_ZFS+="${p_name:-$pool}$p_size$p_alloc$p_free$cap_cell$(state_html "$state")"$'\n' if [ "$state" != "ONLINE" ]; then PROBLEMS+="[ZFS] Pool '$pool' state: $state\n" fi + # Datasets are excluded from the df check below, so pool cap is the + # only capacity alert for ZFS data. + if [ "${p_cap%\%}" -gt "$ZFS_CAP_THRESHOLD" ] 2>/dev/null; then + PROBLEMS+="[ZFS] Pool '$pool' is ${p_cap} full\n" + fi + # Check for errors via zpool status -p (parseable). # Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM. # Flag any vdev that is not ONLINE, or has non-zero error counters. @@ -202,6 +211,27 @@ while IFS= read -r pool; do done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true) done < <(zpool list -H -o name 2>/dev/null || true) +# ─── ZFS Dataset Quotas ────────────────────────────────────────── +# Quotas are hard limits — writes fail at 100% — so warn before the kids +# hit the wall. Lists every dataset with a quota set (with -p, an unset +# quota reads as 0). +STATUS+="\nQuotas:\n" +HTML_QUOTA="" +while IFS=$'\t' read -r ds used quota; do + [ -n "$ds" ] || continue + [ "$quota" -gt 0 ] 2>/dev/null || continue + pct=$(( used * 100 / quota )) + used_h=$(numfmt --to=iec "$used" 2>/dev/null || echo "$used") + quota_h=$(numfmt --to=iec "$quota" 2>/dev/null || echo "$quota") + STATUS+=" $ds: ${pct}% (${used_h}/${quota_h})\n" + weight="" + if [ "$pct" -gt "$QUOTA_THRESHOLD" ]; then + PROBLEMS+="[QUOTA] $ds is at ${pct}% of its ${quota_h} quota\n" + weight=";font-weight:bold" + fi + HTML_QUOTA+="$ds$used_h$quota_h${pct}%"$'\n' +done < <(zfs list -Hp -o name,used,quota -t filesystem 2>/dev/null || true) + # ─── Disk Space ────────────────────────────────────────────────── STATUS+="\nDisk Space:\n" while IFS= read -r line; do @@ -219,7 +249,9 @@ while IFS= read -r line; do fi usage_cell="${usage}%" HTML_SPACE+="$mount$usage_cell$used$size$avail"$'\n' -done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true) +# ZFS datasets are excluded: their df percentages are computed against +# shared pool free space, so they're misleading — pool cap above covers them. +done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs -x zfs 2>/dev/null | tail -n +2 || true) # ─── Output ────────────────────────────────────────────────────── if [ "$ERROR_ONLY" = true ]; then @@ -269,6 +301,9 @@ if [ "$USE_EMAIL" = true ]; then HTML_BODY+="

ZFS Pools

"$'\n' HTML_BODY+=""$'\n' HTML_BODY+="$HTML_ZFS
PoolSizeAllocFreeCapState
"$'\n' + HTML_BODY+="

Quotas

"$'\n' + HTML_BODY+=""$'\n' + HTML_BODY+="$HTML_QUOTA
DatasetUsedQuotaUsed %
"$'\n' HTML_BODY+="

Disk Space

"$'\n' HTML_BODY+=""$'\n' HTML_BODY+="$HTML_SPACE
MountUsed %UsedSizeFree
"$'\n'