zfs monitoring

scripts/disk-health-check.sh

@@ -23,6 +23,8 @@ set -uo pipefail
 MAILTO="timothykim@fastmail.fm"
 HOSTNAME=$(hostname)
 SPACE_THRESHOLD=85  # percent used — alert above this
+ZFS_CAP_THRESHOLD=80  # ZFS write performance degrades on nearly-full pools
+QUOTA_THRESHOLD=85  # percent of dataset quota — alert above this
 
 ERROR_ONLY=false
 USE_EMAIL=false
@@ -174,12 +176,19 @@ while IFS= read -r pool; do
 
     # zpool -H output is tab-separated
     IFS=$'\t' read -r p_name p_size p_alloc p_free p_cap <<< "$pool_info" || true
-    HTML_ZFS+="<tr><td $TD>${p_name:-$pool}</td><td $TDR>$p_size</td><td $TDR>$p_alloc</td><td $TDR>$p_free</td><td $TDR>$p_cap</td><td $TD>$(state_html "$state")</td></tr>"$'\n'
+    cap_cell="<span style=\"color:$(usage_color "${p_cap%\%}")\">$p_cap</span>"
+    HTML_ZFS+="<tr><td $TD>${p_name:-$pool}</td><td $TDR>$p_size</td><td $TDR>$p_alloc</td><td $TDR>$p_free</td><td $TDR>$cap_cell</td><td $TD>$(state_html "$state")</td></tr>"$'\n'
 
     if [ "$state" != "ONLINE" ]; then
         PROBLEMS+="[ZFS] Pool '$pool' state: $state\n"
     fi
 
+    # Datasets are excluded from the df check below, so pool cap is the
+    # only capacity alert for ZFS data.
+    if [ "${p_cap%\%}" -gt "$ZFS_CAP_THRESHOLD" ] 2>/dev/null; then
+        PROBLEMS+="[ZFS] Pool '$pool' is ${p_cap} full\n"
+    fi
+
     # Check for errors via zpool status -p (parseable).
     # Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM.
     # Flag any vdev that is not ONLINE, or has non-zero error counters.
@@ -202,6 +211,27 @@ while IFS= read -r pool; do
     done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true)
 done < <(zpool list -H -o name 2>/dev/null || true)
 
+# ─── ZFS Dataset Quotas ──────────────────────────────────────────
+# Quotas are hard limits — writes fail at 100% — so warn before the kids
+# hit the wall. Lists every dataset with a quota set (with -p, an unset
+# quota reads as 0).
+STATUS+="\nQuotas:\n"
+HTML_QUOTA=""
+while IFS=$'\t' read -r ds used quota; do
+    [ -n "$ds" ] || continue
+    [ "$quota" -gt 0 ] 2>/dev/null || continue
+    pct=$(( used * 100 / quota ))
+    used_h=$(numfmt --to=iec "$used" 2>/dev/null || echo "$used")
+    quota_h=$(numfmt --to=iec "$quota" 2>/dev/null || echo "$quota")
+    STATUS+="  $ds: ${pct}% (${used_h}/${quota_h})\n"
+    weight=""
+    if [ "$pct" -gt "$QUOTA_THRESHOLD" ]; then
+        PROBLEMS+="[QUOTA] $ds is at ${pct}% of its ${quota_h} quota\n"
+        weight=";font-weight:bold"
+    fi
+    HTML_QUOTA+="<tr><td $TD>$ds</td><td $TDR>$used_h</td><td $TDR>$quota_h</td><td $TDR><span style=\"color:$(usage_color "$pct")$weight\">${pct}%</span></td></tr>"$'\n'
+done < <(zfs list -Hp -o name,used,quota -t filesystem 2>/dev/null || true)
+
 # ─── Disk Space ──────────────────────────────────────────────────
 STATUS+="\nDisk Space:\n"
 while IFS= read -r line; do
@@ -219,7 +249,9 @@ while IFS= read -r line; do
     fi
     usage_cell="<span style=\"color:$(usage_color "$usage")$weight\">${usage}%</span>"
     HTML_SPACE+="<tr><td $TD>$mount</td><td $TDR>$usage_cell</td><td $TDR>$used</td><td $TDR>$size</td><td $TDR>$avail</td></tr>"$'\n'
-done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true)
+# ZFS datasets are excluded: their df percentages are computed against
+# shared pool free space, so they're misleading — pool cap above covers them.
+done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs -x zfs 2>/dev/null | tail -n +2 || true)
 
 # ─── Output ──────────────────────────────────────────────────────
 if [ "$ERROR_ONLY" = true ]; then
@@ -269,6 +301,9 @@ if [ "$USE_EMAIL" = true ]; then
         HTML_BODY+="<h3>ZFS Pools</h3><table $TABLE>"$'\n'
         HTML_BODY+="<tr><th $TH>Pool</th><th $TH>Size</th><th $TH>Alloc</th><th $TH>Free</th><th $TH>Cap</th><th $TH>State</th></tr>"$'\n'
         HTML_BODY+="$HTML_ZFS</table>"$'\n'
+        HTML_BODY+="<h3>Quotas</h3><table $TABLE>"$'\n'
+        HTML_BODY+="<tr><th $TH>Dataset</th><th $TH>Used</th><th $TH>Quota</th><th $TH>Used %</th></tr>"$'\n'
+        HTML_BODY+="$HTML_QUOTA</table>"$'\n'
         HTML_BODY+="<h3>Disk Space</h3><table $TABLE>"$'\n'
         HTML_BODY+="<tr><th $TH>Mount</th><th $TH>Used %</th><th $TH>Used</th><th $TH>Size</th><th $TH>Free</th></tr>"$'\n'
         HTML_BODY+="$HTML_SPACE</table>"$'\n'