initial commit

2026-04-07 15:46:18 -04:00
commit bc1b1d1232
43 changed files with 1627 additions and 0 deletions
@@ -0,0 +1,193 @@
+#!/bin/bash
+# server-health-check.sh — Argento health monitoring
+# Checks: SMART disk health, ZFS pool status, disk space
+# Outputs to stdout by default, or sends email with --email.
+# Use --error-only to suppress output when everything is healthy.
+#
+# Usage:
+#   ./server-health-check.sh               # full report to stdout
+#   ./server-health-check.sh --error-only  # only print errors (silent if healthy)
+#   ./server-health-check.sh --email       # full report to email
+#   ./server-health-check.sh --error-only --email # email errors (no email if healthy)
+#
+# Install:
+#   chmod +x /opt/server-health-check.sh
+#   crontab -e:
+#     0 6 * * * /opt/server-health-check.sh --error-only --email  # daily alert
+#     0 8 * * 1 /opt/server-health-check.sh --email               # weekly report
+#
+# Dependencies: smartmontools, zfs, msmtp (as sendmail)
+
+set -uo pipefail
+
+MAILTO="timothykim@fastmail.fm"
+HOSTNAME=$(hostname)
+SPACE_THRESHOLD=85  # percent used — alert above this
+
+ERROR_ONLY=false
+USE_EMAIL=false
+for arg in "$@"; do
+    case "$arg" in
+        --error-only) ERROR_ONLY=true ;;
+        --email) USE_EMAIL=true ;;
+        *) echo "Unknown option: $arg" >&2; exit 1 ;;
+    esac
+done
+
+PROBLEMS=""
+STATUS=""
+
+# ─── SMART Disk Health (SATA/SAS) ───────────────────────────────
+STATUS+="SMART (SATA):\n"
+for disk in /dev/sd?; do
+    [ -b "$disk" ] || continue
+
+    # smartctl returns non-zero exit codes as a bitmask for disk
+    # conditions (standby, threshold exceeded, etc.), so don't
+    # treat non-zero as a script error — capture output and check it.
+    health=$(smartctl -H "$disk" 2>/dev/null | grep "test result" | awk -F': ' '{print $2}') || true
+    health="${health:-UNKNOWN}"
+    temp=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 194 || $1 == 190 {print $10; exit}') || true
+    temp="${temp:-?}"
+    hours=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 9 {print $10; exit}') || true
+    hours="${hours:-?}"
+    model=$(smartctl -i "$disk" 2>/dev/null | awk -F': +' '/Device Model|Model Number/{print $2; exit}') || true
+    model="${model:-unknown}"
+
+    STATUS+="  $disk ($model): $health, ${temp}C, ${hours}h\n"
+
+    if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
+        PROBLEMS+="[SMART] $disk: overall health check FAILED ($health)\n"
+    fi
+
+    # Check critical SMART attributes
+    # 5   = Reallocated_Sector_Ct
+    # 187 = Reported_Uncorrect
+    # 197 = Current_Pending_Sector
+    # 198 = Offline_Uncorrectable
+    while IFS= read -r line; do
+        id=$(echo "$line" | awk '{print $1}')
+        name=$(echo "$line" | awk '{print $2}')
+        raw=$(echo "$line" | awk '{print $10}')
+        case "$id" in
+            5|187|197|198)
+                if [ "$raw" -gt 0 ] 2>/dev/null; then
+                    PROBLEMS+="[SMART] $disk: $name = $raw (should be 0)\n"
+                fi
+                ;;
+        esac
+    done < <(smartctl -A "$disk" 2>/dev/null | awk 'NR>7 && /^[[:space:]]*[0-9]/' || true)
+done
+
+# ─── SMART Disk Health (NVMe) ───────────────────────────────────
+STATUS+="\nSMART (NVMe):\n"
+for disk in /dev/nvme[0-9]*; do
+    # Match only controller devices (nvme0, nvme1), not namespaces (nvme0n1)
+    [[ "$disk" =~ ^/dev/nvme[0-9]+$ ]] || continue
+    [ -c "$disk" ] || continue
+
+    ns="${disk}n1"  # namespace device for smartctl
+    [ -b "$ns" ] || continue
+
+    health=$(smartctl -H "$ns" 2>/dev/null | grep "SMART overall-health" | awk -F': ' '{print $2}') || true
+    health="${health:-UNKNOWN}"
+    model=$(smartctl -i "$ns" 2>/dev/null | awk -F': +' '/Model Number/{print $2; exit}') || true
+    model="${model:-unknown}"
+    temp=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Temperature:/{print $2; exit}') || true
+    temp="${temp:-?}"
+    hours=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Power On Hours/{print $2; exit}') || true
+    hours="${hours:-?}"
+    # NVMe percentage used — 100% means full rated write endurance consumed
+    pct_used=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Percentage Used/{print $2; exit}' | tr -d '%') || true
+
+    STATUS+="  $ns ($model): $health, ${temp}, ${hours}h\n"
+
+    if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
+        PROBLEMS+="[SMART] $ns: overall health check FAILED ($health)\n"
+    fi
+
+    if [ -n "$pct_used" ] && [ "$pct_used" -gt 90 ] 2>/dev/null; then
+        PROBLEMS+="[SMART] $ns: NVMe endurance ${pct_used}% used (>90%)\n"
+    fi
+done
+
+# ─── ZFS Pool Health ─────────────────────────────────────────────
+STATUS+="\nZFS Pools:\n"
+while IFS= read -r pool; do
+    [ -n "$pool" ] || continue
+    state=$(zpool list -H -o health "$pool" 2>/dev/null) || true
+    pool_info=$(zpool list -H -o name,size,alloc,free,cap "$pool" 2>/dev/null) || true
+    STATUS+="  $pool_info  $state\n"
+
+    if [ "$state" != "ONLINE" ]; then
+        PROBLEMS+="[ZFS] Pool '$pool' state: $state\n"
+    fi
+
+    # Check for errors via zpool status -p (parseable).
+    # Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM.
+    # Flag any vdev that is not ONLINE, or has non-zero error counters.
+    while IFS= read -r vdev_line; do
+        vdev_state=$(echo "$vdev_line" | awk '{print $2}')
+        vdev_read=$(echo "$vdev_line" | awk '{print $3}')
+        vdev_write=$(echo "$vdev_line" | awk '{print $4}')
+        vdev_cksum=$(echo "$vdev_line" | awk '{print $5}')
+        vdev_name=$(echo "$vdev_line" | awk '{print $1}')
+
+        if [ "$vdev_state" != "ONLINE" ] && [ "$vdev_state" != "state" ]; then
+            PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' state: $vdev_state\n"
+        fi
+        for val in "$vdev_read" "$vdev_write" "$vdev_cksum"; do
+            if [ "$val" -gt 0 ] 2>/dev/null; then
+                PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' has errors (R:${vdev_read} W:${vdev_write} C:${vdev_cksum})\n"
+                break
+            fi
+        done
+    done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true)
+done < <(zpool list -H -o name 2>/dev/null || true)
+
+# ─── Disk Space ──────────────────────────────────────────────────
+STATUS+="\nDisk Space:\n"
+while IFS= read -r line; do
+    [ -n "$line" ] || continue
+    usage=$(echo "$line" | awk '{print $5}' | tr -d '%')
+    mount=$(echo "$line" | awk '{print $6}')
+    size=$(echo "$line" | awk '{print $2}')
+    used=$(echo "$line" | awk '{print $3}')
+    avail=$(echo "$line" | awk '{print $4}')
+    STATUS+="  $mount: ${usage}% (${used}/${size}, ${avail} free)\n"
+    if [ "$usage" -gt "$SPACE_THRESHOLD" ] 2>/dev/null; then
+        PROBLEMS+="[SPACE] $mount is ${usage}% full\n"
+    fi
+done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true)
+
+# ─── Output ──────────────────────────────────────────────────────
+if [ "$ERROR_ONLY" = true ]; then
+    if [ -z "$PROBLEMS" ]; then
+        exit 0
+    fi
+    SUBJECT="[ALERT] $HOSTNAME health check failed"
+    BODY="Health check found issues on $HOSTNAME at $(date):\n\n$PROBLEMS\n---\n"
+else
+    if [ -n "$PROBLEMS" ]; then
+        SUBJECT="[REPORT] $HOSTNAME health — ISSUES FOUND"
+    else
+        SUBJECT="[REPORT] $HOSTNAME health — all clear"
+    fi
+    BODY="Health report for $HOSTNAME at $(date):\n\n"
+    if [ -n "$PROBLEMS" ]; then
+        BODY+="*** ISSUES ***\n\n$PROBLEMS\n"
+    fi
+    BODY+="--- Status ---\n\n$STATUS\n---\n"
+fi
+
+if [ "$USE_EMAIL" = true ]; then
+    {
+        echo "Subject: $SUBJECT"
+        echo "From: $HOSTNAME <root@$HOSTNAME>"
+        echo "To: $MAILTO"
+        echo ""
+        printf '%b' "$BODY"
+    } | sendmail "$MAILTO"
+else
+    printf '%b' "$BODY"
+fi