initial commit
This commit is contained in:
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Daily USB backup. Dumps DBs for consistency, then rsyncs everything.
|
||||
set -euo pipefail
|
||||
|
||||
FAILURES=()
|
||||
|
||||
# Check the USB drive is actually mounted
|
||||
if ! mountpoint -q /mnt/backup; then
|
||||
echo "FATAL: /mnt/backup is not mounted" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Dump Nextcloud PostgreSQL for consistency
|
||||
if docker exec nextcloud-db pg_dumpall -U nextcloud > /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp; then
|
||||
mv /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp /opt/argento/docker/nextcloud/db/pg_dumpall.sql
|
||||
else
|
||||
rm -f /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp
|
||||
FAILURES+=("nextcloud pg_dump")
|
||||
fi
|
||||
|
||||
# Dump Gitea SQLite for consistency
|
||||
sqlite3 /opt/argento/docker/gitea/data/gitea/gitea.db \
|
||||
".backup /opt/argento/docker/gitea/data/gitea/gitea.db.bak" \
|
||||
|| FAILURES+=("gitea sqlite backup")
|
||||
|
||||
# Dump Jellyfin SQLite DBs for consistency
|
||||
for db in jellyfin library; do
|
||||
sqlite3 /opt/argento/docker/jellyfin/config/data/${db}.db \
|
||||
".backup /opt/argento/docker/jellyfin/config/data/${db}.db.bak" \
|
||||
|| FAILURES+=("jellyfin ${db} sqlite backup")
|
||||
done
|
||||
|
||||
# Single rsync covers everything: configs, app data, media, DB dumps
|
||||
RSYNC_OPTS="-a --delete"
|
||||
[ -t 1 ] && RSYNC_OPTS="$RSYNC_OPTS --info=progress2"
|
||||
rsync $RSYNC_OPTS /opt/argento/ /mnt/backup/argento/ \
|
||||
|| FAILURES+=("rsync argento")
|
||||
|
||||
if [ ${#FAILURES[@]} -gt 0 ]; then
|
||||
echo "BACKUP FAILED: ${FAILURES[*]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Backup completed successfully at $(date)"
|
||||
Executable
+193
@@ -0,0 +1,193 @@
|
||||
#!/bin/bash
|
||||
# server-health-check.sh — Argento health monitoring
|
||||
# Checks: SMART disk health, ZFS pool status, disk space
|
||||
# Outputs to stdout by default, or sends email with --email.
|
||||
# Use --error-only to suppress output when everything is healthy.
|
||||
#
|
||||
# Usage:
|
||||
# ./server-health-check.sh # full report to stdout
|
||||
# ./server-health-check.sh --error-only # only print errors (silent if healthy)
|
||||
# ./server-health-check.sh --email # full report to email
|
||||
# ./server-health-check.sh --error-only --email # email errors (no email if healthy)
|
||||
#
|
||||
# Install:
|
||||
# chmod +x /opt/server-health-check.sh
|
||||
# crontab -e:
|
||||
# 0 6 * * * /opt/server-health-check.sh --error-only --email # daily alert
|
||||
# 0 8 * * 1 /opt/server-health-check.sh --email # weekly report
|
||||
#
|
||||
# Dependencies: smartmontools, zfs, msmtp (as sendmail)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
MAILTO="timothykim@fastmail.fm"
|
||||
HOSTNAME=$(hostname)
|
||||
SPACE_THRESHOLD=85 # percent used — alert above this
|
||||
|
||||
ERROR_ONLY=false
|
||||
USE_EMAIL=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--error-only) ERROR_ONLY=true ;;
|
||||
--email) USE_EMAIL=true ;;
|
||||
*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
PROBLEMS=""
|
||||
STATUS=""
|
||||
|
||||
# ─── SMART Disk Health (SATA/SAS) ───────────────────────────────
|
||||
STATUS+="SMART (SATA):\n"
|
||||
for disk in /dev/sd?; do
|
||||
[ -b "$disk" ] || continue
|
||||
|
||||
# smartctl returns non-zero exit codes as a bitmask for disk
|
||||
# conditions (standby, threshold exceeded, etc.), so don't
|
||||
# treat non-zero as a script error — capture output and check it.
|
||||
health=$(smartctl -H "$disk" 2>/dev/null | grep "test result" | awk -F': ' '{print $2}') || true
|
||||
health="${health:-UNKNOWN}"
|
||||
temp=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 194 || $1 == 190 {print $10; exit}') || true
|
||||
temp="${temp:-?}"
|
||||
hours=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 9 {print $10; exit}') || true
|
||||
hours="${hours:-?}"
|
||||
model=$(smartctl -i "$disk" 2>/dev/null | awk -F': +' '/Device Model|Model Number/{print $2; exit}') || true
|
||||
model="${model:-unknown}"
|
||||
|
||||
STATUS+=" $disk ($model): $health, ${temp}C, ${hours}h\n"
|
||||
|
||||
if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
|
||||
PROBLEMS+="[SMART] $disk: overall health check FAILED ($health)\n"
|
||||
fi
|
||||
|
||||
# Check critical SMART attributes
|
||||
# 5 = Reallocated_Sector_Ct
|
||||
# 187 = Reported_Uncorrect
|
||||
# 197 = Current_Pending_Sector
|
||||
# 198 = Offline_Uncorrectable
|
||||
while IFS= read -r line; do
|
||||
id=$(echo "$line" | awk '{print $1}')
|
||||
name=$(echo "$line" | awk '{print $2}')
|
||||
raw=$(echo "$line" | awk '{print $10}')
|
||||
case "$id" in
|
||||
5|187|197|198)
|
||||
if [ "$raw" -gt 0 ] 2>/dev/null; then
|
||||
PROBLEMS+="[SMART] $disk: $name = $raw (should be 0)\n"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done < <(smartctl -A "$disk" 2>/dev/null | awk 'NR>7 && /^[[:space:]]*[0-9]/' || true)
|
||||
done
|
||||
|
||||
# ─── SMART Disk Health (NVMe) ───────────────────────────────────
|
||||
STATUS+="\nSMART (NVMe):\n"
|
||||
for disk in /dev/nvme[0-9]*; do
|
||||
# Match only controller devices (nvme0, nvme1), not namespaces (nvme0n1)
|
||||
[[ "$disk" =~ ^/dev/nvme[0-9]+$ ]] || continue
|
||||
[ -c "$disk" ] || continue
|
||||
|
||||
ns="${disk}n1" # namespace device for smartctl
|
||||
[ -b "$ns" ] || continue
|
||||
|
||||
health=$(smartctl -H "$ns" 2>/dev/null | grep "SMART overall-health" | awk -F': ' '{print $2}') || true
|
||||
health="${health:-UNKNOWN}"
|
||||
model=$(smartctl -i "$ns" 2>/dev/null | awk -F': +' '/Model Number/{print $2; exit}') || true
|
||||
model="${model:-unknown}"
|
||||
temp=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Temperature:/{print $2; exit}') || true
|
||||
temp="${temp:-?}"
|
||||
hours=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Power On Hours/{print $2; exit}') || true
|
||||
hours="${hours:-?}"
|
||||
# NVMe percentage used — 100% means full rated write endurance consumed
|
||||
pct_used=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Percentage Used/{print $2; exit}' | tr -d '%') || true
|
||||
|
||||
STATUS+=" $ns ($model): $health, ${temp}, ${hours}h\n"
|
||||
|
||||
if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then
|
||||
PROBLEMS+="[SMART] $ns: overall health check FAILED ($health)\n"
|
||||
fi
|
||||
|
||||
if [ -n "$pct_used" ] && [ "$pct_used" -gt 90 ] 2>/dev/null; then
|
||||
PROBLEMS+="[SMART] $ns: NVMe endurance ${pct_used}% used (>90%)\n"
|
||||
fi
|
||||
done
|
||||
|
||||
# ─── ZFS Pool Health ─────────────────────────────────────────────
|
||||
STATUS+="\nZFS Pools:\n"
|
||||
while IFS= read -r pool; do
|
||||
[ -n "$pool" ] || continue
|
||||
state=$(zpool list -H -o health "$pool" 2>/dev/null) || true
|
||||
pool_info=$(zpool list -H -o name,size,alloc,free,cap "$pool" 2>/dev/null) || true
|
||||
STATUS+=" $pool_info $state\n"
|
||||
|
||||
if [ "$state" != "ONLINE" ]; then
|
||||
PROBLEMS+="[ZFS] Pool '$pool' state: $state\n"
|
||||
fi
|
||||
|
||||
# Check for errors via zpool status -p (parseable).
|
||||
# Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM.
|
||||
# Flag any vdev that is not ONLINE, or has non-zero error counters.
|
||||
while IFS= read -r vdev_line; do
|
||||
vdev_state=$(echo "$vdev_line" | awk '{print $2}')
|
||||
vdev_read=$(echo "$vdev_line" | awk '{print $3}')
|
||||
vdev_write=$(echo "$vdev_line" | awk '{print $4}')
|
||||
vdev_cksum=$(echo "$vdev_line" | awk '{print $5}')
|
||||
vdev_name=$(echo "$vdev_line" | awk '{print $1}')
|
||||
|
||||
if [ "$vdev_state" != "ONLINE" ] && [ "$vdev_state" != "state" ]; then
|
||||
PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' state: $vdev_state\n"
|
||||
fi
|
||||
for val in "$vdev_read" "$vdev_write" "$vdev_cksum"; do
|
||||
if [ "$val" -gt 0 ] 2>/dev/null; then
|
||||
PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' has errors (R:${vdev_read} W:${vdev_write} C:${vdev_cksum})\n"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true)
|
||||
done < <(zpool list -H -o name 2>/dev/null || true)
|
||||
|
||||
# ─── Disk Space ──────────────────────────────────────────────────
|
||||
STATUS+="\nDisk Space:\n"
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] || continue
|
||||
usage=$(echo "$line" | awk '{print $5}' | tr -d '%')
|
||||
mount=$(echo "$line" | awk '{print $6}')
|
||||
size=$(echo "$line" | awk '{print $2}')
|
||||
used=$(echo "$line" | awk '{print $3}')
|
||||
avail=$(echo "$line" | awk '{print $4}')
|
||||
STATUS+=" $mount: ${usage}% (${used}/${size}, ${avail} free)\n"
|
||||
if [ "$usage" -gt "$SPACE_THRESHOLD" ] 2>/dev/null; then
|
||||
PROBLEMS+="[SPACE] $mount is ${usage}% full\n"
|
||||
fi
|
||||
done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true)
|
||||
|
||||
# ─── Output ──────────────────────────────────────────────────────
|
||||
if [ "$ERROR_ONLY" = true ]; then
|
||||
if [ -z "$PROBLEMS" ]; then
|
||||
exit 0
|
||||
fi
|
||||
SUBJECT="[ALERT] $HOSTNAME health check failed"
|
||||
BODY="Health check found issues on $HOSTNAME at $(date):\n\n$PROBLEMS\n---\n"
|
||||
else
|
||||
if [ -n "$PROBLEMS" ]; then
|
||||
SUBJECT="[REPORT] $HOSTNAME health — ISSUES FOUND"
|
||||
else
|
||||
SUBJECT="[REPORT] $HOSTNAME health — all clear"
|
||||
fi
|
||||
BODY="Health report for $HOSTNAME at $(date):\n\n"
|
||||
if [ -n "$PROBLEMS" ]; then
|
||||
BODY+="*** ISSUES ***\n\n$PROBLEMS\n"
|
||||
fi
|
||||
BODY+="--- Status ---\n\n$STATUS\n---\n"
|
||||
fi
|
||||
|
||||
if [ "$USE_EMAIL" = true ]; then
|
||||
{
|
||||
echo "Subject: $SUBJECT"
|
||||
echo "From: $HOSTNAME <root@$HOSTNAME>"
|
||||
echo "To: $MAILTO"
|
||||
echo ""
|
||||
printf '%b' "$BODY"
|
||||
} | sendmail "$MAILTO"
|
||||
else
|
||||
printf '%b' "$BODY"
|
||||
fi
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# Issue SSL certificates via certbot webroot through the nginx container.
|
||||
#
|
||||
# This script ONLY issues certs. It does not touch the real nginx confs.
|
||||
# Nginx must be running (even with minimal config) to serve ACME challenges.
|
||||
#
|
||||
# Provisioning order:
|
||||
# 1. Start nginx with no app confs (just security-headers.inc)
|
||||
# 2. Run this script to issue certs
|
||||
# 3. Start all app containers (so upstreams are resolvable)
|
||||
# 4. Copy app confs into conf.d and reload nginx
|
||||
#
|
||||
# For adding a cert to a running server, apps are already up — just run
|
||||
# this script, add the conf, and reload nginx.
|
||||
#
|
||||
# Usage: ./issue-cert.sh <domain> [domain...]
|
||||
# Example: ./issue-cert.sh nextcloud.timothykim.net git.timothykim.net
|
||||
set -euo pipefail
|
||||
|
||||
CONF_DIR="/opt/argento/docker/nginx/conf.d"
|
||||
WEBROOT="/opt/argento/docker/nginx/certbot/www"
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "Usage: $0 <domain> [domain...]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$WEBROOT"
|
||||
|
||||
for domain in "$@"; do
|
||||
echo "=== Issuing cert for $domain ==="
|
||||
|
||||
# Write temporary HTTP-only config
|
||||
cat > "$CONF_DIR/$domain.tmp.conf" <<NGINXCONF
|
||||
server {
|
||||
listen 80;
|
||||
listen [::]:80;
|
||||
server_name $domain;
|
||||
location /.well-known/acme-challenge/ { root /var/www/certbot; }
|
||||
location / { return 444; }
|
||||
}
|
||||
NGINXCONF
|
||||
|
||||
trap 'rm -f "$CONF_DIR/$domain.tmp.conf"' EXIT
|
||||
docker exec nginx nginx -t
|
||||
docker exec nginx nginx -s reload
|
||||
|
||||
certbot certonly --webroot -w "$WEBROOT" \
|
||||
--non-interactive --agree-tos -m timothykim@fastmail.fm \
|
||||
--cert-name "$domain" -d "$domain"
|
||||
|
||||
rm -f "$CONF_DIR/$domain.tmp.conf"
|
||||
trap - EXIT
|
||||
|
||||
echo "Certificate issued for $domain"
|
||||
done
|
||||
|
||||
echo "=== Done ==="
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
# Copies system config files into the repo and commits if anything changed.
|
||||
# File mapping is defined in system/tracked-configs.
|
||||
# Intended to run via cron (e.g., daily).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="/opt/argento"
|
||||
|
||||
# Copy each tracked config into the repo
|
||||
while IFS=: read -r src dest; do
|
||||
[ -z "$src" ] && continue
|
||||
[[ "$src" = \#* ]] && continue
|
||||
if [ -f "$src" ]; then
|
||||
cp "$src" "$REPO/$dest"
|
||||
fi
|
||||
done < "$REPO/system/tracked-configs"
|
||||
|
||||
sed -i 's/^password .*/password REDACTED/' "$REPO/system/msmtprc"
|
||||
|
||||
# Sync root crontab
|
||||
crontab -l > "$REPO/system/root-crontab" 2>/dev/null || true
|
||||
|
||||
cd "$REPO"
|
||||
|
||||
# Stage and check if anything changed
|
||||
git add system/
|
||||
if git diff --cached --quiet; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "auto: sync system configs"
|
||||
git push || echo "Push failed (remote ahead?) — committed locally, will push next time."
|
||||
Reference in New Issue
Block a user