From bc1b1d12329e65f5d1eee8ad7169cf0d710c0327 Mon Sep 17 00:00:00 2001 From: Timothy Kim Date: Tue, 7 Apr 2026 15:46:18 -0400 Subject: [PATCH] initial commit --- .gitignore | 3 + CLAUDE.md | 36 +++ README.md | 88 +++++++ RUNBOOK.md | 231 ++++++++++++++++++ docker/garage/.gitignore | 4 + docker/garage/compose.yaml | 39 +++ docker/garage/garage.toml | 20 ++ docker/gitea/.gitignore | 4 + docker/gitea/compose.yaml | 43 ++++ docker/gitea/runner-config.yaml | 25 ++ docker/jellyfin/.gitignore | 4 + docker/jellyfin/compose.yaml | 34 +++ docker/jellyfin/jellyfin-monitor.sh | 21 ++ docker/minecraft/.gitignore | 3 + docker/minecraft/compose.yaml | 35 +++ docker/nextcloud/.gitignore | 5 + docker/nextcloud/Dockerfile | 12 + docker/nextcloud/compose.yaml | 97 ++++++++ docker/nextcloud/upgrade.sh | 32 +++ docker/nginx/compose.yaml | 22 ++ docker/nginx/conf.d/00-default.conf | 20 ++ docker/nginx/conf.d/_template.conf.example | 48 ++++ .../nginx/conf.d/argento.timothykim.net.conf | 46 ++++ .../nginx/conf.d/garage.timothykim.net.conf | 35 +++ docker/nginx/conf.d/git.timothykim.net.conf | 41 ++++ .../nginx/conf.d/jellyfin.timothykim.net.conf | 64 +++++ .../conf.d/minecraft.thekims.family.conf | 71 ++++++ .../conf.d/nextcloud.timothykim.net.conf | 49 ++++ docker/nginx/conf.d/security-headers.inc | 6 + docker/nginx/nginx.conf | 27 ++ scripts/backup.sh | 44 ++++ scripts/disk-health-check.sh | 193 +++++++++++++++ scripts/issue-cert.sh | 58 +++++ scripts/sync-system.sh | 32 +++ system/daemon.json | 8 + system/docker-daemon.json | 8 + system/firewalld/public.xml | 24 ++ system/msmtprc | 9 + system/root-crontab | 8 + system/sanoid.conf | 21 ++ system/smartd.conf | 1 + system/smb.conf | 50 ++++ system/tracked-configs | 6 + 43 files changed, 1627 insertions(+) create mode 100644 .gitignore create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 RUNBOOK.md create mode 100644 docker/garage/.gitignore create mode 100644 docker/garage/compose.yaml create mode 100644 docker/garage/garage.toml create mode 100644 docker/gitea/.gitignore create mode 100644 docker/gitea/compose.yaml create mode 100644 docker/gitea/runner-config.yaml create mode 100644 docker/jellyfin/.gitignore create mode 100644 docker/jellyfin/compose.yaml create mode 100755 docker/jellyfin/jellyfin-monitor.sh create mode 100644 docker/minecraft/.gitignore create mode 100644 docker/minecraft/compose.yaml create mode 100644 docker/nextcloud/.gitignore create mode 100644 docker/nextcloud/Dockerfile create mode 100644 docker/nextcloud/compose.yaml create mode 100755 docker/nextcloud/upgrade.sh create mode 100644 docker/nginx/compose.yaml create mode 100644 docker/nginx/conf.d/00-default.conf create mode 100644 docker/nginx/conf.d/_template.conf.example create mode 100644 docker/nginx/conf.d/argento.timothykim.net.conf create mode 100644 docker/nginx/conf.d/garage.timothykim.net.conf create mode 100644 docker/nginx/conf.d/git.timothykim.net.conf create mode 100644 docker/nginx/conf.d/jellyfin.timothykim.net.conf create mode 100644 docker/nginx/conf.d/minecraft.thekims.family.conf create mode 100644 docker/nginx/conf.d/nextcloud.timothykim.net.conf create mode 100644 docker/nginx/conf.d/security-headers.inc create mode 100644 docker/nginx/nginx.conf create mode 100755 scripts/backup.sh create mode 100755 scripts/disk-health-check.sh create mode 100755 scripts/issue-cert.sh create mode 100755 scripts/sync-system.sh create mode 100644 system/daemon.json create mode 100644 system/docker-daemon.json create mode 100644 system/firewalld/public.xml create mode 100644 system/msmtprc create mode 100644 system/root-crontab create mode 100644 system/sanoid.conf create mode 100644 system/smartd.conf create mode 100644 system/smb.conf create mode 100644 system/tracked-configs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..da136ce --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +smb/ +**/.env + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..f13241f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,36 @@ +# CLAUDE.md + +## What this repo is + +Git-based provisioning and recovery repo for the "argento" home server (Rocky Linux 9, ZFS, Docker). The repo lives at `/opt/argento/` on the server and contains the actual running configs. This dev copy at `~/dev/argento` is where edits happen, then files are manually copied to the server. + +Recovery steps are in `RUNBOOK.md`. Repo overview and workflows are in `README.md`. + +## Architecture notes not in the docs + +- **Docker shared network**: Created by the nginx compose file. All Docker apps that nginx proxies to join this network as `external: true`. If nginx compose hasn't been started, other apps will fail with "network shared not found". +- **nginx variable upstreams**: `nginx.conf` uses `resolver 127.0.0.11` (Docker DNS). Confs that proxy to container names use `set $upstream_` variables so nginx resolves at request time, not startup. Without this, one down container prevents nginx from starting at all. +- **host.docker.internal**: Used for services running on the host network (Jellyfin, Cockpit). Requires `extra_hosts: ["host.docker.internal:host-gateway"]` in nginx's compose.yaml. +- **Jellyfin uses host network**: Needs direct access to NVIDIA GPU for hardware transcoding. Proxied via `host.docker.internal:8096`. +- **Cockpit (argento.timothykim.net)**: Runs directly on the host (not Docker) on port 9090 with self-signed TLS. The nginx conf inlines security headers instead of including security-headers.inc because Cockpit's internal iframes break with X-Frame-Options: DENY. Cockpit has a known bug where font files return 404 — this is not a proxy issue. +- **Minecraft HTTP-only**: HTTPS causes WebSocket/CORS errors with the MCSManager daemon. No cert issued for this domain. +- **SSL cert issuance chicken-and-egg**: On fresh recovery, certs don't exist so SSL confs can't load. The `.hold` pattern moves SSL confs aside, starts nginx with HTTP-only confs, issues certs, then restores SSL confs. +- **SELinux is enforcing**: Caused issues with samba (symlinks blocked, parent directory traversal blocked). Don't use symlinks for config files — use the sync-system.sh copy approach instead. +- **ZFS parent dataset mountpoints**: Pools have parent datasets (e.g., `nextcloud`, `threeteras`) that create mountpoints like `/mnt/nextcloud`. These are empty and expected — do not delete them. +- **msmtp**: Replaces sendmail for cron MAILTO and smartd email alerts. Configured via `/etc/msmtprc` (Fastmail SMTP). The symlink `/usr/sbin/sendmail -> /usr/bin/msmtp` is set up in recovery step 2. +- **hantim-server**: Sister repo with similar nginx/certbot patterns. Located at `~/dev/hantim-server`. Useful as reference for deploy patterns. + +## Remotes + +- **origin**: Gitea on argento (private) +- **GitHub mirror**: Push mirror configured in Gitea repo settings (survives if argento dies) + +## Working preferences + +- Don't edit compose files in this repo — Timothy edits those directly on the server. Compose changes here get rejected. +- Comments in config files should explain the real WHY for this specific app, not generic descriptions of what a directive does. +- Don't add redundant documentation. If something is already on disk (e.g., crontab contents), don't repeat it in the docs. +- Keep docs linear and non-confusing. Avoid branching flows that duplicate steps. +- When proposing config changes, explain the flags/options — Timothy wants to understand what he's committing, not blindly trust it. +- No emojis in any files. +- Verify suggestions actually work before confidently recommending them. If unsure, say so. diff --git a/README.md b/README.md new file mode 100644 index 0000000..124b618 --- /dev/null +++ b/README.md @@ -0,0 +1,88 @@ +# argento + +Configuration and recovery repo for the argento home server. This repo is the runbook -- it contains the actual config files and documents the manual steps between them. + +- Rocky Linux 9 on NVMe +- ZFS storage (2 pools: `nextcloud` mirror, `threeteras` raidz2) +- Docker apps behind nginx reverse proxy +- Daily USB backup via rsync + +## Repo structure + +``` +scripts/ Backup, cert issuance, health monitoring, system config sync +docker/ Docker app configs (compose files, nginx confs) + nginx/ Reverse proxy + SSL termination + nextcloud/ Nextcloud + PostgreSQL + gitea/ Gitea + CI runner + jellyfin/ Media server + garage/ S3-compatible storage + minecraft/ MCSManager +smb/ Samba share (ZFS mountpoint, not in git) +system/ System config snapshots (smb.conf, sanoid.conf, firewalld, etc.) +``` + +## Key files + +| File | Purpose | +|------|---------| +| `RUNBOOK.md` | Full recovery steps, top to bottom | +| `scripts/backup.sh` | Daily USB backup (DB dumps + rsync) | +| `scripts/issue-cert.sh` | Issue SSL certs via certbot webroot | +| `scripts/disk-health-check.sh` | SMART, ZFS, disk space monitoring (daily alerts + weekly reports via cron) | +| `scripts/sync-system.sh` | Auto-sync system configs to git | +| `system/tracked-configs` | Maps system config paths to repo paths | +| `docker/nginx/conf.d/_template.conf.example` | Template for new nginx app confs | + +## Workflows + +**Compose files** -- edit in `/opt/argento/`, apply, commit: + +```bash +cd /opt/argento/docker/ +vim compose.yaml +docker compose up -d +git add . && git commit && git push +``` + +**System configs** -- edit in place, auto-synced daily: + +```bash +vim /etc/samba/smb.conf +systemctl restart smb +# sync-system.sh runs via cron, or run manually: /opt/argento/scripts/sync-system.sh +``` + +**Nginx configs** -- edit in repo, reload: + +```bash +vim /opt/argento/docker/nginx/conf.d/.conf +docker exec nginx nginx -t +docker exec nginx nginx -s reload +git add . && git commit && git push +``` + +**Adding a new app:** + +1. Create `docker//compose.yaml` with `container_name` and `shared` network +2. `docker compose up -d` +3. `./scripts/issue-cert.sh ` +4. `cp docker/nginx/conf.d/_template.conf.example docker/nginx/conf.d/.conf` and fill in placeholders +5. `docker exec nginx nginx -t && docker exec nginx nginx -s reload` +6. Commit and push + +## Recovery + +See [RUNBOOK.md](RUNBOOK.md) for full disaster recovery steps. + +## What's protected where + +| Data | Protection | Recovery | +|------|-----------|----------| +| Configs (compose, nginx, system) | Git (Gitea + GitHub mirror) | `git clone` | +| Nextcloud files | ZFS mirror + sanoid snapshots + USB | `rsync` from USB | +| Media library | ZFS raidz2 + sanoid snapshots + USB | `rsync` from USB | +| Nextcloud DB (PostgreSQL) | ZFS dataset + `pg_dumpall` + USB | `rsync` from USB | +| Garage S3 data | 2-node replication + USB | Replication or USB | +| Gitea repos + DB | SQLite `.backup` + USB | `rsync` from USB | +| SSL certificates | Re-issued from Let's Encrypt | `./scripts/issue-cert.sh` | diff --git a/RUNBOOK.md b/RUNBOOK.md new file mode 100644 index 0000000..054cc5b --- /dev/null +++ b/RUNBOOK.md @@ -0,0 +1,231 @@ +# Argento Recovery Runbook + +Full top-to-bottom rebuild. Follow these steps in order. + +## 1. Install Rocky Linux 9 + +Standard minimal install on NVMe. + +## 2. Repos and base packages + +```bash +# EPEL +dnf install -y epel-release + +# Docker CE +dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo + +# ZFS — must use zfs-testing repo (stable repo's version can't import the existing pools) +dnf install -y https://zfsonlinux.org/epel/zfs-release-2-3$(rpm --eval "%{dist}").noarch.rpm +dnf config-manager --enable zfs-testing +dnf install -y kernel-devel zfs # uses DKMS — kernel-devel is required + +# NVIDIA (CUDA repo — provides nvidia-driver and container toolkit) +dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo + +# Install everything +# Sanoid (not in EPEL — COPR repo, see https://github.com/jimsalterjrs/sanoid for manual install) +dnf copr enable -y orrisroot/zfs + +dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-buildx-plugin \ + certbot msmtp smartmontools \ + jq git sanoid samba \ + nvidia-driver nvidia-container-toolkit \ + cockpit + +# Load ZFS kernel module +modprobe zfs + +# Configure NVIDIA container runtime for Docker +nvidia-ctk runtime configure --runtime=docker + +# Set up msmtp as sendmail replacement (for cron MAILTO and smartd alerts) +ln -sf /usr/bin/msmtp /usr/sbin/sendmail +``` + +## 3. Enable services + +```bash +systemctl enable --now docker smb firewalld cockpit.socket sshd smartd +systemctl enable --now zfs-import-cache zfs-mount zfs-share zfs-zed +systemctl enable --now sanoid.timer +``` + +## 4. Firewall + +```bash +cp /opt/argento/system/firewalld/public.xml /etc/firewalld/zones/public.xml +firewall-cmd --reload +``` + +## 5. Mount USB backup drive + +```bash +echo 'LABEL=argento-backup /mnt/backup ext4 defaults,nofail 0 2' >> /etc/fstab +mkdir -p /mnt/backup +mount /mnt/backup +``` + +## 6. Get the repo + +```bash +git clone https://github.com//argento-server.git /opt/argento + +# If GitHub is unavailable, clone from the Gitea bare repo on the USB backup: +# git clone /mnt/backup/argento/docker/gitea/data/git/repositories/timothykim/argento.git /opt/argento +``` + +Clone first -- ZFS datasets will mount into this directory tree in the next step. + +## 7. ZFS pools + data restore + +**If ZFS disks are intact (OS drive failure only)** -- the common case: + +```bash +# ZFS stores pool metadata on the disks — import auto-detects them +# Run "zpool import" (no args) first to list available pools +zpool import nextcloud +zpool import threeteras + +# Verify mountpoints (update if the old /opt/docker/ paths are still set): +zfs set mountpoint=/opt/argento/docker/nextcloud/data nextcloud/data +zfs set mountpoint=/opt/argento/smb/media threeteras/media +zfs set mountpoint=/opt/argento/docker/garage/data threeteras/garage + +zpool status # verify pools are ONLINE + +# Restore non-ZFS app data (Gitea, Minecraft, Jellyfin config) from USB: +rsync -a --exclude='docker/nextcloud/data/' --exclude='docker/garage/data/' --exclude='smb/' \ + /mnt/backup/argento/ /opt/argento/ +``` + +**If ZFS disks are lost (full rebuild):** + +**Important:** The pools use feature flags that require ZFS from the `zfs-testing` repo (step 2). If `zpool import` fails with a feature flag error, verify `zfs-testing` is enabled and you have ZFS 2.4.0+. + +```bash +# Find disk IDs (use these instead of /dev/sdX — they're stable across reboots): +ls -l /dev/disk/by-id/ | grep -v part + +# Nextcloud pool — mirror of 2x 7.3TB drives +zpool create nextcloud mirror /dev/disk/by-id/ /dev/disk/by-id/ +zfs set compression=lz4 nextcloud +zfs create -o mountpoint=/opt/argento/docker/nextcloud/data nextcloud/data + +# Threeteras pool — raidz2 of 6x 2.7TB drives +zpool create threeteras raidz2 /dev/disk/by-id/ ... /dev/disk/by-id/ +zfs set compression=lz4 threeteras +zfs create -o mountpoint=/opt/argento/smb/media threeteras/media +zfs create -o mountpoint=/opt/argento/docker/garage/data threeteras/garage + +zpool status # verify pools are ONLINE + +# Restore everything from USB (ZFS mountpoints are set, data lands in the right place): +rsync -a /mnt/backup/argento/ /opt/argento/ +``` + +## 8. System configs + +```bash +cp /opt/argento/system/docker-daemon.json /etc/docker/daemon.json +cp /opt/argento/system/smb.conf /etc/samba/smb.conf +cp /opt/argento/system/sanoid.conf /etc/sanoid/sanoid.conf +cp /opt/argento/system/msmtprc /etc/msmtprc +chmod 600 /etc/msmtprc +cp /opt/argento/system/smartd.conf /etc/smartd.conf +systemctl restart docker smb smartd +crontab /opt/argento/system/root-crontab +``` + +## 9. Nginx + SSL + +```bash +# Create shared network +docker network create shared + +# Move SSL confs aside (nginx can't load them — cert files don't exist yet) +# HTTP-only confs (minecraft) and non-SSL files (.inc, .example) can stay +cd /opt/argento/docker/nginx +mkdir -p conf.d/.hold certbot/www +for f in conf.d/*.conf; do + grep -q ssl_certificate "$f" && mv "$f" conf.d/.hold/ +done + +# Start nginx (only HTTP-only confs and the default catch-all are loaded) +docker compose up -d + +# Issue certs (add any new domains that have been added since this was written) +/opt/argento/scripts/issue-cert.sh \ + nextcloud.timothykim.net git.timothykim.net jellyfin.timothykim.net \ + argento.timothykim.net garage.timothykim.net + +# Restore SSL confs now that certs exist +mv conf.d/.hold/*.conf conf.d/ +rmdir conf.d/.hold +docker exec nginx nginx -t && docker exec nginx nginx -s reload + +# Set up cert renewal +mkdir -p /etc/letsencrypt/renewal-hooks/deploy +cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK' +#!/bin/bash +docker exec nginx nginx -s reload +HOOK +chmod +x /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh +systemctl enable --now certbot-renew.timer + +# Verify renewal works +certbot renew --dry-run +``` + +## 10. Docker apps + +Deploy each app (nginx is already running and will return 502 until they start): + +```bash +# Add any new apps that have been added since this was written +for app in nextcloud gitea jellyfin garage minecraft; do + cd /opt/argento/docker/$app + docker compose pull + docker compose build + docker compose up -d + echo "$app started" +done +docker ps # verify all containers are healthy +``` + +## 11. Router + +Set up port forwarding on the home router: + +- 80 -> 192.168.2.122:80 +- 443 -> 192.168.2.122:443 +- 222 -> 192.168.2.122:222 (Gitea SSH) +- 3901 -> 192.168.2.122:3901 (Garage RPC) +- 25500-25600 -> 192.168.2.122:25500-25600 (Minecraft) + +## 12. DNS + +CNAME records pointing to the DDNS hostname. These don't need updating when the IP changes -- only the DDNS record does. + +| Record | Type | Value | +|--------|------|-------| +| `nextcloud.timothykim.net` | CNAME | `argento.ddns.net` | +| `git.timothykim.net` | CNAME | `argento.ddns.net` | +| `jellyfin.timothykim.net` | CNAME | `argento.ddns.net` | +| `argento.timothykim.net` | CNAME | `argento.ddns.net` | +| `garage.timothykim.net` | CNAME | `argento.ddns.net` | +| `minecraft.thekims.family` | CNAME | `argento.ddns.net` | + +## 13. Verify + +```bash +curl -I https://nextcloud.timothykim.net +curl -I https://git.timothykim.net +curl -I https://jellyfin.timothykim.net +curl -I https://argento.timothykim.net +curl -I https://garage.timothykim.net +curl -I http://minecraft.thekims.family + +# Verify SMART, ZFS, and disk space are healthy +/opt/argento/scripts/disk-health-check.sh +``` diff --git a/docker/garage/.gitignore b/docker/garage/.gitignore new file mode 100644 index 0000000..3466aed --- /dev/null +++ b/docker/garage/.gitignore @@ -0,0 +1,4 @@ +data/ +meta/ +.env + diff --git a/docker/garage/compose.yaml b/docker/garage/compose.yaml new file mode 100644 index 0000000..8dfb8d5 --- /dev/null +++ b/docker/garage/compose.yaml @@ -0,0 +1,39 @@ +services: + garage: + image: dxflrs/garage:v2.2.0 + container_name: garage + restart: unless-stopped + networks: + - shared + ports: + - '3900:3900' + - '3901:3901' + - '3902:3902' + - '3903:3903' + volumes: + - ./data:/var/lib/garage/data + - ./meta:/var/lib/garage/meta + - ./garage.toml:/etc/garage.toml:ro + environment: + GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET} + GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN} + + garage-webui: + image: khairul169/garage-webui + container_name: garage-webui + restart: unless-stopped + networks: + - shared + environment: + API_BASE_URL: http://garage:3903 + S3_ENDPOINT_URL: http://garage:3900 + S3_REGION: garage + AUTH_USER_PASS: ${AUTH_USER_PASS} + API_ADMIN_KEY: ${GARAGE_ADMIN_TOKEN} + volumes: + - ./garage.toml:/etc/garage.toml:ro + +networks: + shared: + external: true + diff --git a/docker/garage/garage.toml b/docker/garage/garage.toml new file mode 100644 index 0000000..5e5ad7c --- /dev/null +++ b/docker/garage/garage.toml @@ -0,0 +1,20 @@ +metadata_dir = "/var/lib/garage/meta" +data_dir = "/var/lib/garage/data" + +db_engine = "sqlite" +replication_factor = 2 +compression_level = 1 + +rpc_bind_addr = "[::]:3901" +rpc_public_addr = "garage.timothykim.net:3901" +rpc_secret = "" + +[s3_api] +s3_region = "garage" +api_bind_addr = "[::]:3900" + +[s3_web] +bind_addr = "[::]:3902" +root_domain = ".web.garage" +index = "index.html" + diff --git a/docker/gitea/.gitignore b/docker/gitea/.gitignore new file mode 100644 index 0000000..05aec79 --- /dev/null +++ b/docker/gitea/.gitignore @@ -0,0 +1,4 @@ +data/ +runner-data/ +.env + diff --git a/docker/gitea/compose.yaml b/docker/gitea/compose.yaml new file mode 100644 index 0000000..2d45f22 --- /dev/null +++ b/docker/gitea/compose.yaml @@ -0,0 +1,43 @@ +name: gitea + +networks: + gitea: + external: false + shared: + external: true + +services: + server: + image: docker.gitea.com/gitea:latest + container_name: gitea + restart: unless-stopped + environment: + - USER_UID=1000 + - USER_GID=1000 + networks: + - gitea + - shared + volumes: + - ./data:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + ports: + - "222:22" + + runner: + image: gitea/act_runner:latest + container_name: gitea_runner + restart: unless-stopped + depends_on: + - server + networks: + - gitea + environment: + - GITEA_INSTANCE_URL=https://git.timothykim.net + - GITEA_RUNNER_REGISTRATION_TOKEN=${GITEA_RUNNER_REGISTRATION_TOKEN} + - CONFIG_FILE=/config.yaml + volumes: + - ./runner-data:/data + - /var/run/docker.sock:/var/run/docker.sock + - ./runner-config.yaml:/config.yaml:ro + diff --git a/docker/gitea/runner-config.yaml b/docker/gitea/runner-config.yaml new file mode 100644 index 0000000..c16c493 --- /dev/null +++ b/docker/gitea/runner-config.yaml @@ -0,0 +1,25 @@ +log: + level: info + +runner: + file: .runner + capacity: 1 + env_file: .env + timeout: 3h + insecure: false + +cache: + enabled: true + dir: "" + +container: + network: "gitea_gitea" + extra_hosts: + - "git.timothykim.net:host-gateway" + privileged: false + valid_volumes: + - "**" + +host: + workdir_parent: "" + diff --git a/docker/jellyfin/.gitignore b/docker/jellyfin/.gitignore new file mode 100644 index 0000000..393a67c --- /dev/null +++ b/docker/jellyfin/.gitignore @@ -0,0 +1,4 @@ +cache/ +config/ +jellyfin-monitor.log + diff --git a/docker/jellyfin/compose.yaml b/docker/jellyfin/compose.yaml new file mode 100644 index 0000000..c371efa --- /dev/null +++ b/docker/jellyfin/compose.yaml @@ -0,0 +1,34 @@ +services: + jellyfin: + image: jellyfin/jellyfin + container_name: jellyfin + network_mode: 'host' + volumes: + - ./config:/config + - ./cache:/cache + - /opt/argento/smb/media/Movies:/media/movies + - /opt/argento/smb/media/TV:/media/tv + - /dev/dri:/dev/dri:rw + runtime: nvidia + #deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: all + # capabilities: [gpu] + restart: 'unless-stopped' + # Optional - alternative address used for autodiscovery + environment: + - JELLYFIN_PublishedServerUrl=https://jellyfin.timothykim.net + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,video,utility + # Optional - may be necessary for docker healthcheck to pass if running in host network mode + extra_hosts: + - 'host.docker.internal:host-gateway' + healthcheck: + test: ["CMD-SHELL", "nvidia-smi > /dev/null 2>&1 || exit 1"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 30s diff --git a/docker/jellyfin/jellyfin-monitor.sh b/docker/jellyfin/jellyfin-monitor.sh new file mode 100755 index 0000000..3cd02da --- /dev/null +++ b/docker/jellyfin/jellyfin-monitor.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Check GPU status in Jellyfin container +output=$(docker exec jellyfin nvidia-smi 2>&1) +exit_code=$? + +# Log file location +LOG_FILE="/opt/docker/jellyfin/jellyfin-monitor.log" + +# Check if command failed or output contains "error" (case insensitive) +if [ $exit_code -ne 0 ] || echo "$output" | grep -iq "error"; then + echo "$(date): GPU error detected." | tee -a "$LOG_FILE" + echo $output | tee -a "$LOG_FILE" + echo "$(date): Restarting Jellyfin..." | tee -a "$LOG_FILE" + cd /opt/docker/jellyfin/ + docker compose restart + echo "$(date): Jellyfin restarted successfully" | tee -a "$LOG_FILE" +else + echo "$(date): GPU status OK" +fi + diff --git a/docker/minecraft/.gitignore b/docker/minecraft/.gitignore new file mode 100644 index 0000000..ddedee8 --- /dev/null +++ b/docker/minecraft/.gitignore @@ -0,0 +1,3 @@ +daemon/ +web/ + diff --git a/docker/minecraft/compose.yaml b/docker/minecraft/compose.yaml new file mode 100644 index 0000000..7b99771 --- /dev/null +++ b/docker/minecraft/compose.yaml @@ -0,0 +1,35 @@ +services: + web: + image: githubyumao/mcsmanager-web:latest + container_name: mcsmanager-web + restart: unless-stopped + networks: + - shared + extra_hosts: + - "minecraft.thekims.family:host-gateway" + volumes: + - /etc/localtime:/etc/localtime:ro + - ./web/data:/opt/mcsmanager/web/data + - ./web/logs:/opt/mcsmanager/web/logs + + daemon: + image: githubyumao/mcsmanager-daemon:latest + container_name: mcsmanager-daemon + restart: unless-stopped + networks: + - shared + expose: + - "24444" + ports: + - "25500-25600:25500-25600" + environment: + - MCSM_DOCKER_WORKSPACE_PATH=./daemon/data/InstanceData + volumes: + - /etc/localtime:/etc/localtime:ro + - ./daemon/data:/opt/mcsmanager/daemon/data + - ./daemon/logs:/opt/mcsmanager/daemon/logs + - /var/run/docker.sock:/var/run/docker.sock + +networks: + shared: + external: true diff --git a/docker/nextcloud/.gitignore b/docker/nextcloud/.gitignore new file mode 100644 index 0000000..54a58fd --- /dev/null +++ b/docker/nextcloud/.gitignore @@ -0,0 +1,5 @@ +app/ +certs/ +data/ +db/ +.env diff --git a/docker/nextcloud/Dockerfile b/docker/nextcloud/Dockerfile new file mode 100644 index 0000000..d8ebf7b --- /dev/null +++ b/docker/nextcloud/Dockerfile @@ -0,0 +1,12 @@ +FROM nextcloud:latest + +# Install smbclient and required dependencies +RUN set -x && apt-get update \ + # PHP Module smbclient + && apt-get install -y smbclient libsmbclient-dev \ + && pecl install smbclient \ + && docker-php-ext-enable smbclient \ + # ffmpeg + && apt-get install -y ffmpeg \ + && apt-get clean \ + diff --git a/docker/nextcloud/compose.yaml b/docker/nextcloud/compose.yaml new file mode 100644 index 0000000..fe813b4 --- /dev/null +++ b/docker/nextcloud/compose.yaml @@ -0,0 +1,97 @@ +services: + nextcloud: +# image: nextcloud:latest + build: + context: . + dockerfile: Dockerfile + container_name: nextcloud + restart: unless-stopped + depends_on: + - postgres + - redis + - imaginary + volumes: + - ./app:/var/www/html + - ./data:/var/www/html/data + environment: + - POSTGRES_DB=nextcloud + - POSTGRES_USER=nextcloud + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_HOST=postgres + - NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.timothykim.net + - REDIS_HOST=redis + - SMTP_HOST=smtp.fastmail.com + - SMTP_SECURE=ssl + - SMTP_PORT=465 + - SMTP_AUTHTYPE=LOGIN + - SMTP_NAME=timothykim@fastmail.fm + - SMTP_PASSWORD=${SMTP_PASSWORD} + - MAIL_FROM_ADDRESS=nextcloud.timothykim.net + - MAIL_DOMAIN=fastmail.com + - PHP_MEMORY_LIMIT=8G + - PHP_UPLOAD_LIMIT=20G + - APACHE_BODY_LIMIT=0 + networks: + - nextcloud_net + - internal_net + - shared + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost/status.php"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 120s + + appapi-harp: + image: ghcr.io/nextcloud/nextcloud-appapi-harp:release + container_name: appapi-harp + hostname: appapi-harp + restart: unless-stopped + depends_on: + nextcloud: + condition: service_healthy + environment: + - HP_SHARED_KEY=${HP_SHARED_KEY} + - NC_INSTANCE_URL=https://nextcloud.timothykim.net + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./certs:/certs + networks: + - nextcloud_net + + postgres: + image: postgres:17-alpine + container_name: nextcloud-db + restart: unless-stopped + volumes: + - ./db:/var/lib/postgresql/data + environment: + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_USER=nextcloud + - POSTGRES_DB=nextcloud + networks: + - internal_net + + redis: + image: redis:alpine + container_name: nextcloud-redis + restart: unless-stopped + networks: + - internal_net + + + imaginary: + image: nextcloud/aio-imaginary:latest + container_name: nextcloud-imaginary + restart: unless-stopped + environment: + - PORT=9000 + networks: + - internal_net + +networks: + internal_net: + nextcloud_net: + external: true + shared: + external: true diff --git a/docker/nextcloud/upgrade.sh b/docker/nextcloud/upgrade.sh new file mode 100755 index 0000000..97db6c2 --- /dev/null +++ b/docker/nextcloud/upgrade.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +echo "Pulling latest Nextcloud image..." +docker pull nextcloud:latest + +echo "Stopping containers..." +docker compose down + +echo "Rebuilding Nextcloud container..." +docker compose build --no-cache nextcloud + +echo "Starting containers..." +docker compose up -d + +echo "Waiting for Nextcloud to be ready..." +sleep 10 + +echo "Running Nextcloud upgrade..." +docker compose exec -u www-data nextcloud php occ upgrade + +echo "Checking status..." +docker compose exec -u www-data nextcloud php occ status + +echo "Running checks..." +docker compose exec -u www-data nextcloud php occ check + +echo "Adding missing database indices..." +docker compose exec -u www-data nextcloud php occ db:add-missing-indices + +echo "Nextcloud update complete!" diff --git a/docker/nginx/compose.yaml b/docker/nginx/compose.yaml new file mode 100644 index 0000000..3ce47ed --- /dev/null +++ b/docker/nginx/compose.yaml @@ -0,0 +1,22 @@ +services: + nginx: + image: nginx:alpine + restart: unless-stopped + container_name: nginx + networks: + - shared + extra_hosts: + - "host.docker.internal:host-gateway" + ports: + - '80:80' + - '443:443' + - '24444:24444' # for mcsmanager-daemon + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + - ./conf.d:/etc/nginx/conf.d:ro + - /etc/letsencrypt:/etc/letsencrypt:ro + - ./certbot/www:/var/www/certbot:ro + +networks: + shared: + name: shared diff --git a/docker/nginx/conf.d/00-default.conf b/docker/nginx/conf.d/00-default.conf new file mode 100644 index 0000000..aa8bcf0 --- /dev/null +++ b/docker/nginx/conf.d/00-default.conf @@ -0,0 +1,20 @@ +# Drop connections with unknown/missing Host headers +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + return 444; +} + +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + http2 on; + server_name _; + + ssl_certificate /etc/letsencrypt/live/argento.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/argento.timothykim.net/privkey.pem; + + return 444; +} + diff --git a/docker/nginx/conf.d/_template.conf.example b/docker/nginx/conf.d/_template.conf.example new file mode 100644 index 0000000..2a9a5fa --- /dev/null +++ b/docker/nginx/conf.d/_template.conf.example @@ -0,0 +1,48 @@ +# Template for new app confs. +# 1. Copy this file: cp _template.conf.example .conf +# 2. Replace DOMAIN with the actual domain +# 3. Replace UPSTREAM_NAME and CONTAINER:PORT with the app's container and port +# 4. Run issue-cert.sh if cert doesn't exist yet +# 5. Copy to argento, nginx -t && nginx -s reload + +server { + listen 80; + listen [::]:80; + server_name DOMAIN; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name DOMAIN; + + ssl_certificate /etc/letsencrypt/live/DOMAIN/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/DOMAIN/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + # Use a variable so nginx starts even if the upstream is down + set $upstream_UPSTREAM_NAME http://CONTAINER:PORT; + + location / { + proxy_pass $upstream_UPSTREAM_NAME; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Uncomment if the app uses WebSockets + # proxy_http_version 1.1; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection "upgrade"; + } +} diff --git a/docker/nginx/conf.d/argento.timothykim.net.conf b/docker/nginx/conf.d/argento.timothykim.net.conf new file mode 100644 index 0000000..b794720 --- /dev/null +++ b/docker/nginx/conf.d/argento.timothykim.net.conf @@ -0,0 +1,46 @@ +# Cockpit runs directly on the host (not Docker), proxied via host.docker.internal. +# Cockpit serves its own self-signed TLS on port 9090, so proxy_pass uses https://. +# nginx defaults proxy_ssl_verify to off, so the self-signed cert is accepted. +server { + listen 80; + listen [::]:80; + server_name argento.timothykim.net; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name argento.timothykim.net; + + ssl_certificate /etc/letsencrypt/live/argento.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/argento.timothykim.net/privkey.pem; + + # Security headers are inlined (not included) because Cockpit uses iframes + # internally and X-Frame-Options: DENY from security-headers.inc breaks it + add_header Strict-Transport-Security "max-age=63072000; preload" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + add_header Permissions-Policy "geolocation=(), microphone=(), camera=()" always; + + location / { + proxy_pass https://host.docker.internal:9090; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Cockpit uses WebSockets for terminal, system monitoring, and all UI interaction + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} diff --git a/docker/nginx/conf.d/garage.timothykim.net.conf b/docker/nginx/conf.d/garage.timothykim.net.conf new file mode 100644 index 0000000..2c78f05 --- /dev/null +++ b/docker/nginx/conf.d/garage.timothykim.net.conf @@ -0,0 +1,35 @@ +server { + listen 80; + listen [::]:80; + server_name garage.timothykim.net; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name garage.timothykim.net; + + ssl_certificate /etc/letsencrypt/live/garage.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/garage.timothykim.net/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + set $upstream_garage http://garage-webui:3909; + + location / { + proxy_pass $upstream_garage; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/docker/nginx/conf.d/git.timothykim.net.conf b/docker/nginx/conf.d/git.timothykim.net.conf new file mode 100644 index 0000000..59e1378 --- /dev/null +++ b/docker/nginx/conf.d/git.timothykim.net.conf @@ -0,0 +1,41 @@ +server { + listen 80; + listen [::]:80; + server_name git.timothykim.net; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name git.timothykim.net; + + ssl_certificate /etc/letsencrypt/live/git.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/git.timothykim.net/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + # Git push/LFS operations can be large + client_max_body_size 100M; + + set $upstream_gitea http://gitea:3000; + + location / { + proxy_pass $upstream_gitea; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # Gitea API token auth (used by CI runners and git CLI) arrives as + # Authorization: Basic/Bearer — pass it through explicitly + proxy_set_header Authorization $http_authorization; + } +} diff --git a/docker/nginx/conf.d/jellyfin.timothykim.net.conf b/docker/nginx/conf.d/jellyfin.timothykim.net.conf new file mode 100644 index 0000000..4e1f347 --- /dev/null +++ b/docker/nginx/conf.d/jellyfin.timothykim.net.conf @@ -0,0 +1,64 @@ +# Jellyfin runs on the host network (not Docker shared network) because it +# needs direct access to hardware transcoding devices. Proxied via +# host.docker.internal instead of container name. +server { + listen 80; + listen [::]:80; + server_name jellyfin.timothykim.net; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name jellyfin.timothykim.net; + + ssl_certificate /etc/letsencrypt/live/jellyfin.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/jellyfin.timothykim.net/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + # Poster and metadata image uploads + client_max_body_size 20M; + + location / { + proxy_pass http://host.docker.internal:8096; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Disable buffering when the nginx proxy gets very resource heavy upon streaming + proxy_buffering off; + + # Jellyfin uses WebSockets for real-time playback state and remote control + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Dedicated WebSocket endpoint for SyncPlay and client communication + location /socket { + proxy_pass http://host.docker.internal:8096; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # sync play compatibility? + proxy_set_header X-Forwarded-Protocol $scheme; + proxy_set_header X-Forwarded-Host $http_host; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } +} diff --git a/docker/nginx/conf.d/minecraft.thekims.family.conf b/docker/nginx/conf.d/minecraft.thekims.family.conf new file mode 100644 index 0000000..a1e0693 --- /dev/null +++ b/docker/nginx/conf.d/minecraft.thekims.family.conf @@ -0,0 +1,71 @@ +# HTTP-only: HTTPS causes WebSocket/CORS errors with the MCSManager daemon connection. +# No ACME challenge block needed since there's no cert to renew. +server { + listen 80; + listen [::]:80; + server_name minecraft.thekims.family; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name minecraft.thekims.family; + + ssl_certificate /etc/letsencrypt/live/minecraft.thekims.family/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/minecraft.thekims.family/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + set $mcsmanager_web http://mcsmanager-web:23333; + + location / { + proxy_pass $mcsmanager_web; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + + # MCSManager uses WebSockets for real-time console/terminal + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + +} + +server { + listen 24444 ssl; + listen [::]:24444 ssl; + server_name minecraft.thekims.family; + + ssl_certificate /etc/letsencrypt/live/minecraft.thekims.family/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/minecraft.thekims.family/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + set $mcsmanager_daemon http://mcsmanager-daemon:24444; + + location / { + proxy_pass $mcsmanager_daemon; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # MCSManager uses WebSockets for real-time console/terminal + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + +} diff --git a/docker/nginx/conf.d/nextcloud.timothykim.net.conf b/docker/nginx/conf.d/nextcloud.timothykim.net.conf new file mode 100644 index 0000000..086c20d --- /dev/null +++ b/docker/nginx/conf.d/nextcloud.timothykim.net.conf @@ -0,0 +1,49 @@ +server { + listen 80; + listen [::]:80; + server_name nextcloud.timothykim.net; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; + server_name nextcloud.timothykim.net; + + ssl_certificate /etc/letsencrypt/live/nextcloud.timothykim.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/nextcloud.timothykim.net/privkey.pem; + + include /etc/nginx/conf.d/security-headers.inc; + + # Nextcloud file uploads can be up to 20G + client_max_body_size 20G; + # Stream uploads directly to Nextcloud instead of buffering to disk first + proxy_request_buffering off; + + set $upstream_nextcloud http://nextcloud:80; + + location / { + proxy_pass $upstream_nextcloud; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Nextcloud requires these redirects for CalDAV/CardDAV client discovery + location /.well-known/carddav { + return 301 /remote.php/dav; + } + + location /.well-known/caldav { + return 301 /remote.php/dav; + } +} diff --git a/docker/nginx/conf.d/security-headers.inc b/docker/nginx/conf.d/security-headers.inc new file mode 100644 index 0000000..d4d3bc8 --- /dev/null +++ b/docker/nginx/conf.d/security-headers.inc @@ -0,0 +1,6 @@ +add_header Strict-Transport-Security "max-age=63072000; preload" always; +add_header X-Content-Type-Options "nosniff" always; +add_header X-Frame-Options "DENY" always; +add_header Referrer-Policy "strict-origin-when-cross-origin" always; +add_header Permissions-Policy "geolocation=(), microphone=(), camera=()" always; + diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf new file mode 100644 index 0000000..cd882e6 --- /dev/null +++ b/docker/nginx/nginx.conf @@ -0,0 +1,27 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_session_tickets off; + resolver 127.0.0.11 valid=30s; + resolver_timeout 5s; + + include /etc/nginx/conf.d/*.conf; +} + diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..ba78915 --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Daily USB backup. Dumps DBs for consistency, then rsyncs everything. +set -euo pipefail + +FAILURES=() + +# Check the USB drive is actually mounted +if ! mountpoint -q /mnt/backup; then + echo "FATAL: /mnt/backup is not mounted" >&2 + exit 1 +fi + +# Dump Nextcloud PostgreSQL for consistency +if docker exec nextcloud-db pg_dumpall -U nextcloud > /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp; then + mv /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp /opt/argento/docker/nextcloud/db/pg_dumpall.sql +else + rm -f /opt/argento/docker/nextcloud/db/pg_dumpall.sql.tmp + FAILURES+=("nextcloud pg_dump") +fi + +# Dump Gitea SQLite for consistency +sqlite3 /opt/argento/docker/gitea/data/gitea/gitea.db \ + ".backup /opt/argento/docker/gitea/data/gitea/gitea.db.bak" \ + || FAILURES+=("gitea sqlite backup") + +# Dump Jellyfin SQLite DBs for consistency +for db in jellyfin library; do + sqlite3 /opt/argento/docker/jellyfin/config/data/${db}.db \ + ".backup /opt/argento/docker/jellyfin/config/data/${db}.db.bak" \ + || FAILURES+=("jellyfin ${db} sqlite backup") +done + +# Single rsync covers everything: configs, app data, media, DB dumps +RSYNC_OPTS="-a --delete" +[ -t 1 ] && RSYNC_OPTS="$RSYNC_OPTS --info=progress2" +rsync $RSYNC_OPTS /opt/argento/ /mnt/backup/argento/ \ + || FAILURES+=("rsync argento") + +if [ ${#FAILURES[@]} -gt 0 ]; then + echo "BACKUP FAILED: ${FAILURES[*]}" >&2 + exit 1 +fi + +echo "Backup completed successfully at $(date)" diff --git a/scripts/disk-health-check.sh b/scripts/disk-health-check.sh new file mode 100755 index 0000000..3c37548 --- /dev/null +++ b/scripts/disk-health-check.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# server-health-check.sh — Argento health monitoring +# Checks: SMART disk health, ZFS pool status, disk space +# Outputs to stdout by default, or sends email with --email. +# Use --error-only to suppress output when everything is healthy. +# +# Usage: +# ./server-health-check.sh # full report to stdout +# ./server-health-check.sh --error-only # only print errors (silent if healthy) +# ./server-health-check.sh --email # full report to email +# ./server-health-check.sh --error-only --email # email errors (no email if healthy) +# +# Install: +# chmod +x /opt/server-health-check.sh +# crontab -e: +# 0 6 * * * /opt/server-health-check.sh --error-only --email # daily alert +# 0 8 * * 1 /opt/server-health-check.sh --email # weekly report +# +# Dependencies: smartmontools, zfs, msmtp (as sendmail) + +set -uo pipefail + +MAILTO="timothykim@fastmail.fm" +HOSTNAME=$(hostname) +SPACE_THRESHOLD=85 # percent used — alert above this + +ERROR_ONLY=false +USE_EMAIL=false +for arg in "$@"; do + case "$arg" in + --error-only) ERROR_ONLY=true ;; + --email) USE_EMAIL=true ;; + *) echo "Unknown option: $arg" >&2; exit 1 ;; + esac +done + +PROBLEMS="" +STATUS="" + +# ─── SMART Disk Health (SATA/SAS) ─────────────────────────────── +STATUS+="SMART (SATA):\n" +for disk in /dev/sd?; do + [ -b "$disk" ] || continue + + # smartctl returns non-zero exit codes as a bitmask for disk + # conditions (standby, threshold exceeded, etc.), so don't + # treat non-zero as a script error — capture output and check it. + health=$(smartctl -H "$disk" 2>/dev/null | grep "test result" | awk -F': ' '{print $2}') || true + health="${health:-UNKNOWN}" + temp=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 194 || $1 == 190 {print $10; exit}') || true + temp="${temp:-?}" + hours=$(smartctl -A "$disk" 2>/dev/null | awk '$1 == 9 {print $10; exit}') || true + hours="${hours:-?}" + model=$(smartctl -i "$disk" 2>/dev/null | awk -F': +' '/Device Model|Model Number/{print $2; exit}') || true + model="${model:-unknown}" + + STATUS+=" $disk ($model): $health, ${temp}C, ${hours}h\n" + + if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then + PROBLEMS+="[SMART] $disk: overall health check FAILED ($health)\n" + fi + + # Check critical SMART attributes + # 5 = Reallocated_Sector_Ct + # 187 = Reported_Uncorrect + # 197 = Current_Pending_Sector + # 198 = Offline_Uncorrectable + while IFS= read -r line; do + id=$(echo "$line" | awk '{print $1}') + name=$(echo "$line" | awk '{print $2}') + raw=$(echo "$line" | awk '{print $10}') + case "$id" in + 5|187|197|198) + if [ "$raw" -gt 0 ] 2>/dev/null; then + PROBLEMS+="[SMART] $disk: $name = $raw (should be 0)\n" + fi + ;; + esac + done < <(smartctl -A "$disk" 2>/dev/null | awk 'NR>7 && /^[[:space:]]*[0-9]/' || true) +done + +# ─── SMART Disk Health (NVMe) ─────────────────────────────────── +STATUS+="\nSMART (NVMe):\n" +for disk in /dev/nvme[0-9]*; do + # Match only controller devices (nvme0, nvme1), not namespaces (nvme0n1) + [[ "$disk" =~ ^/dev/nvme[0-9]+$ ]] || continue + [ -c "$disk" ] || continue + + ns="${disk}n1" # namespace device for smartctl + [ -b "$ns" ] || continue + + health=$(smartctl -H "$ns" 2>/dev/null | grep "SMART overall-health" | awk -F': ' '{print $2}') || true + health="${health:-UNKNOWN}" + model=$(smartctl -i "$ns" 2>/dev/null | awk -F': +' '/Model Number/{print $2; exit}') || true + model="${model:-unknown}" + temp=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Temperature:/{print $2; exit}') || true + temp="${temp:-?}" + hours=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Power On Hours/{print $2; exit}') || true + hours="${hours:-?}" + # NVMe percentage used — 100% means full rated write endurance consumed + pct_used=$(smartctl -A "$ns" 2>/dev/null | awk -F': +' '/Percentage Used/{print $2; exit}' | tr -d '%') || true + + STATUS+=" $ns ($model): $health, ${temp}, ${hours}h\n" + + if [ "$health" != "PASSED" ] && [ "$health" != "UNKNOWN" ]; then + PROBLEMS+="[SMART] $ns: overall health check FAILED ($health)\n" + fi + + if [ -n "$pct_used" ] && [ "$pct_used" -gt 90 ] 2>/dev/null; then + PROBLEMS+="[SMART] $ns: NVMe endurance ${pct_used}% used (>90%)\n" + fi +done + +# ─── ZFS Pool Health ───────────────────────────────────────────── +STATUS+="\nZFS Pools:\n" +while IFS= read -r pool; do + [ -n "$pool" ] || continue + state=$(zpool list -H -o health "$pool" 2>/dev/null) || true + pool_info=$(zpool list -H -o name,size,alloc,free,cap "$pool" 2>/dev/null) || true + STATUS+=" $pool_info $state\n" + + if [ "$state" != "ONLINE" ]; then + PROBLEMS+="[ZFS] Pool '$pool' state: $state\n" + fi + + # Check for errors via zpool status -p (parseable). + # Parse the per-vdev lines: columns are NAME STATE READ WRITE CKSUM. + # Flag any vdev that is not ONLINE, or has non-zero error counters. + while IFS= read -r vdev_line; do + vdev_state=$(echo "$vdev_line" | awk '{print $2}') + vdev_read=$(echo "$vdev_line" | awk '{print $3}') + vdev_write=$(echo "$vdev_line" | awk '{print $4}') + vdev_cksum=$(echo "$vdev_line" | awk '{print $5}') + vdev_name=$(echo "$vdev_line" | awk '{print $1}') + + if [ "$vdev_state" != "ONLINE" ] && [ "$vdev_state" != "state" ]; then + PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' state: $vdev_state\n" + fi + for val in "$vdev_read" "$vdev_write" "$vdev_cksum"; do + if [ "$val" -gt 0 ] 2>/dev/null; then + PROBLEMS+="[ZFS] Pool '$pool' vdev '$vdev_name' has errors (R:${vdev_read} W:${vdev_write} C:${vdev_cksum})\n" + break + fi + done + done < <(zpool status -p "$pool" 2>/dev/null | awk '/NAME.*STATE.*READ/{found=1; next} found && /^[[:space:]]+[^ ]/{print} /^$/{found=0}' || true) +done < <(zpool list -H -o name 2>/dev/null || true) + +# ─── Disk Space ────────────────────────────────────────────────── +STATUS+="\nDisk Space:\n" +while IFS= read -r line; do + [ -n "$line" ] || continue + usage=$(echo "$line" | awk '{print $5}' | tr -d '%') + mount=$(echo "$line" | awk '{print $6}') + size=$(echo "$line" | awk '{print $2}') + used=$(echo "$line" | awk '{print $3}') + avail=$(echo "$line" | awk '{print $4}') + STATUS+=" $mount: ${usage}% (${used}/${size}, ${avail} free)\n" + if [ "$usage" -gt "$SPACE_THRESHOLD" ] 2>/dev/null; then + PROBLEMS+="[SPACE] $mount is ${usage}% full\n" + fi +done < <(df -h --output=source,size,used,avail,pcent,target -x tmpfs -x devtmpfs -x overlay -x efivarfs 2>/dev/null | tail -n +2 || true) + +# ─── Output ────────────────────────────────────────────────────── +if [ "$ERROR_ONLY" = true ]; then + if [ -z "$PROBLEMS" ]; then + exit 0 + fi + SUBJECT="[ALERT] $HOSTNAME health check failed" + BODY="Health check found issues on $HOSTNAME at $(date):\n\n$PROBLEMS\n---\n" +else + if [ -n "$PROBLEMS" ]; then + SUBJECT="[REPORT] $HOSTNAME health — ISSUES FOUND" + else + SUBJECT="[REPORT] $HOSTNAME health — all clear" + fi + BODY="Health report for $HOSTNAME at $(date):\n\n" + if [ -n "$PROBLEMS" ]; then + BODY+="*** ISSUES ***\n\n$PROBLEMS\n" + fi + BODY+="--- Status ---\n\n$STATUS\n---\n" +fi + +if [ "$USE_EMAIL" = true ]; then + { + echo "Subject: $SUBJECT" + echo "From: $HOSTNAME " + echo "To: $MAILTO" + echo "" + printf '%b' "$BODY" + } | sendmail "$MAILTO" +else + printf '%b' "$BODY" +fi diff --git a/scripts/issue-cert.sh b/scripts/issue-cert.sh new file mode 100755 index 0000000..9f64f00 --- /dev/null +++ b/scripts/issue-cert.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Issue SSL certificates via certbot webroot through the nginx container. +# +# This script ONLY issues certs. It does not touch the real nginx confs. +# Nginx must be running (even with minimal config) to serve ACME challenges. +# +# Provisioning order: +# 1. Start nginx with no app confs (just security-headers.inc) +# 2. Run this script to issue certs +# 3. Start all app containers (so upstreams are resolvable) +# 4. Copy app confs into conf.d and reload nginx +# +# For adding a cert to a running server, apps are already up — just run +# this script, add the conf, and reload nginx. +# +# Usage: ./issue-cert.sh [domain...] +# Example: ./issue-cert.sh nextcloud.timothykim.net git.timothykim.net +set -euo pipefail + +CONF_DIR="/opt/argento/docker/nginx/conf.d" +WEBROOT="/opt/argento/docker/nginx/certbot/www" + +if [ $# -eq 0 ]; then + echo "Usage: $0 [domain...]" + exit 1 +fi + +mkdir -p "$WEBROOT" + +for domain in "$@"; do + echo "=== Issuing cert for $domain ===" + + # Write temporary HTTP-only config + cat > "$CONF_DIR/$domain.tmp.conf" < "$REPO/system/root-crontab" 2>/dev/null || true + +cd "$REPO" + +# Stage and check if anything changed +git add system/ +if git diff --cached --quiet; then + exit 0 +fi + +git commit -m "auto: sync system configs" +git push || echo "Push failed (remote ahead?) — committed locally, will push next time." diff --git a/system/daemon.json b/system/daemon.json new file mode 100644 index 0000000..3c40a98 --- /dev/null +++ b/system/daemon.json @@ -0,0 +1,8 @@ +{ + "runtimes": { + "nvidia": { + "args": [], + "path": "nvidia-container-runtime" + } + } +} \ No newline at end of file diff --git a/system/docker-daemon.json b/system/docker-daemon.json new file mode 100644 index 0000000..3c40a98 --- /dev/null +++ b/system/docker-daemon.json @@ -0,0 +1,8 @@ +{ + "runtimes": { + "nvidia": { + "args": [], + "path": "nvidia-container-runtime" + } + } +} \ No newline at end of file diff --git a/system/firewalld/public.xml b/system/firewalld/public.xml new file mode 100644 index 0000000..3f25509 --- /dev/null +++ b/system/firewalld/public.xml @@ -0,0 +1,24 @@ + + + Public + For use in public areas. You do not trust the other computers on networks to not harm your computer. Only selected incoming connections are accepted. + + + + + + + + + + + + + + + + + + + + diff --git a/system/msmtprc b/system/msmtprc new file mode 100644 index 0000000..63bcbc9 --- /dev/null +++ b/system/msmtprc @@ -0,0 +1,9 @@ +account default +host smtp.fastmail.com +port 465 +tls on +tls_starttls off +auth on +user timothykim@fastmail.fm +password REDACTED +from argento@fastmail.com diff --git a/system/root-crontab b/system/root-crontab new file mode 100644 index 0000000..7e7c966 --- /dev/null +++ b/system/root-crontab @@ -0,0 +1,8 @@ +MAILTO=timothykim@fastmail.com +*/5 * * * * docker exec -u www-data nextcloud php /var/www/html/cron.php +1 * * * * /opt/argento/docker/jellyfin/jellyfin-monitor.sh +0 2 * * * /opt/argento/scripts/sync-system.sh +0 3 * * * /opt/argento/scripts/backup.sh +0 3 * * 0 docker system prune -f --volumes > /dev/null +0 6 * * * /opt/argento/scripts/server-health-check.sh --error-only --email +0 8 * * 1 /opt/argento/scripts/server-health-check.sh --email diff --git a/system/sanoid.conf b/system/sanoid.conf new file mode 100644 index 0000000..dfd3786 --- /dev/null +++ b/system/sanoid.conf @@ -0,0 +1,21 @@ +[nextcloud/data] + use_template = production + hourly = 24 + daily = 30 + monthly = 3 + +[threeteras/media] + use_template = production + hourly = 0 + daily = 30 + monthly = 6 + +[threeteras/garage] + use_template = production + hourly = 0 + daily = 14 + monthly = 0 + +[template_production] + autosnap = yes + autoprune = yes diff --git a/system/smartd.conf b/system/smartd.conf new file mode 100644 index 0000000..e7e814a --- /dev/null +++ b/system/smartd.conf @@ -0,0 +1 @@ +DEVICESCAN -a -o on -S on -n standby,q -s (S/../.././02|L/../../6/03) -m timothykim@fastmail.com -M exec /usr/bin/msmtp diff --git a/system/smb.conf b/system/smb.conf new file mode 100644 index 0000000..e0c26ae --- /dev/null +++ b/system/smb.conf @@ -0,0 +1,50 @@ +# See smb.conf.example for a more detailed config file or +# read the smb.conf manpage. +# Run 'testparm' to verify the config is correct after +# you modified it. +# +# Note: +# SMB1 is disabled by default. This means clients without support for SMB2 or +# SMB3 are no longer able to connect to smbd (by default). + +[global] + workgroup = SAMBA + security = user + include = registry + passdb backend = tdbsam +# printing = cups +# printcap name = cups +# load printers = yes +# cups options = raw + +[media] + path = /opt/argento/smb/media + force group = sambagroup + #valid users = timothykim + browseable = yes + writable = yes + read only = no + guest ok = no + + +#[homes] +# comment = Home Directories +# valid users = %S, %D%w%S +# browseable = No +# read only = No +# inherit acls = Yes +# +#[printers] +# comment = All Printers +# path = /var/tmp +# printable = Yes +# create mask = 0600 +# browseable = No +# +#[print$] +# comment = Printer Drivers +# path = /var/lib/samba/drivers +# write list = @printadmin root +# force group = @printadmin +# create mask = 0664 +# directory mask = 0775 diff --git a/system/tracked-configs b/system/tracked-configs new file mode 100644 index 0000000..efa135c --- /dev/null +++ b/system/tracked-configs @@ -0,0 +1,6 @@ +/etc/samba/smb.conf:system/smb.conf +/etc/sanoid/sanoid.conf:system/sanoid.conf +/etc/docker/daemon.json:system/docker-daemon.json +/etc/firewalld/zones/public.xml:system/firewalld/public.xml +/etc/msmtprc:system/msmtprc +/etc/smartd.conf:system/smartd.conf