self-heal standalone cert renewal configs on provision

fix e2e tests for garage v2 and uptimerobot v3 api changes
remove kgfamily.com
2026-06-08 11:35:37 -04:00 · 2026-04-10 16:25:54 -04:00 · 2026-04-10 16:21:58 -04:00 · 2026-04-10 16:20:28 -04:00 · 2026-04-10 16:17:42 -04:00 · 2026-04-10 16:14:29 -04:00
51 changed files with 1580 additions and 607 deletions
@@ -0,0 +1,19 @@
+name: Deploy beszel
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/beszel/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-beszel
@@ -0,0 +1,19 @@
+name: Deploy carolpreschool.com
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/carolpreschool.com/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-carolpreschool.com
@@ -1,10 +1,12 @@
 name: Deploy garage

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/garage/**'
+      - '.gitea/workflows/deploy-garage.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-garage
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-garage
@@ -0,0 +1,19 @@
+name: Deploy goatcounter
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/goatcounter/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-goatcounter
@@ -1,10 +1,12 @@
 name: Deploy haanmind.net

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/haanmind.net/**'
+      - '.gitea/workflows/deploy-haanmind.net.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-haanmind.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-haanmind.net
@@ -1,10 +1,12 @@
 name: Deploy hantim.net

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/hantim.net/**'
+      - '.gitea/workflows/deploy-hantim.net.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hantim.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hantim.net
@@ -0,0 +1,19 @@
+name: Deploy hcsuzuki.music
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/hcsuzuki.music/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.music
@@ -1,10 +1,12 @@
 name: Deploy hcsuzuki.net

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/hcsuzuki.net/**'
+      - '.gitea/workflows/deploy-hcsuzuki.net.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.net
@@ -1,10 +1,12 @@
 name: Deploy nginx

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/nginx/**'
+      - '.gitea/workflows/deploy-nginx.yml'

 jobs:
  deploy:
@@ -13,17 +15,8 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-nginx
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-nginx

-  test:
-    needs: deploy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-      - run: pip install pytest
-      - run: pytest tests/ -v
@@ -1,10 +1,12 @@
 name: Deploy thekims.family

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/thekims.family/**'
+      - '.gitea/workflows/deploy-thekims.family.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-thekims.family
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-thekims.family
@@ -1,10 +1,12 @@
 name: Deploy timothykim.net

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'docker/timothykim.net/**'
+      - '.gitea/workflows/deploy-timothykim.yml'

 jobs:
  deploy:
@@ -13,6 +15,7 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-timothykim.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-timothykim.net
@@ -1,10 +1,12 @@
 name: Provision server

 on:
+  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - 'scripts/configure.sh'
+      - '.gitea/workflows/provision.yml'

 jobs:
  provision:
@@ -13,6 +15,7 @@ jobs:
      - name: Provision via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} provision
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} provision
@@ -5,3 +5,5 @@ docker/nginx/certs/
 docker/nginx/certbot/
 docker/garage/data/
 docker/garage/meta/
+SECURITY.md
+__pycache__/
@@ -91,8 +91,8 @@ client sends a command string (e.g., `deploy-example.com`), and `deploy.sh`
 parses it to determine the action:

 - `deploy-<app>` — deploy a specific app
- `deploy-nginx` — deploy nginx (with cert handling)
- `cert-<domain>` — issue a cert only
+- `deploy-nginx` — deploy nginx (fails if certs are missing)
+- `cert-<domain>` — issue a cert only (via `certbot --webroot`, zero downtime; subdomains like `garage.hantim.net` skip the `www.` variant)
 - `provision` — run `configure.sh`

 ## SSL certificate handling
@@ -101,18 +101,16 @@ Cert names are derived from nginx conf filenames (e.g., `example.com.conf` →
 cert `example.com`). All `server_name` values in the conf are included in the
 cert as SANs.

-**New domain (nginx not yet running):**
-1. `deploy.sh` detects missing cert files
-2. Stops nginx temporarily
-3. Issues cert via `certbot --standalone`
-4. Starts nginx
-5. On error, cleans up the temp conf file (trap)
+**New domain:** issue cert before deploying nginx config:
+1. Run `app.sh cert <domain>` (uses `certbot --webroot`, zero downtime)
+2. Deploy nginx config — `deploy.sh` verifies certs exist, fails if missing

-**New domain (nginx already running):**
-1. Issues cert via `certbot --webroot` using the ACME challenge directory
-2. Zero downtime
-
-**Renewal:** certbot timer/cron runs daily, deploy hook reloads nginx.
+**Renewal:** certbot timer/cron runs daily, deploy hook reloads nginx. All certs
+must use the `webroot` authenticator so nginx can keep port 80; certs issued by
+the old `--standalone` flow (removed in `de1d60b`) can never renew because nginx
+holds port 80. `configure.sh` self-heals this on every `provision`: it scans
+`/etc/letsencrypt/renewal/*.conf` and reissues any `standalone` cert via the
+`cert-<domain>` webroot path (best-effort; warns and exits non-zero on failure).

 ## `.env.keys` mechanism

@@ -140,6 +138,19 @@ If any secret is missing, deploy fails with an error (not a warning).
 - **One domain per conf file** — `deploy.sh` derives the cert name from the
  filename and collects all `server_name` values for that cert

+## GoatCounter (analytics)
+
+GoatCounter runs at `goatcounter.hantim.net` for privacy-friendly web analytics.
+
+**Script injection:** nginx injects the GoatCounter tracking script into all
+HTML responses via `sub_filter`. The shared include `conf.d/goatcounter.inc`
+is added to each site's main HTTPS server block. New sites get it automatically
+via `app.sh`.
+
+**Config:** `docker/goatcounter/compose.yaml` uses the `baethon/goatcounter`
+image. Password is stored in Bitwarden (`hantim-goatcounter-password`) and
+injected via the `.env.keys` mechanism.
+
 ## URL routing

 - All HTTP → HTTPS redirect
@@ -168,7 +179,7 @@ Client -> nginx (443) -> /media/ location -> garage:3902
 runs envsubst at container start. Values come from `.env` generated by the
 `.env.keys` mechanism.

-**Bucket setup** (done by `new-app.sh` via admin API v2):
+**Bucket setup** (done by `app.sh` via admin API v2):
 1. `POST /v2/CreateBucket` with `globalAlias: <domain>`
 2. `POST /v2/AllowBucketKey` to grant media-key read/write access
 3. `POST /v2/UpdateBucket` to enable website access
@@ -182,6 +193,7 @@ Dockerfile              # Copies static/ into nginx:alpine, applies nginx.conf
 nginx.conf              # Per-app HTTP config (port 80, routing, cache headers)
 static/                 # HTML/CSS/JS content
 static/media/           # Local media files (gitignored)
+media.sh                # Sync media with Garage (pull/push)
 .gitea/workflows/build.yml  # Build image, push to registry, SSH deploy
 ```

@@ -8,12 +8,12 @@ for technical deep-dives, USECASES.md for expected behaviors.
 - `scripts/bootstrap.sh` -- First-time server setup (manual, uses Bitwarden)
 - `scripts/configure.sh` -- Idempotent server config (firewall, certbot, start services; CI-safe)
 - `scripts/deploy.sh` -- SSH-triggered deploy + cert issuance + provision (deploy-*, cert-*, provision commands)
- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, all)
- `tools/new-app.sh` -- Wrapper for `app.sh all` (backwards compat)
- `tools/new-service.sh` -- Scaffold a new Docker Compose service (compose + workflow)
+- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, monitor, all)
+- `tools/service.sh` -- Manage Docker services by subcommand (dns, files, nginx, cert, all)
 - `tools/remove-app.sh` -- Remove a static site (local files + Gitea repo)
 - `docker/nginx/conf.d/` -- Per-app nginx server blocks
 - `docker/<domain>/compose.yml` -- Per-app compose files
+- `docker/goatcounter/` -- GoatCounter analytics (compose.yaml, .env.keys)
 - `docker/garage/` -- Garage S3 object storage (Dockerfile, garage.toml, .env.keys)
 - `.gitea/workflows/` -- Per-app deploy workflows + provision.yml

@@ -36,10 +36,50 @@ for technical deep-dives, USECASES.md for expected behaviors.

 ## Build and test

-No build step. No tests. Verify changes by:
-1. Reading scripts and checking idempotency
-2. Walking through each use case in USECASES.md
-3. Cross-checking docs against scripts
+No build step. Run e2e tests with:
+
+    uvx pytest tests/test_app_e2e.py -v -x
+
+### Prerequisites
+
+- `uv` installed (`curl -LsSf https://astral.sh/uv/install.sh | sh`)
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- bws token: set `BWS_ACCESS_TOKEN` env var or save to `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP (check https://my.vultr.com/settings/#settingsapi)
+
+### Test structure
+
+- `tests/conftest.py` -- Fixtures: preflight checks, bws secrets, API helper, cleanup
+- `tests/test_app_e2e.py` -- E2E tests for `tools/app.sh` against real services
+- `tests/test_security_headers.py` -- Verify security headers on live sites
+- `tests/test_default_server.py` -- Verify unknown Host headers are dropped
+- `tests/test_tls.py` -- Verify TLS hardening
+
+### E2E tests (`test_app_e2e.py`)
+
+Tests use `kgfamily.com` as a throwaway test domain. Two phases:
+
+1. **TestAppSubcommands** -- Runs each `app.sh` subcommand individually (dns, repo,
+   files, cert, garage, build, monitor), verifies side effects via API, then cleans up.
+   Does not test `verify` (needs full deploy flow).
+2. **TestAppAll** -- Runs `app.sh all kgfamily.com`, verifies everything including
+   site liveness and git commit, then cleans up (including reverting the commit+push).
+
+Cleanup is done directly via APIs (Vultr, Gitea, Garage, UptimeRobot) + local file
+deletion. Does NOT use `remove-app.sh` to avoid coupling. Runs before and after each
+phase to handle leftovers from crashed runs.
+
+### Known issue
+
+- Vultr API keys can be IP-restricted. If `test_dns` fails with HTTP 401, add the
+  machine's IP to the Vultr API access control list.
+
+### Other tests
+
+The security/TLS/default-server tests run against live sites and are triggered
+automatically after nginx deploy. They can also be run locally:
+
+    uvx pytest tests/ -v --ignore=tests/test_app_e2e.py

 ## Commit style

@@ -26,10 +26,10 @@ nodes.
 scripts/
  bootstrap.sh         # First-time server setup (manual, uses Bitwarden)
  configure.sh         # Idempotent server config (CI-safe)
-  deploy.sh            # Deploy + cert issuance + provision (called via SSH)
+  deploy.sh            # Deploy + provision (called via SSH)
 tools/
-  new-app.sh           # Add a new static site (run from dev machine)
-  new-service.sh       # Add a new Docker service (run from dev machine)
+  app.sh               # Add a new static site (run from dev machine)
+  service.sh           # Manage Docker services by subcommand (dns, files, nginx, cert, all)
  remove-app.sh        # Remove a static site (run from dev machine)
 docker/
  nginx/               # Reverse proxy (nginx:alpine) + SSL config
@@ -38,6 +38,7 @@ docker/
    compose.yml
  <domain>/            # Per-app compose files (e.g. example.com)
    compose.yml
+  goatcounter/         # GoatCounter analytics
  garage/              # Garage S3-compatible object storage
    compose.yml
    Dockerfile
@@ -73,12 +74,12 @@ server. Everything is idempotent.
 ## Adding a new static site

 ```bash
-./tools/new-app.sh <domain>
+./tools/app.sh all <domain>
 ```

 This single command handles everything: DNS records (Vultr), Gitea repo
 (from `static-site-template`), nginx/compose/workflow configs, SSL cert,
-Garage media bucket, first build, and verification.
+Garage media bucket, first build, verification, and UptimeRobot monitoring.

 **Prerequisites:** domain nameservers pointed to Vultr, `bws`/`jq`/`dig`
 installed, `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`.
@@ -163,19 +164,55 @@ reconnects automatically.

 ## Testing

-Tests live in `tests/` and run with pytest:
+Tests live in `tests/` and run with [uv](https://docs.astral.sh/uv/):

 ```bash
-pip install pytest
-pytest tests/ -v
+# Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Live site tests (security headers, TLS, default server)
+uvx pytest tests/ -v --ignore=tests/test_app_e2e.py
+
+# E2E tests for tools/app.sh (creates real resources)
+uvx pytest tests/test_app_e2e.py -v -x
 ```

-Tests auto-discover sites from `docker/nginx/conf.d/*.conf` — no list to
-maintain when adding new sites.
+### Live site tests

-The `deploy-nginx.yml` workflow runs tests automatically after each nginx
+These run against live sites and verify infrastructure invariants:
+
+- **`test_security_headers.py`** — Checks that all sites including
+  `security-headers.inc` return the expected headers (HSTS, X-Frame-Options,
+  etc.). Sites are auto-discovered from `docker/nginx/conf.d/*.conf`.
+- **`test_tls.py`** — Verifies TLS 1.3 and 1.2 work, TLS 1.1 is rejected.
+- **`test_default_server.py`** — Verifies unknown Host headers get dropped
+  (connection reset) on both HTTP and HTTPS.
+
+The `deploy-nginx.yml` workflow runs these automatically after each nginx
 deploy. If tests fail, Gitea sends an email notification.

+### E2E tests
+
+`test_app_e2e.py` tests `tools/app.sh` against real services (Vultr, Gitea,
+Garage, UptimeRobot). Uses `kgfamily.com` as a throwaway test domain.
+
+**Prerequisites:**
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP
+  (check https://my.vultr.com/settings/#settingsapi)
+
+**Structure:**
+
+1. **TestAppSubcommands** — Runs each `app.sh` subcommand individually
+   (dns, repo, files, cert, garage, build, monitor), verifies side effects
+   via API, then cleans up.
+2. **TestAppAll** — Runs `app.sh all kgfamily.com`, verifies the full flow
+   including site liveness and git commit, then cleans up.
+
+Cleanup runs before and after each phase via APIs (Vultr, Gitea, Garage,
+UptimeRobot) + local file deletion. Does not use `remove-app.sh` to avoid
+coupling.
+
 ## Secrets

 ### Bitwarden Secrets Manager
@@ -186,14 +223,16 @@ Project: `hantim`. Fetched via `bws` CLI on the server.
 |---|---|---|
 | `hantim-ci-registry-push` | Gitea token for Docker registry | `bootstrap.sh` |
 | `hantim-deploy-ssh-public-key` | Deploy user's SSH public key | `bootstrap.sh` |
-| `hantim-deploy-ssh-private-key` | Deploy user's SSH private key | `new-app.sh` |
-| `hantim-new-app-script` | Gitea API token for creating repos | `new-app.sh` |
-| `hantim-vultr-api-key` | Vultr API key for DNS management | `new-app.sh` |
+| `hantim-deploy-ssh-private-key` | Deploy user's SSH private key | `app.sh` |
+| `hantim-new-app-script` | Gitea API token for creating repos | `app.sh` |
+| `hantim-vultr-api-key` | Vultr API key for DNS management | `app.sh` |
 | `hantim-garage-rpc-secret` | Garage cluster RPC secret | `deploy.sh` |
 | `hantim-garage-argento-node-id` | Argento's Garage node ID + address | `deploy.sh` |
-| `hantim-garage-admin-token` | Garage admin API token | `deploy.sh`, `new-app.sh` |
-| `hantim-garage-media-key-id` | S3 access key ID for media uploads | `new-app.sh`, `aws` CLI |
+| `hantim-garage-admin-token` | Garage admin API token | `deploy.sh`, `app.sh` |
+| `hantim-garage-media-key-id` | S3 access key ID for media uploads | `app.sh`, `aws` CLI |
 | `hantim-garage-media-secret-key` | S3 secret key for media uploads | `aws` CLI |
+| `hantim-uptimerobot-api-key` | UptimeRobot API key for monitors | `app.sh` |
+| `hantim-goatcounter-password` | GoatCounter admin password | `deploy.sh` |

 Machine accounts: `hantim-server` (token at `/etc/bws-token`), `hantim-ci` (reserved).

@@ -211,5 +250,12 @@ Stored in the `hantim` Gitea org for direct use in CI workflows:
 | Name | Type | Purpose |
 |---|---|---|
 | `DEPLOY_HOST` | Variable | Server IP or hostname |
+| `DEPLOY_HOST_KEY` | Variable | Server SSH host public key (for host key pinning) |
 | `DEPLOY_SSH_KEY` | Secret | SSH private key for the deploy user |
 | `CI_REGISTRY_TOKEN` | Secret | Gitea token for Docker registry login |
+
+After reprovisioning hantim (new host key), update `DEPLOY_HOST_KEY`:
+
+```bash
+ssh-keyscan -t ed25519 hantim.net 2>/dev/null
+```
@@ -1,407 +0,0 @@
-# Security Analysis
-
-Vulnerability assessment of the hantim-server infrastructure. Each finding
-includes severity, affected files, explanation, and a proposed fix. Sorted
-from highest to lowest severity.
-
---
-
-## HIGH
-
-### ~~1. Security headers silently dropped on all HTTPS responses~~ FIXED
-
-Fixed in `cc07a34`. All security headers moved to shared snippet
-(`security-headers.inc`) included from each HTTPS server block.
-Post-deploy test in `tests/test_security_headers.py` verifies all headers.
-
---
-
-### ~~2. Garage admin API exposed to the internet with only token auth~~ FIXED
-
-Fixed in `cce91c0`. Admin API restricted to home IP via nginx
-`allow`/`deny`. Update IP if it changes: `dig +short argento.ddns.net`.
-
---
-
-### ~~3. Docker group membership gives deploy user root-equivalent access~~ FIXED
-
-Fixed in `2099464`. Removed `usermod -aG docker deploy` from
-`bootstrap.sh`. Deploy user removed from docker group on live server.
-
---
-
-### ~~4. All secrets fetched on every deploy (bws secret list in a loop)~~ FIXED
-
-Fixed in `33a2383`. `.env.keys` now uses secret UUIDs. `deploy.sh` fetches
-each secret individually via `bws secret get <uuid>`.
-
---
-
-### 5. Garage RPC port open to the internet — ACCEPTED
-
-**Affected:** `docker/garage/compose.yml:12`, `scripts/configure.sh:20`
-
-Port 3901 is bound to the host and opened in the firewall for cluster
-replication with argento. Firewalld cannot resolve DDNS hostnames
-dynamically, and argento's IP changes with the home ISP. The RPC secret
-(long random string from Bitwarden) is the primary protection.
-
-**Risk accepted:** restricting by IP would break replication on IP change.
-The RPC secret provides adequate protection. Downgraded from HIGH to
-MEDIUM.
-
---
-
-## MEDIUM
-
-### 6. No SSL/TLS hardening in nginx
-
-**Affected:** `docker/nginx/nginx.conf`, all `conf.d/*.conf`
-
-No `ssl_protocols`, `ssl_ciphers`, `ssl_session_cache`, or OCSP stapling
-is configured. The nginx:alpine image's defaults may allow TLS 1.0/1.1 and
-weak cipher suites depending on the version.
-
-Certbot writes `options-ssl-nginx.conf` to `/etc/letsencrypt/` (which is
-mounted into the container), but no conf file includes it.
-
-**Fix:** Add TLS hardening to `nginx.conf` inside the `http` block:
-
-```nginx
-ssl_protocols TLSv1.2 TLSv1.3;
-ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
-ssl_prefer_server_ciphers off;
-ssl_session_cache shared:SSL:10m;
-ssl_session_timeout 1d;
-ssl_session_tickets off;
-ssl_stapling on;
-ssl_stapling_verify on;
-resolver 1.1.1.1 8.8.8.8 valid=300s;
-resolver_timeout 5s;
-```
-
-Or include certbot's config:
-
-```nginx
-include /etc/letsencrypt/options-ssl-nginx.conf;
-ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
-```
-
---
-
-### 7. No default server block to catch unknown Host headers
-
-**Affected:** `docker/nginx/conf.d/`
-
-Without a catch-all server block, requests with unknown `Host` headers are
-served by whichever conf file nginx loads first (alphabetically). This
-leaks which sites are hosted and can be used for reconnaissance.
-
-**Fix:** Add a default server block:
-
-```nginx
-# docker/nginx/conf.d/00-default.conf
-server {
-    listen 80 default_server;
-    listen [::]:80 default_server;
-    listen 443 ssl default_server;
-    listen [::]:443 ssl default_server;
-    http2 on;
-    server_name _;
-
-    ssl_certificate     /etc/letsencrypt/live/hantim.net/fullchain.pem;
-    ssl_certificate_key /etc/letsencrypt/live/hantim.net/privkey.pem;
-
-    return 444;
-}
-```
-
-The `444` status silently drops the connection.
-
---
-
-### 8. No rate limiting on any endpoint
-
-**Affected:** `docker/nginx/nginx.conf`, all `conf.d/*.conf`
-
-No `limit_req` or `limit_conn` zones are defined. The server is vulnerable
-to brute-force attacks against the S3 and admin APIs, and to connection
-exhaustion / slowloris attacks.
-
-**Fix:** Add rate limiting zones to `nginx.conf`:
-
-```nginx
-http {
-    limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
-    limit_req_zone $binary_remote_addr zone=api:10m rate=2r/s;
-    ...
-}
-```
-
-Apply them in server blocks:
-
-```nginx
-# In s3.hantim.net.conf and garage.hantim.net.conf:
-limit_req zone=api burst=20 nodelay;
-
-# In site server blocks:
-limit_req zone=general burst=30 nodelay;
-```
-
---
-
-### 9. envsubst replaces all environment variables in garage.toml
-
-**Affected:** `docker/garage/Dockerfile:8`
-
-```dockerfile
-CMD envsubst < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server
-```
-
-`envsubst` without arguments replaces every `$VAR` or `${VAR}` pattern. If
-any other environment variable (e.g., `$HOME`, `$PATH`) matches a pattern
-in the template, it will be silently substituted. This is fragile and could
-break the config in unexpected ways.
-
-**Fix:** Specify the variables explicitly:
-
-```dockerfile
-CMD envsubst '${RPC_SECRET} ${ARGENTO_NODE_ID} ${ADMIN_TOKEN}' < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server
-```
-
---
-
-### 10. StrictHostKeyChecking=accept-new trusts first connection blindly
-
-**Affected:** `.gitea/workflows/*.yml`, `tools/app.sh:367`
-
-All SSH connections use `-o StrictHostKeyChecking=accept-new`, which trusts
-whatever host key is presented on first contact. A man-in-the-middle during
-the first CI run (or after server reprovisioning) could intercept the
-deploy connection and capture the SSH private key.
-
-**Fix:** Pin the server's host key in workflows. Add a step before SSH:
-
-```yaml
- name: Configure SSH
-  run: |
-    mkdir -p ~/.ssh
-    echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
-    chmod 600 ~/.ssh/deploy_key
-    echo "${{ vars.DEPLOY_HOST_KEY }}" >> ~/.ssh/known_hosts
-    ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-nginx
-```
-
-Store the server's host public key as a Gitea variable (`DEPLOY_HOST_KEY`).
-
---
-
-### 11. API tokens visible in process listings
-
-**Affected:** `tools/app.sh` (lines 134, 139, 225, etc.)
-
-Secrets passed as curl header arguments appear in `ps aux` and
-`/proc/<pid>/cmdline` for the duration of the curl call:
-
-```bash
-VULTR_AUTH="Authorization: Bearer $VULTR_API_KEY"
-curl -s "$VULTR_API/domains/$APP" -H "$VULTR_AUTH"
-```
-
-**Fix:** Use curl's `--header @-` or `-H @<file>` to read headers from
-stdin/file instead of command arguments:
-
-```bash
-VULTR_AUTH_FILE=$(mktemp)
-echo "Authorization: Bearer $VULTR_API_KEY" > "$VULTR_AUTH_FILE"
-chmod 600 "$VULTR_AUTH_FILE"
-trap "rm -f $VULTR_AUTH_FILE" EXIT
-curl -s "$VULTR_API/domains/$APP" -H "@$VULTR_AUTH_FILE"
-```
-
-Or use a `.netrc` file for authentication.
-
---
-
-### 12. No container resource limits
-
-**Affected:** All `docker/*/compose.yml`
-
-No memory or CPU limits are set on any container. A runaway container
-(memory leak, fork bomb, or compromised process) can consume all host
-resources and take down every service.
-
-**Fix:** Add resource limits to compose files:
-
-```yaml
-services:
-  app:
-    ...
-    deploy:
-      resources:
-        limits:
-          memory: 128M
-          cpus: '0.5'
-```
-
-For nginx and garage, use higher limits appropriate to their workload.
-
---
-
-### 13. Brief nginx downtime during initial cert issuance
-
-**Affected:** `scripts/deploy.sh:87-114`
-
-When deploying nginx with new domains that need certs, the script stops
-nginx, runs `certbot --standalone`, then starts nginx. During this window
-(potentially minutes if multiple certs are needed), all sites are down.
-
-**Fix:** Issue certs via the `cert-<domain>` SSH command (which uses
-`--webroot` and causes zero downtime) before deploying the nginx config.
-The `app.sh` already does this (`cmd_cert` runs before nginx gets the new
-config). For the standalone path, consider using certbot's `--preferred-challenges dns`
-or pre-issuing certs in a separate step.
-
---
-
-### 14. Hardcoded Bitwarden secret UUIDs
-
-**Affected:** `scripts/bootstrap.sh:21-22`
-
-```bash
-SECRET_REGISTRY_TOKEN="43f4a77d-7bb4-49ad-8044-b411016c5c6e"
-SECRET_DEPLOY_PUBKEY="945760dd-90e2-46e5-98ad-b411016e3f3b"
-```
-
-UUIDs identify specific secrets. If this repo is exposed, an attacker
-knows exactly which secret IDs to target. Using `bws secret list` with
-name lookup (like the rest of the codebase does) is more consistent and
-doesn't leak UUIDs.
-
-**Fix:** Use name-based lookup like `deploy.sh` and `app.sh` do:
-
-```bash
-REGISTRY_TOKEN=$(bws secret list | jq -r '.[] | select(.key == "hantim-ci-registry-push") | .value')
-DEPLOY_PUBKEY=$(bws secret list | jq -r '.[] | select(.key == "hantim-deploy-ssh-public-key") | .value')
-```
-
---
-
-### 15. certbot registers without email
-
-**Affected:** `scripts/deploy.sh:29, 109`
-
-```bash
-certbot certonly --standalone --non-interactive --agree-tos \
-    --register-unsafely-without-email ...
-```
-
-Without an email, Let's Encrypt cannot send expiration warnings. If the
-certbot renewal timer/cron breaks silently, certificates expire without
-notice, causing site outages.
-
-**Fix:** Register with an email:
-
-```bash
-certbot certonly --standalone --non-interactive --agree-tos \
-    -m admin@hantim.net ...
-```
-
---
-
-## LOW
-
-### 16. Certbot renewal hook has no error handling
-
-**Affected:** `scripts/configure.sh:26-30`
-
-```bash
-cat > /etc/letsencrypt/renewal-hooks/deploy/reload-nginx.sh <<'HOOK'
-#!/bin/bash
-docker exec nginx nginx -s reload
-HOOK
-```
-
-If the nginx container is down when certs renew, the reload fails silently.
-The new cert won't take effect until nginx is manually restarted.
-
-**Fix:**
-
-```bash
-#!/bin/bash
-set -e
-if ! docker exec nginx nginx -s reload; then
-    logger -t certbot "ERROR: Failed to reload nginx after cert renewal"
-    exit 1
-fi
-```
-
---
-
-### 17. No audit logging for deploy operations
-
-**Affected:** `scripts/deploy.sh`
-
-There is no logging of who triggered a deploy, what changed, or when
-secrets were accessed. If the server is compromised, there's no audit trail.
-
-**Fix:** Add logging to deploy.sh:
-
-```bash
-logger -t hantim-deploy "deploy started: cmd=$CMD user=$(whoami) ssh_client=${SSH_CLIENT:-local}"
-```
-
-And at the end:
-
-```bash
-logger -t hantim-deploy "deploy finished: cmd=$CMD"
-```
-
---
-
-### 18. git pull without explicit branch
-
-**Affected:** `scripts/deploy.sh:39, 51`
-
-```bash
-git pull
-```
-
-If the repo's default tracking branch changes or is misconfigured, `git
-pull` could pull the wrong branch.
-
-**Fix:**
-
-```bash
-git pull origin main
-```
-
---
-
-## Positive security practices already in place
-
- SSH `command=` restriction limits deploy user to specific commands
- Deploy user has `nologin` shell and scoped sudoers entry
- All scripts use `set -euo pipefail`
- HTTPS enforced everywhere with HTTP-to-HTTPS redirects
- HSTS with preload on all main server blocks
- `.env` files are `chmod 600` and gitignored
- BWS token file is `chmod 600`
- Docker network isolation: only nginx binds host ports
- Input validation on domain names with regex in `deploy.sh` and `app.sh`
- Idempotent scripts safe to re-run
- Certbot auto-renewal configured via timer or cron
- `nginx -t` config test before applying changes
-
---
-
-## Recommended priority
-
-1. ~~Fix security headers (item 1)~~ DONE
-2. ~~Restrict Garage admin API access (item 2)~~ DONE
-3. ~~Remove deploy user from docker group (item 3)~~ DONE
-4. ~~Fix bws secret list loop (item 4)~~ DONE
-5. ~~Garage RPC port (item 5)~~ ACCEPTED
-6. Add SSL/TLS hardening (item 6)
-7. Add default server block (item 7)
-8. Everything else as time permits
@@ -38,7 +38,7 @@ for stateless static sites.

 ## 3. Add a new static site

-**Trigger:** `./tools/new-app.sh example.com` from dev machine.
+**Trigger:** `./tools/app.sh all example.com` from dev machine.

 **Prerequisites:** domain nameservers on Vultr, `bws`/`jq`/`dig` installed.

@@ -50,6 +50,7 @@ for stateless static sites.
 - Garage media bucket created with media-key access
 - First build triggered
 - `https://www.example.com` responds within 3 minutes
+- UptimeRobot HTTPS monitor created (skipped if already exists)
 - Site ready for customization via git clone + push

 ## 4. Update app code
@@ -88,12 +89,43 @@ for stateless static sites.
 - Existing certs skipped
 - Services restarted

-## 8. Upload media files
+## 8. Add a new Docker service

-**Trigger:** `aws s3 cp` or `aws s3 sync` to `s3://example.com/`.
+**Trigger:** `./tools/service.sh all <name> <port>` from dev machine.
+
+**Prerequisites:** `bws`/`jq`/`dig` installed.
+
+**Expected outcome:**
+- DNS A record created for `<name>.hantim.net`
+- Compose file and deploy workflow scaffolded
+- Nginx conf created proxying `<name>.hantim.net` to port `<port>`
+- SSL cert issued for `<name>.hantim.net` (no www variant)
+- Service ready for customization (edit compose.yml image, add `.env.keys` if needed)
+
+## 9. Analytics tracking
+
+**Trigger:** automatic — nginx injects the GoatCounter script into all HTML responses.
+
+**Expected outcome:**
+- Every site includes the GoatCounter tracking script (injected via `sub_filter`)
+- Page views are recorded at `goatcounter.hantim.net`
+- New sites provisioned with `app.sh` include tracking automatically
+- GoatCounter dashboard itself is not tracked
+
+## 10. Upload media files
+
+**Trigger:** `aws s3 cp` or `aws s3 sync` to `s3://example.com/`, or `./media.sh push` from a site repo.

 **Expected outcome:**
 - File uploaded via S3 API (`s3.hantim.net`)
 - Replicated to both Garage nodes (hantim + argento)
 - Accessible at `https://www.example.com/media/<path>`
- Local development: `static/media/` (gitignored) serves at `/media/`
+
+## 11. Local development with media files
+
+**Trigger:** `./media.sh pull` from a site repo.
+
+**Expected outcome:**
+- Media files downloaded from Garage to `static/media/` (gitignored)
+- Files served locally at `/media/` by the app container's nginx
+- To upload local changes back to Garage: `./media.sh push`
@@ -0,0 +1,4 @@
+# hantim-beszel-public-key
+KEY=1b5b742b-42db-4a03-a8ec-b42600150be4
+# hantim-beszel-token
+TOKEN=6666398f-b936-4781-954b-b426001528cb
@@ -0,0 +1,7 @@
+# !create
+beszel_data/
+# !create
+beszel_socket/
+# !create
+beszel_agent_data/
+
@@ -0,0 +1,33 @@
+services:
+  beszel:
+    image: henrygd/beszel
+    container_name: beszel
+    restart: unless-stopped
+    environment:
+      APP_URL: https://beszel.hantim.net
+    ports:
+      - 127.0.0.1:8090:8090
+    volumes:
+      - ./beszel_data:/beszel_data
+      - ./beszel_socket:/beszel_socket
+    networks:
+      - shared
+
+  beszel-agent:
+    image: henrygd/beszel-agent
+    container_name: beszel-agent
+    restart: unless-stopped
+    network_mode: host
+    volumes:
+      - ./beszel_agent_data:/var/lib/beszel-agent
+      - ./beszel_socket:/beszel_socket
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    env_file: .env
+    environment:
+      LISTEN: /beszel_socket/beszel.sock
+      HUB_URL: http://localhost:8090
+
+networks:
+  shared:
+    external: true
+
@@ -0,0 +1,11 @@
+services:
+  app:
+    image: git.timothykim.net/hantim/carolpreschool.com:latest
+    restart: unless-stopped
+    container_name: carolpreschool_com
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true
@@ -5,4 +5,4 @@ FROM base
 COPY --from=dxflrs/garage:v2.2.0 /garage /garage
 COPY garage.toml /etc/garage.toml.tpl

-CMD envsubst < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server
+CMD envsubst '${RPC_SECRET} ${ARGENTO_NODE_ID} ${ADMIN_TOKEN}' < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server
@@ -0,0 +1,2 @@
+# hantim-goatcounter-password
+GOATCOUNTER_PASSWORD=1b02fe2e-7acb-4111-918a-b42701546299
@@ -0,0 +1,3 @@
+# !create
+db/
+.env
@@ -0,0 +1,17 @@
+services:
+  goatcounter:
+    image: baethon/goatcounter
+    restart: unless-stopped
+    container_name: goatcounter
+    env_file: .env
+    environment:
+      GOATCOUNTER_DOMAIN: goatcounter.hantim.net
+      GOATCOUNTER_EMAIL: timothykim@fastmail.fm
+    volumes:
+      - ./db:/goatcounter/db
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true
@@ -0,0 +1,11 @@
+services:
+  app:
+    image: git.timothykim.net/hantim/hcsuzuki.music:latest
+    restart: unless-stopped
+    container_name: hcsuzuki_music
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true
@@ -1,4 +1,4 @@
-# Drop connections with unknown Host headers
+# Drop connections with unknown/missing Host headers
 server {
    listen 80 default_server;
    listen [::]:80 default_server;
@@ -0,0 +1,33 @@
+server {
+    listen 80;
+    listen [::]:80;
+    server_name beszel.hantim.net;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://beszel.hantim.net$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name beszel.hantim.net;
+
+    ssl_certificate     /etc/letsencrypt/live/beszel.hantim.net/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/beszel.hantim.net/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://beszel:8090;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}
@@ -0,0 +1,54 @@
+# Redirect HTTP to HTTPS, bare domain to www
+server {
+    listen 80;
+    listen [::]:80;
+    server_name carolpreschool.com www.carolpreschool.com;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://www.carolpreschool.com$request_uri;
+    }
+}
+
+# Redirect bare HTTPS domain to www
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name carolpreschool.com;
+
+    ssl_certificate     /etc/letsencrypt/live/carolpreschool.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/carolpreschool.com/privkey.pem;
+
+    return 301 https://www.carolpreschool.com$request_uri;
+}
+
+# Main site
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name www.carolpreschool.com;
+
+    ssl_certificate     /etc/letsencrypt/live/carolpreschool.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/carolpreschool.com/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
+
+    location /media/ {
+        proxy_pass http://garage:3902/;
+        proxy_set_header Host carolpreschool.com.web.garage;
+    }
+
+    location / {
+        proxy_pass http://carolpreschool_com:80;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}
@@ -31,5 +31,6 @@ server {
        deny all;

        proxy_pass http://garage:3903;
+        proxy_hide_header X-Frame-Options;
    }
 }
@@ -0,0 +1,33 @@
+server {
+    listen 80;
+    listen [::]:80;
+    server_name goatcounter.hantim.net;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://goatcounter.hantim.net$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name goatcounter.hantim.net;
+
+    ssl_certificate     /etc/letsencrypt/live/goatcounter.hantim.net/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/goatcounter.hantim.net/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://goatcounter:8080;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}
@@ -0,0 +1,2 @@
+sub_filter '</body>' '<script>window.goatcounter={path:function(p){return location.host+p}}</script><script data-goatcounter="https://goatcounter.hantim.net/count" async src="//goatcounter.hantim.net/count.js"></script></body>';
+sub_filter_once on;
@@ -37,6 +37,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/haanmind.net/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -37,6 +37,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/hantim.net/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -0,0 +1,54 @@
+# Redirect HTTP to HTTPS, bare domain to www
+server {
+    listen 80;
+    listen [::]:80;
+    server_name hcsuzuki.music www.hcsuzuki.music;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://www.hcsuzuki.music$request_uri;
+    }
+}
+
+# Redirect bare HTTPS domain to www
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name hcsuzuki.music;
+
+    ssl_certificate     /etc/letsencrypt/live/hcsuzuki.music/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.music/privkey.pem;
+
+    return 301 https://www.hcsuzuki.music$request_uri;
+}
+
+# Main site
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name www.hcsuzuki.music;
+
+    ssl_certificate     /etc/letsencrypt/live/hcsuzuki.music/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.music/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
+
+    location /media/ {
+        proxy_pass http://garage:3902/;
+        proxy_set_header Host hcsuzuki.music.web.garage;
+    }
+
+    location / {
+        proxy_pass http://hcsuzuki_music:80;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}
@@ -37,6 +37,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.net/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -31,5 +31,6 @@ server {
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
    }
 }
@@ -37,6 +37,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/thekims.family/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -37,6 +37,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/timothykim.net/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -19,8 +19,8 @@ http {
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 1d;
    ssl_session_tickets off;
-    ssl_stapling on;
-    ssl_stapling_verify on;
+    #ssl_stapling on;
+    #ssl_stapling_verify on;
    resolver 1.1.1.1 8.8.8.8 valid=300s;
    resolver_timeout 5s;

@@ -99,7 +99,7 @@ fi
 echo "==> Setting up deploy SSH key..."
 mkdir -p /home/deploy/.ssh
 chmod 700 /home/deploy/.ssh
-echo "command=\"sudo /opt/hantim/scripts/deploy.sh \$SSH_ORIGINAL_COMMAND\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $DEPLOY_PUBKEY" > /home/deploy/.ssh/authorized_keys
+echo "command=\"sudo /opt/hantim/scripts/deploy.sh \\\"\$SSH_ORIGINAL_COMMAND\\\"\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $DEPLOY_PUBKEY" > /home/deploy/.ssh/authorized_keys
 chmod 600 /home/deploy/.ssh/authorized_keys
 chown -R deploy:deploy /home/deploy/.ssh

@@ -37,6 +37,25 @@ fi
 echo "==> Deploying services..."
 # Deploy nginx first (creates the shared network)
 bash "$REPO_DIR/scripts/deploy.sh" deploy-nginx
+
+# Self-heal renewal config drift: certs issued by the old --standalone flow
+# (removed in de1d60b) can never renew because nginx now holds port 80. Reissue
+# any such cert via the canonical webroot path. nginx must be up (above) to
+# serve the ACME challenge. Best-effort: a single failure warns but does not
+# block the rest of the provision.
+echo "==> Healing standalone cert renewal configs..."
+heal_failed=false
+for conf in /etc/letsencrypt/renewal/*.conf; do
+    [ -e "$conf" ] || continue
+    grep -q '^authenticator = standalone' "$conf" || continue
+    cert_name=$(basename "$conf" .conf)
+    echo "  $cert_name uses standalone; reissuing via webroot..."
+    if ! bash "$REPO_DIR/scripts/deploy.sh" "cert-$cert_name"; then
+        echo "  WARNING: failed to reissue $cert_name; leaving standalone config in place"
+        heal_failed=true
+    fi
+done
+
 # Deploy all other apps
 for app in "$REPO_DIR"/docker/*/; do
    app_name=$(basename "$app")
@@ -46,3 +65,8 @@ for app in "$REPO_DIR"/docker/*/; do
 done

 echo "==> Done."
+
+if [ "$heal_failed" = true ]; then
+    echo "ERROR: one or more standalone certs could not be reissued (see warnings above)."
+    exit 1
+fi
@@ -6,12 +6,21 @@ CMD="${SSH_ORIGINAL_COMMAND:-${1:-}}"
 if [[ "$CMD" =~ ^cert-[a-zA-Z0-9._-]+$ ]]; then
    APP="${CMD#cert-}"

+    # Subdomains (e.g., garage.hantim.net) don't have www variants
+    if [[ "$APP" == *.*.* ]]; then
+        CERT_DOMAINS="-d $APP"
+        SERVER_NAMES="$APP"
+    else
+        CERT_DOMAINS="-d $APP -d www.$APP"
+        SERVER_NAMES="$APP www.$APP"
+    fi
+
    # Write temporary HTTP-only config so nginx can serve ACME challenges
    cat > "/opt/hantim/docker/nginx/conf.d/$APP.conf" <<NGINXCONF
 server {
    listen 80;
    listen [::]:80;
-    server_name $APP www.$APP;
+    server_name $SERVER_NAMES;

    location /.well-known/acme-challenge/ {
        root /var/www/certbot;
@@ -23,20 +32,21 @@ server {
 }
 NGINXCONF

-    trap 'rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"' EXIT
+    trap 'git -C /opt/hantim restore "docker/nginx/conf.d/$APP.conf" 2>/dev/null || rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"; docker exec nginx nginx -t && docker exec nginx nginx -s reload' EXIT
    docker exec nginx nginx -t && docker exec nginx nginx -s reload
    certbot certonly --webroot -w /opt/hantim/docker/nginx/certbot/www \
-        --non-interactive --agree-tos --register-unsafely-without-email \
-        --cert-name "$APP" -d "$APP" -d "www.$APP"
-    rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"
+        --non-interactive --agree-tos -m timothykim@fastmail.fm \
+        --cert-name "$APP" $CERT_DOMAINS
+    git -C /opt/hantim restore "docker/nginx/conf.d/$APP.conf" 2>/dev/null || rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"
    trap - EXIT
+    docker exec nginx nginx -t && docker exec nginx nginx -s reload
    echo "Certificate issued for $APP"
    exit 0
 fi

 if [ "$CMD" = "provision" ]; then
    cd /opt/hantim
-    git pull
+    git pull origin main
    bash scripts/configure.sh
    exit 0
 fi
@@ -48,7 +58,13 @@ fi

 APP="${CMD#deploy-}"
 cd /opt/hantim
-git pull
+git pull origin main
+
+# Re-exec so the running script reflects any changes just pulled
+if [ -z "${DEPLOY_REEXEC:-}" ]; then
+    export DEPLOY_REEXEC=1
+    exec /opt/hantim/scripts/deploy.sh "$CMD"
+fi

 # Generate .env from bws if this service declares .env.keys
 if [ -f "docker/$APP/.env.keys" ] && [ -f /etc/bws-token ]; then
@@ -76,6 +92,26 @@ if [ -f "docker/$APP/.env.keys" ] && [ -f /etc/bws-token ]; then
 fi

 cd "docker/$APP"
+
+# Create directories marked with "# !create" in .gitignore
+if [ -f .gitignore ]; then
+    create_next=false
+    while IFS= read -r line || [ -n "$line" ]; do
+        if [ "$line" = "# !create" ]; then
+            create_next=true
+            continue
+        fi
+        if [ "$create_next" = true ] && [ -n "$line" ]; then
+            dir="${line%/}"
+            if [ -n "$dir" ]; then
+                mkdir -p "$dir"
+                echo "Created directory: $dir"
+            fi
+            create_next=false
+        fi
+    done < .gitignore
+fi
+
 if grep -q '^\s*build:' compose.yml 2>/dev/null; then
    docker compose build
 else
@@ -86,33 +122,15 @@ else
 fi

 if [ "$APP" = "nginx" ]; then
-    # Issue certs for any new domains (brief nginx downtime while certbot binds port 80)
-    missing_certs=false
-    for conf in /opt/hantim/docker/nginx/conf.d/*.conf; do
+    # Fail loudly if any certs are missing — issue them first with: app.sh cert <domain>
+    for conf in /opt/hantim/docker/nginx/conf.d/*.*.conf; do
        cert_name=$(basename "$conf" .conf)
        if [ ! -d "/etc/letsencrypt/live/$cert_name" ]; then
-            missing_certs=true
-            break
+            echo "ERROR: Missing certificate for $cert_name"
+            echo "  Run: app.sh cert $cert_name"
+            exit 1
        fi
    done
-    if [ "$missing_certs" = true ]; then
-        docker stop nginx 2>/dev/null || true
-        trap 'docker start nginx 2>/dev/null || true' EXIT
-        for conf in /opt/hantim/docker/nginx/conf.d/*.conf; do
-            cert_name=$(basename "$conf" .conf)
-            if [ ! -d "/etc/letsencrypt/live/$cert_name" ]; then
-                domains=$(grep -oP 'server_name\s+\K[^;]+' "$conf" | tr ' ' '\n' | sort -u)
-                domain_args=""
-                for d in $domains; do
-                    domain_args="$domain_args -d $d"
-                done
-                echo "Issuing cert for $cert_name..."
-                certbot certonly --standalone --non-interactive --agree-tos \
-                    --register-unsafely-without-email --cert-name "$cert_name" $domain_args
-            fi
-        done
-        trap - EXIT
-    fi
    # Test config before applying — a bad config would take down all sites
    docker compose run --rm -T nginx nginx -t
 fi
@@ -0,0 +1,266 @@
+"""Shared fixtures and helpers for e2e tests."""
+
+import json
+import os
+import shutil
+import subprocess
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import pytest
+
+TEST_DOMAIN = "kgfamily.com"
+CONTAINER_NAME = TEST_DOMAIN.replace(".", "_")
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+APP_SH = os.path.join(REPO_ROOT, "tools", "app.sh")
+
+VULTR_API = "https://api.vultr.com/v2"
+GITEA_URL = "https://git.timothykim.net"
+GITEA_ORG = "hantim"
+GARAGE_API = "https://garage.hantim.net"
+UPTIMEROBOT_API = "https://api.uptimerobot.com/v3"
+
+
+def api(method, url, headers=None, data=None):
+    """HTTP request helper. Returns (status_code, parsed_json_or_None)."""
+    hdrs = dict(headers or {})
+    body = None
+    if data is not None:
+        body = json.dumps(data).encode()
+        hdrs.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, method=method, headers=hdrs, data=body)
+    try:
+        resp = urllib.request.urlopen(req, timeout=30)
+        raw = resp.read().decode()
+        return resp.status, json.loads(raw) if raw.strip() else None
+    except urllib.error.HTTPError as e:
+        raw = e.read().decode()
+        try:
+            return e.code, json.loads(raw) if raw.strip() else None
+        except json.JSONDecodeError:
+            return e.code, None
+
+
+def _bws_env():
+    """Return env dict with BWS_ACCESS_TOKEN set from token file if needed."""
+    env = os.environ.copy()
+    if not env.get("BWS_ACCESS_TOKEN") and os.path.isfile(BWS_TOKEN_FILE):
+        env["BWS_ACCESS_TOKEN"] = open(BWS_TOKEN_FILE).read().strip()
+    return env
+
+
+def run_app(command, domain=TEST_DOMAIN, timeout=300):
+    """Run app.sh with given command and domain. Returns CompletedProcess."""
+    return subprocess.run(
+        [APP_SH, command, domain],
+        capture_output=True, text=True, timeout=timeout,
+        cwd=REPO_ROOT,
+    )
+
+
+BWS_TOKEN_FILE = os.path.join(
+    os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+    "hantim", "bws-token",
+)
+
+REQUIRED_TOOLS = ["bws", "jq", "dig", "ssh", "curl", "git"]
+
+REQUIRED_SECRETS = [
+    "hantim-vultr-api-key",
+    "hantim-new-app-script",
+    "hantim-deploy-ssh-private-key",
+    "hantim-garage-admin-token",
+    "hantim-garage-media-key-id",
+    "hantim-uptimerobot-api-key",
+]
+
+
+@pytest.fixture(scope="session", autouse=True)
+def preflight():
+    """Check that all required tools and credentials are available."""
+    errors = []
+
+    for tool in REQUIRED_TOOLS:
+        if shutil.which(tool) is None:
+            errors.append(f"missing CLI tool: {tool}")
+
+    if not os.environ.get("BWS_ACCESS_TOKEN") and not os.path.isfile(BWS_TOKEN_FILE):
+        errors.append(
+            f"no bws token: set BWS_ACCESS_TOKEN or create {BWS_TOKEN_FILE}"
+        )
+
+    if errors:
+        pytest.exit(
+            "Preflight failed:\n  " + "\n  ".join(errors),
+            returncode=1,
+        )
+
+
+@pytest.fixture(scope="session")
+def secrets(preflight):
+    """Fetch all bws secrets once for the session."""
+    result = subprocess.run(
+        ["bws", "secret", "list"],
+        capture_output=True, text=True, env=_bws_env(),
+    )
+    if result.returncode != 0:
+        pytest.exit(
+            f"bws secret list failed (is the token valid?):\n{result.stderr}",
+            returncode=1,
+        )
+
+    all_secrets = json.loads(result.stdout)
+    by_key = {s["key"]: s["value"] for s in all_secrets}
+
+    missing = [k for k in REQUIRED_SECRETS if k not in by_key]
+    if missing:
+        pytest.exit(
+            f"Missing secrets in Bitwarden: {', '.join(missing)}",
+            returncode=1,
+        )
+
+    return by_key
+
+
+def cleanup(secrets, revert_git=False):
+    """Best-effort cleanup of all TEST_DOMAIN resources."""
+    errors = []
+
+    # --- UptimeRobot monitor ---
+    try:
+        key = secrets["hantim-uptimerobot-api-key"]
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET",
+            f"{UPTIMEROBOT_API}/monitors?"
+            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
+            headers=auth,
+        )
+        if status == 200 and data:
+            for m in data.get("data", []):
+                if m.get("url") == f"https://www.{TEST_DOMAIN}":
+                    api("DELETE", f"{UPTIMEROBOT_API}/monitors/{m['id']}", headers=auth)
+    except Exception as e:
+        errors.append(f"UptimeRobot: {e}")
+
+    # --- Garage bucket ---
+    try:
+        token = secrets["hantim-garage-admin-token"]
+        auth = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=auth,
+        )
+        if status == 200 and data and data.get("id"):
+            bid = data["id"]
+            # Deny media key access
+            media_key = secrets.get("hantim-garage-media-key-id", "")
+            if media_key:
+                api(
+                    "POST",
+                    f"{GARAGE_API}/v2/DenyBucketKey",
+                    headers=auth,
+                    data={
+                        "bucketId": bid,
+                        "accessKeyId": media_key,
+                        "permissions": {"read": True, "write": True, "owner": False},
+                    },
+                )
+            # Delete bucket directly (Garage rejects alias removal when
+            # it's the bucket's only alias, so skip RemoveBucketAlias)
+            api(
+                "POST",
+                f"{GARAGE_API}/v2/DeleteBucket?"
+                + urllib.parse.urlencode({"id": bid}),
+                headers={"Authorization": f"Bearer {token}"},
+            )
+    except Exception as e:
+        errors.append(f"Garage: {e}")
+
+    # --- Gitea repo ---
+    try:
+        token = secrets["hantim-new-app-script"]
+        api(
+            "DELETE",
+            f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}",
+            headers={"Authorization": f"token {token}"},
+        )
+    except Exception as e:
+        errors.append(f"Gitea: {e}")
+
+    # --- Local files ---
+    local_paths = [
+        os.path.join(REPO_ROOT, "docker", TEST_DOMAIN),
+        os.path.join(REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml"),
+        os.path.join(REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf"),
+    ]
+
+    if revert_git:
+        # Files may be tracked — check if last commit added them
+        try:
+            log = subprocess.run(
+                ["git", "log", "--format=%s", "-1"],
+                capture_output=True, text=True, cwd=REPO_ROOT,
+            )
+            if log.stdout.strip() == f"add {TEST_DOMAIN}":
+                removed_any = False
+                for f in [
+                    f"docker/{TEST_DOMAIN}",
+                    f".gitea/workflows/deploy-{TEST_DOMAIN}.yml",
+                    f"docker/nginx/conf.d/{TEST_DOMAIN}.conf",
+                ]:
+                    full = os.path.join(REPO_ROOT, f)
+                    if os.path.exists(full):
+                        subprocess.run(
+                            ["git", "rm", "-rf", f],
+                            cwd=REPO_ROOT,
+                            capture_output=True,
+                        )
+                        removed_any = True
+                if removed_any:
+                    subprocess.run(
+                        ["git", "commit", "-m", f"remove {TEST_DOMAIN}"],
+                        cwd=REPO_ROOT, check=True, capture_output=True,
+                    )
+                    subprocess.run(
+                        ["git", "push"],
+                        cwd=REPO_ROOT, check=True, capture_output=True,
+                    )
+        except Exception as e:
+            errors.append(f"Git revert: {e}")
+
+    # Always try to remove untracked local files too
+    for path in local_paths:
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            elif os.path.isfile(path):
+                os.remove(path)
+        except FileNotFoundError:
+            pass
+        except Exception as e:
+            errors.append(f"Local {path}: {e}")
+
+    # --- Vultr DNS records (keep zone to avoid slow re-creation) ---
+    try:
+        key = secrets["hantim-vultr-api-key"]
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
+        )
+        if status == 200 and data:
+            for r in data.get("records", []):
+                if r["type"] == "A" and r["name"] in ("", "www"):
+                    api(
+                        "DELETE",
+                        f"{VULTR_API}/domains/{TEST_DOMAIN}/records/{r['id']}",
+                        headers=auth,
+                    )
+    except Exception as e:
+        errors.append(f"Vultr: {e}")
+
+    if errors:
+        print(f"Cleanup warnings: {'; '.join(errors)}")
@@ -0,0 +1,236 @@
+"""End-to-end tests for tools/app.sh against real services.
+
+Test domain: kgfamily.com
+Run: uvx pytest tests/test_app_e2e.py -v -x
+"""
+
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import pytest
+
+from conftest import (
+    TEST_DOMAIN,
+    CONTAINER_NAME,
+    REPO_ROOT,
+    VULTR_API,
+    GITEA_URL,
+    GITEA_ORG,
+    GARAGE_API,
+    UPTIMEROBOT_API,
+    api,
+    run_app,
+    cleanup,
+)
+
+
+class TestAppSubcommands:
+    """Test each app.sh subcommand individually, in order."""
+
+    @pytest.fixture(autouse=True, scope="class")
+    def _cleanup(self, secrets):
+        cleanup(secrets, revert_git=True)
+        yield
+        cleanup(secrets, revert_git=False)
+
+    def test_dns(self, secrets):
+        result = run_app("dns")
+        assert result.returncode == 0, f"app.sh dns failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {"Authorization": f"Bearer {secrets['hantim-vultr-api-key']}"}
+        status, data = api("GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth)
+        assert status == 200, f"Vultr API returned {status}"
+
+        records = data["records"]
+        bare_a = [r for r in records if r["type"] == "A" and r["name"] == ""]
+        www_a = [r for r in records if r["type"] == "A" and r["name"] == "www"]
+
+        assert len(bare_a) == 1, f"Expected 1 bare A record, got {len(bare_a)}"
+        assert len(www_a) == 1, f"Expected 1 www A record, got {len(www_a)}"
+        assert bare_a[0]["data"] == www_a[0]["data"], "Bare and www should point to same IP"
+
+    def test_repo(self, secrets):
+        result = run_app("repo")
+        assert result.returncode == 0, f"app.sh repo failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {"Authorization": f"token {secrets['hantim-new-app-script']}"}
+        status, data = api("GET", f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}", headers=auth)
+        assert status == 200, f"Repo not found (HTTP {status})"
+        assert data["name"] == TEST_DOMAIN
+
+    def test_files(self):
+        result = run_app("files")
+        assert result.returncode == 0, f"app.sh files failed:\n{result.stdout}\n{result.stderr}"
+
+        # compose.yml
+        compose_path = os.path.join(REPO_ROOT, "docker", TEST_DOMAIN, "compose.yml")
+        assert os.path.isfile(compose_path)
+        compose = open(compose_path).read()
+        assert f"image: git.timothykim.net/hantim/{TEST_DOMAIN}:latest" in compose
+        assert f"container_name: {CONTAINER_NAME}" in compose
+        assert "shared" in compose
+
+        # deploy workflow
+        workflow_path = os.path.join(
+            REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml"
+        )
+        assert os.path.isfile(workflow_path)
+        workflow = open(workflow_path).read()
+        assert f"docker/{TEST_DOMAIN}/**" in workflow
+        assert f"deploy-{TEST_DOMAIN}" in workflow
+
+        # nginx conf
+        conf_path = os.path.join(
+            REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf"
+        )
+        assert os.path.isfile(conf_path)
+        conf = open(conf_path).read()
+        assert f"server_name {TEST_DOMAIN} www.{TEST_DOMAIN}" in conf
+        assert f"server_name www.{TEST_DOMAIN}" in conf
+        assert "security-headers.inc" in conf
+        assert f"proxy_pass http://{CONTAINER_NAME}:80" in conf
+
+    def test_cert(self):
+        result = run_app("cert", timeout=180)
+        assert result.returncode == 0, f"app.sh cert failed:\n{result.stdout}\n{result.stderr}"
+
+    def test_garage(self, secrets):
+        result = run_app("garage")
+        assert result.returncode == 0, f"app.sh garage failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {
+            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
+        }
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=auth,
+        )
+        assert status == 200, f"Bucket not found (HTTP {status})"
+        assert data["id"], "Bucket has no ID"
+        assert TEST_DOMAIN in data.get("globalAliases", []), "Global alias not set"
+        assert data.get("websiteAccess") is True, f"Website access not enabled: {data.get('websiteAccess')}"
+
+    def test_build(self, secrets):
+        result = run_app("build", timeout=180)
+        assert result.returncode == 0, f"app.sh build failed:\n{result.stdout}\n{result.stderr}"
+
+    def test_monitor(self, secrets):
+        result = run_app("monitor")
+        assert result.returncode == 0, f"app.sh monitor failed:\n{result.stdout}\n{result.stderr}"
+
+        # UptimeRobot API may have brief eventual consistency after create
+        auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=auth,
+            )
+            assert status == 200, f"UptimeRobot API returned {status}"
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls, (
+            f"Monitor not found. URLs: {urls}\n"
+            f"app.sh output:\n{result.stdout}"
+        )
+
+
+class TestAppAll:
+    """Test the full app.sh all flow end-to-end."""
+
+    @pytest.fixture(autouse=True, scope="class")
+    def _cleanup(self, secrets):
+        cleanup(secrets, revert_git=True)
+        yield
+        cleanup(secrets, revert_git=True)
+
+    def test_all(self, secrets):
+        result = run_app("all", timeout=900)
+        assert result.returncode == 0, f"app.sh all failed:\n{result.stdout}\n{result.stderr}"
+
+        # --- DNS ---
+        auth = {"Authorization": f"Bearer {secrets['hantim-vultr-api-key']}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
+        )
+        assert status == 200
+        records = data["records"]
+        assert any(r["type"] == "A" and r["name"] == "" for r in records)
+        assert any(r["type"] == "A" and r["name"] == "www" for r in records)
+
+        # --- Gitea repo ---
+        gitea_auth = {"Authorization": f"token {secrets['hantim-new-app-script']}"}
+        status, _ = api(
+            "GET",
+            f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}",
+            headers=gitea_auth,
+        )
+        assert status == 200, "Gitea repo not found"
+
+        # --- Local files ---
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, "docker", TEST_DOMAIN, "compose.yml")
+        )
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml")
+        )
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf")
+        )
+
+        # --- Git commit was pushed ---
+        log = subprocess.run(
+            ["git", "log", "--format=%s", "-1"],
+            capture_output=True, text=True, cwd=REPO_ROOT,
+        )
+        assert log.stdout.strip() == f"add {TEST_DOMAIN}"
+
+        # --- Garage bucket ---
+        garage_auth = {
+            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
+        }
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=garage_auth,
+        )
+        assert status == 200, "Garage bucket not found"
+
+        # --- Site is live ---
+        # Use server IP from verified DNS records to bypass negative cache
+        server_ip = next(
+            r["data"] for r in records if r["type"] == "A" and r["name"] == ""
+        )
+        curl = subprocess.run(
+            ["curl", "-sf", "--resolve", f"www.{TEST_DOMAIN}:443:{server_ip}",
+             "-o", "/dev/null", "-w", "%{http_code}",
+             f"https://www.{TEST_DOMAIN}"],
+            capture_output=True, text=True, timeout=10,
+        )
+        assert curl.returncode == 0, f"Site not reachable (HTTP {curl.stdout})"
+
+        # --- UptimeRobot monitor ---
+        ur_auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=ur_auth,
+            )
+            assert status == 200
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls
@@ -26,6 +26,7 @@ def test_tls12_works():
        assert s.version() == "TLSv1.2"


+@pytest.mark.filterwarnings("ignore::DeprecationWarning")
 def test_tls11_rejected():
    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    ctx.minimum_version = ssl.TLSVersion.TLSv1_1
@@ -28,7 +28,8 @@ usage() {
    echo "  garage    Create Garage media bucket and grant access"
    echo "  build     Trigger initial build workflow"
    echo "  verify    Check if site is live"
-    echo "  all       Run all steps (dns, repo, files, cert, garage, commit, build, verify)"
+    echo "  monitor   Create UptimeRobot HTTPS monitor"
+    echo "  all       Run all steps (dns, repo, files, cert, garage, commit, build, verify, monitor)"
    echo ""
    echo "Requires: bws, jq, dig"
    exit 1
@@ -252,16 +253,22 @@ cmd_repo() {
 }

 cmd_files() {
+    local all_exist=true
    for f in "docker/$APP/compose.yml" ".gitea/workflows/deploy-$APP.yml" "docker/nginx/conf.d/$APP.conf"; do
-        if [ -e "$REPO_ROOT/$f" ]; then
-            echo "Error: $f already exists."
-            exit 1
+        if [ ! -e "$REPO_ROOT/$f" ]; then
+            all_exist=false
+            break
        fi
    done
+    if [ "$all_exist" = true ]; then
+        echo "  All files already exist, skipping."
+        return
+    fi

-    echo "==> Creating docker/$APP/compose.yml..."
-    mkdir -p "$REPO_ROOT/docker/$APP"
-    cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
+    if [ ! -e "$REPO_ROOT/docker/$APP/compose.yml" ]; then
+        echo "==> Creating docker/$APP/compose.yml..."
+        mkdir -p "$REPO_ROOT/docker/$APP"
+        cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
 services:
  app:
    image: git.timothykim.net/hantim/$APP:latest
@@ -274,10 +281,14 @@ networks:
  shared:
    external: true
 EOF
+    else
+        echo "  docker/$APP/compose.yml already exists, skipping."
+    fi

-    echo "==> Creating .gitea/workflows/deploy-$APP.yml..."
-    mkdir -p "$REPO_ROOT/.gitea/workflows"
-    cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
+    if [ ! -e "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" ]; then
+        echo "==> Creating .gitea/workflows/deploy-$APP.yml..."
+        mkdir -p "$REPO_ROOT/.gitea/workflows"
+        cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
 name: Deploy APP_PLACEHOLDER

 on:
@@ -293,14 +304,19 @@ jobs:
      - name: Deploy via SSH
        run: |
          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
 OUTER
-    sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
+        sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
+    else
+        echo "  .gitea/workflows/deploy-$APP.yml already exists, skipping."
+    fi

-    echo "==> Creating docker/nginx/conf.d/$APP.conf..."
-    cat > "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" <<NGINX
+    if [ ! -e "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" ]; then
+        echo "==> Creating docker/nginx/conf.d/$APP.conf..."
+        cat > "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" <<NGINX
 # Redirect HTTP to HTTPS, bare domain to www
 server {
    listen 80;
@@ -340,6 +356,7 @@ server {
    ssl_certificate_key /etc/letsencrypt/live/$APP/privkey.pem;

    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;

    location /media/ {
        proxy_pass http://garage:3902/;
@@ -355,8 +372,9 @@ server {
    }
 }
 NGINX
-
-    echo "  Files created. Run 'git add' and commit when ready."
+    else
+        echo "  docker/nginx/conf.d/$APP.conf already exists, skipping."
+    fi
 }

 cmd_cert() {
@@ -386,10 +404,8 @@ cmd_garage() {

    BUCKET_ID=$(echo "$BUCKET_RESP" | jq -r '.id')
    if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
-        BUCKET_ID=$(curl -s -X POST "$GARAGE_API/v2/GetBucketInfo" \
-            -H "$GARAGE_AUTH" \
-            -H "Content-Type: application/json" \
-            -d "{\"globalAlias\": \"$APP\"}" | jq -r '.id')
+        BUCKET_ID=$(curl -s "$GARAGE_API/v2/GetBucketInfo?globalAlias=$APP" \
+            -H "$GARAGE_AUTH" | jq -r '.id')
        if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
            echo "Error: Failed to create or find bucket for $APP"
            echo "$BUCKET_RESP"
@@ -466,9 +482,52 @@ cmd_verify() {
    echo "    - The container is running on the server: docker ps"
 }

+cmd_monitor() {
+    require_bws
+
+    echo "==> Fetching UptimeRobot API key..."
+    UPTIMEROBOT_API_KEY=$(bws_get "hantim-uptimerobot-api-key")
+    UPTIMEROBOT_API="https://api.uptimerobot.com/v3"
+    UPTIMEROBOT_AUTH="Authorization: Bearer $UPTIMEROBOT_API_KEY"
+
+    MONITOR_URL="https://www.$APP"
+
+    echo "==> Checking for existing UptimeRobot monitor for $MONITOR_URL..."
+    EXISTING=$(curl -s -G "$UPTIMEROBOT_API/monitors" \
+        -H "$UPTIMEROBOT_AUTH" \
+        --data-urlencode "search=$APP")
+
+    MATCH=$(echo "$EXISTING" | jq -r --arg url "$MONITOR_URL" \
+        '.data[]? | select(.url == $url) | .id')
+    if [ -n "$MATCH" ]; then
+        echo "  Monitor already exists (id: $MATCH), skipping."
+        return
+    fi
+
+    echo "==> Fetching alert contacts..."
+    ALERT_CONTACTS=$(curl -s "$UPTIMEROBOT_API/user/alert-contacts" \
+        -H "$UPTIMEROBOT_AUTH" | jq '[.[] | {alertContactId: .id, threshold: 5, recurrence: 30}]')
+
+    echo "==> Creating UptimeRobot HTTPS monitor for $MONITOR_URL..."
+    RESP=$(curl -s -X POST "$UPTIMEROBOT_API/monitors" \
+        -H "$UPTIMEROBOT_AUTH" \
+        -H "Content-Type: application/json" \
+        -d "$(jq -n --arg name "$APP" --arg url "$MONITOR_URL" --argjson contacts "$ALERT_CONTACTS" \
+            '{friendlyName: $name, url: $url, type: 1, interval: 300, timeout: 30, assignedAlertContacts: $contacts, tagNames: ["hantim"]}')")
+
+    MONITOR_ID=$(echo "$RESP" | jq -r '.id // empty')
+    if [ -n "$MONITOR_ID" ]; then
+        echo "  Monitor created (id: $MONITOR_ID)."
+    else
+        ERROR_MSG=$(echo "$RESP" | jq -r 'if .message then (.message | if type == "array" then join("; ") else . end) else .error // "unknown error" end')
+        echo "  Warning: Failed to create monitor: $ERROR_MSG"
+    fi
+}
+
 cmd_all() {
    cmd_dns
    cmd_repo
+    cmd_cert
    cmd_files

    echo "==> Committing and pushing hantim-server..."
@@ -481,10 +540,10 @@ cmd_all() {
        git push
    fi

-    cmd_cert
    cmd_garage
    cmd_build
    cmd_verify
+    cmd_monitor
 }

 # --- Dispatch ---
@@ -497,6 +556,7 @@ case "$COMMAND" in
    garage)  cmd_garage ;;
    build)   cmd_build ;;
    verify)  cmd_verify ;;
+    monitor) cmd_monitor ;;
    all)     cmd_all ;;
    *)
        echo "Unknown command: $COMMAND"
@@ -1,13 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# Thin wrapper — delegates to app.sh all
-if [ -z "${1:-}" ]; then
-    echo "Usage: ./tools/new-app.sh <domain>"
-    echo "  Example: ./tools/new-app.sh example.com"
-    echo ""
-    echo "For individual steps, use: ./tools/app.sh <command> <domain>"
-    exit 1
-fi
-
-exec "$(dirname "$0")/app.sh" all "$1"
@@ -1,72 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-if [ -z "${1:-}" ]; then
-    echo "Usage: ./tools/new-service.sh <name>"
-    echo "  Example: ./tools/new-service.sh garage"
-    echo ""
-    echo "Creates docker/<name>/compose.yml and .gitea/workflows/deploy-<name>.yml"
-    exit 1
-fi
-
-APP="$1"
-
-if ! [[ "$APP" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
-    echo "Error: name must be alphanumeric (hyphens, dots, underscores allowed)."
-    exit 1
-fi
-
-REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-
-if [ -d "$REPO_ROOT/docker/$APP" ]; then
-    echo "Error: docker/$APP already exists."
-    exit 1
-fi
-
-echo "Creating docker/$APP/compose.yml..."
-mkdir -p "$REPO_ROOT/docker/$APP"
-cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
-services:
-  $APP:
-    image: TODO
-    restart: unless-stopped
-    container_name: $APP
-    networks:
-      - shared
-
-networks:
-  shared:
-    external: true
-EOF
-
-echo "Creating .gitea/workflows/deploy-$APP.yml..."
-mkdir -p "$REPO_ROOT/.gitea/workflows"
-cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
-name: Deploy APP_PLACEHOLDER
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - 'docker/APP_PLACEHOLDER/**'
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy via SSH
-        run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
-OUTER
-sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
-
-echo "Done. Edit docker/$APP/compose.yml, then commit and push."
-echo ""
-echo "If this service needs secrets:"
-echo "  1. Add secrets to Bitwarden Secrets Manager"
-echo "  2. Map them in docker/$APP/.env.keys (format: ENV_VAR=bws-secret-name)"
-echo "  3. Add env_file: .env to compose.yml"
-echo "  configure.sh will generate the .env file on the server automatically."
@@ -0,0 +1,367 @@
+#!/bin/bash
+set -euo pipefail
+
+usage() {
+    echo "Usage: ./tools/service.sh <command> <name>"
+    echo ""
+    echo "Commands:"
+    echo "  dns <name>          Create A record for <name>.hantim.net"
+    echo "  files <name>        Create compose.yml and deploy workflow"
+    echo "  nginx <name> <port> Create nginx conf for <name>.hantim.net"
+    echo "  cert <name>         Issue SSL certificate for <name>.hantim.net"
+    echo "  all <name> <port>   Run dns + files + nginx + cert"
+    echo ""
+    echo "Examples:"
+    echo "  ./tools/service.sh files garage"
+    echo "  ./tools/service.sh dns garage        # garage.hantim.net -> server IP"
+    echo "  ./tools/service.sh nginx garage 3903 # create nginx conf"
+    echo "  ./tools/service.sh cert garage       # issue SSL cert"
+    echo "  ./tools/service.sh all garage 3903"
+    echo ""
+    echo "Requires: bws, jq, dig (for dns/cert)"
+    exit 1
+}
+
+COMMAND="${1:-}"
+
+if [ -z "$COMMAND" ]; then
+    usage
+fi
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+# --- Dependency checks ---
+
+require_bws() {
+    if ! command -v bws &>/dev/null; then
+        echo "Error: Bitwarden Secrets Manager CLI (bws) is not installed."
+        echo "  https://github.com/bitwarden/sdk-sm/releases"
+        exit 1
+    fi
+    if ! command -v jq &>/dev/null; then
+        echo "Error: jq is not installed."
+        exit 1
+    fi
+
+    BWS_TOKEN_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hantim/bws-token"
+    if [ -z "${BWS_ACCESS_TOKEN:-}" ]; then
+        if [ -f "$BWS_TOKEN_FILE" ]; then
+            BWS_ACCESS_TOKEN=$(cat "$BWS_TOKEN_FILE")
+        else
+            echo "==> Bitwarden Secrets Manager access token required."
+            echo "    Generate one for your machine account at:"
+            echo "    https://vault.bitwarden.com/#/sm/machine-accounts"
+            read -rp "    Paste access token: " BWS_ACCESS_TOKEN
+            if [ -z "$BWS_ACCESS_TOKEN" ]; then
+                echo "ERROR: Access token cannot be empty."
+                exit 1
+            fi
+            mkdir -p "$(dirname "$BWS_TOKEN_FILE")"
+            echo "$BWS_ACCESS_TOKEN" > "$BWS_TOKEN_FILE"
+            chmod 600 "$BWS_TOKEN_FILE"
+        fi
+    fi
+    export BWS_ACCESS_TOKEN
+}
+
+bws_get() {
+    local name="$1"
+    local value
+    value=$(bws secret list | jq -r --arg name "$name" '.[] | select(.key == $name) | .value')
+    if [ -z "$value" ]; then
+        echo "ERROR: Secret '$name' not found in Bitwarden Secrets Manager." >&2
+        exit 1
+    fi
+    echo "$value"
+}
+
+require_dig() {
+    if ! command -v dig &>/dev/null; then
+        echo "Error: dig is not installed."
+        echo "  On macOS: brew install bind"
+        echo "  On Linux: dnf install bind-utils"
+        exit 1
+    fi
+}
+
+resolve_server_ip() {
+    require_dig
+    SERVER_IP=$(dig +short hantim.net | head -1)
+    if [ -z "$SERVER_IP" ]; then
+        echo "Error: Could not resolve hantim.net to get server IP."
+        exit 1
+    fi
+}
+
+setup_deploy_key() {
+    require_bws
+    echo "==> Fetching deploy key..."
+    DEPLOY_KEY=$(bws_get "hantim-deploy-ssh-private-key")
+    DEPLOY_KEY_FILE=$(mktemp)
+    echo "$DEPLOY_KEY" > "$DEPLOY_KEY_FILE"
+    chmod 600 "$DEPLOY_KEY_FILE"
+    trap "rm -f $DEPLOY_KEY_FILE" EXIT
+}
+
+validate_name() {
+    local name="$1"
+    if [ -z "$name" ]; then
+        echo "Error: service name is required."
+        echo ""
+        usage
+    fi
+    if ! [[ "$name" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
+        echo "Error: name must be alphanumeric (hyphens, dots, underscores allowed)."
+        exit 1
+    fi
+}
+
+# --- Commands ---
+
+cmd_dns() {
+    local name="$1"
+    local zone="hantim.net"
+    local fqdn="$name.$zone"
+
+    require_dig
+    require_bws
+
+    SERVER_IP=$(dig +short "$zone" | head -1)
+    if [ -z "$SERVER_IP" ]; then
+        echo "Error: Could not resolve $zone to get server IP."
+        exit 1
+    fi
+
+    echo "==> Fetching Vultr API key..."
+    VULTR_API_KEY=$(bws_get "hantim-vultr-api-key")
+    VULTR_API="https://api.vultr.com/v2"
+    VULTR_AUTH="Authorization: Bearer $VULTR_API_KEY"
+
+    echo "==> Checking existing A records for $name in $zone..."
+    RECORDS=$(curl -s "$VULTR_API/domains/$zone/records" -H "$VULTR_AUTH")
+
+    echo "$RECORDS" | jq -r --arg sub "$name" \
+        '.records[] | select(.type == "A" and .name == $sub) | .id' | while read -r id; do
+        echo "  Deleting existing A record for $fqdn (id: $id)"
+        curl -s -X DELETE "$VULTR_API/domains/$zone/records/$id" -H "$VULTR_AUTH"
+    done
+
+    echo "  Creating A record: $fqdn -> $SERVER_IP"
+    RESP=$(mktemp)
+    HTTP_CODE=$(curl -s -o "$RESP" -w "%{http_code}" \
+        -X POST "$VULTR_API/domains/$zone/records" \
+        -H "$VULTR_AUTH" \
+        -H "Content-Type: application/json" \
+        -d "{\"name\": \"$name\", \"type\": \"A\", \"data\": \"$SERVER_IP\", \"ttl\": 3600}")
+    if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "204" ]; then
+        echo "Error: Failed to create A record (HTTP $HTTP_CODE)"
+        cat "$RESP"
+        rm -f "$RESP"
+        exit 1
+    fi
+    rm -f "$RESP"
+
+    echo "==> Verifying DNS propagation for $fqdn..."
+    for i in $(seq 1 12); do
+        RESOLVED=$(dig +short "$fqdn" @ns1.vultr.com 2>/dev/null | head -1)
+        if [ "$RESOLVED" = "$SERVER_IP" ]; then
+            echo "  DNS is live: $fqdn -> $SERVER_IP"
+            return
+        fi
+        if [ "$i" = "12" ]; then
+            echo "  Warning: $fqdn did not resolve to $SERVER_IP after 60 seconds. It may take longer to propagate."
+            return
+        fi
+        sleep 5
+    done
+}
+
+cmd_files() {
+    local name="$1"
+
+    local all_exist=true
+    for f in "docker/$name/compose.yml" ".gitea/workflows/deploy-$name.yml"; do
+        if [ ! -e "$REPO_ROOT/$f" ]; then
+            all_exist=false
+            break
+        fi
+    done
+    if [ "$all_exist" = true ]; then
+        echo "  All files already exist, skipping."
+        return
+    fi
+
+    if [ ! -e "$REPO_ROOT/docker/$name/compose.yml" ]; then
+        echo "==> Creating docker/$name/compose.yml..."
+        mkdir -p "$REPO_ROOT/docker/$name"
+        touch "$REPO_ROOT/docker/$name/.gitignore"
+        cat > "$REPO_ROOT/docker/$name/compose.yml" <<EOF
+services:
+  $name:
+    image: TODO
+    restart: unless-stopped
+    container_name: $name
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true
+EOF
+    else
+        echo "  docker/$name/compose.yml already exists, skipping."
+    fi
+
+    if [ ! -e "$REPO_ROOT/.gitea/workflows/deploy-$name.yml" ]; then
+        echo "==> Creating .gitea/workflows/deploy-$name.yml..."
+        mkdir -p "$REPO_ROOT/.gitea/workflows"
+        cat > "$REPO_ROOT/.gitea/workflows/deploy-$name.yml" <<'OUTER'
+name: Deploy APP_PLACEHOLDER
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/APP_PLACEHOLDER/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
+OUTER
+        sed -i "s/APP_PLACEHOLDER/$name/g" "$REPO_ROOT/.gitea/workflows/deploy-$name.yml"
+    else
+        echo "  .gitea/workflows/deploy-$name.yml already exists, skipping."
+    fi
+
+    echo ""
+    echo "Done. Edit docker/$name/compose.yml, then commit and push."
+    echo ""
+    echo "If this service needs secrets:"
+    echo "  1. Add secrets to Bitwarden Secrets Manager"
+    echo "  2. Map them in docker/$name/.env.keys (format: ENV_VAR=bws-secret-name)"
+    echo "  3. Add env_file: .env to compose.yml"
+    echo "  configure.sh will generate the .env file on the server automatically."
+}
+
+cmd_nginx() {
+    local name="$1"
+    local port="$2"
+    local fqdn="$name.hantim.net"
+
+    if ! [[ "$port" =~ ^[0-9]+$ ]]; then
+        echo "Error: port must be a number."
+        exit 1
+    fi
+
+    if [ -e "$REPO_ROOT/docker/nginx/conf.d/$fqdn.conf" ]; then
+        echo "  docker/nginx/conf.d/$fqdn.conf already exists, skipping."
+        return
+    fi
+
+    echo "==> Creating docker/nginx/conf.d/$fqdn.conf..."
+    cat > "$REPO_ROOT/docker/nginx/conf.d/$fqdn.conf" <<NGINX
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $fqdn;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://$fqdn\$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name $fqdn;
+
+    ssl_certificate     /etc/letsencrypt/live/$fqdn/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/$fqdn/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://$name:$port;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}
+NGINX
+}
+
+cmd_cert() {
+    local name="$1"
+    local fqdn="$name.hantim.net"
+
+    resolve_server_ip
+    setup_deploy_key
+
+    echo "==> Issuing SSL certificate for $fqdn via deploy user..."
+    ssh -o StrictHostKeyChecking=accept-new -i "$DEPLOY_KEY_FILE" deploy@"$SERVER_IP" "cert-$fqdn"
+    echo "  Certificate issued."
+}
+
+cmd_all() {
+    local name="$1"
+    local port="$2"
+
+    cmd_dns "$name"
+    cmd_files "$name"
+    cmd_nginx "$name" "$port"
+    cmd_cert "$name"
+}
+
+# --- Dispatch ---
+
+case "$COMMAND" in
+    dns)
+        validate_name "${2:-}"
+        cmd_dns "${2:-}"
+        ;;
+    files)
+        validate_name "${2:-}"
+        cmd_files "${2:-}"
+        ;;
+    nginx)
+        validate_name "${2:-}"
+        if [ -z "${3:-}" ]; then
+            echo "Error: port is required for nginx command."
+            echo "  Usage: ./tools/service.sh nginx <name> <port>"
+            exit 1
+        fi
+        cmd_nginx "${2:-}" "${3:-}"
+        ;;
+    cert)
+        validate_name "${2:-}"
+        cmd_cert "${2:-}"
+        ;;
+    all)
+        validate_name "${2:-}"
+        if [ -z "${3:-}" ]; then
+            echo "Error: port is required for all command."
+            echo "  Usage: ./tools/service.sh all <name> <port>"
+            exit 1
+        fi
+        cmd_all "${2:-}" "${3:-}"
+        ;;
+    *)
+        echo "Unknown command: $COMMAND"
+        echo ""
+        usage
+        ;;
+esac
Author	SHA1	Message	Date
timothykim	0f951281d5	self-heal standalone cert renewal configs on provision Provision server / provision (push) Successful in 22s Details	2026-06-08 11:35:37 -04:00
timothykim	f093a66580	fix e2e tests for garage v2 and uptimerobot v3 api changes	2026-04-10 16:25:54 -04:00
timothykim	f48ad0e21e	remove kgfamily.com Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 16:21:58 -04:00
timothykim	1f41f51430	add kgfamily.com Deploy kgfamily.com / deploy (push) Successful in 2s Details Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 16:20:28 -04:00
timothykim	50ee47917c	remove kgfamily.com Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 16:17:42 -04:00
timothykim	a303e55de6	add kgfamily.com Deploy kgfamily.com / deploy (push) Successful in 2s Details Deploy nginx / deploy (push) Successful in 2s Details	2026-04-10 16:14:29 -04:00
timothykim	bc41880155	remove kgfamily.com Deploy nginx / deploy (push) Successful in 2s Details	2026-04-10 16:08:19 -04:00
timothykim	59f0a367ef	add kgfamily.com Deploy kgfamily.com / deploy (push) Successful in 1s Details Deploy nginx / deploy (push) Successful in 2s Details	2026-04-10 16:04:54 -04:00
timothykim	7366f11323	remove kgfamily.com Deploy nginx / deploy (push) Successful in 4s Details	2026-04-10 15:53:08 -04:00
timothykim	3678b37133	add kgfamily.com Deploy kgfamily.com / deploy (push) Successful in 3s Details Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 15:49:54 -04:00
timothykim	37ad2405b3	remove kgfamily.com Deploy nginx / deploy (push) Successful in 2s Details	2026-04-10 15:43:07 -04:00
timothykim	6a0792e1fd	add kgfamily.com Deploy kgfamily.com / deploy (push) Successful in 3s Details Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 15:40:02 -04:00
timothykim	1f3ddc8024	suppress expected TLS 1.1 deprecation warning in test	2026-04-10 15:11:18 -04:00
timothykim	514061936b	strip upstream X-Frame-Options from service proxies Deploy nginx / deploy (push) Successful in 3s Details	2026-04-10 15:08:59 -04:00
timothykim	5e85de9130	fix deploy race condition and harden security Deploy nginx / deploy (push) Successful in 5s Details - reorder app.sh to run cert before commit/push (fixes workflow failure on first run) - quote $SSH_ORIGINAL_COMMAND in authorized_keys to prevent command injection - add port validation in service.sh nginx command - pin host key in deploy-beszel.yml workflow - fix append vs overwrite for known_hosts in deploy-nginx.yml	2026-04-10 15:00:10 -04:00
timothykim	900900a95b	remove duplicate sub_filter_types text/html Deploy nginx / deploy (push) Successful in 3s Details	2026-04-09 17:48:16 -04:00
timothykim	0869be8814	gitignore beszel_agent_data Deploy beszel / deploy (push) Successful in 3s Details	2026-04-09 17:46:28 -04:00
timothykim	ba04116fb2	add carolpreschool.com Deploy nginx / deploy (push) Failing after 2s Details Deploy carolpreschool.com / deploy (push) Successful in 3s Details	2026-04-09 17:40:51 -04:00
timothykim	ff828ecc44	add domain prefix to goatcounter paths Deploy nginx / deploy (push) Successful in 3s Details	2026-04-09 17:35:00 -04:00
timothykim	449b763621	add goatcounter tracking to all sites via nginx sub_filter Deploy nginx / deploy (push) Successful in 4s Details	2026-04-09 16:54:35 -04:00
timothykim	553b77788d	Merge branch 'goatcounter' Deploy goatcounter / deploy (push) Successful in 3s Details	2026-04-09 16:40:16 -04:00
timothykim	f5f08c48ac	adds secret	2026-04-09 16:40:11 -04:00
timothykim	cf92394e3f	update to garage sync	2026-04-09 16:04:35 -04:00
timothykim	a8a3acff84	goat counter	2026-04-09 14:26:17 -04:00
timothykim	ee0c58ad28	adds goatcounter Deploy goatcounter / deploy (push) Successful in 8s Details Deploy nginx / deploy (push) Failing after 1s Details	2026-04-08 20:23:22 -04:00
timothykim	4b32e358a1	add beszel agent with socket connection and env.keys Deploy beszel / deploy (push) Successful in 3s Details	2026-04-08 18:43:50 -04:00
timothykim	79996d02a4	update beszel configuration Deploy beszel / deploy (push) Successful in 5s Details Deploy nginx / deploy (push) Successful in 3s Details	2026-04-07 21:07:56 -04:00
timothykim	54d6c03bd1	add nginx and cert subcommands to service.sh	2026-04-07 20:59:58 -04:00
timothykim	0656d245d5	remove test workflow Deploy nginx / deploy (push) Successful in 6s Details	2026-04-07 20:33:12 -04:00
timothykim	d2b9d0bce9	fix idempotency in app.sh, service.sh and nginx reload in cert handler	2026-04-07 15:56:59 -04:00
timothykim	fcc720ee7b	fix cert handler deleting nginx conf file	2026-03-28 17:48:09 -04:00
timothykim	123af3a269	remove ssl stapling Deploy nginx / deploy (push) Successful in 4s Details Deploy nginx / test (push) Failing after 7s Details	2026-03-28 17:15:12 -04:00
timothykim	cf26235d70	add hcsuzuki.music Deploy hcsuzuki.music / deploy (push) Successful in 28s Details Deploy nginx / deploy (push) Failing after 4s Details Deploy nginx / test (push) Has been skipped Details	2026-03-28 13:37:49 -04:00
timothykim	111969fa9c	e2e test	2026-03-27 16:19:21 -04:00
timothykim	c87f76a78a	rename new-service.sh to service.sh with subcommands	2026-03-27 12:28:32 -04:00
timothykim	313fb4655f	updated workflow Deploy beszel / deploy (push) Successful in 4s Details	2026-03-26 20:02:09 -04:00
timothykim	81151f90aa	update the deploy and new service	2026-03-26 20:01:06 -04:00
timothykim	78773110fc	update beszel compose	2026-03-26 23:49:41 +00:00
timothykim	8762339713	add beszel	2026-03-26 19:45:19 -04:00
timothykim	697de3e199	add uptimerobot monitor subcommand to app.sh	2026-03-26 13:13:41 -04:00
timothykim	43fc8c2e7b	remove new-app.sh, update docs to reference app.sh directly	2026-03-20 20:14:16 -04:00
timothykim	787e747fa4	fix deploy re-exec not forwarding command argument	2026-03-20 20:10:32 -04:00
timothykim	7b1a888db0	add workflow_dispatch and self-trigger paths to all workflows Deploy haanmind.net / deploy (push) Failing after 1s Details Deploy hantim.net / deploy (push) Failing after 1s Details Deploy hcsuzuki.net / deploy (push) Failing after 1s Details Deploy nginx / deploy (push) Failing after 1s Details Deploy nginx / test (push) Has been skipped Details Deploy thekims.family / deploy (push) Failing after 1s Details Deploy timothykim.net / deploy (push) Failing after 1s Details Provision server / provision (push) Successful in 11s Details Deploy garage / deploy (push) Successful in 5s Details	2026-03-20 20:03:11 -04:00
timothykim	a37872489c	add certbot email, pin git pull to origin main	2026-03-20 19:57:45 -04:00
timothykim	de1d60b1db	remove standalone certbot fallback, fail on missing certs	2026-03-20 19:50:44 -04:00
timothykim	c8c79a542a	re-exec deploy.sh after git pull to pick up changes Deploy nginx / deploy (push) Successful in 3s Details Deploy nginx / test (push) Successful in 7s Details	2026-03-20 17:01:36 -04:00
timothykim	b2bb8fa5eb	clarify default server block comment Deploy nginx / deploy (push) Successful in 4s Details Deploy nginx / test (push) Successful in 8s Details	2026-03-20 16:57:08 -04:00
timothykim	729eadee33	use ..conf glob to match only domain conf files for cert issuance	2026-03-20 16:55:52 -04:00
timothykim	4b39eaaf06	skip default server block in cert issuance, remove ssh debug lines	2026-03-20 16:53:47 -04:00
timothykim	085b0bdd1b	clean up trailing newline in nginx.conf Deploy nginx / deploy (push) Failing after 4s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:51:09 -04:00
timothykim	020e7b2950	retrigger nginx deploy for host key test Deploy nginx / deploy (push) Failing after 5s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:43:54 -04:00
timothykim	9a9df25742	retrigger nginx deploy for host key debug Deploy nginx / deploy (push) Failing after 0s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:41:27 -04:00
timothykim	6e2ec87ccf	debug host key pinning in nginx workflow	2026-03-20 16:40:37 -04:00
timothykim	77e08f34cf	retrigger nginx deploy for host key test Deploy nginx / deploy (push) Failing after 0s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:38:40 -04:00
timothykim	061213bf7e	retrigger nginx deploy for host key test Deploy nginx / deploy (push) Failing after 0s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:35:34 -04:00
timothykim	03e45dc6f2	trigger nginx deploy to test host key pinning Deploy nginx / deploy (push) Failing after 0s Details Deploy nginx / test (push) Has been skipped Details	2026-03-20 16:33:15 -04:00
timothykim	16ea39811d	document deploy host key variable in readme	2026-03-20 16:31:54 -04:00
timothykim	7daf7a267b	pin ssh host key in all workflows	2026-03-20 16:31:03 -04:00
timothykim	5abfca19b3	restrict envsubst to expected variables only Deploy garage / deploy (push) Failing after 2s Details	2026-03-20 16:25:23 -04:00
timothykim	9e3996a72b	remove security.md from repo, add to gitignore	2026-03-20 16:17:23 -04:00
timothykim	163aefddee	add default server block to drop unknown host headers	2026-03-20 16:12:06 -04:00
timothykim	3a1ef3a167	fix tls tests to load system ca certificates	2026-03-20 16:07:32 -04:00
timothykim	cce9b5c4c4	add ssl/tls hardening to nginx	2026-03-20 16:05:41 -04:00
timothykim	2bcf67dcc4	fetch secrets by uuid instead of listing all	2026-03-20 15:51:43 -04:00
timothykim	b19e194c0b	remove deploy user from docker group	2026-03-20 15:38:11 -04:00
timothykim	1a7f823a87	restrict garage admin api to home ip	2026-03-20 15:33:36 -04:00
timothykim	0ca8394f8a	add testing section to readme	2026-03-20 15:22:58 -04:00
timothykim	e767ab3235	test -> tests	2026-03-20 15:20:29 -04:00
timothykim	50ab425e8f	fix test directory path in deploy-nginx workflow	2026-03-20 15:19:41 -04:00
timothykim	b7439b23e3	fix nginx security headers not being applied to https responses nginx add_header inheritance meant X-Content-Type-Options and X-Frame-Options were silently dropped from all https server blocks. moved all security headers into a shared snippet (security-headers.inc) included per server block. added pytest-based header verification that auto-discovers sites from conf files.	2026-03-20 15:11:56 -04:00