self-heal standalone cert renewal configs on provision

- reorder app.sh to run cert before commit/push (fixes workflow failure on first run)
- quote $SSH_ORIGINAL_COMMAND in authorized_keys to prevent command injection
- add port validation in service.sh nginx command
- pin host key in deploy-beszel.yml workflow
- fix append vs overwrite for known_hosts in deploy-nginx.yml

.gitea/workflows/deploy-beszel.yml

@@ -0,0 +1,19 @@
+name: Deploy beszel
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/beszel/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-beszel

.gitea/workflows/deploy-carolpreschool.com.yml

@@ -0,0 +1,19 @@
+name: Deploy carolpreschool.com
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/carolpreschool.com/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-carolpreschool.com

.gitea/workflows/deploy-garage.yml

@@ -1,10 +1,12 @@
 name: Deploy garage
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/garage/**'
+      - '.gitea/workflows/deploy-garage.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-garage
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-garage

.gitea/workflows/deploy-goatcounter.yml

@@ -0,0 +1,19 @@
+name: Deploy goatcounter
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/goatcounter/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-goatcounter

.gitea/workflows/deploy-haanmind.net.yml

@@ -1,10 +1,12 @@
 name: Deploy haanmind.net
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/haanmind.net/**'
+      - '.gitea/workflows/deploy-haanmind.net.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-haanmind.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-haanmind.net

.gitea/workflows/deploy-hantim.net.yml

@@ -1,10 +1,12 @@
 name: Deploy hantim.net
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/hantim.net/**'
+      - '.gitea/workflows/deploy-hantim.net.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hantim.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hantim.net

.gitea/workflows/deploy-hcsuzuki.music.yml

@@ -0,0 +1,19 @@
+name: Deploy hcsuzuki.music
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/hcsuzuki.music/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.music

.gitea/workflows/deploy-hcsuzuki.net.yml

@@ -1,10 +1,12 @@
 name: Deploy hcsuzuki.net
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/hcsuzuki.net/**'
+      - '.gitea/workflows/deploy-hcsuzuki.net.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-hcsuzuki.net

.gitea/workflows/deploy-nginx.yml

@@ -1,10 +1,12 @@
 name: Deploy nginx
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/nginx/**'
+      - '.gitea/workflows/deploy-nginx.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,8 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-nginx
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-nginx
+

.gitea/workflows/deploy-thekims.family.yml

@@ -1,10 +1,12 @@
 name: Deploy thekims.family
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/thekims.family/**'
+      - '.gitea/workflows/deploy-thekims.family.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-thekims.family
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-thekims.family

.gitea/workflows/deploy-timothykim.yml

@@ -1,10 +1,12 @@
 name: Deploy timothykim.net
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'docker/timothykim.net/**'
+      - '.gitea/workflows/deploy-timothykim.yml'
 
 jobs:
   deploy:
@@ -13,6 +15,7 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-timothykim.net
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-timothykim.net

.gitea/workflows/provision.yml

@@ -1,10 +1,12 @@
 name: Provision server
 
 on:
+  workflow_dispatch:
   push:
     branches: [main]
     paths:
       - 'scripts/configure.sh'
+      - '.gitea/workflows/provision.yml'
 
 jobs:
   provision:
@@ -13,6 +15,7 @@ jobs:
       - name: Provision via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} provision
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} provision

.gitignore

@@ -5,3 +5,5 @@ docker/nginx/certs/
 docker/nginx/certbot/
 docker/garage/data/
 docker/garage/meta/
+SECURITY.md
+__pycache__/

ARCHITECTURE.md

@@ -91,8 +91,8 @@ client sends a command string (e.g., `deploy-example.com`), and `deploy.sh`
 parses it to determine the action:
 
 - `deploy-<app>` — deploy a specific app
-- `deploy-nginx` — deploy nginx (with cert handling)
+- `deploy-nginx` — deploy nginx (fails if certs are missing)
-- `cert-<domain>` — issue a cert only
+- `cert-<domain>` — issue a cert only (via `certbot --webroot`, zero downtime; subdomains like `garage.hantim.net` skip the `www.` variant)
 - `provision` — run `configure.sh`
 
 ## SSL certificate handling
@@ -101,25 +101,24 @@ Cert names are derived from nginx conf filenames (e.g., `example.com.conf` →
 cert `example.com`). All `server_name` values in the conf are included in the
 cert as SANs.
 
-**New domain (nginx not yet running):**
+**New domain:** issue cert before deploying nginx config:
-1. `deploy.sh` detects missing cert files
+1. Run `app.sh cert <domain>` (uses `certbot --webroot`, zero downtime)
-2. Stops nginx temporarily
+2. Deploy nginx config — `deploy.sh` verifies certs exist, fails if missing
-3. Issues cert via `certbot --standalone`
-4. Starts nginx
-5. On error, cleans up the temp conf file (trap)
 
-**New domain (nginx already running):**
+**Renewal:** certbot timer/cron runs daily, deploy hook reloads nginx. All certs
-1. Issues cert via `certbot --webroot` using the ACME challenge directory
+must use the `webroot` authenticator so nginx can keep port 80; certs issued by
-2. Zero downtime
+the old `--standalone` flow (removed in `de1d60b`) can never renew because nginx
-
+holds port 80. `configure.sh` self-heals this on every `provision`: it scans
-**Renewal:** certbot timer/cron runs daily, deploy hook reloads nginx.
+`/etc/letsencrypt/renewal/*.conf` and reissues any `standalone` cert via the
+`cert-<domain>` webroot path (best-effort; warns and exits non-zero on failure).
 
 ## `.env.keys` mechanism
 
 Services needing secrets declare them in `.env.keys`:
 
 ```
-ENV_VAR=bws-secret-name
+# bws-secret-name
+ENV_VAR=bws-secret-uuid
 ```
 
 During deploy, `deploy.sh`:
@@ -139,6 +138,19 @@ If any secret is missing, deploy fails with an error (not a warning).
 - **One domain per conf file** — `deploy.sh` derives the cert name from the
   filename and collects all `server_name` values for that cert
 
+## GoatCounter (analytics)
+
+GoatCounter runs at `goatcounter.hantim.net` for privacy-friendly web analytics.
+
+**Script injection:** nginx injects the GoatCounter tracking script into all
+HTML responses via `sub_filter`. The shared include `conf.d/goatcounter.inc`
+is added to each site's main HTTPS server block. New sites get it automatically
+via `app.sh`.
+
+**Config:** `docker/goatcounter/compose.yaml` uses the `baethon/goatcounter`
+image. Password is stored in Bitwarden (`hantim-goatcounter-password`) and
+injected via the `.env.keys` mechanism.
+
 ## URL routing
 
 - All HTTP → HTTPS redirect
@@ -167,7 +179,7 @@ Client -> nginx (443) -> /media/ location -> garage:3902
 runs envsubst at container start. Values come from `.env` generated by the
 `.env.keys` mechanism.
 
-**Bucket setup** (done by `new-app.sh` via admin API v2):
+**Bucket setup** (done by `app.sh` via admin API v2):
 1. `POST /v2/CreateBucket` with `globalAlias: <domain>`
 2. `POST /v2/AllowBucketKey` to grant media-key read/write access
 3. `POST /v2/UpdateBucket` to enable website access
@@ -181,6 +193,7 @@ Dockerfile              # Copies static/ into nginx:alpine, applies nginx.conf
 nginx.conf              # Per-app HTTP config (port 80, routing, cache headers)
 static/                 # HTML/CSS/JS content
 static/media/           # Local media files (gitignored)
+media.sh                # Sync media with Garage (pull/push)
 .gitea/workflows/build.yml  # Build image, push to registry, SSH deploy
 ```
 

CLAUDE.md

@@ -8,12 +8,12 @@ for technical deep-dives, USECASES.md for expected behaviors.
 - `scripts/bootstrap.sh` -- First-time server setup (manual, uses Bitwarden)
 - `scripts/configure.sh` -- Idempotent server config (firewall, certbot, start services; CI-safe)
 - `scripts/deploy.sh` -- SSH-triggered deploy + cert issuance + provision (deploy-*, cert-*, provision commands)
-- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, all)
+- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, monitor, all)
-- `tools/new-app.sh` -- Wrapper for `app.sh all` (backwards compat)
+- `tools/service.sh` -- Manage Docker services by subcommand (dns, files, nginx, cert, all)
-- `tools/new-service.sh` -- Scaffold a new Docker Compose service (compose + workflow)
 - `tools/remove-app.sh` -- Remove a static site (local files + Gitea repo)
 - `docker/nginx/conf.d/` -- Per-app nginx server blocks
 - `docker/<domain>/compose.yml` -- Per-app compose files
+- `docker/goatcounter/` -- GoatCounter analytics (compose.yaml, .env.keys)
 - `docker/garage/` -- Garage S3 object storage (Dockerfile, garage.toml, .env.keys)
 - `.gitea/workflows/` -- Per-app deploy workflows + provision.yml
 
@@ -36,10 +36,50 @@ for technical deep-dives, USECASES.md for expected behaviors.
 
 ## Build and test
 
-No build step. No tests. Verify changes by:
+No build step. Run e2e tests with:
-1. Reading scripts and checking idempotency
+
-2. Walking through each use case in USECASES.md
+    uvx pytest tests/test_app_e2e.py -v -x
-3. Cross-checking docs against scripts
+
+### Prerequisites
+
+- `uv` installed (`curl -LsSf https://astral.sh/uv/install.sh | sh`)
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- bws token: set `BWS_ACCESS_TOKEN` env var or save to `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP (check https://my.vultr.com/settings/#settingsapi)
+
+### Test structure
+
+- `tests/conftest.py` -- Fixtures: preflight checks, bws secrets, API helper, cleanup
+- `tests/test_app_e2e.py` -- E2E tests for `tools/app.sh` against real services
+- `tests/test_security_headers.py` -- Verify security headers on live sites
+- `tests/test_default_server.py` -- Verify unknown Host headers are dropped
+- `tests/test_tls.py` -- Verify TLS hardening
+
+### E2E tests (`test_app_e2e.py`)
+
+Tests use `kgfamily.com` as a throwaway test domain. Two phases:
+
+1. **TestAppSubcommands** -- Runs each `app.sh` subcommand individually (dns, repo,
+   files, cert, garage, build, monitor), verifies side effects via API, then cleans up.
+   Does not test `verify` (needs full deploy flow).
+2. **TestAppAll** -- Runs `app.sh all kgfamily.com`, verifies everything including
+   site liveness and git commit, then cleans up (including reverting the commit+push).
+
+Cleanup is done directly via APIs (Vultr, Gitea, Garage, UptimeRobot) + local file
+deletion. Does NOT use `remove-app.sh` to avoid coupling. Runs before and after each
+phase to handle leftovers from crashed runs.
+
+### Known issue
+
+- Vultr API keys can be IP-restricted. If `test_dns` fails with HTTP 401, add the
+  machine's IP to the Vultr API access control list.
+
+### Other tests
+
+The security/TLS/default-server tests run against live sites and are triggered
+automatically after nginx deploy. They can also be run locally:
+
+    uvx pytest tests/ -v --ignore=tests/test_app_e2e.py
 
 ## Commit style
 

README.md

@@ -26,10 +26,10 @@ nodes.
 scripts/
   bootstrap.sh         # First-time server setup (manual, uses Bitwarden)
   configure.sh         # Idempotent server config (CI-safe)
-  deploy.sh            # Deploy + cert issuance + provision (called via SSH)
+  deploy.sh            # Deploy + provision (called via SSH)
 tools/
-  new-app.sh           # Add a new static site (run from dev machine)
+  app.sh               # Add a new static site (run from dev machine)
-  new-service.sh       # Add a new Docker service (run from dev machine)
+  service.sh           # Manage Docker services by subcommand (dns, files, nginx, cert, all)
   remove-app.sh        # Remove a static site (run from dev machine)
 docker/
   nginx/               # Reverse proxy (nginx:alpine) + SSL config
@@ -38,11 +38,13 @@ docker/
     compose.yml
   <domain>/            # Per-app compose files (e.g. example.com)
     compose.yml
+  goatcounter/         # GoatCounter analytics
   garage/              # Garage S3-compatible object storage
     compose.yml
     Dockerfile
     garage.toml
     .env.keys
+tests/                 # pytest integration tests (run post-deploy)
 .gitea/workflows/      # Per-app deploy workflows + provision.yml
 ```
 
@@ -72,12 +74,12 @@ server. Everything is idempotent.
 ## Adding a new static site
 
 ```bash
-./tools/new-app.sh <domain>
+./tools/app.sh all <domain>
 ```
 
 This single command handles everything: DNS records (Vultr), Gitea repo
 (from `static-site-template`), nginx/compose/workflow configs, SSL cert,
-Garage media bucket, first build, and verification.
+Garage media bucket, first build, verification, and UptimeRobot monitoring.
 
 **Prerequisites:** domain nameservers pointed to Vultr, `bws`/`jq`/`dig`
 installed, `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`.
@@ -160,6 +162,57 @@ docker exec garage /garage layout apply --version 1
 If only hantim is reprovisioned, argento retains the layout and hantim
 reconnects automatically.
 
+## Testing
+
+Tests live in `tests/` and run with [uv](https://docs.astral.sh/uv/):
+
+```bash
+# Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Live site tests (security headers, TLS, default server)
+uvx pytest tests/ -v --ignore=tests/test_app_e2e.py
+
+# E2E tests for tools/app.sh (creates real resources)
+uvx pytest tests/test_app_e2e.py -v -x
+```
+
+### Live site tests
+
+These run against live sites and verify infrastructure invariants:
+
+- **`test_security_headers.py`** — Checks that all sites including
+  `security-headers.inc` return the expected headers (HSTS, X-Frame-Options,
+  etc.). Sites are auto-discovered from `docker/nginx/conf.d/*.conf`.
+- **`test_tls.py`** — Verifies TLS 1.3 and 1.2 work, TLS 1.1 is rejected.
+- **`test_default_server.py`** — Verifies unknown Host headers get dropped
+  (connection reset) on both HTTP and HTTPS.
+
+The `deploy-nginx.yml` workflow runs these automatically after each nginx
+deploy. If tests fail, Gitea sends an email notification.
+
+### E2E tests
+
+`test_app_e2e.py` tests `tools/app.sh` against real services (Vultr, Gitea,
+Garage, UptimeRobot). Uses `kgfamily.com` as a throwaway test domain.
+
+**Prerequisites:**
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP
+  (check https://my.vultr.com/settings/#settingsapi)
+
+**Structure:**
+
+1. **TestAppSubcommands** — Runs each `app.sh` subcommand individually
+   (dns, repo, files, cert, garage, build, monitor), verifies side effects
+   via API, then cleans up.
+2. **TestAppAll** — Runs `app.sh all kgfamily.com`, verifies the full flow
+   including site liveness and git commit, then cleans up.
+
+Cleanup runs before and after each phase via APIs (Vultr, Gitea, Garage,
+UptimeRobot) + local file deletion. Does not use `remove-app.sh` to avoid
+coupling.
+
 ## Secrets
 
 ### Bitwarden Secrets Manager
@@ -170,22 +223,25 @@ Project: `hantim`. Fetched via `bws` CLI on the server.
 |---|---|---|
 | `hantim-ci-registry-push` | Gitea token for Docker registry | `bootstrap.sh` |
 | `hantim-deploy-ssh-public-key` | Deploy user's SSH public key | `bootstrap.sh` |
-| `hantim-deploy-ssh-private-key` | Deploy user's SSH private key | `new-app.sh` |
+| `hantim-deploy-ssh-private-key` | Deploy user's SSH private key | `app.sh` |
-| `hantim-new-app-script` | Gitea API token for creating repos | `new-app.sh` |
+| `hantim-new-app-script` | Gitea API token for creating repos | `app.sh` |
-| `hantim-vultr-api-key` | Vultr API key for DNS management | `new-app.sh` |
+| `hantim-vultr-api-key` | Vultr API key for DNS management | `app.sh` |
 | `hantim-garage-rpc-secret` | Garage cluster RPC secret | `deploy.sh` |
 | `hantim-garage-argento-node-id` | Argento's Garage node ID + address | `deploy.sh` |
-| `hantim-garage-admin-token` | Garage admin API token | `deploy.sh`, `new-app.sh` |
+| `hantim-garage-admin-token` | Garage admin API token | `deploy.sh`, `app.sh` |
-| `hantim-garage-media-key-id` | S3 access key ID for media uploads | `new-app.sh`, `aws` CLI |
+| `hantim-garage-media-key-id` | S3 access key ID for media uploads | `app.sh`, `aws` CLI |
 | `hantim-garage-media-secret-key` | S3 secret key for media uploads | `aws` CLI |
+| `hantim-uptimerobot-api-key` | UptimeRobot API key for monitors | `app.sh` |
+| `hantim-goatcounter-password` | GoatCounter admin password | `deploy.sh` |
 
 Machine accounts: `hantim-server` (token at `/etc/bws-token`), `hantim-ci` (reserved).
 
 ### Service secrets (`.env.keys`)
 
 Services that need secrets declare them in `.env.keys` (format:
-`ENV_VAR=bws-secret-name`, one per line). `deploy.sh` fetches each secret
+`ENV_VAR=bws-secret-uuid`, one per line, with `# secret-name` comments).
-from bws and generates `.env` before starting the service.
+`deploy.sh` fetches each secret by UUID from bws and generates `.env`
+before starting the service.
 
 ### Gitea org-level secrets
 
@@ -194,5 +250,12 @@ Stored in the `hantim` Gitea org for direct use in CI workflows:
 | Name | Type | Purpose |
 |---|---|---|
 | `DEPLOY_HOST` | Variable | Server IP or hostname |
+| `DEPLOY_HOST_KEY` | Variable | Server SSH host public key (for host key pinning) |
 | `DEPLOY_SSH_KEY` | Secret | SSH private key for the deploy user |
 | `CI_REGISTRY_TOKEN` | Secret | Gitea token for Docker registry login |
+
+After reprovisioning hantim (new host key), update `DEPLOY_HOST_KEY`:
+
+```bash
+ssh-keyscan -t ed25519 hantim.net 2>/dev/null
+```

USECASES.md

@@ -38,7 +38,7 @@ for stateless static sites.
 
 ## 3. Add a new static site
 
-**Trigger:** `./tools/new-app.sh example.com` from dev machine.
+**Trigger:** `./tools/app.sh all example.com` from dev machine.
 
 **Prerequisites:** domain nameservers on Vultr, `bws`/`jq`/`dig` installed.
 
@@ -50,6 +50,7 @@ for stateless static sites.
 - Garage media bucket created with media-key access
 - First build triggered
 - `https://www.example.com` responds within 3 minutes
+- UptimeRobot HTTPS monitor created (skipped if already exists)
 - Site ready for customization via git clone + push
 
 ## 4. Update app code
@@ -88,12 +89,43 @@ for stateless static sites.
 - Existing certs skipped
 - Services restarted
 
-## 8. Upload media files
+## 8. Add a new Docker service
 
-**Trigger:** `aws s3 cp` or `aws s3 sync` to `s3://example.com/`.
+**Trigger:** `./tools/service.sh all <name> <port>` from dev machine.
+
+**Prerequisites:** `bws`/`jq`/`dig` installed.
+
+**Expected outcome:**
+- DNS A record created for `<name>.hantim.net`
+- Compose file and deploy workflow scaffolded
+- Nginx conf created proxying `<name>.hantim.net` to port `<port>`
+- SSL cert issued for `<name>.hantim.net` (no www variant)
+- Service ready for customization (edit compose.yml image, add `.env.keys` if needed)
+
+## 9. Analytics tracking
+
+**Trigger:** automatic — nginx injects the GoatCounter script into all HTML responses.
+
+**Expected outcome:**
+- Every site includes the GoatCounter tracking script (injected via `sub_filter`)
+- Page views are recorded at `goatcounter.hantim.net`
+- New sites provisioned with `app.sh` include tracking automatically
+- GoatCounter dashboard itself is not tracked
+
+## 10. Upload media files
+
+**Trigger:** `aws s3 cp` or `aws s3 sync` to `s3://example.com/`, or `./media.sh push` from a site repo.
 
 **Expected outcome:**
 - File uploaded via S3 API (`s3.hantim.net`)
 - Replicated to both Garage nodes (hantim + argento)
 - Accessible at `https://www.example.com/media/<path>`
-- Local development: `static/media/` (gitignored) serves at `/media/`
+
+## 11. Local development with media files
+
+**Trigger:** `./media.sh pull` from a site repo.
+
+**Expected outcome:**
+- Media files downloaded from Garage to `static/media/` (gitignored)
+- Files served locally at `/media/` by the app container's nginx
+- To upload local changes back to Garage: `./media.sh push`

docker/beszel/.env.keys

@@ -0,0 +1,4 @@
+# hantim-beszel-public-key
+KEY=1b5b742b-42db-4a03-a8ec-b42600150be4
+# hantim-beszel-token
+TOKEN=6666398f-b936-4781-954b-b426001528cb

docker/beszel/.gitignore

@@ -0,0 +1,7 @@
+# !create
+beszel_data/
+# !create
+beszel_socket/
+# !create
+beszel_agent_data/
+

docker/beszel/compose.yaml

@@ -0,0 +1,33 @@
+services:
+  beszel:
+    image: henrygd/beszel
+    container_name: beszel
+    restart: unless-stopped
+    environment:
+      APP_URL: https://beszel.hantim.net
+    ports:
+      - 127.0.0.1:8090:8090
+    volumes:
+      - ./beszel_data:/beszel_data
+      - ./beszel_socket:/beszel_socket
+    networks:
+      - shared
+
+  beszel-agent:
+    image: henrygd/beszel-agent
+    container_name: beszel-agent
+    restart: unless-stopped
+    network_mode: host
+    volumes:
+      - ./beszel_agent_data:/var/lib/beszel-agent
+      - ./beszel_socket:/beszel_socket
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    env_file: .env
+    environment:
+      LISTEN: /beszel_socket/beszel.sock
+      HUB_URL: http://localhost:8090
+
+networks:
+  shared:
+    external: true
+

docker/carolpreschool.com/compose.yml

@@ -0,0 +1,11 @@
+services:
+  app:
+    image: git.timothykim.net/hantim/carolpreschool.com:latest
+    restart: unless-stopped
+    container_name: carolpreschool_com
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true

docker/garage/.env.keys

@@ -1,3 +1,6 @@
-RPC_SECRET=hantim-garage-rpc-secret
+# hantim-garage-rpc-secret
-ARGENTO_NODE_ID=hantim-garage-argento-node-id
+RPC_SECRET=076a4798-4180-4190-9212-b41200ff08e3
-ADMIN_TOKEN=hantim-garage-admin-token
+# hantim-garage-argento-node-id
+ARGENTO_NODE_ID=98454088-faf7-4d36-b98a-b41200ff52dd
+# hantim-garage-admin-token
+ADMIN_TOKEN=5565abc1-cd9f-451e-97a2-b412015dbb53

docker/garage/Dockerfile

@@ -5,4 +5,4 @@ FROM base
 COPY --from=dxflrs/garage:v2.2.0 /garage /garage
 COPY garage.toml /etc/garage.toml.tpl
 
-CMD envsubst < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server
+CMD envsubst '${RPC_SECRET} ${ARGENTO_NODE_ID} ${ADMIN_TOKEN}' < /etc/garage.toml.tpl > /etc/garage.toml && exec /garage server

docker/goatcounter/.env.keys

@@ -0,0 +1,2 @@
+# hantim-goatcounter-password
+GOATCOUNTER_PASSWORD=1b02fe2e-7acb-4111-918a-b42701546299

docker/goatcounter/.gitignore

@@ -0,0 +1,3 @@
+# !create
+db/
+.env

docker/goatcounter/compose.yaml

@@ -0,0 +1,17 @@
+services:
+  goatcounter:
+    image: baethon/goatcounter
+    restart: unless-stopped
+    container_name: goatcounter
+    env_file: .env
+    environment:
+      GOATCOUNTER_DOMAIN: goatcounter.hantim.net
+      GOATCOUNTER_EMAIL: timothykim@fastmail.fm
+    volumes:
+      - ./db:/goatcounter/db
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true

docker/hcsuzuki.music/compose.yml

@@ -0,0 +1,11 @@
+services:
+  app:
+    image: git.timothykim.net/hantim/hcsuzuki.music:latest
+    restart: unless-stopped
+    container_name: hcsuzuki_music
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true

docker/nginx/conf.d/00-default.conf

@@ -0,0 +1,19 @@
+# Drop connections with unknown/missing Host headers
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+    server_name _;
+    return 444;
+}
+
+server {
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+    http2 on;
+    server_name _;
+
+    ssl_certificate     /etc/letsencrypt/live/hantim.net/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/hantim.net/privkey.pem;
+
+    return 444;
+}

docker/nginx/conf.d/beszel.hantim.net.conf

@@ -0,0 +1,33 @@
+server {
+    listen 80;
+    listen [::]:80;
+    server_name beszel.hantim.net;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://beszel.hantim.net$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name beszel.hantim.net;
+
+    ssl_certificate     /etc/letsencrypt/live/beszel.hantim.net/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/beszel.hantim.net/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://beszel:8090;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}

docker/nginx/conf.d/carolpreschool.com.conf

@@ -0,0 +1,54 @@
+# Redirect HTTP to HTTPS, bare domain to www
+server {
+    listen 80;
+    listen [::]:80;
+    server_name carolpreschool.com www.carolpreschool.com;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://www.carolpreschool.com$request_uri;
+    }
+}
+
+# Redirect bare HTTPS domain to www
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name carolpreschool.com;
+
+    ssl_certificate     /etc/letsencrypt/live/carolpreschool.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/carolpreschool.com/privkey.pem;
+
+    return 301 https://www.carolpreschool.com$request_uri;
+}
+
+# Main site
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name www.carolpreschool.com;
+
+    ssl_certificate     /etc/letsencrypt/live/carolpreschool.com/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/carolpreschool.com/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
+
+    location /media/ {
+        proxy_pass http://garage:3902/;
+        proxy_set_header Host carolpreschool.com.web.garage;
+    }
+
+    location / {
+        proxy_pass http://carolpreschool_com:80;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}

docker/nginx/conf.d/garage.hantim.net.conf

@@ -22,9 +22,15 @@ server {
     ssl_certificate     /etc/letsencrypt/live/garage.hantim.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/garage.hantim.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
 
     location / {
+        # restrict to argento (home IP)
+        # update if IP changes: dig +short argento.ddns.net
+        allow 173.79.207.76;
+        deny all;
+
         proxy_pass http://garage:3903;
+        proxy_hide_header X-Frame-Options;
     }
 }

docker/nginx/conf.d/goatcounter.hantim.net.conf

@@ -0,0 +1,33 @@
+server {
+    listen 80;
+    listen [::]:80;
+    server_name goatcounter.hantim.net;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://goatcounter.hantim.net$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name goatcounter.hantim.net;
+
+    ssl_certificate     /etc/letsencrypt/live/goatcounter.hantim.net/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/goatcounter.hantim.net/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://goatcounter:8080;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}

docker/nginx/conf.d/goatcounter.inc

@@ -0,0 +1,2 @@
+sub_filter '</body>' '<script>window.goatcounter={path:function(p){return location.host+p}}</script><script data-goatcounter="https://goatcounter.hantim.net/count" async src="//goatcounter.hantim.net/count.js"></script></body>';
+sub_filter_once on;

docker/nginx/conf.d/haanmind.net.conf

@@ -36,7 +36,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/haanmind.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/haanmind.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;

docker/nginx/conf.d/hantim.net.conf

@@ -36,7 +36,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/hantim.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/hantim.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;

docker/nginx/conf.d/hcsuzuki.music.conf

@@ -0,0 +1,54 @@
+# Redirect HTTP to HTTPS, bare domain to www
+server {
+    listen 80;
+    listen [::]:80;
+    server_name hcsuzuki.music www.hcsuzuki.music;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://www.hcsuzuki.music$request_uri;
+    }
+}
+
+# Redirect bare HTTPS domain to www
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name hcsuzuki.music;
+
+    ssl_certificate     /etc/letsencrypt/live/hcsuzuki.music/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.music/privkey.pem;
+
+    return 301 https://www.hcsuzuki.music$request_uri;
+}
+
+# Main site
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name www.hcsuzuki.music;
+
+    ssl_certificate     /etc/letsencrypt/live/hcsuzuki.music/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.music/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
+
+    location /media/ {
+        proxy_pass http://garage:3902/;
+        proxy_set_header Host hcsuzuki.music.web.garage;
+    }
+
+    location / {
+        proxy_pass http://hcsuzuki_music:80;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}

docker/nginx/conf.d/hcsuzuki.net.conf

@@ -36,7 +36,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/hcsuzuki.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/hcsuzuki.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;

docker/nginx/conf.d/s3.hantim.net.conf

@@ -22,7 +22,7 @@ server {
     ssl_certificate     /etc/letsencrypt/live/s3.hantim.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/s3.hantim.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
 
     client_max_body_size 100M;
 
@@ -31,5 +31,6 @@ server {
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
     }
 }

docker/nginx/conf.d/security-headers.inc

@@ -0,0 +1,5 @@
+add_header Strict-Transport-Security "max-age=63072000; preload" always;
+add_header X-Content-Type-Options "nosniff" always;
+add_header X-Frame-Options "DENY" always;
+add_header Referrer-Policy "strict-origin-when-cross-origin" always;
+add_header Permissions-Policy "geolocation=(), microphone=(), camera=()" always;

docker/nginx/conf.d/thekims.family.conf

@@ -36,7 +36,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/thekims.family/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/thekims.family/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;

docker/nginx/conf.d/timothykim.net.conf

@@ -36,7 +36,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/timothykim.net/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/timothykim.net/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;

docker/nginx/nginx.conf

@@ -13,8 +13,16 @@ http {
     access_log /var/log/nginx/access.log;
     error_log  /var/log/nginx/error.log;
 
-    add_header X-Content-Type-Options nosniff;
+    ssl_protocols TLSv1.2 TLSv1.3;
-    add_header X-Frame-Options DENY;
+    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384;
+    ssl_prefer_server_ciphers off;
+    ssl_session_cache shared:SSL:10m;
+    ssl_session_timeout 1d;
+    ssl_session_tickets off;
+    #ssl_stapling on;
+    #ssl_stapling_verify on;
+    resolver 1.1.1.1 8.8.8.8 valid=300s;
+    resolver_timeout 5s;
 
     include /etc/nginx/conf.d/*.conf;
 }

scripts/bootstrap.sh

@@ -95,12 +95,11 @@ if ! id deploy &>/dev/null; then
     echo "==> Creating deploy user..."
     useradd -r -s /usr/sbin/nologin deploy
 fi
-usermod -aG docker deploy
 
 echo "==> Setting up deploy SSH key..."
 mkdir -p /home/deploy/.ssh
 chmod 700 /home/deploy/.ssh
-echo "command=\"sudo /opt/hantim/scripts/deploy.sh \$SSH_ORIGINAL_COMMAND\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $DEPLOY_PUBKEY" > /home/deploy/.ssh/authorized_keys
+echo "command=\"sudo /opt/hantim/scripts/deploy.sh \\\"\$SSH_ORIGINAL_COMMAND\\\"\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty $DEPLOY_PUBKEY" > /home/deploy/.ssh/authorized_keys
 chmod 600 /home/deploy/.ssh/authorized_keys
 chown -R deploy:deploy /home/deploy/.ssh
 

scripts/configure.sh

@@ -37,6 +37,25 @@ fi
 echo "==> Deploying services..."
 # Deploy nginx first (creates the shared network)
 bash "$REPO_DIR/scripts/deploy.sh" deploy-nginx
+
+# Self-heal renewal config drift: certs issued by the old --standalone flow
+# (removed in de1d60b) can never renew because nginx now holds port 80. Reissue
+# any such cert via the canonical webroot path. nginx must be up (above) to
+# serve the ACME challenge. Best-effort: a single failure warns but does not
+# block the rest of the provision.
+echo "==> Healing standalone cert renewal configs..."
+heal_failed=false
+for conf in /etc/letsencrypt/renewal/*.conf; do
+    [ -e "$conf" ] || continue
+    grep -q '^authenticator = standalone' "$conf" || continue
+    cert_name=$(basename "$conf" .conf)
+    echo "  $cert_name uses standalone; reissuing via webroot..."
+    if ! bash "$REPO_DIR/scripts/deploy.sh" "cert-$cert_name"; then
+        echo "  WARNING: failed to reissue $cert_name; leaving standalone config in place"
+        heal_failed=true
+    fi
+done
+
 # Deploy all other apps
 for app in "$REPO_DIR"/docker/*/; do
     app_name=$(basename "$app")
@@ -46,3 +65,8 @@ for app in "$REPO_DIR"/docker/*/; do
 done
 
 echo "==> Done."
+
+if [ "$heal_failed" = true ]; then
+    echo "ERROR: one or more standalone certs could not be reissued (see warnings above)."
+    exit 1
+fi

scripts/deploy.sh

@@ -6,12 +6,21 @@ CMD="${SSH_ORIGINAL_COMMAND:-${1:-}}"
 if [[ "$CMD" =~ ^cert-[a-zA-Z0-9._-]+$ ]]; then
     APP="${CMD#cert-}"
 
+    # Subdomains (e.g., garage.hantim.net) don't have www variants
+    if [[ "$APP" == *.*.* ]]; then
+        CERT_DOMAINS="-d $APP"
+        SERVER_NAMES="$APP"
+    else
+        CERT_DOMAINS="-d $APP -d www.$APP"
+        SERVER_NAMES="$APP www.$APP"
+    fi
+
     # Write temporary HTTP-only config so nginx can serve ACME challenges
     cat > "/opt/hantim/docker/nginx/conf.d/$APP.conf" <<NGINXCONF
 server {
     listen 80;
     listen [::]:80;
-    server_name $APP www.$APP;
+    server_name $SERVER_NAMES;
 
     location /.well-known/acme-challenge/ {
         root /var/www/certbot;
@@ -23,20 +32,21 @@ server {
 }
 NGINXCONF
 
-    trap 'rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"' EXIT
+    trap 'git -C /opt/hantim restore "docker/nginx/conf.d/$APP.conf" 2>/dev/null || rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"; docker exec nginx nginx -t && docker exec nginx nginx -s reload' EXIT
     docker exec nginx nginx -t && docker exec nginx nginx -s reload
     certbot certonly --webroot -w /opt/hantim/docker/nginx/certbot/www \
-        --non-interactive --agree-tos --register-unsafely-without-email \
+        --non-interactive --agree-tos -m timothykim@fastmail.fm \
-        --cert-name "$APP" -d "$APP" -d "www.$APP"
+        --cert-name "$APP" $CERT_DOMAINS
-    rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"
+    git -C /opt/hantim restore "docker/nginx/conf.d/$APP.conf" 2>/dev/null || rm -f "/opt/hantim/docker/nginx/conf.d/$APP.conf"
     trap - EXIT
+    docker exec nginx nginx -t && docker exec nginx nginx -s reload
     echo "Certificate issued for $APP"
     exit 0
 fi
 
 if [ "$CMD" = "provision" ]; then
     cd /opt/hantim
-    git pull
+    git pull origin main
     bash scripts/configure.sh
     exit 0
 fi
@@ -48,7 +58,13 @@ fi
 
 APP="${CMD#deploy-}"
 cd /opt/hantim
-git pull
+git pull origin main
+
+# Re-exec so the running script reflects any changes just pulled
+if [ -z "${DEPLOY_REEXEC:-}" ]; then
+    export DEPLOY_REEXEC=1
+    exec /opt/hantim/scripts/deploy.sh "$CMD"
+fi
 
 # Generate .env from bws if this service declares .env.keys
 if [ -f "docker/$APP/.env.keys" ] && [ -f /etc/bws-token ]; then
@@ -57,11 +73,12 @@ if [ -f "docker/$APP/.env.keys" ] && [ -f /etc/bws-token ]; then
     env_content=""
     while IFS= read -r line || [ -n "$line" ]; do
         [ -z "$line" ] && continue
+        [[ "$line" = \#* ]] && continue
         var_name="${line%%=*}"
-        secret_key="${line#*=}"
+        secret_id="${line#*=}"
-        value=$(bws secret list | jq -r --arg key "$secret_key" '.[] | select(.key == $key) | .value')
+        value=$(bws secret get "$secret_id" | jq -r .value)
         if [ -z "$value" ]; then
-            echo "ERROR: Secret '$secret_key' not found in bws."
+            echo "ERROR: Secret '$secret_id' not found in bws."
             echo "  Check that the secret exists and the machine account has access."
             exit 1
         fi
@@ -75,6 +92,26 @@ if [ -f "docker/$APP/.env.keys" ] && [ -f /etc/bws-token ]; then
 fi
 
 cd "docker/$APP"
+
+# Create directories marked with "# !create" in .gitignore
+if [ -f .gitignore ]; then
+    create_next=false
+    while IFS= read -r line || [ -n "$line" ]; do
+        if [ "$line" = "# !create" ]; then
+            create_next=true
+            continue
+        fi
+        if [ "$create_next" = true ] && [ -n "$line" ]; then
+            dir="${line%/}"
+            if [ -n "$dir" ]; then
+                mkdir -p "$dir"
+                echo "Created directory: $dir"
+            fi
+            create_next=false
+        fi
+    done < .gitignore
+fi
+
 if grep -q '^\s*build:' compose.yml 2>/dev/null; then
     docker compose build
 else
@@ -85,33 +122,15 @@ else
 fi
 
 if [ "$APP" = "nginx" ]; then
-    # Issue certs for any new domains (brief nginx downtime while certbot binds port 80)
+    # Fail loudly if any certs are missing — issue them first with: app.sh cert <domain>
-    missing_certs=false
+    for conf in /opt/hantim/docker/nginx/conf.d/*.*.conf; do
-    for conf in /opt/hantim/docker/nginx/conf.d/*.conf; do
         cert_name=$(basename "$conf" .conf)
         if [ ! -d "/etc/letsencrypt/live/$cert_name" ]; then
-            missing_certs=true
+            echo "ERROR: Missing certificate for $cert_name"
-            break
+            echo "  Run: app.sh cert $cert_name"
+            exit 1
         fi
     done
-    if [ "$missing_certs" = true ]; then
-        docker stop nginx 2>/dev/null || true
-        trap 'docker start nginx 2>/dev/null || true' EXIT
-        for conf in /opt/hantim/docker/nginx/conf.d/*.conf; do
-            cert_name=$(basename "$conf" .conf)
-            if [ ! -d "/etc/letsencrypt/live/$cert_name" ]; then
-                domains=$(grep -oP 'server_name\s+\K[^;]+' "$conf" | tr ' ' '\n' | sort -u)
-                domain_args=""
-                for d in $domains; do
-                    domain_args="$domain_args -d $d"
-                done
-                echo "Issuing cert for $cert_name..."
-                certbot certonly --standalone --non-interactive --agree-tos \
-                    --register-unsafely-without-email --cert-name "$cert_name" $domain_args
-            fi
-        done
-        trap - EXIT
-    fi
     # Test config before applying — a bad config would take down all sites
     docker compose run --rm -T nginx nginx -t
 fi

tests/conftest.py

@@ -0,0 +1,266 @@
+"""Shared fixtures and helpers for e2e tests."""
+
+import json
+import os
+import shutil
+import subprocess
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import pytest
+
+TEST_DOMAIN = "kgfamily.com"
+CONTAINER_NAME = TEST_DOMAIN.replace(".", "_")
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+APP_SH = os.path.join(REPO_ROOT, "tools", "app.sh")
+
+VULTR_API = "https://api.vultr.com/v2"
+GITEA_URL = "https://git.timothykim.net"
+GITEA_ORG = "hantim"
+GARAGE_API = "https://garage.hantim.net"
+UPTIMEROBOT_API = "https://api.uptimerobot.com/v3"
+
+
+def api(method, url, headers=None, data=None):
+    """HTTP request helper. Returns (status_code, parsed_json_or_None)."""
+    hdrs = dict(headers or {})
+    body = None
+    if data is not None:
+        body = json.dumps(data).encode()
+        hdrs.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, method=method, headers=hdrs, data=body)
+    try:
+        resp = urllib.request.urlopen(req, timeout=30)
+        raw = resp.read().decode()
+        return resp.status, json.loads(raw) if raw.strip() else None
+    except urllib.error.HTTPError as e:
+        raw = e.read().decode()
+        try:
+            return e.code, json.loads(raw) if raw.strip() else None
+        except json.JSONDecodeError:
+            return e.code, None
+
+
+def _bws_env():
+    """Return env dict with BWS_ACCESS_TOKEN set from token file if needed."""
+    env = os.environ.copy()
+    if not env.get("BWS_ACCESS_TOKEN") and os.path.isfile(BWS_TOKEN_FILE):
+        env["BWS_ACCESS_TOKEN"] = open(BWS_TOKEN_FILE).read().strip()
+    return env
+
+
+def run_app(command, domain=TEST_DOMAIN, timeout=300):
+    """Run app.sh with given command and domain. Returns CompletedProcess."""
+    return subprocess.run(
+        [APP_SH, command, domain],
+        capture_output=True, text=True, timeout=timeout,
+        cwd=REPO_ROOT,
+    )
+
+
+BWS_TOKEN_FILE = os.path.join(
+    os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
+    "hantim", "bws-token",
+)
+
+REQUIRED_TOOLS = ["bws", "jq", "dig", "ssh", "curl", "git"]
+
+REQUIRED_SECRETS = [
+    "hantim-vultr-api-key",
+    "hantim-new-app-script",
+    "hantim-deploy-ssh-private-key",
+    "hantim-garage-admin-token",
+    "hantim-garage-media-key-id",
+    "hantim-uptimerobot-api-key",
+]
+
+
+@pytest.fixture(scope="session", autouse=True)
+def preflight():
+    """Check that all required tools and credentials are available."""
+    errors = []
+
+    for tool in REQUIRED_TOOLS:
+        if shutil.which(tool) is None:
+            errors.append(f"missing CLI tool: {tool}")
+
+    if not os.environ.get("BWS_ACCESS_TOKEN") and not os.path.isfile(BWS_TOKEN_FILE):
+        errors.append(
+            f"no bws token: set BWS_ACCESS_TOKEN or create {BWS_TOKEN_FILE}"
+        )
+
+    if errors:
+        pytest.exit(
+            "Preflight failed:\n  " + "\n  ".join(errors),
+            returncode=1,
+        )
+
+
+@pytest.fixture(scope="session")
+def secrets(preflight):
+    """Fetch all bws secrets once for the session."""
+    result = subprocess.run(
+        ["bws", "secret", "list"],
+        capture_output=True, text=True, env=_bws_env(),
+    )
+    if result.returncode != 0:
+        pytest.exit(
+            f"bws secret list failed (is the token valid?):\n{result.stderr}",
+            returncode=1,
+        )
+
+    all_secrets = json.loads(result.stdout)
+    by_key = {s["key"]: s["value"] for s in all_secrets}
+
+    missing = [k for k in REQUIRED_SECRETS if k not in by_key]
+    if missing:
+        pytest.exit(
+            f"Missing secrets in Bitwarden: {', '.join(missing)}",
+            returncode=1,
+        )
+
+    return by_key
+
+
+def cleanup(secrets, revert_git=False):
+    """Best-effort cleanup of all TEST_DOMAIN resources."""
+    errors = []
+
+    # --- UptimeRobot monitor ---
+    try:
+        key = secrets["hantim-uptimerobot-api-key"]
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET",
+            f"{UPTIMEROBOT_API}/monitors?"
+            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
+            headers=auth,
+        )
+        if status == 200 and data:
+            for m in data.get("data", []):
+                if m.get("url") == f"https://www.{TEST_DOMAIN}":
+                    api("DELETE", f"{UPTIMEROBOT_API}/monitors/{m['id']}", headers=auth)
+    except Exception as e:
+        errors.append(f"UptimeRobot: {e}")
+
+    # --- Garage bucket ---
+    try:
+        token = secrets["hantim-garage-admin-token"]
+        auth = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=auth,
+        )
+        if status == 200 and data and data.get("id"):
+            bid = data["id"]
+            # Deny media key access
+            media_key = secrets.get("hantim-garage-media-key-id", "")
+            if media_key:
+                api(
+                    "POST",
+                    f"{GARAGE_API}/v2/DenyBucketKey",
+                    headers=auth,
+                    data={
+                        "bucketId": bid,
+                        "accessKeyId": media_key,
+                        "permissions": {"read": True, "write": True, "owner": False},
+                    },
+                )
+            # Delete bucket directly (Garage rejects alias removal when
+            # it's the bucket's only alias, so skip RemoveBucketAlias)
+            api(
+                "POST",
+                f"{GARAGE_API}/v2/DeleteBucket?"
+                + urllib.parse.urlencode({"id": bid}),
+                headers={"Authorization": f"Bearer {token}"},
+            )
+    except Exception as e:
+        errors.append(f"Garage: {e}")
+
+    # --- Gitea repo ---
+    try:
+        token = secrets["hantim-new-app-script"]
+        api(
+            "DELETE",
+            f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}",
+            headers={"Authorization": f"token {token}"},
+        )
+    except Exception as e:
+        errors.append(f"Gitea: {e}")
+
+    # --- Local files ---
+    local_paths = [
+        os.path.join(REPO_ROOT, "docker", TEST_DOMAIN),
+        os.path.join(REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml"),
+        os.path.join(REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf"),
+    ]
+
+    if revert_git:
+        # Files may be tracked — check if last commit added them
+        try:
+            log = subprocess.run(
+                ["git", "log", "--format=%s", "-1"],
+                capture_output=True, text=True, cwd=REPO_ROOT,
+            )
+            if log.stdout.strip() == f"add {TEST_DOMAIN}":
+                removed_any = False
+                for f in [
+                    f"docker/{TEST_DOMAIN}",
+                    f".gitea/workflows/deploy-{TEST_DOMAIN}.yml",
+                    f"docker/nginx/conf.d/{TEST_DOMAIN}.conf",
+                ]:
+                    full = os.path.join(REPO_ROOT, f)
+                    if os.path.exists(full):
+                        subprocess.run(
+                            ["git", "rm", "-rf", f],
+                            cwd=REPO_ROOT,
+                            capture_output=True,
+                        )
+                        removed_any = True
+                if removed_any:
+                    subprocess.run(
+                        ["git", "commit", "-m", f"remove {TEST_DOMAIN}"],
+                        cwd=REPO_ROOT, check=True, capture_output=True,
+                    )
+                    subprocess.run(
+                        ["git", "push"],
+                        cwd=REPO_ROOT, check=True, capture_output=True,
+                    )
+        except Exception as e:
+            errors.append(f"Git revert: {e}")
+
+    # Always try to remove untracked local files too
+    for path in local_paths:
+        try:
+            if os.path.isdir(path):
+                shutil.rmtree(path)
+            elif os.path.isfile(path):
+                os.remove(path)
+        except FileNotFoundError:
+            pass
+        except Exception as e:
+            errors.append(f"Local {path}: {e}")
+
+    # --- Vultr DNS records (keep zone to avoid slow re-creation) ---
+    try:
+        key = secrets["hantim-vultr-api-key"]
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
+        )
+        if status == 200 and data:
+            for r in data.get("records", []):
+                if r["type"] == "A" and r["name"] in ("", "www"):
+                    api(
+                        "DELETE",
+                        f"{VULTR_API}/domains/{TEST_DOMAIN}/records/{r['id']}",
+                        headers=auth,
+                    )
+    except Exception as e:
+        errors.append(f"Vultr: {e}")
+
+    if errors:
+        print(f"Cleanup warnings: {'; '.join(errors)}")

tests/test_app_e2e.py

@@ -0,0 +1,236 @@
+"""End-to-end tests for tools/app.sh against real services.
+
+Test domain: kgfamily.com
+Run: uvx pytest tests/test_app_e2e.py -v -x
+"""
+
+import os
+import subprocess
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+import pytest
+
+from conftest import (
+    TEST_DOMAIN,
+    CONTAINER_NAME,
+    REPO_ROOT,
+    VULTR_API,
+    GITEA_URL,
+    GITEA_ORG,
+    GARAGE_API,
+    UPTIMEROBOT_API,
+    api,
+    run_app,
+    cleanup,
+)
+
+
+class TestAppSubcommands:
+    """Test each app.sh subcommand individually, in order."""
+
+    @pytest.fixture(autouse=True, scope="class")
+    def _cleanup(self, secrets):
+        cleanup(secrets, revert_git=True)
+        yield
+        cleanup(secrets, revert_git=False)
+
+    def test_dns(self, secrets):
+        result = run_app("dns")
+        assert result.returncode == 0, f"app.sh dns failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {"Authorization": f"Bearer {secrets['hantim-vultr-api-key']}"}
+        status, data = api("GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth)
+        assert status == 200, f"Vultr API returned {status}"
+
+        records = data["records"]
+        bare_a = [r for r in records if r["type"] == "A" and r["name"] == ""]
+        www_a = [r for r in records if r["type"] == "A" and r["name"] == "www"]
+
+        assert len(bare_a) == 1, f"Expected 1 bare A record, got {len(bare_a)}"
+        assert len(www_a) == 1, f"Expected 1 www A record, got {len(www_a)}"
+        assert bare_a[0]["data"] == www_a[0]["data"], "Bare and www should point to same IP"
+
+    def test_repo(self, secrets):
+        result = run_app("repo")
+        assert result.returncode == 0, f"app.sh repo failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {"Authorization": f"token {secrets['hantim-new-app-script']}"}
+        status, data = api("GET", f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}", headers=auth)
+        assert status == 200, f"Repo not found (HTTP {status})"
+        assert data["name"] == TEST_DOMAIN
+
+    def test_files(self):
+        result = run_app("files")
+        assert result.returncode == 0, f"app.sh files failed:\n{result.stdout}\n{result.stderr}"
+
+        # compose.yml
+        compose_path = os.path.join(REPO_ROOT, "docker", TEST_DOMAIN, "compose.yml")
+        assert os.path.isfile(compose_path)
+        compose = open(compose_path).read()
+        assert f"image: git.timothykim.net/hantim/{TEST_DOMAIN}:latest" in compose
+        assert f"container_name: {CONTAINER_NAME}" in compose
+        assert "shared" in compose
+
+        # deploy workflow
+        workflow_path = os.path.join(
+            REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml"
+        )
+        assert os.path.isfile(workflow_path)
+        workflow = open(workflow_path).read()
+        assert f"docker/{TEST_DOMAIN}/**" in workflow
+        assert f"deploy-{TEST_DOMAIN}" in workflow
+
+        # nginx conf
+        conf_path = os.path.join(
+            REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf"
+        )
+        assert os.path.isfile(conf_path)
+        conf = open(conf_path).read()
+        assert f"server_name {TEST_DOMAIN} www.{TEST_DOMAIN}" in conf
+        assert f"server_name www.{TEST_DOMAIN}" in conf
+        assert "security-headers.inc" in conf
+        assert f"proxy_pass http://{CONTAINER_NAME}:80" in conf
+
+    def test_cert(self):
+        result = run_app("cert", timeout=180)
+        assert result.returncode == 0, f"app.sh cert failed:\n{result.stdout}\n{result.stderr}"
+
+    def test_garage(self, secrets):
+        result = run_app("garage")
+        assert result.returncode == 0, f"app.sh garage failed:\n{result.stdout}\n{result.stderr}"
+
+        auth = {
+            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
+        }
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=auth,
+        )
+        assert status == 200, f"Bucket not found (HTTP {status})"
+        assert data["id"], "Bucket has no ID"
+        assert TEST_DOMAIN in data.get("globalAliases", []), "Global alias not set"
+        assert data.get("websiteAccess") is True, f"Website access not enabled: {data.get('websiteAccess')}"
+
+    def test_build(self, secrets):
+        result = run_app("build", timeout=180)
+        assert result.returncode == 0, f"app.sh build failed:\n{result.stdout}\n{result.stderr}"
+
+    def test_monitor(self, secrets):
+        result = run_app("monitor")
+        assert result.returncode == 0, f"app.sh monitor failed:\n{result.stdout}\n{result.stderr}"
+
+        # UptimeRobot API may have brief eventual consistency after create
+        auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=auth,
+            )
+            assert status == 200, f"UptimeRobot API returned {status}"
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls, (
+            f"Monitor not found. URLs: {urls}\n"
+            f"app.sh output:\n{result.stdout}"
+        )
+
+
+class TestAppAll:
+    """Test the full app.sh all flow end-to-end."""
+
+    @pytest.fixture(autouse=True, scope="class")
+    def _cleanup(self, secrets):
+        cleanup(secrets, revert_git=True)
+        yield
+        cleanup(secrets, revert_git=True)
+
+    def test_all(self, secrets):
+        result = run_app("all", timeout=900)
+        assert result.returncode == 0, f"app.sh all failed:\n{result.stdout}\n{result.stderr}"
+
+        # --- DNS ---
+        auth = {"Authorization": f"Bearer {secrets['hantim-vultr-api-key']}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
+        )
+        assert status == 200
+        records = data["records"]
+        assert any(r["type"] == "A" and r["name"] == "" for r in records)
+        assert any(r["type"] == "A" and r["name"] == "www" for r in records)
+
+        # --- Gitea repo ---
+        gitea_auth = {"Authorization": f"token {secrets['hantim-new-app-script']}"}
+        status, _ = api(
+            "GET",
+            f"{GITEA_URL}/api/v1/repos/{GITEA_ORG}/{TEST_DOMAIN}",
+            headers=gitea_auth,
+        )
+        assert status == 200, "Gitea repo not found"
+
+        # --- Local files ---
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, "docker", TEST_DOMAIN, "compose.yml")
+        )
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, ".gitea", "workflows", f"deploy-{TEST_DOMAIN}.yml")
+        )
+        assert os.path.isfile(
+            os.path.join(REPO_ROOT, "docker", "nginx", "conf.d", f"{TEST_DOMAIN}.conf")
+        )
+
+        # --- Git commit was pushed ---
+        log = subprocess.run(
+            ["git", "log", "--format=%s", "-1"],
+            capture_output=True, text=True, cwd=REPO_ROOT,
+        )
+        assert log.stdout.strip() == f"add {TEST_DOMAIN}"
+
+        # --- Garage bucket ---
+        garage_auth = {
+            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
+        }
+        status, data = api(
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
+            headers=garage_auth,
+        )
+        assert status == 200, "Garage bucket not found"
+
+        # --- Site is live ---
+        # Use server IP from verified DNS records to bypass negative cache
+        server_ip = next(
+            r["data"] for r in records if r["type"] == "A" and r["name"] == ""
+        )
+        curl = subprocess.run(
+            ["curl", "-sf", "--resolve", f"www.{TEST_DOMAIN}:443:{server_ip}",
+             "-o", "/dev/null", "-w", "%{http_code}",
+             f"https://www.{TEST_DOMAIN}"],
+            capture_output=True, text=True, timeout=10,
+        )
+        assert curl.returncode == 0, f"Site not reachable (HTTP {curl.stdout})"
+
+        # --- UptimeRobot monitor ---
+        ur_auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=ur_auth,
+            )
+            assert status == 200
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls

tests/test_default_server.py

@@ -0,0 +1,29 @@
+"""Test that unknown Host headers are dropped by the default server block."""
+
+import socket
+import ssl
+
+import pytest
+
+
+HOST = "www.hantim.net"
+
+
+def test_unknown_host_https_dropped():
+    """HTTPS request with a fake Host header should be dropped (connection reset)."""
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    sock = socket.create_connection((HOST, 443))
+    ssock = ctx.wrap_socket(sock, server_hostname=HOST)
+    ssock.sendall(b"GET / HTTP/1.1\r\nHost: evil.example.com\r\n\r\n")
+    resp = ssock.recv(4096)
+    assert resp == b"", f"Expected empty response, got {resp[:100]}"
+
+
+def test_unknown_host_http_dropped():
+    """HTTP request with a fake Host header should be dropped (connection reset)."""
+    sock = socket.create_connection((HOST, 80))
+    sock.sendall(b"GET / HTTP/1.1\r\nHost: evil.example.com\r\n\r\n")
+    resp = sock.recv(4096)
+    assert resp == b"", f"Expected empty response, got {resp[:100]}"

tests/test_security_headers.py

@@ -0,0 +1,60 @@
+"""Test that security headers are present on all HTTPS sites."""
+
+import glob
+import os
+import re
+import urllib.request
+import ssl
+
+import pytest
+
+CONF_DIR = os.path.join(os.path.dirname(__file__), "..", "docker", "nginx", "conf.d")
+
+
+def discover_sites():
+    """Build HTTPS URLs from nginx conf files that include security-headers.inc."""
+    sites = []
+    for conf in sorted(glob.glob(os.path.join(CONF_DIR, "*.conf"))):
+        text = open(conf).read()
+        if "security-headers.inc" not in text:
+            continue
+        # Use the first server_name in the block that includes the headers
+        for block in re.split(r"(?=server\s*\{)", text):
+            if "security-headers.inc" in block:
+                m = re.search(r"server_name\s+([^;]+);", block)
+                if m:
+                    sites.append(f"https://{m.group(1).split()[0]}")
+                    break
+    return sites
+
+
+SITES = discover_sites()
+
+EXPECTED_HEADERS = {
+    "Strict-Transport-Security": "max-age=63072000; preload",
+    "X-Content-Type-Options": "nosniff",
+    "X-Frame-Options": "DENY",
+    "Referrer-Policy": "strict-origin-when-cross-origin",
+    "Permissions-Policy": "geolocation=(), microphone=(), camera=()",
+}
+
+
+def get_headers(url):
+    ctx = ssl.create_default_context()
+    req = urllib.request.Request(url, method="HEAD")
+    try:
+        resp = urllib.request.urlopen(req, context=ctx, timeout=10)
+        return dict(resp.headers)
+    except urllib.error.HTTPError as e:
+        return dict(e.headers)
+
+
+@pytest.mark.parametrize("url", SITES, ids=lambda u: u.split("//")[1])
+def test_security_headers(url):
+    headers = get_headers(url)
+    missing = []
+    for name, expected in EXPECTED_HEADERS.items():
+        actual = headers.get(name)
+        if actual != expected:
+            missing.append(f"{name}: expected '{expected}', got '{actual}'")
+    assert not missing, "\n".join(missing)

tests/test_tls.py

@@ -0,0 +1,35 @@
+"""Test that TLS is properly hardened on all HTTPS sites."""
+
+import socket
+import ssl
+
+import pytest
+
+HOST = "www.hantim.net"
+
+
+def test_tls13_works():
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.load_default_certs()
+    ctx.minimum_version = ssl.TLSVersion.TLSv1_3
+    ctx.maximum_version = ssl.TLSVersion.TLSv1_3
+    with ctx.wrap_socket(socket.create_connection((HOST, 443)), server_hostname=HOST) as s:
+        assert s.version() == "TLSv1.3"
+
+
+def test_tls12_works():
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.load_default_certs()
+    ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+    ctx.maximum_version = ssl.TLSVersion.TLSv1_2
+    with ctx.wrap_socket(socket.create_connection((HOST, 443)), server_hostname=HOST) as s:
+        assert s.version() == "TLSv1.2"
+
+
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")
+def test_tls11_rejected():
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.minimum_version = ssl.TLSVersion.TLSv1_1
+    ctx.maximum_version = ssl.TLSVersion.TLSv1_1
+    with pytest.raises((ssl.SSLError, OSError)):
+        ctx.wrap_socket(socket.create_connection((HOST, 443)), server_hostname=HOST)

tools/app.sh

@@ -28,7 +28,8 @@ usage() {
     echo "  garage    Create Garage media bucket and grant access"
     echo "  build     Trigger initial build workflow"
     echo "  verify    Check if site is live"
-    echo "  all       Run all steps (dns, repo, files, cert, garage, commit, build, verify)"
+    echo "  monitor   Create UptimeRobot HTTPS monitor"
+    echo "  all       Run all steps (dns, repo, files, cert, garage, commit, build, verify, monitor)"
     echo ""
     echo "Requires: bws, jq, dig"
     exit 1
@@ -252,16 +253,22 @@ cmd_repo() {
 }
 
 cmd_files() {
+    local all_exist=true
     for f in "docker/$APP/compose.yml" ".gitea/workflows/deploy-$APP.yml" "docker/nginx/conf.d/$APP.conf"; do
-        if [ -e "$REPO_ROOT/$f" ]; then
+        if [ ! -e "$REPO_ROOT/$f" ]; then
-            echo "Error: $f already exists."
+            all_exist=false
-            exit 1
+            break
         fi
     done
+    if [ "$all_exist" = true ]; then
+        echo "  All files already exist, skipping."
+        return
+    fi
 
-    echo "==> Creating docker/$APP/compose.yml..."
+    if [ ! -e "$REPO_ROOT/docker/$APP/compose.yml" ]; then
-    mkdir -p "$REPO_ROOT/docker/$APP"
+        echo "==> Creating docker/$APP/compose.yml..."
-    cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
+        mkdir -p "$REPO_ROOT/docker/$APP"
+        cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
 services:
   app:
     image: git.timothykim.net/hantim/$APP:latest
@@ -274,10 +281,14 @@ networks:
   shared:
     external: true
 EOF
+    else
+        echo "  docker/$APP/compose.yml already exists, skipping."
+    fi
 
-    echo "==> Creating .gitea/workflows/deploy-$APP.yml..."
+    if [ ! -e "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" ]; then
-    mkdir -p "$REPO_ROOT/.gitea/workflows"
+        echo "==> Creating .gitea/workflows/deploy-$APP.yml..."
-    cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
+        mkdir -p "$REPO_ROOT/.gitea/workflows"
+        cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
 name: Deploy APP_PLACEHOLDER
 
 on:
@@ -293,14 +304,19 @@ jobs:
       - name: Deploy via SSH
         run: |
           mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
           echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
           chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
 OUTER
-    sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
+        sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
+    else
+        echo "  .gitea/workflows/deploy-$APP.yml already exists, skipping."
+    fi
 
-    echo "==> Creating docker/nginx/conf.d/$APP.conf..."
+    if [ ! -e "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" ]; then
-    cat > "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" <<NGINX
+        echo "==> Creating docker/nginx/conf.d/$APP.conf..."
+        cat > "$REPO_ROOT/docker/nginx/conf.d/$APP.conf" <<NGINX
 # Redirect HTTP to HTTPS, bare domain to www
 server {
     listen 80;
@@ -339,7 +355,8 @@ server {
     ssl_certificate     /etc/letsencrypt/live/$APP/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/$APP/privkey.pem;
 
-    add_header Strict-Transport-Security "max-age=63072000; preload" always;
+    include /etc/nginx/conf.d/security-headers.inc;
+    include /etc/nginx/conf.d/goatcounter.inc;
 
     location /media/ {
         proxy_pass http://garage:3902/;
@@ -355,8 +372,9 @@ server {
     }
 }
 NGINX
-
+    else
-    echo "  Files created. Run 'git add' and commit when ready."
+        echo "  docker/nginx/conf.d/$APP.conf already exists, skipping."
+    fi
 }
 
 cmd_cert() {
@@ -386,10 +404,8 @@ cmd_garage() {
 
     BUCKET_ID=$(echo "$BUCKET_RESP" | jq -r '.id')
     if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
-        BUCKET_ID=$(curl -s -X POST "$GARAGE_API/v2/GetBucketInfo" \
+        BUCKET_ID=$(curl -s "$GARAGE_API/v2/GetBucketInfo?globalAlias=$APP" \
-            -H "$GARAGE_AUTH" \
+            -H "$GARAGE_AUTH" | jq -r '.id')
-            -H "Content-Type: application/json" \
-            -d "{\"globalAlias\": \"$APP\"}" | jq -r '.id')
         if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
             echo "Error: Failed to create or find bucket for $APP"
             echo "$BUCKET_RESP"
@@ -466,9 +482,52 @@ cmd_verify() {
     echo "    - The container is running on the server: docker ps"
 }
 
+cmd_monitor() {
+    require_bws
+
+    echo "==> Fetching UptimeRobot API key..."
+    UPTIMEROBOT_API_KEY=$(bws_get "hantim-uptimerobot-api-key")
+    UPTIMEROBOT_API="https://api.uptimerobot.com/v3"
+    UPTIMEROBOT_AUTH="Authorization: Bearer $UPTIMEROBOT_API_KEY"
+
+    MONITOR_URL="https://www.$APP"
+
+    echo "==> Checking for existing UptimeRobot monitor for $MONITOR_URL..."
+    EXISTING=$(curl -s -G "$UPTIMEROBOT_API/monitors" \
+        -H "$UPTIMEROBOT_AUTH" \
+        --data-urlencode "search=$APP")
+
+    MATCH=$(echo "$EXISTING" | jq -r --arg url "$MONITOR_URL" \
+        '.data[]? | select(.url == $url) | .id')
+    if [ -n "$MATCH" ]; then
+        echo "  Monitor already exists (id: $MATCH), skipping."
+        return
+    fi
+
+    echo "==> Fetching alert contacts..."
+    ALERT_CONTACTS=$(curl -s "$UPTIMEROBOT_API/user/alert-contacts" \
+        -H "$UPTIMEROBOT_AUTH" | jq '[.[] | {alertContactId: .id, threshold: 5, recurrence: 30}]')
+
+    echo "==> Creating UptimeRobot HTTPS monitor for $MONITOR_URL..."
+    RESP=$(curl -s -X POST "$UPTIMEROBOT_API/monitors" \
+        -H "$UPTIMEROBOT_AUTH" \
+        -H "Content-Type: application/json" \
+        -d "$(jq -n --arg name "$APP" --arg url "$MONITOR_URL" --argjson contacts "$ALERT_CONTACTS" \
+            '{friendlyName: $name, url: $url, type: 1, interval: 300, timeout: 30, assignedAlertContacts: $contacts, tagNames: ["hantim"]}')")
+
+    MONITOR_ID=$(echo "$RESP" | jq -r '.id // empty')
+    if [ -n "$MONITOR_ID" ]; then
+        echo "  Monitor created (id: $MONITOR_ID)."
+    else
+        ERROR_MSG=$(echo "$RESP" | jq -r 'if .message then (.message | if type == "array" then join("; ") else . end) else .error // "unknown error" end')
+        echo "  Warning: Failed to create monitor: $ERROR_MSG"
+    fi
+}
+
 cmd_all() {
     cmd_dns
     cmd_repo
+    cmd_cert
     cmd_files
 
     echo "==> Committing and pushing hantim-server..."
@@ -481,10 +540,10 @@ cmd_all() {
         git push
     fi
 
-    cmd_cert
     cmd_garage
     cmd_build
     cmd_verify
+    cmd_monitor
 }
 
 # --- Dispatch ---
@@ -497,6 +556,7 @@ case "$COMMAND" in
     garage)  cmd_garage ;;
     build)   cmd_build ;;
     verify)  cmd_verify ;;
+    monitor) cmd_monitor ;;
     all)     cmd_all ;;
     *)
         echo "Unknown command: $COMMAND"

tools/new-app.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# Thin wrapper — delegates to app.sh all
-if [ -z "${1:-}" ]; then
-    echo "Usage: ./tools/new-app.sh <domain>"
-    echo "  Example: ./tools/new-app.sh example.com"
-    echo ""
-    echo "For individual steps, use: ./tools/app.sh <command> <domain>"
-    exit 1
-fi
-
-exec "$(dirname "$0")/app.sh" all "$1"

tools/new-service.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-if [ -z "${1:-}" ]; then
-    echo "Usage: ./tools/new-service.sh <name>"
-    echo "  Example: ./tools/new-service.sh garage"
-    echo ""
-    echo "Creates docker/<name>/compose.yml and .gitea/workflows/deploy-<name>.yml"
-    exit 1
-fi
-
-APP="$1"
-
-if ! [[ "$APP" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
-    echo "Error: name must be alphanumeric (hyphens, dots, underscores allowed)."
-    exit 1
-fi
-
-REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-
-if [ -d "$REPO_ROOT/docker/$APP" ]; then
-    echo "Error: docker/$APP already exists."
-    exit 1
-fi
-
-echo "Creating docker/$APP/compose.yml..."
-mkdir -p "$REPO_ROOT/docker/$APP"
-cat > "$REPO_ROOT/docker/$APP/compose.yml" <<EOF
-services:
-  $APP:
-    image: TODO
-    restart: unless-stopped
-    container_name: $APP
-    networks:
-      - shared
-
-networks:
-  shared:
-    external: true
-EOF
-
-echo "Creating .gitea/workflows/deploy-$APP.yml..."
-mkdir -p "$REPO_ROOT/.gitea/workflows"
-cat > "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml" <<'OUTER'
-name: Deploy APP_PLACEHOLDER
-
-on:
-  push:
-    branches: [main]
-    paths:
-      - 'docker/APP_PLACEHOLDER/**'
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Deploy via SSH
-        run: |
-          mkdir -p ~/.ssh
-          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          ssh -o StrictHostKeyChecking=accept-new -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
-OUTER
-sed -i "s/APP_PLACEHOLDER/$APP/g" "$REPO_ROOT/.gitea/workflows/deploy-$APP.yml"
-
-echo "Done. Edit docker/$APP/compose.yml, then commit and push."
-echo ""
-echo "If this service needs secrets:"
-echo "  1. Add secrets to Bitwarden Secrets Manager"
-echo "  2. Map them in docker/$APP/.env.keys (format: ENV_VAR=bws-secret-name)"
-echo "  3. Add env_file: .env to compose.yml"
-echo "  configure.sh will generate the .env file on the server automatically."

tools/service.sh

@@ -0,0 +1,367 @@
+#!/bin/bash
+set -euo pipefail
+
+usage() {
+    echo "Usage: ./tools/service.sh <command> <name>"
+    echo ""
+    echo "Commands:"
+    echo "  dns <name>          Create A record for <name>.hantim.net"
+    echo "  files <name>        Create compose.yml and deploy workflow"
+    echo "  nginx <name> <port> Create nginx conf for <name>.hantim.net"
+    echo "  cert <name>         Issue SSL certificate for <name>.hantim.net"
+    echo "  all <name> <port>   Run dns + files + nginx + cert"
+    echo ""
+    echo "Examples:"
+    echo "  ./tools/service.sh files garage"
+    echo "  ./tools/service.sh dns garage        # garage.hantim.net -> server IP"
+    echo "  ./tools/service.sh nginx garage 3903 # create nginx conf"
+    echo "  ./tools/service.sh cert garage       # issue SSL cert"
+    echo "  ./tools/service.sh all garage 3903"
+    echo ""
+    echo "Requires: bws, jq, dig (for dns/cert)"
+    exit 1
+}
+
+COMMAND="${1:-}"
+
+if [ -z "$COMMAND" ]; then
+    usage
+fi
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
+# --- Dependency checks ---
+
+require_bws() {
+    if ! command -v bws &>/dev/null; then
+        echo "Error: Bitwarden Secrets Manager CLI (bws) is not installed."
+        echo "  https://github.com/bitwarden/sdk-sm/releases"
+        exit 1
+    fi
+    if ! command -v jq &>/dev/null; then
+        echo "Error: jq is not installed."
+        exit 1
+    fi
+
+    BWS_TOKEN_FILE="${XDG_CONFIG_HOME:-$HOME/.config}/hantim/bws-token"
+    if [ -z "${BWS_ACCESS_TOKEN:-}" ]; then
+        if [ -f "$BWS_TOKEN_FILE" ]; then
+            BWS_ACCESS_TOKEN=$(cat "$BWS_TOKEN_FILE")
+        else
+            echo "==> Bitwarden Secrets Manager access token required."
+            echo "    Generate one for your machine account at:"
+            echo "    https://vault.bitwarden.com/#/sm/machine-accounts"
+            read -rp "    Paste access token: " BWS_ACCESS_TOKEN
+            if [ -z "$BWS_ACCESS_TOKEN" ]; then
+                echo "ERROR: Access token cannot be empty."
+                exit 1
+            fi
+            mkdir -p "$(dirname "$BWS_TOKEN_FILE")"
+            echo "$BWS_ACCESS_TOKEN" > "$BWS_TOKEN_FILE"
+            chmod 600 "$BWS_TOKEN_FILE"
+        fi
+    fi
+    export BWS_ACCESS_TOKEN
+}
+
+bws_get() {
+    local name="$1"
+    local value
+    value=$(bws secret list | jq -r --arg name "$name" '.[] | select(.key == $name) | .value')
+    if [ -z "$value" ]; then
+        echo "ERROR: Secret '$name' not found in Bitwarden Secrets Manager." >&2
+        exit 1
+    fi
+    echo "$value"
+}
+
+require_dig() {
+    if ! command -v dig &>/dev/null; then
+        echo "Error: dig is not installed."
+        echo "  On macOS: brew install bind"
+        echo "  On Linux: dnf install bind-utils"
+        exit 1
+    fi
+}
+
+resolve_server_ip() {
+    require_dig
+    SERVER_IP=$(dig +short hantim.net | head -1)
+    if [ -z "$SERVER_IP" ]; then
+        echo "Error: Could not resolve hantim.net to get server IP."
+        exit 1
+    fi
+}
+
+setup_deploy_key() {
+    require_bws
+    echo "==> Fetching deploy key..."
+    DEPLOY_KEY=$(bws_get "hantim-deploy-ssh-private-key")
+    DEPLOY_KEY_FILE=$(mktemp)
+    echo "$DEPLOY_KEY" > "$DEPLOY_KEY_FILE"
+    chmod 600 "$DEPLOY_KEY_FILE"
+    trap "rm -f $DEPLOY_KEY_FILE" EXIT
+}
+
+validate_name() {
+    local name="$1"
+    if [ -z "$name" ]; then
+        echo "Error: service name is required."
+        echo ""
+        usage
+    fi
+    if ! [[ "$name" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
+        echo "Error: name must be alphanumeric (hyphens, dots, underscores allowed)."
+        exit 1
+    fi
+}
+
+# --- Commands ---
+
+cmd_dns() {
+    local name="$1"
+    local zone="hantim.net"
+    local fqdn="$name.$zone"
+
+    require_dig
+    require_bws
+
+    SERVER_IP=$(dig +short "$zone" | head -1)
+    if [ -z "$SERVER_IP" ]; then
+        echo "Error: Could not resolve $zone to get server IP."
+        exit 1
+    fi
+
+    echo "==> Fetching Vultr API key..."
+    VULTR_API_KEY=$(bws_get "hantim-vultr-api-key")
+    VULTR_API="https://api.vultr.com/v2"
+    VULTR_AUTH="Authorization: Bearer $VULTR_API_KEY"
+
+    echo "==> Checking existing A records for $name in $zone..."
+    RECORDS=$(curl -s "$VULTR_API/domains/$zone/records" -H "$VULTR_AUTH")
+
+    echo "$RECORDS" | jq -r --arg sub "$name" \
+        '.records[] | select(.type == "A" and .name == $sub) | .id' | while read -r id; do
+        echo "  Deleting existing A record for $fqdn (id: $id)"
+        curl -s -X DELETE "$VULTR_API/domains/$zone/records/$id" -H "$VULTR_AUTH"
+    done
+
+    echo "  Creating A record: $fqdn -> $SERVER_IP"
+    RESP=$(mktemp)
+    HTTP_CODE=$(curl -s -o "$RESP" -w "%{http_code}" \
+        -X POST "$VULTR_API/domains/$zone/records" \
+        -H "$VULTR_AUTH" \
+        -H "Content-Type: application/json" \
+        -d "{\"name\": \"$name\", \"type\": \"A\", \"data\": \"$SERVER_IP\", \"ttl\": 3600}")
+    if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "204" ]; then
+        echo "Error: Failed to create A record (HTTP $HTTP_CODE)"
+        cat "$RESP"
+        rm -f "$RESP"
+        exit 1
+    fi
+    rm -f "$RESP"
+
+    echo "==> Verifying DNS propagation for $fqdn..."
+    for i in $(seq 1 12); do
+        RESOLVED=$(dig +short "$fqdn" @ns1.vultr.com 2>/dev/null | head -1)
+        if [ "$RESOLVED" = "$SERVER_IP" ]; then
+            echo "  DNS is live: $fqdn -> $SERVER_IP"
+            return
+        fi
+        if [ "$i" = "12" ]; then
+            echo "  Warning: $fqdn did not resolve to $SERVER_IP after 60 seconds. It may take longer to propagate."
+            return
+        fi
+        sleep 5
+    done
+}
+
+cmd_files() {
+    local name="$1"
+
+    local all_exist=true
+    for f in "docker/$name/compose.yml" ".gitea/workflows/deploy-$name.yml"; do
+        if [ ! -e "$REPO_ROOT/$f" ]; then
+            all_exist=false
+            break
+        fi
+    done
+    if [ "$all_exist" = true ]; then
+        echo "  All files already exist, skipping."
+        return
+    fi
+
+    if [ ! -e "$REPO_ROOT/docker/$name/compose.yml" ]; then
+        echo "==> Creating docker/$name/compose.yml..."
+        mkdir -p "$REPO_ROOT/docker/$name"
+        touch "$REPO_ROOT/docker/$name/.gitignore"
+        cat > "$REPO_ROOT/docker/$name/compose.yml" <<EOF
+services:
+  $name:
+    image: TODO
+    restart: unless-stopped
+    container_name: $name
+    networks:
+      - shared
+
+networks:
+  shared:
+    external: true
+EOF
+    else
+        echo "  docker/$name/compose.yml already exists, skipping."
+    fi
+
+    if [ ! -e "$REPO_ROOT/.gitea/workflows/deploy-$name.yml" ]; then
+        echo "==> Creating .gitea/workflows/deploy-$name.yml..."
+        mkdir -p "$REPO_ROOT/.gitea/workflows"
+        cat > "$REPO_ROOT/.gitea/workflows/deploy-$name.yml" <<'OUTER'
+name: Deploy APP_PLACEHOLDER
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'docker/APP_PLACEHOLDER/**'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Deploy via SSH
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ vars.DEPLOY_HOST_KEY }}" > ~/.ssh/known_hosts
+          echo "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/deploy_key
+          chmod 600 ~/.ssh/deploy_key
+          ssh -o StrictHostKeyChecking=yes -i ~/.ssh/deploy_key deploy@${{ vars.DEPLOY_HOST }} deploy-APP_PLACEHOLDER
+OUTER
+        sed -i "s/APP_PLACEHOLDER/$name/g" "$REPO_ROOT/.gitea/workflows/deploy-$name.yml"
+    else
+        echo "  .gitea/workflows/deploy-$name.yml already exists, skipping."
+    fi
+
+    echo ""
+    echo "Done. Edit docker/$name/compose.yml, then commit and push."
+    echo ""
+    echo "If this service needs secrets:"
+    echo "  1. Add secrets to Bitwarden Secrets Manager"
+    echo "  2. Map them in docker/$name/.env.keys (format: ENV_VAR=bws-secret-name)"
+    echo "  3. Add env_file: .env to compose.yml"
+    echo "  configure.sh will generate the .env file on the server automatically."
+}
+
+cmd_nginx() {
+    local name="$1"
+    local port="$2"
+    local fqdn="$name.hantim.net"
+
+    if ! [[ "$port" =~ ^[0-9]+$ ]]; then
+        echo "Error: port must be a number."
+        exit 1
+    fi
+
+    if [ -e "$REPO_ROOT/docker/nginx/conf.d/$fqdn.conf" ]; then
+        echo "  docker/nginx/conf.d/$fqdn.conf already exists, skipping."
+        return
+    fi
+
+    echo "==> Creating docker/nginx/conf.d/$fqdn.conf..."
+    cat > "$REPO_ROOT/docker/nginx/conf.d/$fqdn.conf" <<NGINX
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $fqdn;
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+
+    location / {
+        return 301 https://$fqdn\$request_uri;
+    }
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    http2 on;
+    server_name $fqdn;
+
+    ssl_certificate     /etc/letsencrypt/live/$fqdn/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/$fqdn/privkey.pem;
+
+    include /etc/nginx/conf.d/security-headers.inc;
+
+    location / {
+        proxy_pass http://$name:$port;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_hide_header X-Frame-Options;
+    }
+}
+NGINX
+}
+
+cmd_cert() {
+    local name="$1"
+    local fqdn="$name.hantim.net"
+
+    resolve_server_ip
+    setup_deploy_key
+
+    echo "==> Issuing SSL certificate for $fqdn via deploy user..."
+    ssh -o StrictHostKeyChecking=accept-new -i "$DEPLOY_KEY_FILE" deploy@"$SERVER_IP" "cert-$fqdn"
+    echo "  Certificate issued."
+}
+
+cmd_all() {
+    local name="$1"
+    local port="$2"
+
+    cmd_dns "$name"
+    cmd_files "$name"
+    cmd_nginx "$name" "$port"
+    cmd_cert "$name"
+}
+
+# --- Dispatch ---
+
+case "$COMMAND" in
+    dns)
+        validate_name "${2:-}"
+        cmd_dns "${2:-}"
+        ;;
+    files)
+        validate_name "${2:-}"
+        cmd_files "${2:-}"
+        ;;
+    nginx)
+        validate_name "${2:-}"
+        if [ -z "${3:-}" ]; then
+            echo "Error: port is required for nginx command."
+            echo "  Usage: ./tools/service.sh nginx <name> <port>"
+            exit 1
+        fi
+        cmd_nginx "${2:-}" "${3:-}"
+        ;;
+    cert)
+        validate_name "${2:-}"
+        cmd_cert "${2:-}"
+        ;;
+    all)
+        validate_name "${2:-}"
+        if [ -z "${3:-}" ]; then
+            echo "Error: port is required for all command."
+            echo "  Usage: ./tools/service.sh all <name> <port>"
+            exit 1
+        fi
+        cmd_all "${2:-}" "${3:-}"
+        ;;
+    *)
+        echo "Unknown command: $COMMAND"
+        echo ""
+        usage
+        ;;
+esac