fix e2e tests for garage v2 and uptimerobot v3 api changes

CLAUDE.md

@@ -8,7 +8,7 @@ for technical deep-dives, USECASES.md for expected behaviors.
 - `scripts/bootstrap.sh` -- First-time server setup (manual, uses Bitwarden)
 - `scripts/configure.sh` -- Idempotent server config (firewall, certbot, start services; CI-safe)
 - `scripts/deploy.sh` -- SSH-triggered deploy + cert issuance + provision (deploy-*, cert-*, provision commands)
-- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, all)
+- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, monitor, all)
 - `tools/service.sh` -- Manage Docker services by subcommand (dns, files, nginx, cert, all)
 - `tools/remove-app.sh` -- Remove a static site (local files + Gitea repo)
 - `docker/nginx/conf.d/` -- Per-app nginx server blocks

README.md

@@ -164,19 +164,55 @@ reconnects automatically.
 
 ## Testing
 
-Tests live in `tests/` and run with pytest:
+Tests live in `tests/` and run with [uv](https://docs.astral.sh/uv/):
 
 ```bash
 # Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
-uvx pytest tests/ -v
+
+# Live site tests (security headers, TLS, default server)
+uvx pytest tests/ -v --ignore=tests/test_app_e2e.py
+
+# E2E tests for tools/app.sh (creates real resources)
+uvx pytest tests/test_app_e2e.py -v -x
 ```
 
-Tests auto-discover sites from `docker/nginx/conf.d/*.conf` — no list to
+### Live site tests
-maintain when adding new sites.
 
-The `deploy-nginx.yml` workflow runs tests automatically after each nginx
+These run against live sites and verify infrastructure invariants:
+
+- **`test_security_headers.py`** — Checks that all sites including
+  `security-headers.inc` return the expected headers (HSTS, X-Frame-Options,
+  etc.). Sites are auto-discovered from `docker/nginx/conf.d/*.conf`.
+- **`test_tls.py`** — Verifies TLS 1.3 and 1.2 work, TLS 1.1 is rejected.
+- **`test_default_server.py`** — Verifies unknown Host headers get dropped
+  (connection reset) on both HTTP and HTTPS.
+
+The `deploy-nginx.yml` workflow runs these automatically after each nginx
 deploy. If tests fail, Gitea sends an email notification.
 
+### E2E tests
+
+`test_app_e2e.py` tests `tools/app.sh` against real services (Vultr, Gitea,
+Garage, UptimeRobot). Uses `kgfamily.com` as a throwaway test domain.
+
+**Prerequisites:**
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP
+  (check https://my.vultr.com/settings/#settingsapi)
+
+**Structure:**
+
+1. **TestAppSubcommands** — Runs each `app.sh` subcommand individually
+   (dns, repo, files, cert, garage, build, monitor), verifies side effects
+   via API, then cleans up.
+2. **TestAppAll** — Runs `app.sh all kgfamily.com`, verifies the full flow
+   including site liveness and git commit, then cleans up.
+
+Cleanup runs before and after each phase via APIs (Vultr, Gitea, Garage,
+UptimeRobot) + local file deletion. Does not use `remove-app.sh` to avoid
+coupling.
+
 ## Secrets
 
 ### Bitwarden Secrets Manager

tests/conftest.py

@@ -138,7 +138,7 @@ def cleanup(secrets, revert_git=False):
             headers=auth,
         )
         if status == 200 and data:
-            for m in data.get("monitors", []):
+            for m in data.get("data", []):
                 if m.get("url") == f"https://www.{TEST_DOMAIN}":
                     api("DELETE", f"{UPTIMEROBOT_API}/monitors/{m['id']}", headers=auth)
     except Exception as e:
@@ -149,10 +149,10 @@ def cleanup(secrets, revert_git=False):
         token = secrets["hantim-garage-admin-token"]
         auth = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
         status, data = api(
-            "POST",
+            "GET",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
             headers=auth,
-            data={"globalAlias": TEST_DOMAIN},
         )
         if status == 200 and data and data.get("id"):
             bid = data["id"]
@@ -169,14 +169,8 @@ def cleanup(secrets, revert_git=False):
                         "permissions": {"read": True, "write": True, "owner": False},
                     },
                 )
-            # Remove global alias
+            # Delete bucket directly (Garage rejects alias removal when
-            api(
+            # it's the bucket's only alias, so skip RemoveBucketAlias)
-                "POST",
-                f"{GARAGE_API}/v2/RemoveBucketAlias",
-                headers=auth,
-                data={"bucketId": bid, "globalAlias": TEST_DOMAIN},
-            )
-            # Delete bucket (id is a query param, not body)
             api(
                 "POST",
                 f"{GARAGE_API}/v2/DeleteBucket?"
@@ -250,13 +244,20 @@ def cleanup(secrets, revert_git=False):
         except Exception as e:
             errors.append(f"Local {path}: {e}")
 
-    # --- Vultr DNS zone ---
+    # --- Vultr DNS records (keep zone to avoid slow re-creation) ---
     try:
         key = secrets["hantim-vultr-api-key"]
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
+        )
+        if status == 200 and data:
+            for r in data.get("records", []):
+                if r["type"] == "A" and r["name"] in ("", "www"):
                     api(
                         "DELETE",
-            f"{VULTR_API}/domains/{TEST_DOMAIN}",
+                        f"{VULTR_API}/domains/{TEST_DOMAIN}/records/{r['id']}",
-            headers={"Authorization": f"Bearer {key}"},
+                        headers=auth,
                     )
     except Exception as e:
         errors.append(f"Vultr: {e}")

tests/test_app_e2e.py

@@ -5,8 +5,8 @@ Run: uvx pytest tests/test_app_e2e.py -v -x
 """
 
 import os
-import ssl
 import subprocess
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -104,19 +104,17 @@ class TestAppSubcommands:
 
         auth = {
             "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
-            "Content-Type": "application/json",
         }
         status, data = api(
-            "POST",
+            "GET",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
             headers=auth,
-            data={"globalAlias": TEST_DOMAIN},
         )
         assert status == 200, f"Bucket not found (HTTP {status})"
         assert data["id"], "Bucket has no ID"
         assert TEST_DOMAIN in data.get("globalAliases", []), "Global alias not set"
-        website = data.get("websiteAccess", {})
+        assert data.get("websiteAccess") is True, f"Website access not enabled: {data.get('websiteAccess')}"
-        assert website.get("enabled") is True, f"Website access not enabled: {website}"
 
     def test_build(self, secrets):
         result = run_app("build", timeout=180)
@@ -126,16 +124,24 @@ class TestAppSubcommands:
         result = run_app("monitor")
         assert result.returncode == 0, f"app.sh monitor failed:\n{result.stdout}\n{result.stderr}"
 
+        # UptimeRobot API may have brief eventual consistency after create
         auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
             status, data = api(
                 "GET",
-            f"{UPTIMEROBOT_API}/monitors?"
+                f"{UPTIMEROBOT_API}/monitors",
-            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
                 headers=auth,
             )
             assert status == 200, f"UptimeRobot API returned {status}"
-        urls = [m["url"] for m in data.get("monitors", [])]
+            urls = [m["url"] for m in data.get("data", [])]
-        assert f"https://www.{TEST_DOMAIN}" in urls, f"Monitor not found. URLs: {urls}"
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls, (
+            f"Monitor not found. URLs: {urls}\n"
+            f"app.sh output:\n{result.stdout}"
+        )
 
 
 class TestAppAll:
@@ -191,35 +197,40 @@ class TestAppAll:
         # --- Garage bucket ---
         garage_auth = {
             "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
-            "Content-Type": "application/json",
         }
         status, data = api(
-            "POST",
+            "GET",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
             headers=garage_auth,
-            data={"globalAlias": TEST_DOMAIN},
         )
         assert status == 200, "Garage bucket not found"
 
         # --- Site is live ---
-        ctx = ssl.create_default_context()
+        # Use server IP from verified DNS records to bypass negative cache
-        req = urllib.request.Request(f"https://www.{TEST_DOMAIN}", method="HEAD")
+        server_ip = next(
-        try:
+            r["data"] for r in records if r["type"] == "A" and r["name"] == ""
-            resp = urllib.request.urlopen(req, context=ctx, timeout=10)
+        )
-            assert resp.status in (200, 301, 302)
+        curl = subprocess.run(
-        except urllib.error.HTTPError:
+            ["curl", "-sf", "--resolve", f"www.{TEST_DOMAIN}:443:{server_ip}",
-            pass  # any HTTP response means the site is up
+             "-o", "/dev/null", "-w", "%{http_code}",
-        except urllib.error.URLError as e:
+             f"https://www.{TEST_DOMAIN}"],
-            pytest.fail(f"Site not reachable: {e}")
+            capture_output=True, text=True, timeout=10,
+        )
+        assert curl.returncode == 0, f"Site not reachable (HTTP {curl.stdout})"
 
         # --- UptimeRobot monitor ---
         ur_auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
             status, data = api(
                 "GET",
-            f"{UPTIMEROBOT_API}/monitors?"
+                f"{UPTIMEROBOT_API}/monitors",
-            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
                 headers=ur_auth,
             )
             assert status == 200
-        urls = [m["url"] for m in data.get("monitors", [])]
+            urls = [m["url"] for m in data.get("data", [])]
-        assert f"https://www.{TEST_DOMAIN}" in urls
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls

tools/app.sh

@@ -404,10 +404,8 @@ cmd_garage() {
 
     BUCKET_ID=$(echo "$BUCKET_RESP" | jq -r '.id')
     if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
-        BUCKET_ID=$(curl -s -X POST "$GARAGE_API/v2/GetBucketInfo" \
+        BUCKET_ID=$(curl -s "$GARAGE_API/v2/GetBucketInfo?globalAlias=$APP" \
-            -H "$GARAGE_AUTH" \
+            -H "$GARAGE_AUTH" | jq -r '.id')
-            -H "Content-Type: application/json" \
-            -d "{\"globalAlias\": \"$APP\"}" | jq -r '.id')
         if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
             echo "Error: Failed to create or find bucket for $APP"
             echo "$BUCKET_RESP"
@@ -500,7 +498,7 @@ cmd_monitor() {
         --data-urlencode "search=$APP")
 
     MATCH=$(echo "$EXISTING" | jq -r --arg url "$MONITOR_URL" \
-        '.monitors[]? | select(.url == $url) | .id')
+        '.data[]? | select(.url == $url) | .id')
     if [ -n "$MATCH" ]; then
         echo "  Monitor already exists (id: $MATCH), skipping."
         return