fix e2e tests for garage v2 and uptimerobot v3 api changes

2026-04-10 16:25:54 -04:00
parent f48ad0e21e
commit f093a66580
5 changed files with 111 additions and 65 deletions
@@ -8,7 +8,7 @@ for technical deep-dives, USECASES.md for expected behaviors.
 - `scripts/bootstrap.sh` -- First-time server setup (manual, uses Bitwarden)
 - `scripts/configure.sh` -- Idempotent server config (firewall, certbot, start services; CI-safe)
 - `scripts/deploy.sh` -- SSH-triggered deploy + cert issuance + provision (deploy-*, cert-*, provision commands)
- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, all)
+- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, monitor, all)
 - `tools/service.sh` -- Manage Docker services by subcommand (dns, files, nginx, cert, all)
 - `tools/remove-app.sh` -- Remove a static site (local files + Gitea repo)
 - `docker/nginx/conf.d/` -- Per-app nginx server blocks
@@ -164,19 +164,55 @@ reconnects automatically.

 ## Testing

-Tests live in `tests/` and run with pytest:
+Tests live in `tests/` and run with [uv](https://docs.astral.sh/uv/):

 ```bash
 # Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh
-uvx pytest tests/ -v
+
+# Live site tests (security headers, TLS, default server)
+uvx pytest tests/ -v --ignore=tests/test_app_e2e.py
+
+# E2E tests for tools/app.sh (creates real resources)
+uvx pytest tests/test_app_e2e.py -v -x
 ```

-Tests auto-discover sites from `docker/nginx/conf.d/*.conf` — no list to
-maintain when adding new sites.
+### Live site tests

-The `deploy-nginx.yml` workflow runs tests automatically after each nginx
+These run against live sites and verify infrastructure invariants:
+
+- **`test_security_headers.py`** — Checks that all sites including
+  `security-headers.inc` return the expected headers (HSTS, X-Frame-Options,
+  etc.). Sites are auto-discovered from `docker/nginx/conf.d/*.conf`.
+- **`test_tls.py`** — Verifies TLS 1.3 and 1.2 work, TLS 1.1 is rejected.
+- **`test_default_server.py`** — Verifies unknown Host headers get dropped
+  (connection reset) on both HTTP and HTTPS.
+
+The `deploy-nginx.yml` workflow runs these automatically after each nginx
 deploy. If tests fail, Gitea sends an email notification.

+### E2E tests
+
+`test_app_e2e.py` tests `tools/app.sh` against real services (Vultr, Gitea,
+Garage, UptimeRobot). Uses `kgfamily.com` as a throwaway test domain.
+
+**Prerequisites:**
+- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git`
+- `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token`
+- Vultr API key must allow requests from the machine's IP
+  (check https://my.vultr.com/settings/#settingsapi)
+
+**Structure:**
+
+1. **TestAppSubcommands** — Runs each `app.sh` subcommand individually
+   (dns, repo, files, cert, garage, build, monitor), verifies side effects
+   via API, then cleans up.
+2. **TestAppAll** — Runs `app.sh all kgfamily.com`, verifies the full flow
+   including site liveness and git commit, then cleans up.
+
+Cleanup runs before and after each phase via APIs (Vultr, Gitea, Garage,
+UptimeRobot) + local file deletion. Does not use `remove-app.sh` to avoid
+coupling.
+
 ## Secrets

 ### Bitwarden Secrets Manager
@@ -138,7 +138,7 @@ def cleanup(secrets, revert_git=False):
            headers=auth,
        )
        if status == 200 and data:
-            for m in data.get("monitors", []):
+            for m in data.get("data", []):
                if m.get("url") == f"https://www.{TEST_DOMAIN}":
                    api("DELETE", f"{UPTIMEROBOT_API}/monitors/{m['id']}", headers=auth)
    except Exception as e:
@@ -149,10 +149,10 @@ def cleanup(secrets, revert_git=False):
        token = secrets["hantim-garage-admin-token"]
        auth = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
        status, data = api(
-            "POST",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
            headers=auth,
-            data={"globalAlias": TEST_DOMAIN},
        )
        if status == 200 and data and data.get("id"):
            bid = data["id"]
@@ -169,14 +169,8 @@ def cleanup(secrets, revert_git=False):
                        "permissions": {"read": True, "write": True, "owner": False},
                    },
                )
-            # Remove global alias
-            api(
-                "POST",
-                f"{GARAGE_API}/v2/RemoveBucketAlias",
-                headers=auth,
-                data={"bucketId": bid, "globalAlias": TEST_DOMAIN},
-            )
-            # Delete bucket (id is a query param, not body)
+            # Delete bucket directly (Garage rejects alias removal when
+            # it's the bucket's only alias, so skip RemoveBucketAlias)
            api(
                "POST",
                f"{GARAGE_API}/v2/DeleteBucket?"
@@ -250,14 +244,21 @@ def cleanup(secrets, revert_git=False):
        except Exception as e:
            errors.append(f"Local {path}: {e}")

-    # --- Vultr DNS zone ---
+    # --- Vultr DNS records (keep zone to avoid slow re-creation) ---
    try:
        key = secrets["hantim-vultr-api-key"]
-        api(
-            "DELETE",
-            f"{VULTR_API}/domains/{TEST_DOMAIN}",
-            headers={"Authorization": f"Bearer {key}"},
+        auth = {"Authorization": f"Bearer {key}"}
+        status, data = api(
+            "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth
        )
+        if status == 200 and data:
+            for r in data.get("records", []):
+                if r["type"] == "A" and r["name"] in ("", "www"):
+                    api(
+                        "DELETE",
+                        f"{VULTR_API}/domains/{TEST_DOMAIN}/records/{r['id']}",
+                        headers=auth,
+                    )
    except Exception as e:
        errors.append(f"Vultr: {e}")

@@ -5,8 +5,8 @@ Run: uvx pytest tests/test_app_e2e.py -v -x
 """

 import os
-import ssl
 import subprocess
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -104,19 +104,17 @@ class TestAppSubcommands:

        auth = {
            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
-            "Content-Type": "application/json",
        }
        status, data = api(
-            "POST",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
            headers=auth,
-            data={"globalAlias": TEST_DOMAIN},
        )
        assert status == 200, f"Bucket not found (HTTP {status})"
        assert data["id"], "Bucket has no ID"
        assert TEST_DOMAIN in data.get("globalAliases", []), "Global alias not set"
-        website = data.get("websiteAccess", {})
-        assert website.get("enabled") is True, f"Website access not enabled: {website}"
+        assert data.get("websiteAccess") is True, f"Website access not enabled: {data.get('websiteAccess')}"

    def test_build(self, secrets):
        result = run_app("build", timeout=180)
@@ -126,16 +124,24 @@ class TestAppSubcommands:
        result = run_app("monitor")
        assert result.returncode == 0, f"app.sh monitor failed:\n{result.stdout}\n{result.stderr}"

+        # UptimeRobot API may have brief eventual consistency after create
        auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
-        status, data = api(
-            "GET",
-            f"{UPTIMEROBOT_API}/monitors?"
-            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
-            headers=auth,
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=auth,
+            )
+            assert status == 200, f"UptimeRobot API returned {status}"
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls, (
+            f"Monitor not found. URLs: {urls}\n"
+            f"app.sh output:\n{result.stdout}"
        )
-        assert status == 200, f"UptimeRobot API returned {status}"
-        urls = [m["url"] for m in data.get("monitors", [])]
-        assert f"https://www.{TEST_DOMAIN}" in urls, f"Monitor not found. URLs: {urls}"


 class TestAppAll:
@@ -191,35 +197,40 @@ class TestAppAll:
        # --- Garage bucket ---
        garage_auth = {
            "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}",
-            "Content-Type": "application/json",
        }
        status, data = api(
-            "POST",
-            f"{GARAGE_API}/v2/GetBucketInfo",
+            "GET",
+            f"{GARAGE_API}/v2/GetBucketInfo?"
+            + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}),
            headers=garage_auth,
-            data={"globalAlias": TEST_DOMAIN},
        )
        assert status == 200, "Garage bucket not found"

        # --- Site is live ---
-        ctx = ssl.create_default_context()
-        req = urllib.request.Request(f"https://www.{TEST_DOMAIN}", method="HEAD")
-        try:
-            resp = urllib.request.urlopen(req, context=ctx, timeout=10)
-            assert resp.status in (200, 301, 302)
-        except urllib.error.HTTPError:
-            pass  # any HTTP response means the site is up
-        except urllib.error.URLError as e:
-            pytest.fail(f"Site not reachable: {e}")
+        # Use server IP from verified DNS records to bypass negative cache
+        server_ip = next(
+            r["data"] for r in records if r["type"] == "A" and r["name"] == ""
+        )
+        curl = subprocess.run(
+            ["curl", "-sf", "--resolve", f"www.{TEST_DOMAIN}:443:{server_ip}",
+             "-o", "/dev/null", "-w", "%{http_code}",
+             f"https://www.{TEST_DOMAIN}"],
+            capture_output=True, text=True, timeout=10,
+        )
+        assert curl.returncode == 0, f"Site not reachable (HTTP {curl.stdout})"

        # --- UptimeRobot monitor ---
        ur_auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"}
-        status, data = api(
-            "GET",
-            f"{UPTIMEROBOT_API}/monitors?"
-            + urllib.parse.urlencode({"search": TEST_DOMAIN}),
-            headers=ur_auth,
-        )
-        assert status == 200
-        urls = [m["url"] for m in data.get("monitors", [])]
-        assert f"https://www.{TEST_DOMAIN}" in urls
+        target = f"https://www.{TEST_DOMAIN}"
+        for attempt in range(3):
+            status, data = api(
+                "GET",
+                f"{UPTIMEROBOT_API}/monitors",
+                headers=ur_auth,
+            )
+            assert status == 200
+            urls = [m["url"] for m in data.get("data", [])]
+            if target in urls:
+                break
+            time.sleep(2)
+        assert target in urls
@@ -404,10 +404,8 @@ cmd_garage() {

    BUCKET_ID=$(echo "$BUCKET_RESP" | jq -r '.id')
    if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
-        BUCKET_ID=$(curl -s -X POST "$GARAGE_API/v2/GetBucketInfo" \
-            -H "$GARAGE_AUTH" \
-            -H "Content-Type: application/json" \
-            -d "{\"globalAlias\": \"$APP\"}" | jq -r '.id')
+        BUCKET_ID=$(curl -s "$GARAGE_API/v2/GetBucketInfo?globalAlias=$APP" \
+            -H "$GARAGE_AUTH" | jq -r '.id')
        if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then
            echo "Error: Failed to create or find bucket for $APP"
            echo "$BUCKET_RESP"
@@ -500,7 +498,7 @@ cmd_monitor() {
        --data-urlencode "search=$APP")

    MATCH=$(echo "$EXISTING" | jq -r --arg url "$MONITOR_URL" \
-        '.monitors[]? | select(.url == $url) | .id')
+        '.data[]? | select(.url == $url) | .id')
    if [ -n "$MATCH" ]; then
        echo "  Monitor already exists (id: $MATCH), skipping."
        return