From f093a66580a419c54201f5869aadf2a46c242e45 Mon Sep 17 00:00:00 2001 From: Timothy Kim Date: Fri, 10 Apr 2026 16:25:54 -0400 Subject: [PATCH] fix e2e tests for garage v2 and uptimerobot v3 api changes --- CLAUDE.md | 2 +- README.md | 46 ++++++++++++++++++++--- tests/conftest.py | 35 +++++++++--------- tests/test_app_e2e.py | 85 ++++++++++++++++++++++++------------------- tools/app.sh | 8 ++-- 5 files changed, 111 insertions(+), 65 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 781f63c..04e81c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ for technical deep-dives, USECASES.md for expected behaviors. - `scripts/bootstrap.sh` -- First-time server setup (manual, uses Bitwarden) - `scripts/configure.sh` -- Idempotent server config (firewall, certbot, start services; CI-safe) - `scripts/deploy.sh` -- SSH-triggered deploy + cert issuance + provision (deploy-*, cert-*, provision commands) -- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, all) +- `tools/app.sh` -- App provisioning by subcommand (dns, repo, files, cert, garage, build, verify, monitor, all) - `tools/service.sh` -- Manage Docker services by subcommand (dns, files, nginx, cert, all) - `tools/remove-app.sh` -- Remove a static site (local files + Gitea repo) - `docker/nginx/conf.d/` -- Per-app nginx server blocks diff --git a/README.md b/README.md index b224af3..a1a11ee 100644 --- a/README.md +++ b/README.md @@ -164,19 +164,55 @@ reconnects automatically. ## Testing -Tests live in `tests/` and run with pytest: +Tests live in `tests/` and run with [uv](https://docs.astral.sh/uv/): ```bash # Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh -uvx pytest tests/ -v + +# Live site tests (security headers, TLS, default server) +uvx pytest tests/ -v --ignore=tests/test_app_e2e.py + +# E2E tests for tools/app.sh (creates real resources) +uvx pytest tests/test_app_e2e.py -v -x ``` -Tests auto-discover sites from `docker/nginx/conf.d/*.conf` — no list to -maintain when adding new sites. +### Live site tests -The `deploy-nginx.yml` workflow runs tests automatically after each nginx +These run against live sites and verify infrastructure invariants: + +- **`test_security_headers.py`** — Checks that all sites including + `security-headers.inc` return the expected headers (HSTS, X-Frame-Options, + etc.). Sites are auto-discovered from `docker/nginx/conf.d/*.conf`. +- **`test_tls.py`** — Verifies TLS 1.3 and 1.2 work, TLS 1.1 is rejected. +- **`test_default_server.py`** — Verifies unknown Host headers get dropped + (connection reset) on both HTTP and HTTPS. + +The `deploy-nginx.yml` workflow runs these automatically after each nginx deploy. If tests fail, Gitea sends an email notification. +### E2E tests + +`test_app_e2e.py` tests `tools/app.sh` against real services (Vultr, Gitea, +Garage, UptimeRobot). Uses `kgfamily.com` as a throwaway test domain. + +**Prerequisites:** +- CLI tools: `bws`, `jq`, `dig`, `ssh`, `curl`, `git` +- `BWS_ACCESS_TOKEN` env var or token at `~/.config/hantim/bws-token` +- Vultr API key must allow requests from the machine's IP + (check https://my.vultr.com/settings/#settingsapi) + +**Structure:** + +1. **TestAppSubcommands** — Runs each `app.sh` subcommand individually + (dns, repo, files, cert, garage, build, monitor), verifies side effects + via API, then cleans up. +2. **TestAppAll** — Runs `app.sh all kgfamily.com`, verifies the full flow + including site liveness and git commit, then cleans up. + +Cleanup runs before and after each phase via APIs (Vultr, Gitea, Garage, +UptimeRobot) + local file deletion. Does not use `remove-app.sh` to avoid +coupling. + ## Secrets ### Bitwarden Secrets Manager diff --git a/tests/conftest.py b/tests/conftest.py index a9b4803..b4f2cbb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -138,7 +138,7 @@ def cleanup(secrets, revert_git=False): headers=auth, ) if status == 200 and data: - for m in data.get("monitors", []): + for m in data.get("data", []): if m.get("url") == f"https://www.{TEST_DOMAIN}": api("DELETE", f"{UPTIMEROBOT_API}/monitors/{m['id']}", headers=auth) except Exception as e: @@ -149,10 +149,10 @@ def cleanup(secrets, revert_git=False): token = secrets["hantim-garage-admin-token"] auth = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} status, data = api( - "POST", - f"{GARAGE_API}/v2/GetBucketInfo", + "GET", + f"{GARAGE_API}/v2/GetBucketInfo?" + + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}), headers=auth, - data={"globalAlias": TEST_DOMAIN}, ) if status == 200 and data and data.get("id"): bid = data["id"] @@ -169,14 +169,8 @@ def cleanup(secrets, revert_git=False): "permissions": {"read": True, "write": True, "owner": False}, }, ) - # Remove global alias - api( - "POST", - f"{GARAGE_API}/v2/RemoveBucketAlias", - headers=auth, - data={"bucketId": bid, "globalAlias": TEST_DOMAIN}, - ) - # Delete bucket (id is a query param, not body) + # Delete bucket directly (Garage rejects alias removal when + # it's the bucket's only alias, so skip RemoveBucketAlias) api( "POST", f"{GARAGE_API}/v2/DeleteBucket?" @@ -250,14 +244,21 @@ def cleanup(secrets, revert_git=False): except Exception as e: errors.append(f"Local {path}: {e}") - # --- Vultr DNS zone --- + # --- Vultr DNS records (keep zone to avoid slow re-creation) --- try: key = secrets["hantim-vultr-api-key"] - api( - "DELETE", - f"{VULTR_API}/domains/{TEST_DOMAIN}", - headers={"Authorization": f"Bearer {key}"}, + auth = {"Authorization": f"Bearer {key}"} + status, data = api( + "GET", f"{VULTR_API}/domains/{TEST_DOMAIN}/records", headers=auth ) + if status == 200 and data: + for r in data.get("records", []): + if r["type"] == "A" and r["name"] in ("", "www"): + api( + "DELETE", + f"{VULTR_API}/domains/{TEST_DOMAIN}/records/{r['id']}", + headers=auth, + ) except Exception as e: errors.append(f"Vultr: {e}") diff --git a/tests/test_app_e2e.py b/tests/test_app_e2e.py index 4738eaa..fefd630 100644 --- a/tests/test_app_e2e.py +++ b/tests/test_app_e2e.py @@ -5,8 +5,8 @@ Run: uvx pytest tests/test_app_e2e.py -v -x """ import os -import ssl import subprocess +import time import urllib.error import urllib.parse import urllib.request @@ -104,19 +104,17 @@ class TestAppSubcommands: auth = { "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}", - "Content-Type": "application/json", } status, data = api( - "POST", - f"{GARAGE_API}/v2/GetBucketInfo", + "GET", + f"{GARAGE_API}/v2/GetBucketInfo?" + + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}), headers=auth, - data={"globalAlias": TEST_DOMAIN}, ) assert status == 200, f"Bucket not found (HTTP {status})" assert data["id"], "Bucket has no ID" assert TEST_DOMAIN in data.get("globalAliases", []), "Global alias not set" - website = data.get("websiteAccess", {}) - assert website.get("enabled") is True, f"Website access not enabled: {website}" + assert data.get("websiteAccess") is True, f"Website access not enabled: {data.get('websiteAccess')}" def test_build(self, secrets): result = run_app("build", timeout=180) @@ -126,16 +124,24 @@ class TestAppSubcommands: result = run_app("monitor") assert result.returncode == 0, f"app.sh monitor failed:\n{result.stdout}\n{result.stderr}" + # UptimeRobot API may have brief eventual consistency after create auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"} - status, data = api( - "GET", - f"{UPTIMEROBOT_API}/monitors?" - + urllib.parse.urlencode({"search": TEST_DOMAIN}), - headers=auth, + target = f"https://www.{TEST_DOMAIN}" + for attempt in range(3): + status, data = api( + "GET", + f"{UPTIMEROBOT_API}/monitors", + headers=auth, + ) + assert status == 200, f"UptimeRobot API returned {status}" + urls = [m["url"] for m in data.get("data", [])] + if target in urls: + break + time.sleep(2) + assert target in urls, ( + f"Monitor not found. URLs: {urls}\n" + f"app.sh output:\n{result.stdout}" ) - assert status == 200, f"UptimeRobot API returned {status}" - urls = [m["url"] for m in data.get("monitors", [])] - assert f"https://www.{TEST_DOMAIN}" in urls, f"Monitor not found. URLs: {urls}" class TestAppAll: @@ -191,35 +197,40 @@ class TestAppAll: # --- Garage bucket --- garage_auth = { "Authorization": f"Bearer {secrets['hantim-garage-admin-token']}", - "Content-Type": "application/json", } status, data = api( - "POST", - f"{GARAGE_API}/v2/GetBucketInfo", + "GET", + f"{GARAGE_API}/v2/GetBucketInfo?" + + urllib.parse.urlencode({"globalAlias": TEST_DOMAIN}), headers=garage_auth, - data={"globalAlias": TEST_DOMAIN}, ) assert status == 200, "Garage bucket not found" # --- Site is live --- - ctx = ssl.create_default_context() - req = urllib.request.Request(f"https://www.{TEST_DOMAIN}", method="HEAD") - try: - resp = urllib.request.urlopen(req, context=ctx, timeout=10) - assert resp.status in (200, 301, 302) - except urllib.error.HTTPError: - pass # any HTTP response means the site is up - except urllib.error.URLError as e: - pytest.fail(f"Site not reachable: {e}") + # Use server IP from verified DNS records to bypass negative cache + server_ip = next( + r["data"] for r in records if r["type"] == "A" and r["name"] == "" + ) + curl = subprocess.run( + ["curl", "-sf", "--resolve", f"www.{TEST_DOMAIN}:443:{server_ip}", + "-o", "/dev/null", "-w", "%{http_code}", + f"https://www.{TEST_DOMAIN}"], + capture_output=True, text=True, timeout=10, + ) + assert curl.returncode == 0, f"Site not reachable (HTTP {curl.stdout})" # --- UptimeRobot monitor --- ur_auth = {"Authorization": f"Bearer {secrets['hantim-uptimerobot-api-key']}"} - status, data = api( - "GET", - f"{UPTIMEROBOT_API}/monitors?" - + urllib.parse.urlencode({"search": TEST_DOMAIN}), - headers=ur_auth, - ) - assert status == 200 - urls = [m["url"] for m in data.get("monitors", [])] - assert f"https://www.{TEST_DOMAIN}" in urls + target = f"https://www.{TEST_DOMAIN}" + for attempt in range(3): + status, data = api( + "GET", + f"{UPTIMEROBOT_API}/monitors", + headers=ur_auth, + ) + assert status == 200 + urls = [m["url"] for m in data.get("data", [])] + if target in urls: + break + time.sleep(2) + assert target in urls diff --git a/tools/app.sh b/tools/app.sh index 7630706..5e362e1 100755 --- a/tools/app.sh +++ b/tools/app.sh @@ -404,10 +404,8 @@ cmd_garage() { BUCKET_ID=$(echo "$BUCKET_RESP" | jq -r '.id') if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then - BUCKET_ID=$(curl -s -X POST "$GARAGE_API/v2/GetBucketInfo" \ - -H "$GARAGE_AUTH" \ - -H "Content-Type: application/json" \ - -d "{\"globalAlias\": \"$APP\"}" | jq -r '.id') + BUCKET_ID=$(curl -s "$GARAGE_API/v2/GetBucketInfo?globalAlias=$APP" \ + -H "$GARAGE_AUTH" | jq -r '.id') if [ -z "$BUCKET_ID" ] || [ "$BUCKET_ID" = "null" ]; then echo "Error: Failed to create or find bucket for $APP" echo "$BUCKET_RESP" @@ -500,7 +498,7 @@ cmd_monitor() { --data-urlencode "search=$APP") MATCH=$(echo "$EXISTING" | jq -r --arg url "$MONITOR_URL" \ - '.monitors[]? | select(.url == $url) | .id') + '.data[]? | select(.url == $url) | .id') if [ -n "$MATCH" ]; then echo " Monitor already exists (id: $MATCH), skipping." return