feat: add API automation tests and contract check scripts

.gitignore

@@ -4,4 +4,5 @@ dist/
 .wrangler/
 .dev.vars
 coverage/
+reports/
 *.log

docs/api-automation-tests.md

@@ -0,0 +1,60 @@
+# API Automation Tests
+
+This project has two API automation layers:
+
+- `scripts/api-contract-test.sh` runs OpenAPI contract checks with Schemathesis against the schema at `OPENAPI_SCHEMA_URL`.
+- `tests/auth-live-api.test.ts` runs focused Vitest live scenarios against a deployed or local Better Auth API.
+
+## Local Commands
+
+Run repository tests without network dependencies:
+
+```bash
+pnpm test
+```
+
+Run live smoke scenarios:
+
+```bash
+API_BASE_URL=https://cfw-auth.bowong.cc/api/auth pnpm test tests/auth-live-api.test.ts
+```
+
+Run OpenAPI contract checks:
+
+```bash
+API_BASE_URL=https://cfw-auth.bowong.cc/api/auth pnpm api:test:contract
+```
+
+Use a lower `MAX_EXAMPLES` for pull requests and a higher value for scheduled checks:
+
+```bash
+API_BASE_URL=https://cfw-auth.bowong.cc/api/auth MAX_EXAMPLES=100 pnpm api:test:contract
+```
+
+## Environment
+
+- `API_BASE_URL`: required for live and contract tests. Use the auth base path, for example `https://cfw-auth.bowong.cc/api/auth`.
+- `OPENAPI_SCHEMA_URL`: optional schema URL. Defaults to `https://cfw-auth.bowong.cc/api/auth/open-api/generate-schema`.
+- `OPENAPI_SCHEMA`: optional downloaded schema path. Defaults to `reports/openapi/cfw-auth-openapi.json`.
+- `API_TOKEN`: optional bearer token for protected checks.
+- `RUN_ID`: optional namespace for test data. Defaults to a timestamp-based value.
+- `ALLOW_AUTH_WRITE_TESTS=true`: enables Vitest scenarios that create authentication data.
+- `MAX_EXAMPLES`, `TEST_SEED`, `REQUEST_TIMEOUT`, `SCHEMATHESIS_PHASES`, `SCHEMATHESIS_CHECKS`, `REPORT_DIR`: tune Schemathesis execution.
+
+## Coverage
+
+OpenAPI automation covers static schema linting and schema-driven request/response checks. The default Schemathesis phases are `examples,coverage` to avoid broad fuzzing against production by accident.
+
+Vitest live tests currently cover:
+
+- OpenAPI document availability and expected core paths.
+- `GET /ok` readiness response.
+- anonymous `GET /get-session` behavior.
+- invalid email sign-in rejection.
+- optional namespaced email sign-up when write tests are explicitly enabled.
+
+## Safety
+
+The OpenAPI document includes account, organization, session, API key, and admin write operations. Do not run broad fuzzing or stateful phases against production unless the target is an isolated test environment.
+
+Write scenarios are skipped unless `ALLOW_AUTH_WRITE_TESTS=true`. Generated users use `RUN_ID` and the `example.invalid` domain, but there is no public delete-user cleanup in the default scenario because Better Auth deletion normally requires a valid session and may require verification.

package.json

@@ -11,6 +11,8 @@
     "db:check": "node scripts/check-better-auth-migration.mjs",
     "db:generate": "auth generate --config src/auth.migration.ts --output docs/schema/better-auth-target.sql -y",
     "test": "vitest run",
+    "api:test:live": "vitest run tests/auth-live-api.test.ts",
+    "api:test:contract": "bash scripts/api-contract-test.sh",
     "typecheck": "tsc --noEmit",
     "ready": "pnpm db:check && pnpm typecheck && pnpm test"
   },

scripts/api-contract-test.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${OPENAPI_SCHEMA_URL:=https://cfw-auth.bowong.cc/api/auth/open-api/generate-schema}"
+: "${OPENAPI_SCHEMA:=reports/openapi/cfw-auth-openapi.json}"
+: "${API_BASE_URL:?Set API_BASE_URL, for example https://cfw-auth.bowong.cc/api/auth or http://localhost:8788/api/auth}"
+: "${REPORT_DIR:=reports/schemathesis}"
+: "${MAX_EXAMPLES:=25}"
+: "${TEST_SEED:=12345}"
+: "${REQUEST_TIMEOUT:=10}"
+: "${SCHEMATHESIS_PHASES:=examples,coverage}"
+: "${SCHEMATHESIS_CHECKS:=all}"
+
+mkdir -p "$(dirname "$OPENAPI_SCHEMA")" "$REPORT_DIR"
+
+curl -fsSL "$OPENAPI_SCHEMA_URL" -o "$OPENAPI_SCHEMA"
+
+if command -v npx >/dev/null 2>&1; then
+  npx --yes @redocly/cli lint "$OPENAPI_SCHEMA"
+fi
+
+AUTH_ARGS=()
+if [[ -n "${API_TOKEN:-}" ]]; then
+  AUTH_ARGS=(-H "Authorization: Bearer ${API_TOKEN}")
+fi
+
+SCHEMATHESIS_CMD=()
+if command -v schemathesis >/dev/null 2>&1; then
+  SCHEMATHESIS_CMD=(schemathesis)
+elif command -v uvx >/dev/null 2>&1; then
+  SCHEMATHESIS_CMD=(uvx schemathesis)
+else
+  echo "Install schemathesis or uv first." >&2
+  exit 127
+fi
+
+"${SCHEMATHESIS_CMD[@]}" run "$OPENAPI_SCHEMA" \
+  --url "$API_BASE_URL" \
+  --checks "$SCHEMATHESIS_CHECKS" \
+  --phases "$SCHEMATHESIS_PHASES" \
+  --max-examples "$MAX_EXAMPLES" \
+  --seed "$TEST_SEED" \
+  --request-timeout "$REQUEST_TIMEOUT" \
+  --report junit,har,vcr \
+  --report-dir "$REPORT_DIR" \
+  "${AUTH_ARGS[@]}"

tests/auth-live-api.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, it } from "vitest";
+
+const API_BASE_URL = process.env.API_BASE_URL?.replace(/\/+$/, "");
+const API_TOKEN = process.env.API_TOKEN;
+const ALLOW_AUTH_WRITE_TESTS = process.env.ALLOW_AUTH_WRITE_TESTS === "true";
+const RUN_ID = process.env.RUN_ID ?? `api-test-${Date.now()}`;
+
+const liveDescribe = API_BASE_URL ? describe : describe.skip;
+const writeDescribe = API_BASE_URL && ALLOW_AUTH_WRITE_TESTS ? describe : describe.skip;
+
+interface OpenAPIDocument {
+  openapi: string;
+  info: {
+    title: string;
+  };
+  servers?: Array<{ url: string }>;
+  paths: Record<string, Record<string, unknown>>;
+}
+
+type JsonObject = Record<string, unknown>;
+
+async function requestJson<T = unknown>(
+  path: string,
+  options: RequestInit = {},
+): Promise<{ status: number; headers: Headers; body: T }> {
+  if (!API_BASE_URL) {
+    throw new Error("Set API_BASE_URL before running live API tests.");
+  }
+
+  const headers = new Headers(options.headers);
+  headers.set("accept", "application/json");
+  if (options.body && !headers.has("content-type")) {
+    headers.set("content-type", "application/json");
+  }
+  if (API_TOKEN) {
+    headers.set("authorization", `Bearer ${API_TOKEN}`);
+  }
+
+  const response = await fetch(new URL(`${API_BASE_URL}${path}`), {
+    ...options,
+    headers,
+  });
+  const text = await response.text();
+  const body = text ? (JSON.parse(text) as T) : (undefined as T);
+
+  return { status: response.status, headers: response.headers, body };
+}
+
+function expectJsonResponse(headers: Headers): void {
+  expect(headers.get("content-type") ?? "").toContain("application/json");
+}
+
+liveDescribe("cfw-auth live API contract smoke tests", () => {
+  it("serves the Better Auth OpenAPI document", async () => {
+    const response = await requestJson<OpenAPIDocument>("/open-api/generate-schema");
+
+    expect(response.status).toBe(200);
+    expectJsonResponse(response.headers);
+    expect(response.body).toMatchObject({
+      openapi: expect.stringMatching(/^3\./),
+      info: {
+        title: "Better Auth",
+      },
+    });
+    expect(response.body.paths).toEqual(
+      expect.objectContaining({
+        "/get-session": expect.any(Object),
+        "/ok": expect.any(Object),
+        "/sign-in/email": expect.any(Object),
+        "/sign-up/email": expect.any(Object),
+      }),
+    );
+    expect(Object.keys(response.body.paths).length).toBeGreaterThan(20);
+  });
+
+  it("reports API readiness from the documented ok endpoint", async () => {
+    const response = await requestJson<{ ok: boolean }>("/ok");
+
+    expect(response.status).toBe(200);
+    expectJsonResponse(response.headers);
+    expect(response.body).toEqual({ ok: true });
+  });
+
+  it("returns an explicit anonymous session state without credentials", async () => {
+    const response = await requestJson<null | JsonObject>("/get-session");
+
+    expect(response.status).toBe(200);
+    expectJsonResponse(response.headers);
+    expect(response.body).toBeNull();
+  });
+
+  it("rejects invalid email sign-in without creating account data", async () => {
+    const response = await requestJson<JsonObject>("/sign-in/email", {
+      method: "POST",
+      body: JSON.stringify({
+        email: `missing-${RUN_ID}@example.invalid`,
+        password: "not-the-right-password",
+      }),
+    });
+
+    expect([400, 401, 403]).toContain(response.status);
+    expectJsonResponse(response.headers);
+    expect(response.body).toEqual(
+      expect.objectContaining({
+        message: expect.any(String),
+      }),
+    );
+  });
+});
+
+writeDescribe("cfw-auth live API write scenarios", () => {
+  it("can sign up a namespaced email user when write tests are explicitly enabled", async () => {
+    const email = `${RUN_ID}@example.invalid`;
+    const response = await requestJson<{ token?: string | null; user?: JsonObject }>(
+      "/sign-up/email",
+      {
+        method: "POST",
+        body: JSON.stringify({
+          name: `API Test ${RUN_ID}`,
+          email,
+          password: "TestPassword123!",
+          rememberMe: false,
+        }),
+      },
+    );
+
+    expect(response.status).toBe(200);
+    expectJsonResponse(response.headers);
+    expect(response.body.user).toEqual(
+      expect.objectContaining({
+        id: expect.any(String),
+        email,
+        name: `API Test ${RUN_ID}`,
+        emailVerified: expect.any(Boolean),
+      }),
+    );
+  });
+});