From c0c997f796c542c64f8a562c53f03b94b3add74e Mon Sep 17 00:00:00 2001 From: ilia Date: Wed, 15 Jul 2026 08:56:31 -0400 Subject: [PATCH] test: timing budgets + CI hardening (artifact v3 pin, npm cache retry) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes three items from the outstanding e2e/CI gap list: - Timing budgets: timings.measure() only ever recorded durations for the (still-unwired) Pushgateway export — nothing failed CI when a step got slow. Add e2e/timing-budgets.ts (expectWithinBudget + shared BUDGET_MS buckets) and wire it into every measure() call site across the suite. Mail-wait steps are deliberately left unbudgeted (external mail-trap delivery latency, not a code performance signal). - actions/upload-artifact@v4 doesn't work against this Gitea/act runner's artifact backend — pin to v3 for the e2e failure-report upload. - Shared act_runner npm cache has corrupted platform-native tarballs before (@next/swc-linux-x64-musl) and reds viewer-unit/admin-unit/e2e with no product bug involved. All three npm ci steps now retry once after `npm cache clean --force` on first failure. Verified: full local suite green against DEV (37 passed, 6 skipped, no budget assertion failures) before wiring into CI. --- .gitea/workflows/ci.yml | 31 ++++++++++++++-- ROADMAP.md | 8 +++- e2e/README.md | 1 + e2e/tests/admin.manage-users-actions.spec.ts | 11 ++++++ e2e/tests/admin.manage-users.spec.ts | 4 ++ e2e/tests/api.auth.spec.ts | 3 ++ e2e/tests/api.catalog.spec.ts | 9 +++++ e2e/tests/api.fastapi-login.spec.ts | 3 ++ e2e/tests/api.health.spec.ts | 2 + e2e/tests/api.role-permissions.spec.ts | 13 +++++++ e2e/tests/auth.setup.ts | 3 ++ e2e/tests/auth.signout.spec.ts | 4 ++ e2e/tests/forgot-password.mail.spec.ts | 3 ++ e2e/tests/gallery.filters.authed.spec.ts | 3 ++ e2e/tests/gallery.search-filters.spec.ts | 6 +++ e2e/tests/idle.logout.spec.ts | 2 + e2e/tests/network-errors.spec.ts | 5 +++ e2e/tests/register.verify.mail.spec.ts | 2 + e2e/tests/upload.smoke.spec.ts | 4 ++ e2e/tests/viewer.session.spec.ts | 2 + e2e/tests/viewer.write-gates.spec.ts | 3 ++ e2e/timing-budgets.ts | 39 ++++++++++++++++++++ 22 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 e2e/timing-budgets.ts diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 278b07b..bf17eaf 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -90,7 +90,15 @@ jobs: cache-dependency-path: viewer-frontend/package-lock.json - name: Install deps - run: npm ci + # act_runner's shared npm cache has corrupted platform-native tarballs + # before (@next/swc-linux-x64-musl) and reds this job with no product + # bug involved — clear the cache and retry once before giving up. + run: | + npm ci || { + echo "npm ci failed, clearing npm cache and retrying once..." + npm cache clean --force + npm ci + } - name: Run Vitest run: npm test @@ -115,7 +123,14 @@ jobs: cache-dependency-path: admin-frontend/package-lock.json - name: Install deps - run: npm ci + # Same shared act_runner npm cache corruption as viewer-unit — retry + # once with a clean cache rather than redding CI for an infra flake. + run: | + npm ci || { + echo "npm ci failed, clearing npm cache and retrying once..." + npm cache clean --force + npm ci + } - name: Run Vitest run: npm test @@ -143,7 +158,13 @@ jobs: git config --global url."https://oauth2:${{ secrets.PLAYKIT_GIT_TOKEN }}@git.levkin.ca/".insteadOf "https://git.levkin.ca/" - name: Install e2e deps - run: npm ci + # Same shared act_runner npm cache corruption as viewer-unit/admin-unit. + run: | + npm ci || { + echo "npm ci failed, clearing npm cache and retrying once..." + npm cache clean --force + npm ci + } - name: Install Chromium run: npx playwright install --with-deps chromium @@ -184,7 +205,9 @@ jobs: - name: Upload report if: failure() - uses: actions/upload-artifact@v4 + # v4 breaks on this Gitea/act runner (act_runner's artifact API doesn't + # speak v4's backend protocol) — pin v3. + uses: actions/upload-artifact@v3 with: name: playwright-report path: | diff --git a/ROADMAP.md b/ROADMAP.md index 7eadc29..62efa45 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -26,6 +26,9 @@ Living plan for product quality, auth/email reliability, and automation. - [x] **PROD smoke spec** — `prod.smoke.spec.ts`, opt-in via `PROD_BASE_URL`; skips (no-op) until the PROD LXC exists — see Ops/docs debt - [x] **NextAuth (viewer) write gates** — third user store (auth DB, `hasWriteAccess=false`) provisioned via `provision-punimtag-e2e-user.py`; `viewer.write-gates.spec.ts` proves 403 (viewer) vs pass-through (admin) on `POST /api/faces/{id}/identify` - [x] **Manage Users real CRUD + causal cross-session e2e** — `admin.manage-users-actions.spec.ts`: admin creates/edits/deletes a user through the actual panel (the older spec only opened/closed it), and deactivating a live user immediately revokes their *already open* session (jwt callback re-checks `isActive`); disposable `e2e-manage-test-*` accounts, always cleaned up +- [x] **Timing budgets** — `e2e/timing-budgets.ts` `expectWithinBudget()` gates every `timings.measure()` call site against a shared `BUDGET_MS` bucket (`api`/`uiAction`/`uiLogin`/`heavyCrud`); a step getting order-of-magnitude slower now reds the test instead of only feeding an unwatched sample +- [x] **CI: `actions/upload-artifact@v4` pinned to `v3`** — v4 doesn't work against this Gitea/act runner's artifact backend; report upload on e2e failure was silently broken +- [x] **CI: npm cache corruption retry** — `viewer-unit`/`admin-unit`/`e2e` all retry `npm ci` once after `npm cache clean --force` on first failure (shared act_runner cache has corrupted `@next/swc-linux-x64-musl` before, redding CI with no product bug) - [x] ROADMAP (this file) ## Next (near-term) @@ -35,6 +38,7 @@ Living plan for product quality, auth/email reliability, and automation. - [ ] **Wire QA/PROD SMTP on live guests** when LXCs 9102/9103 exist (`make punimtag-sync-smtp ENV=qa|prod`) — confirmed via `pct list` on `10.0.10.201`: only DEV (`9101`) exists today, so this (and PROD smoke actually running, and the PROD `NEXTAUTH_URL` hostname) stays dormant-but-ready until QA/PROD LXCs are provisioned - [ ] **Stop seeding `admin@admin.com` in docs** as the day-to-day login; keep bootstrap scripts but point operators at Vaultwarden `PunimTag e2e` - [ ] Set the new `E2E_VIEWER_EMAIL`/`E2E_VIEWER_PASSWORD` Gitea secrets from Infisical/Vaultwarden if CI doesn't already have them synced (script sets Gitea directly, so likely already OK — verify on next CI run) +- [ ] **git-ci-01 still OOM-kills under multi-repo bursts even at 8 GB** — bumped LXC 241 3→8 GB (ansible PR #128) after act_runner OOM-killed mid-job; that fix landed, but **pve10 the hypervisor itself is overcommitted** (~75 GB allocated across guests vs 62 GB physical RAM, host swap observed 7.9/8.0 GB used, ~8-9 GB free) — a burst of concurrent CI across `ansible`/`punimtag`/`playkit`/`maCopy` OOM-killed act_runner three times in 13 minutes on 2026-07-15 even with the larger allocation. Real fix is host-wide: audit over-allocated idle guests (e.g. `automationlab` 4 GB stopped, `immich`/`paperless` at 6 GB each) and either trim or move some guests to pve201. Out of scope for a CI-runner-only fix — flagging for a deliberate pass. - [ ] **DEV LXC (9101) redeploy** — `pct exec 9101` shows viewer-frontend checked out at `e54c609`, **19 commits behind** `dev` HEAD (missing e.g. `c011c80` accessibility/aria-label pass). Two uncommitted local files from an earlier session (`e2e/tests/admin.manage-users-actions.spec.ts`, `e2e/pages/ManageUsersPanel.ts` — real Manage-Users CRUD + a causal cross-session deactivation check) time out because the deployed UI's row action buttons lack the `aria-label`s those specs target; every other spec here is unaffected since it doesn't depend on that commit. Redeploy DEV, then commit + verify those two files. ### Playkit adoption (pin is `v0.3.1` today) @@ -46,13 +50,13 @@ Living plan for product quality, auth/email reliability, and automation. | `playkitFailureArtifacts()` | yes | `playwright.config.ts` | | Mailpit / `createMailInbox` | yes | mail specs | | `interceptNetworkCall` / `startNetworkErrorMonitor` | yes | both in use — `network-errors.spec.ts`, `upload.smoke.spec.ts`, `gallery.search-filters.spec.ts` | -| Timing → Pushgateway | yes API | **not wired** — needs `make deploy-observability` + CI `PLAYKIT_METRICS_*` | +| Timing → Pushgateway | yes API | **not wired** — needs Pushgateway deployed (ansible `deploy/observability/`) + CI `PLAYKIT_METRICS_*` env | ### Product / coverage gaps - [ ] **PROD smoke actually running** — spec exists (`prod.smoke.spec.ts`) and is verified against a real public host, but stays skipped until `PROD_BASE_URL` is set (blocked on the LXC existing — see Ops/docs debt) - [ ] Consider a non-admin **manager/editor**-role FastAPI + NextAuth user if finer-grained role coverage is ever needed (today's viewer/admin split covers the binary write-access gate) -- [ ] **No latency/perf budget assertions anywhere** — `timings.measure()` records every step's duration but nothing ever asserts on it (no `expect(duration).toBeLessThan(...)`); this is separate from the Pushgateway wiring gap above — even once metrics are pushed, nothing fails CI for being slow. Also no Core Web Vitals (LCP/CLS/INP) collection. +- [x] ~~No latency/perf budget assertions anywhere~~ — closed via `expectWithinBudget()` (see "Now — shipped / in flight"). Still no Core Web Vitals (LCP/CLS/INP) collection — separate gap, not addressed. ### Deferred (soak policy) diff --git a/e2e/README.md b/e2e/README.md index 7458c15..619d3d2 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -64,6 +64,7 @@ not enabled in this CI job yet. 18. NextAuth (browser-session) write gates: viewer (`hasWriteAccess=false`) gets 403, admin gets past the gate, on `POST /api/faces/{id}/identify` (`viewer.write-gates.spec.ts`) — distinct from the FastAPI role-permission gates in #12 19. PROD smoke (`prod.smoke.spec.ts`, opt-in via `PROD_BASE_URL`): `/api/health` + login page load on the public PROD host — dormant until the PROD LXC exists (see ROADMAP) 20. Manage Users real CRUD + causal cross-session effect (`admin.manage-users-actions.spec.ts`): admin creates/edits/deletes a user through the actual panel (not just open/close), and deactivating a user via the UI immediately revokes that user's *already open* NextAuth session (jwt callback re-checks `isActive`, Kolby #57 fix) — every run creates a disposable `e2e-manage-test-*` account and deletes it in a `finally`, never touching the shared punimtagdev/MirrorMatch DB's real accounts +21. Timing budgets (`timing-budgets.ts`): every `timings.measure()` call site is followed by `expectWithinBudget()` against a shared `BUDGET_MS` bucket — a step getting order-of-magnitude slower reds the test, not just a Pushgateway sample nobody's watching See repo root [`ROADMAP.md`](../ROADMAP.md) for gaps and next steps. diff --git a/e2e/tests/admin.manage-users-actions.spec.ts b/e2e/tests/admin.manage-users-actions.spec.ts index 43e56cd..f4f9b31 100644 --- a/e2e/tests/admin.manage-users-actions.spec.ts +++ b/e2e/tests/admin.manage-users-actions.spec.ts @@ -1,6 +1,7 @@ import type { Page } from '@playwright/test'; import { test, expect } from '../fixtures'; import { LoginPage } from '../pages/LoginPage'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Real Manage Users CRUD through the admin UI (`admin.manage-users.spec.ts` @@ -62,6 +63,8 @@ test.describe('admin manage users: create/edit/delete @smoke', () => { await timings.measure('create_user', () => manageUsersPanel.createUser({ email, password: THROWAWAY_PASSWORD, name: 'E2E Throwaway' }), ); + // Generous bucket — shared-CI DEV LXC under concurrent load (see file header). + expectWithinBudget(timings, 'create_user', BUDGET_MS.heavyCrud); expect(await manageUsersPanel.statusBadgeText(email)).toMatch(/active/i); expect(await manageUsersPanel.roleBadgeText(email)).toMatch(/user/i); expect(await manageUsersPanel.writeAccessText(email)).toMatch(/no/i); @@ -71,9 +74,11 @@ test.describe('admin manage users: create/edit/delete @smoke', () => { await manageUsersPanel.setWriteAccess(true); await manageUsersPanel.saveEdit(); }); + expectWithinBudget(timings, 'edit_user', BUDGET_MS.heavyCrud); expect(await manageUsersPanel.writeAccessText(email)).toMatch(/yes/i); await timings.measure('delete_user', () => manageUsersPanel.deleteUser(email)); + expectWithinBudget(timings, 'delete_user', BUDGET_MS.heavyCrud); await manageUsersPanel.waitForRowGone(email); await accountMenu.closeManageUsers(); @@ -112,6 +117,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => { await timings.measure('create_throwaway', () => manageUsersPanel.createUser({ email, password: THROWAWAY_PASSWORD, name: 'E2E Throwaway' }), ); + expectWithinBudget(timings, 'create_throwaway', BUDGET_MS.heavyCrud); await accountMenu.closeManageUsers(); // Independent second session/actor — the throwaway user, logged in @@ -126,6 +132,9 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => { await userPage.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 }); await userPage.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); }); + // Login flows on this file have hit ~90s under a concurrent CI burst + // (see comment above) — budget matches the file's own 120s headroom. + expectWithinBudget(timings, 'throwaway_login', BUDGET_MS.heavyCrud); const preRes = await userPage.request.get(`${playkitConfig.baseUrl}/api/auth/session`); const preBody = await preRes.json(); @@ -139,6 +148,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => { await manageUsersPanel.setActive(false); await manageUsersPanel.saveEdit(); }); + expectWithinBudget(timings, 'deactivate_user', BUDGET_MS.heavyCrud); // The table defaults to (and refetches with) the "Active only" filter, // so the just-deactivated row drops out of view immediately — switch to // "All" before reading its badge, or this hangs forever waiting for a @@ -154,6 +164,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => { const postBody = await postRes.json(); expect(postBody?.user).toBeFalsy(); }); + expectWithinBudget(timings, 'session_revoked', BUDGET_MS.uiAction); } finally { await userContext?.close(); await deleteUserByEmail(page, playkitConfig.baseUrl, email).catch(() => undefined); diff --git a/e2e/tests/admin.manage-users.spec.ts b/e2e/tests/admin.manage-users.spec.ts index 3077ba9..58386d7 100644 --- a/e2e/tests/admin.manage-users.spec.ts +++ b/e2e/tests/admin.manage-users.spec.ts @@ -1,5 +1,6 @@ import { assertPublicHost, waitForUrlHost } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Kolby #57 class: Manage Users overlay must close cleanly before sign-out @@ -23,14 +24,17 @@ test.describe('admin users @smoke', () => { await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 }); await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); }); + expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin); await timings.measure('manage_users', async () => { await accountMenu.openManageUsers(); await expect(page.getByRole('heading', { name: /Manage Users/i })).toBeVisible(); await accountMenu.closeManageUsers(); }); + expectWithinBudget(timings, 'manage_users', BUDGET_MS.uiAction); await timings.measure('sign_out', () => accountMenu.signOut()); + expectWithinBudget(timings, 'sign_out', BUDGET_MS.uiAction); await waitForUrlHost(page, playkitConfig.expectedHost); expect(page.url()).not.toMatch(/10\.\d+\.\d+\.\d+/); await expect(page.getByRole('button', { name: /Sign in/i })).toBeVisible({ diff --git a/e2e/tests/api.auth.spec.ts b/e2e/tests/api.auth.spec.ts index 0945702..2e7b38a 100644 --- a/e2e/tests/api.auth.spec.ts +++ b/e2e/tests/api.auth.spec.ts @@ -1,10 +1,12 @@ import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; test.describe('api auth @smoke', () => { test('protected /api/v1/auth/me returns 401 without token', async ({ api, timings }) => { const res = await timings.measure('api_me_unauth', () => api.get('/api/v1/auth/me', { expectedStatus: 401 }), ); + expectWithinBudget(timings, 'api_me_unauth', BUDGET_MS.api); expect(res.status).toBe(401); expect(res.data).toMatchObject({ detail: expect.stringMatching(/not authenticated/i) }); }); @@ -13,6 +15,7 @@ test.describe('api auth @smoke', () => { const res = await timings.measure('api_photos_unauth', () => api.get('/api/v1/photos', { expectedStatus: 401 }), ); + expectWithinBudget(timings, 'api_photos_unauth', BUDGET_MS.api); expect(res.status).toBe(401); }); }); diff --git a/e2e/tests/api.catalog.spec.ts b/e2e/tests/api.catalog.spec.ts index 747d426..ddc3f83 100644 --- a/e2e/tests/api.catalog.spec.ts +++ b/e2e/tests/api.catalog.spec.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; const PeopleList = z.object({ items: z.array(z.object({ id: z.number() }).passthrough()).min(1), @@ -23,6 +24,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_people', () => api.get('/api/v1/people', { expectedStatus: 200, schema: PeopleList }), ); + expectWithinBudget(timings, 'api_people', BUDGET_MS.api); expect(res.data.items[0]).toEqual(expect.objectContaining({ id: expect.any(Number) })); }); @@ -33,6 +35,7 @@ test.describe('api catalog @smoke', () => { schema: ItemsList, }), ); + expectWithinBudget(timings, 'api_people_faces', BUDGET_MS.api); expect(Array.isArray(res.data.items)).toBe(true); }); @@ -40,6 +43,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_tags', () => api.get('/api/v1/tags', { expectedStatus: 200, schema: TagsList }), ); + expectWithinBudget(timings, 'api_tags', BUDGET_MS.api); expect(res.data.items[0]).toEqual(expect.objectContaining({ id: expect.any(Number) })); }); @@ -47,6 +51,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_login_empty', () => api.post('/api/v1/auth/login', { body: {}, expectedStatus: 422 }), ); + expectWithinBudget(timings, 'api_login_empty', BUDGET_MS.api); expect(res.status).toBe(422); }); @@ -57,6 +62,7 @@ test.describe('api catalog @smoke', () => { expectedStatus: 401, }), ); + expectWithinBudget(timings, 'api_login_bad', BUDGET_MS.api); expect(res.status).toBe(401); expect(res.data).toMatchObject({ detail: expect.stringMatching(/incorrect|password|username/i), @@ -67,6 +73,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_users_unauth', () => api.get('/api/v1/users', { expectedStatus: 401 }), ); + expectWithinBudget(timings, 'api_users_unauth', BUDGET_MS.api); expect(res.status).toBe(401); }); @@ -74,6 +81,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_roles_unauth', () => api.get('/api/v1/role-permissions', { expectedStatus: 401 }), ); + expectWithinBudget(timings, 'api_roles_unauth', BUDGET_MS.api); expect(res.status).toBe(401); }); @@ -81,6 +89,7 @@ test.describe('api catalog @smoke', () => { const res = await timings.measure('api_photo_missing', () => api.get('/api/v1/photos/999999999', { expectedStatus: [401, 404] }), ); + expectWithinBudget(timings, 'api_photo_missing', BUDGET_MS.api); expect([401, 404]).toContain(res.status); }); }); diff --git a/e2e/tests/api.fastapi-login.spec.ts b/e2e/tests/api.fastapi-login.spec.ts index b2c18d7..f0ef36d 100644 --- a/e2e/tests/api.fastapi-login.spec.ts +++ b/e2e/tests/api.fastapi-login.spec.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * FastAPI `/api/v1/auth/login` uses a *separate* user DB from NextAuth. @@ -35,12 +36,14 @@ test.describe('fastapi login @smoke', () => { schema: TokenResponse, }), ); + expectWithinBudget(timings, 'api_login', BUDGET_MS.api); expect(res.data.access_token).toBeTruthy(); const authed = api.withAuthBearer(res.data.access_token); const me = await timings.measure('api_me', () => authed.get('/api/v1/auth/me', { expectedStatus: 200, schema: UserResponse }), ); + expectWithinBudget(timings, 'api_me', BUDGET_MS.api); expect(me.status).toBe(200); expect(me.data.username).toBe(username); }); diff --git a/e2e/tests/api.health.spec.ts b/e2e/tests/api.health.spec.ts index e04ba1d..8853e69 100644 --- a/e2e/tests/api.health.spec.ts +++ b/e2e/tests/api.health.spec.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; const HealthSchema = z.object({ status: z.literal('ok') }); @@ -8,6 +9,7 @@ test.describe('api @smoke', () => { const res = await timings.measure('api_health', () => api.get('/health', { expectedStatus: 200, schema: HealthSchema }), ); + expectWithinBudget(timings, 'api_health', BUDGET_MS.api); expect(res.data).toEqual({ status: 'ok' }); }); }); diff --git a/e2e/tests/api.role-permissions.spec.ts b/e2e/tests/api.role-permissions.spec.ts index d4659be..1b48ee5 100644 --- a/e2e/tests/api.role-permissions.spec.ts +++ b/e2e/tests/api.role-permissions.spec.ts @@ -1,5 +1,6 @@ import type { ApiClient } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * FastAPI role-permission write gates: `admin`-role users pass @@ -28,11 +29,14 @@ test.describe('role permissions @smoke', () => { test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required'); const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass)); + expectWithinBudget(timings, 'login_admin', BUDGET_MS.api); const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass)); + expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api); const viewerRes = await timings.measure('viewer_users', () => api.withAuthBearer(viewerToken).get('/api/v1/users', { expectedStatus: 403 }), ); + expectWithinBudget(timings, 'viewer_users', BUDGET_MS.api); expect(viewerRes.status).toBe(403); expect(viewerRes.data).toMatchObject({ detail: expect.stringMatching(/admin/i) }); @@ -41,6 +45,7 @@ test.describe('role permissions @smoke', () => { expectedStatus: 200, }), ); + expectWithinBudget(timings, 'admin_users', BUDGET_MS.api); expect(adminRes.status).toBe(200); expect(Array.isArray(adminRes.data.items)).toBe(true); }); @@ -52,16 +57,20 @@ test.describe('role permissions @smoke', () => { test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required'); const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass)); + expectWithinBudget(timings, 'login_admin', BUDGET_MS.api); const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass)); + expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api); const viewerRes = await timings.measure('viewer_role_perms', () => api.withAuthBearer(viewerToken).get('/api/v1/role-permissions', { expectedStatus: 403 }), ); + expectWithinBudget(timings, 'viewer_role_perms', BUDGET_MS.api); expect(viewerRes.status).toBe(403); const adminRes = await timings.measure('admin_role_perms', () => api.withAuthBearer(adminToken).get('/api/v1/role-permissions', { expectedStatus: 200 }), ); + expectWithinBudget(timings, 'admin_role_perms', BUDGET_MS.api); expect(adminRes.status).toBe(200); expect(adminRes.data).toHaveProperty('permissions'); }); @@ -73,7 +82,9 @@ test.describe('role permissions @smoke', () => { test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required'); const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass)); + expectWithinBudget(timings, 'login_admin', BUDGET_MS.api); const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass)); + expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api); // Use a photo id that cannot exist — proves the write *gate*, not deletion. // The admin dependency runs before the handler body, so this never touches @@ -86,6 +97,7 @@ test.describe('role permissions @smoke', () => { expectedStatus: 403, }), ); + expectWithinBudget(timings, 'viewer_bulk_delete', BUDGET_MS.api); expect(viewerRes.status).toBe(403); const adminRes = await timings.measure('admin_bulk_delete', () => @@ -94,6 +106,7 @@ test.describe('role permissions @smoke', () => { expectedStatus: 200, }), ); + expectWithinBudget(timings, 'admin_bulk_delete', BUDGET_MS.api); expect(adminRes.status).toBe(200); expect(adminRes.data).toMatchObject({ deleted_count: 0, missing_photo_ids: [nonExistentId] }); }); diff --git a/e2e/tests/auth.setup.ts b/e2e/tests/auth.setup.ts index fef3b62..d4a72cc 100644 --- a/e2e/tests/auth.setup.ts +++ b/e2e/tests/auth.setup.ts @@ -1,6 +1,7 @@ import { assertPublicHost, saveStorageState, waitForUrlHost } from '@levkin/playkit'; import { test as setup } from '../fixtures'; import path from 'node:path'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; const authFile = path.join(__dirname, '../.auth/admin.json'); const viewerAuthFile = path.join(__dirname, '../.auth/viewer.json'); @@ -16,6 +17,7 @@ setup('authenticate e2e admin', async ({ page, playkitConfig, loginPage, e2eCred await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); await waitForUrlHost(page, playkitConfig.expectedHost); }); + expectWithinBudget(timings, 'setup_login', BUDGET_MS.uiLogin); await saveStorageState(page, authFile); }); @@ -35,6 +37,7 @@ setup( await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); await waitForUrlHost(page, playkitConfig.expectedHost); }); + expectWithinBudget(timings, 'setup_login_viewer', BUDGET_MS.uiLogin); await saveStorageState(page, viewerAuthFile); }, diff --git a/e2e/tests/auth.signout.spec.ts b/e2e/tests/auth.signout.spec.ts index e7fca1b..1f27796 100644 --- a/e2e/tests/auth.signout.spec.ts +++ b/e2e/tests/auth.signout.spec.ts @@ -3,6 +3,7 @@ import { waitForUrlHost, } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; test.describe('auth @smoke', () => { test('login page loads on public host', async ({ @@ -14,6 +15,7 @@ test.describe('auth @smoke', () => { assertPublicHost(playkitConfig.baseUrl); await timings.measure('login_page', () => loginPage.openLogin()); + expectWithinBudget(timings, 'login_page', BUDGET_MS.uiAction); await expect(page.getByRole('heading', { name: /Sign in/i })).toBeVisible(); await expect(page.locator('#email')).toBeVisible(); @@ -42,10 +44,12 @@ test.describe('auth @smoke', () => { await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 }); await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); }); + expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin); await waitForUrlHost(page, playkitConfig.expectedHost); await timings.measure('sign_out', () => accountMenu.signOut()); + expectWithinBudget(timings, 'sign_out', BUDGET_MS.uiAction); // After sign-out, NextAuth must redirect to the public host — never 10.x. await waitForUrlHost(page, playkitConfig.expectedHost); diff --git a/e2e/tests/forgot-password.mail.spec.ts b/e2e/tests/forgot-password.mail.spec.ts index b42d89c..d52a560 100644 --- a/e2e/tests/forgot-password.mail.spec.ts +++ b/e2e/tests/forgot-password.mail.spec.ts @@ -7,6 +7,7 @@ import { type MailtrapMessage, } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Forgot-password must land in the mail trap (Mailpit) with a public reset link. @@ -40,6 +41,7 @@ test.describe('mail @smoke', () => { headers: { 'Content-Type': 'application/json' }, }), ); + expectWithinBudget(timings, 'forgot_password_api', BUDGET_MS.api); expect(res.ok()).toBeTruthy(); const msg = await timings.measure('mail_wait', () => @@ -81,6 +83,7 @@ test.describe('mail @smoke', () => { const after = new Date(Date.now() - 5_000); await timings.measure('open_home', () => page.goto(playkitConfig.baseUrl)); + expectWithinBudget(timings, 'open_home', BUDGET_MS.uiAction); await page.getByRole('button', { name: /Sign in/i }).click(); await page.getByRole('button', { name: /Forgot password/i }).click(); await page.locator('#forgot-email').fill(to); diff --git a/e2e/tests/gallery.filters.authed.spec.ts b/e2e/tests/gallery.filters.authed.spec.ts index e4fee79..b639da9 100644 --- a/e2e/tests/gallery.filters.authed.spec.ts +++ b/e2e/tests/gallery.filters.authed.spec.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Logged-in-only gallery filters: Favorites and People are hidden from the @@ -19,6 +20,7 @@ test.describe('gallery filters (authed) @smoke', () => { await timings.measure('open_search', async () => { await page.goto(`${playkitConfig.baseUrl}/search`); }); + expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction); const favoritesCheckbox = page.locator('#favorites-only'); await expect(favoritesCheckbox).toBeVisible({ timeout: 20_000 }); @@ -54,6 +56,7 @@ test.describe('gallery filters (authed) @smoke', () => { await timings.measure('open_search', async () => { await page.goto(`${playkitConfig.baseUrl}/search`); }); + expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction); await page.getByRole('button', { name: /select people/i }).click(); await page.getByPlaceholder('Search people...').fill(person.first_name); diff --git a/e2e/tests/gallery.search-filters.spec.ts b/e2e/tests/gallery.search-filters.spec.ts index 0438660..b27eca1 100644 --- a/e2e/tests/gallery.search-filters.spec.ts +++ b/e2e/tests/gallery.search-filters.spec.ts @@ -1,6 +1,7 @@ import { interceptNetworkCall } from '@levkin/playkit'; import { z } from 'zod'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Public gallery search (`viewer-frontend` `/api/search`, backed by its own @@ -43,12 +44,14 @@ test.describe('gallery search filters @smoke', () => { ); const tag = tagsRes.data.items[0]; expect(tag).toBeTruthy(); + expectWithinBudget(timings, 'api_tags', BUDGET_MS.api); const res = await timings.measure('search_by_tag', () => page.request.get(`${playkitConfig.baseUrl}/api/search`, { params: { tags: String(tag.id) }, }), ); + expectWithinBudget(timings, 'search_by_tag', BUDGET_MS.api); expect(res.ok()).toBeTruthy(); const body = SearchResponse.parse(await res.json()); for (const photo of body.photos) { @@ -70,12 +73,14 @@ test.describe('gallery search filters @smoke', () => { ); const person = peopleRes.data.items[0]; expect(person).toBeTruthy(); + expectWithinBudget(timings, 'api_people', BUDGET_MS.api); const res = await timings.measure('search_by_person', () => page.request.get(`${playkitConfig.baseUrl}/api/search`, { params: { people: String(person.id) }, }), ); + expectWithinBudget(timings, 'search_by_person', BUDGET_MS.api); expect(res.ok()).toBeTruthy(); const body = SearchResponse.parse(await res.json()); for (const photo of body.photos) { @@ -130,6 +135,7 @@ test.describe('gallery search filters @smoke', () => { await timings.measure('open_search', async () => { await page.goto(`${playkitConfig.baseUrl}/search`); }); + expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction); // Match only the *filtered* request (query includes `tags=`) — the page // also fires an unfiltered `/api/search?page=1&pageSize=30` on mount. diff --git a/e2e/tests/idle.logout.spec.ts b/e2e/tests/idle.logout.spec.ts index cd18692..da3630f 100644 --- a/e2e/tests/idle.logout.spec.ts +++ b/e2e/tests/idle.logout.spec.ts @@ -1,5 +1,6 @@ import { assertPublicHost, waitForUrlHost } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Idle logout must redirect to the public host (same class as #57). @@ -22,6 +23,7 @@ test.describe('auth idle @smoke', () => { await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 }); await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 }); }); + expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin); // Short idle window — requires IdleLogoutHandler e2e_idle_ms support on DEV await page.goto(`${playkitConfig.baseUrl}/?e2e_idle_ms=2500`); diff --git a/e2e/tests/network-errors.spec.ts b/e2e/tests/network-errors.spec.ts index e29920c..adf89fd 100644 --- a/e2e/tests/network-errors.spec.ts +++ b/e2e/tests/network-errors.spec.ts @@ -1,6 +1,7 @@ import path from 'node:path'; import { startNetworkErrorMonitor } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Silent-failure guard: these pages can *look* fine in the UI while a @@ -24,6 +25,7 @@ test.describe('network errors (public pages) @smoke', () => { timeout: 20_000, }); }); + expectWithinBudget(timings, 'open_gallery', BUDGET_MS.uiAction); } finally { net.assertNoErrors(); } @@ -45,6 +47,7 @@ test.describe('network errors (public pages) @smoke', () => { page.getByText(/Found \d+ photos?|No photos found matching your filters/i).first(), ).toBeVisible({ timeout: 20_000 }); }); + expectWithinBudget(timings, 'search_tag_filter', BUDGET_MS.uiAction); } finally { net.assertNoErrors(); } @@ -61,6 +64,7 @@ test.describe('network errors (authed) @smoke', () => { await page.goto(`${playkitConfig.baseUrl}/upload`); await expect(page.getByLabel('Account menu')).toBeVisible({ timeout: 20_000 }); }); + expectWithinBudget(timings, 'open_upload', BUDGET_MS.uiAction); } finally { net.assertNoErrors(); } @@ -77,6 +81,7 @@ test.describe('network errors (authed) @smoke', () => { await expect(page.getByRole('heading', { name: /Manage Users/i })).toBeVisible(); await accountMenu.closeManageUsers(); }); + expectWithinBudget(timings, 'manage_users', BUDGET_MS.uiAction); } finally { net.assertNoErrors(); } diff --git a/e2e/tests/register.verify.mail.spec.ts b/e2e/tests/register.verify.mail.spec.ts index 5585516..645e5f4 100644 --- a/e2e/tests/register.verify.mail.spec.ts +++ b/e2e/tests/register.verify.mail.spec.ts @@ -7,6 +7,7 @@ import { type MailtrapMessage, } from '@levkin/playkit'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Register → confirmation mail in Mailpit → verify link on public host. @@ -38,6 +39,7 @@ test.describe('auth register @mail', () => { }); await page.waitForURL(/\/login/, { timeout: 30_000 }); }); + expectWithinBudget(timings, 'register', BUDGET_MS.uiLogin); await expect(page.getByText(/Account created|check your email|confirm/i).first()).toBeVisible({ timeout: 15_000, diff --git a/e2e/tests/upload.smoke.spec.ts b/e2e/tests/upload.smoke.spec.ts index 374d32f..eb25847 100644 --- a/e2e/tests/upload.smoke.spec.ts +++ b/e2e/tests/upload.smoke.spec.ts @@ -1,6 +1,7 @@ import { assertPublicHost, interceptNetworkCall, waitForUrlHost } from '@levkin/playkit'; import path from 'node:path'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; const tinyPng = path.join(__dirname, '../fixtures/tiny.png'); @@ -18,6 +19,7 @@ test.describe('upload @smoke', () => { await page.goto(`${playkitConfig.baseUrl}/upload`); await waitForUrlHost(page, playkitConfig.expectedHost); }); + expectWithinBudget(timings, 'open_upload', BUDGET_MS.uiAction); expect(new URL(page.url()).pathname).toMatch(/\/upload/); await expect(page.getByLabel('Account menu')).toBeVisible({ timeout: 20_000 }); @@ -52,5 +54,7 @@ test.describe('upload @smoke', () => { .first(); await expect(outcome).toBeVisible({ timeout: 30_000 }); }); + // Network round trip + UI settle — same magnitude as a login flow. + expectWithinBudget(timings, 'upload_file', BUDGET_MS.uiLogin); }); }); diff --git a/e2e/tests/viewer.session.spec.ts b/e2e/tests/viewer.session.spec.ts index 4353ddd..4b5b3a7 100644 --- a/e2e/tests/viewer.session.spec.ts +++ b/e2e/tests/viewer.session.spec.ts @@ -1,6 +1,7 @@ import { assertPublicHost, waitForUrlHost } from '@levkin/playkit'; import path from 'node:path'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * Viewer (NextAuth) session — cookies from storageState. @@ -20,5 +21,6 @@ test.describe('viewer session @smoke', () => { const body = await res.json(); expect(body?.user?.email).toBeTruthy(); }); + expectWithinBudget(timings, 'session', BUDGET_MS.uiAction); }); }); diff --git a/e2e/tests/viewer.write-gates.spec.ts b/e2e/tests/viewer.write-gates.spec.ts index e0622f9..eae204f 100644 --- a/e2e/tests/viewer.write-gates.spec.ts +++ b/e2e/tests/viewer.write-gates.spec.ts @@ -1,5 +1,6 @@ import path from 'node:path'; import { test, expect } from '../fixtures'; +import { BUDGET_MS, expectWithinBudget } from '../timing-budgets'; /** * NextAuth (browser-session) write gates — `session.user.hasWriteAccess` @@ -32,6 +33,7 @@ test.describe('viewer write gates (NextAuth, viewer) @smoke', () => { data: { firstName: 'Test', lastName: 'Viewer' }, }), ); + expectWithinBudget(timings, 'viewer_identify', BUDGET_MS.api); expect(res.status()).toBe(403); const body = await res.json(); expect(body).toMatchObject({ error: expect.stringMatching(/write access/i) }); @@ -51,6 +53,7 @@ test.describe('viewer write gates (NextAuth, admin) @smoke', () => { data: { firstName: 'Test', lastName: 'Admin' }, }), ); + expectWithinBudget(timings, 'admin_identify', BUDGET_MS.api); // Admin clears the write-access gate; a nonexistent face id then 404s — // proves the gate didn't block a legitimate write-access user. expect(res.status()).toBe(404); diff --git a/e2e/timing-budgets.ts b/e2e/timing-budgets.ts new file mode 100644 index 0000000..08d6ddf --- /dev/null +++ b/e2e/timing-budgets.ts @@ -0,0 +1,39 @@ +import { expect } from '@playwright/test'; +import type { TimingCollector } from '@levkin/playkit'; + +/** + * Perf-budget gate for a `timings.measure(name, ...)` step. + * + * `timings.measure()` on its own only *records* durations (for the + * Pushgateway export, see ROADMAP "Timing -> Pushgateway") — nothing ever + * failed CI for a step getting slow. This asserts the most recently + * recorded sample for `name` against `maxMs`, so a regression shows up as a + * red test instead of only a graph nobody's watching. + * + * Budgets are intentionally generous (shared CI runner + DEV LXC under + * concurrent PR load — see docs/guides/ci-runners-and-control.md in the + * ansible repo) — this catches order-of-magnitude regressions, not + * micro-optimization. Pick the closest `BUDGET_MS` bucket rather than + * inventing a new one-off number per call site. + */ +export function expectWithinBudget(timings: TimingCollector, name: string, maxMs: number): void { + const samples = timings.getSamples().filter((sample) => sample.name === name); + const sample = samples[samples.length - 1]; + expect(sample, `no timing sample recorded for "${name}" — call timings.measure() first`).toBeTruthy(); + expect( + sample!.durationMs, + `"${name}" took ${Math.round(sample!.durationMs)}ms, budget is ${maxMs}ms`, + ).toBeLessThanOrEqual(maxMs); +} + +/** Shared budget buckets — see `expectWithinBudget` for rationale. */ +export const BUDGET_MS = { + /** Direct FastAPI/NextAuth-route calls (LAN, no browser). */ + api: 8_000, + /** Single UI interaction: open a page/panel, apply a filter, sign out. */ + uiAction: 20_000, + /** Full login flow (navigate + submit + redirect wait). */ + uiLogin: 45_000, + /** Multi-step UI CRUD (create/edit/deactivate + list refetch). */ + heavyCrud: 60_000, +} as const;