test: timing budgets + CI hardening (artifact v3 pin, npm cache retry) #69
@ -90,7 +90,15 @@ jobs:
|
||||
cache-dependency-path: viewer-frontend/package-lock.json
|
||||
|
||||
- name: Install deps
|
||||
run: npm ci
|
||||
# act_runner's shared npm cache has corrupted platform-native tarballs
|
||||
# before (@next/swc-linux-x64-musl) and reds this job with no product
|
||||
# bug involved — clear the cache and retry once before giving up.
|
||||
run: |
|
||||
npm ci || {
|
||||
echo "npm ci failed, clearing npm cache and retrying once..."
|
||||
npm cache clean --force
|
||||
npm ci
|
||||
}
|
||||
|
||||
- name: Run Vitest
|
||||
run: npm test
|
||||
@ -115,7 +123,14 @@ jobs:
|
||||
cache-dependency-path: admin-frontend/package-lock.json
|
||||
|
||||
- name: Install deps
|
||||
run: npm ci
|
||||
# Same shared act_runner npm cache corruption as viewer-unit — retry
|
||||
# once with a clean cache rather than redding CI for an infra flake.
|
||||
run: |
|
||||
npm ci || {
|
||||
echo "npm ci failed, clearing npm cache and retrying once..."
|
||||
npm cache clean --force
|
||||
npm ci
|
||||
}
|
||||
|
||||
- name: Run Vitest
|
||||
run: npm test
|
||||
@ -143,7 +158,13 @@ jobs:
|
||||
git config --global url."https://oauth2:${{ secrets.PLAYKIT_GIT_TOKEN }}@git.levkin.ca/".insteadOf "https://git.levkin.ca/"
|
||||
|
||||
- name: Install e2e deps
|
||||
run: npm ci
|
||||
# Same shared act_runner npm cache corruption as viewer-unit/admin-unit.
|
||||
run: |
|
||||
npm ci || {
|
||||
echo "npm ci failed, clearing npm cache and retrying once..."
|
||||
npm cache clean --force
|
||||
npm ci
|
||||
}
|
||||
|
||||
- name: Install Chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
@ -184,7 +205,9 @@ jobs:
|
||||
|
||||
- name: Upload report
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
# v4 breaks on this Gitea/act runner (act_runner's artifact API doesn't
|
||||
# speak v4's backend protocol) — pin v3.
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: playwright-report
|
||||
path: |
|
||||
|
||||
@ -26,6 +26,9 @@ Living plan for product quality, auth/email reliability, and automation.
|
||||
- [x] **PROD smoke spec** — `prod.smoke.spec.ts`, opt-in via `PROD_BASE_URL`; skips (no-op) until the PROD LXC exists — see Ops/docs debt
|
||||
- [x] **NextAuth (viewer) write gates** — third user store (auth DB, `hasWriteAccess=false`) provisioned via `provision-punimtag-e2e-user.py`; `viewer.write-gates.spec.ts` proves 403 (viewer) vs pass-through (admin) on `POST /api/faces/{id}/identify`
|
||||
- [x] **Manage Users real CRUD + causal cross-session e2e** — `admin.manage-users-actions.spec.ts`: admin creates/edits/deletes a user through the actual panel (the older spec only opened/closed it), and deactivating a live user immediately revokes their *already open* session (jwt callback re-checks `isActive`); disposable `e2e-manage-test-*` accounts, always cleaned up
|
||||
- [x] **Timing budgets** — `e2e/timing-budgets.ts` `expectWithinBudget()` gates every `timings.measure()` call site against a shared `BUDGET_MS` bucket (`api`/`uiAction`/`uiLogin`/`heavyCrud`); a step getting order-of-magnitude slower now reds the test instead of only feeding an unwatched sample
|
||||
- [x] **CI: `actions/upload-artifact@v4` pinned to `v3`** — v4 doesn't work against this Gitea/act runner's artifact backend; report upload on e2e failure was silently broken
|
||||
- [x] **CI: npm cache corruption retry** — `viewer-unit`/`admin-unit`/`e2e` all retry `npm ci` once after `npm cache clean --force` on first failure (shared act_runner cache has corrupted `@next/swc-linux-x64-musl` before, redding CI with no product bug)
|
||||
- [x] ROADMAP (this file)
|
||||
|
||||
## Next (near-term)
|
||||
@ -35,6 +38,7 @@ Living plan for product quality, auth/email reliability, and automation.
|
||||
- [ ] **Wire QA/PROD SMTP on live guests** when LXCs 9102/9103 exist (`make punimtag-sync-smtp ENV=qa|prod`) — confirmed via `pct list` on `10.0.10.201`: only DEV (`9101`) exists today, so this (and PROD smoke actually running, and the PROD `NEXTAUTH_URL` hostname) stays dormant-but-ready until QA/PROD LXCs are provisioned
|
||||
- [ ] **Stop seeding `admin@admin.com` in docs** as the day-to-day login; keep bootstrap scripts but point operators at Vaultwarden `PunimTag e2e`
|
||||
- [ ] Set the new `E2E_VIEWER_EMAIL`/`E2E_VIEWER_PASSWORD` Gitea secrets from Infisical/Vaultwarden if CI doesn't already have them synced (script sets Gitea directly, so likely already OK — verify on next CI run)
|
||||
- [ ] **git-ci-01 still OOM-kills under multi-repo bursts even at 8 GB** — bumped LXC 241 3→8 GB (ansible PR #128) after act_runner OOM-killed mid-job; that fix landed, but **pve10 the hypervisor itself is overcommitted** (~75 GB allocated across guests vs 62 GB physical RAM, host swap observed 7.9/8.0 GB used, ~8-9 GB free) — a burst of concurrent CI across `ansible`/`punimtag`/`playkit`/`maCopy` OOM-killed act_runner three times in 13 minutes on 2026-07-15 even with the larger allocation. Real fix is host-wide: audit over-allocated idle guests (e.g. `automationlab` 4 GB stopped, `immich`/`paperless` at 6 GB each) and either trim or move some guests to pve201. Out of scope for a CI-runner-only fix — flagging for a deliberate pass.
|
||||
- [ ] **DEV LXC (9101) redeploy** — `pct exec 9101` shows viewer-frontend checked out at `e54c609`, **19 commits behind** `dev` HEAD (missing e.g. `c011c80` accessibility/aria-label pass). Two uncommitted local files from an earlier session (`e2e/tests/admin.manage-users-actions.spec.ts`, `e2e/pages/ManageUsersPanel.ts` — real Manage-Users CRUD + a causal cross-session deactivation check) time out because the deployed UI's row action buttons lack the `aria-label`s those specs target; every other spec here is unaffected since it doesn't depend on that commit. Redeploy DEV, then commit + verify those two files.
|
||||
|
||||
### Playkit adoption (pin is `v0.3.1` today)
|
||||
@ -46,13 +50,13 @@ Living plan for product quality, auth/email reliability, and automation.
|
||||
| `playkitFailureArtifacts()` | yes | `playwright.config.ts` |
|
||||
| Mailpit / `createMailInbox` | yes | mail specs |
|
||||
| `interceptNetworkCall` / `startNetworkErrorMonitor` | yes | both in use — `network-errors.spec.ts`, `upload.smoke.spec.ts`, `gallery.search-filters.spec.ts` |
|
||||
| Timing → Pushgateway | yes API | **not wired** — needs `make deploy-observability` + CI `PLAYKIT_METRICS_*` |
|
||||
| Timing → Pushgateway | yes API | **not wired** — needs Pushgateway deployed (ansible `deploy/observability/`) + CI `PLAYKIT_METRICS_*` env |
|
||||
|
||||
### Product / coverage gaps
|
||||
|
||||
- [ ] **PROD smoke actually running** — spec exists (`prod.smoke.spec.ts`) and is verified against a real public host, but stays skipped until `PROD_BASE_URL` is set (blocked on the LXC existing — see Ops/docs debt)
|
||||
- [ ] Consider a non-admin **manager/editor**-role FastAPI + NextAuth user if finer-grained role coverage is ever needed (today's viewer/admin split covers the binary write-access gate)
|
||||
- [ ] **No latency/perf budget assertions anywhere** — `timings.measure()` records every step's duration but nothing ever asserts on it (no `expect(duration).toBeLessThan(...)`); this is separate from the Pushgateway wiring gap above — even once metrics are pushed, nothing fails CI for being slow. Also no Core Web Vitals (LCP/CLS/INP) collection.
|
||||
- [x] ~~No latency/perf budget assertions anywhere~~ — closed via `expectWithinBudget()` (see "Now — shipped / in flight"). Still no Core Web Vitals (LCP/CLS/INP) collection — separate gap, not addressed.
|
||||
|
||||
### Deferred (soak policy)
|
||||
|
||||
|
||||
@ -64,6 +64,7 @@ not enabled in this CI job yet.
|
||||
18. NextAuth (browser-session) write gates: viewer (`hasWriteAccess=false`) gets 403, admin gets past the gate, on `POST /api/faces/{id}/identify` (`viewer.write-gates.spec.ts`) — distinct from the FastAPI role-permission gates in #12
|
||||
19. PROD smoke (`prod.smoke.spec.ts`, opt-in via `PROD_BASE_URL`): `/api/health` + login page load on the public PROD host — dormant until the PROD LXC exists (see ROADMAP)
|
||||
20. Manage Users real CRUD + causal cross-session effect (`admin.manage-users-actions.spec.ts`): admin creates/edits/deletes a user through the actual panel (not just open/close), and deactivating a user via the UI immediately revokes that user's *already open* NextAuth session (jwt callback re-checks `isActive`, Kolby #57 fix) — every run creates a disposable `e2e-manage-test-*` account and deletes it in a `finally`, never touching the shared punimtagdev/MirrorMatch DB's real accounts
|
||||
21. Timing budgets (`timing-budgets.ts`): every `timings.measure()` call site is followed by `expectWithinBudget()` against a shared `BUDGET_MS` bucket — a step getting order-of-magnitude slower reds the test, not just a Pushgateway sample nobody's watching
|
||||
|
||||
See repo root [`ROADMAP.md`](../ROADMAP.md) for gaps and next steps.
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import type { Page } from '@playwright/test';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { LoginPage } from '../pages/LoginPage';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Real Manage Users CRUD through the admin UI (`admin.manage-users.spec.ts`
|
||||
@ -62,6 +63,8 @@ test.describe('admin manage users: create/edit/delete @smoke', () => {
|
||||
await timings.measure('create_user', () =>
|
||||
manageUsersPanel.createUser({ email, password: THROWAWAY_PASSWORD, name: 'E2E Throwaway' }),
|
||||
);
|
||||
// Generous bucket — shared-CI DEV LXC under concurrent load (see file header).
|
||||
expectWithinBudget(timings, 'create_user', BUDGET_MS.heavyCrud);
|
||||
expect(await manageUsersPanel.statusBadgeText(email)).toMatch(/active/i);
|
||||
expect(await manageUsersPanel.roleBadgeText(email)).toMatch(/user/i);
|
||||
expect(await manageUsersPanel.writeAccessText(email)).toMatch(/no/i);
|
||||
@ -71,9 +74,11 @@ test.describe('admin manage users: create/edit/delete @smoke', () => {
|
||||
await manageUsersPanel.setWriteAccess(true);
|
||||
await manageUsersPanel.saveEdit();
|
||||
});
|
||||
expectWithinBudget(timings, 'edit_user', BUDGET_MS.heavyCrud);
|
||||
expect(await manageUsersPanel.writeAccessText(email)).toMatch(/yes/i);
|
||||
|
||||
await timings.measure('delete_user', () => manageUsersPanel.deleteUser(email));
|
||||
expectWithinBudget(timings, 'delete_user', BUDGET_MS.heavyCrud);
|
||||
await manageUsersPanel.waitForRowGone(email);
|
||||
|
||||
await accountMenu.closeManageUsers();
|
||||
@ -112,6 +117,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => {
|
||||
await timings.measure('create_throwaway', () =>
|
||||
manageUsersPanel.createUser({ email, password: THROWAWAY_PASSWORD, name: 'E2E Throwaway' }),
|
||||
);
|
||||
expectWithinBudget(timings, 'create_throwaway', BUDGET_MS.heavyCrud);
|
||||
await accountMenu.closeManageUsers();
|
||||
|
||||
// Independent second session/actor — the throwaway user, logged in
|
||||
@ -126,6 +132,9 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => {
|
||||
await userPage.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 });
|
||||
await userPage.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
});
|
||||
// Login flows on this file have hit ~90s under a concurrent CI burst
|
||||
// (see comment above) — budget matches the file's own 120s headroom.
|
||||
expectWithinBudget(timings, 'throwaway_login', BUDGET_MS.heavyCrud);
|
||||
|
||||
const preRes = await userPage.request.get(`${playkitConfig.baseUrl}/api/auth/session`);
|
||||
const preBody = await preRes.json();
|
||||
@ -139,6 +148,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => {
|
||||
await manageUsersPanel.setActive(false);
|
||||
await manageUsersPanel.saveEdit();
|
||||
});
|
||||
expectWithinBudget(timings, 'deactivate_user', BUDGET_MS.heavyCrud);
|
||||
// The table defaults to (and refetches with) the "Active only" filter,
|
||||
// so the just-deactivated row drops out of view immediately — switch to
|
||||
// "All" before reading its badge, or this hangs forever waiting for a
|
||||
@ -154,6 +164,7 @@ test.describe('admin manage users: causal cross-session effect @smoke', () => {
|
||||
const postBody = await postRes.json();
|
||||
expect(postBody?.user).toBeFalsy();
|
||||
});
|
||||
expectWithinBudget(timings, 'session_revoked', BUDGET_MS.uiAction);
|
||||
} finally {
|
||||
await userContext?.close();
|
||||
await deleteUserByEmail(page, playkitConfig.baseUrl, email).catch(() => undefined);
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { assertPublicHost, waitForUrlHost } from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Kolby #57 class: Manage Users overlay must close cleanly before sign-out
|
||||
@ -23,14 +24,17 @@ test.describe('admin users @smoke', () => {
|
||||
await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 });
|
||||
await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin);
|
||||
|
||||
await timings.measure('manage_users', async () => {
|
||||
await accountMenu.openManageUsers();
|
||||
await expect(page.getByRole('heading', { name: /Manage Users/i })).toBeVisible();
|
||||
await accountMenu.closeManageUsers();
|
||||
});
|
||||
expectWithinBudget(timings, 'manage_users', BUDGET_MS.uiAction);
|
||||
|
||||
await timings.measure('sign_out', () => accountMenu.signOut());
|
||||
expectWithinBudget(timings, 'sign_out', BUDGET_MS.uiAction);
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
expect(page.url()).not.toMatch(/10\.\d+\.\d+\.\d+/);
|
||||
await expect(page.getByRole('button', { name: /Sign in/i })).toBeVisible({
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
test.describe('api auth @smoke', () => {
|
||||
test('protected /api/v1/auth/me returns 401 without token', async ({ api, timings }) => {
|
||||
const res = await timings.measure('api_me_unauth', () =>
|
||||
api.get('/api/v1/auth/me', { expectedStatus: 401 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_me_unauth', BUDGET_MS.api);
|
||||
expect(res.status).toBe(401);
|
||||
expect(res.data).toMatchObject({ detail: expect.stringMatching(/not authenticated/i) });
|
||||
});
|
||||
@ -13,6 +15,7 @@ test.describe('api auth @smoke', () => {
|
||||
const res = await timings.measure('api_photos_unauth', () =>
|
||||
api.get('/api/v1/photos', { expectedStatus: 401 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_photos_unauth', BUDGET_MS.api);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
const PeopleList = z.object({
|
||||
items: z.array(z.object({ id: z.number() }).passthrough()).min(1),
|
||||
@ -23,6 +24,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_people', () =>
|
||||
api.get<PeopleList>('/api/v1/people', { expectedStatus: 200, schema: PeopleList }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_people', BUDGET_MS.api);
|
||||
expect(res.data.items[0]).toEqual(expect.objectContaining({ id: expect.any(Number) }));
|
||||
});
|
||||
|
||||
@ -33,6 +35,7 @@ test.describe('api catalog @smoke', () => {
|
||||
schema: ItemsList,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_people_faces', BUDGET_MS.api);
|
||||
expect(Array.isArray(res.data.items)).toBe(true);
|
||||
});
|
||||
|
||||
@ -40,6 +43,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_tags', () =>
|
||||
api.get<TagsList>('/api/v1/tags', { expectedStatus: 200, schema: TagsList }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_tags', BUDGET_MS.api);
|
||||
expect(res.data.items[0]).toEqual(expect.objectContaining({ id: expect.any(Number) }));
|
||||
});
|
||||
|
||||
@ -47,6 +51,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_login_empty', () =>
|
||||
api.post('/api/v1/auth/login', { body: {}, expectedStatus: 422 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_login_empty', BUDGET_MS.api);
|
||||
expect(res.status).toBe(422);
|
||||
});
|
||||
|
||||
@ -57,6 +62,7 @@ test.describe('api catalog @smoke', () => {
|
||||
expectedStatus: 401,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_login_bad', BUDGET_MS.api);
|
||||
expect(res.status).toBe(401);
|
||||
expect(res.data).toMatchObject({
|
||||
detail: expect.stringMatching(/incorrect|password|username/i),
|
||||
@ -67,6 +73,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_users_unauth', () =>
|
||||
api.get('/api/v1/users', { expectedStatus: 401 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_users_unauth', BUDGET_MS.api);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
@ -74,6 +81,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_roles_unauth', () =>
|
||||
api.get('/api/v1/role-permissions', { expectedStatus: 401 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_roles_unauth', BUDGET_MS.api);
|
||||
expect(res.status).toBe(401);
|
||||
});
|
||||
|
||||
@ -81,6 +89,7 @@ test.describe('api catalog @smoke', () => {
|
||||
const res = await timings.measure('api_photo_missing', () =>
|
||||
api.get('/api/v1/photos/999999999', { expectedStatus: [401, 404] }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_photo_missing', BUDGET_MS.api);
|
||||
expect([401, 404]).toContain(res.status);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* FastAPI `/api/v1/auth/login` uses a *separate* user DB from NextAuth.
|
||||
@ -35,12 +36,14 @@ test.describe('fastapi login @smoke', () => {
|
||||
schema: TokenResponse,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_login', BUDGET_MS.api);
|
||||
expect(res.data.access_token).toBeTruthy();
|
||||
|
||||
const authed = api.withAuthBearer(res.data.access_token);
|
||||
const me = await timings.measure('api_me', () =>
|
||||
authed.get<UserResponse>('/api/v1/auth/me', { expectedStatus: 200, schema: UserResponse }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_me', BUDGET_MS.api);
|
||||
expect(me.status).toBe(200);
|
||||
expect(me.data.username).toBe(username);
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { z } from 'zod';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
const HealthSchema = z.object({ status: z.literal('ok') });
|
||||
|
||||
@ -8,6 +9,7 @@ test.describe('api @smoke', () => {
|
||||
const res = await timings.measure('api_health', () =>
|
||||
api.get('/health', { expectedStatus: 200, schema: HealthSchema }),
|
||||
);
|
||||
expectWithinBudget(timings, 'api_health', BUDGET_MS.api);
|
||||
expect(res.data).toEqual({ status: 'ok' });
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import type { ApiClient } from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* FastAPI role-permission write gates: `admin`-role users pass
|
||||
@ -28,11 +29,14 @@ test.describe('role permissions @smoke', () => {
|
||||
test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required');
|
||||
|
||||
const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass));
|
||||
expectWithinBudget(timings, 'login_admin', BUDGET_MS.api);
|
||||
const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass));
|
||||
expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api);
|
||||
|
||||
const viewerRes = await timings.measure('viewer_users', () =>
|
||||
api.withAuthBearer(viewerToken).get('/api/v1/users', { expectedStatus: 403 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'viewer_users', BUDGET_MS.api);
|
||||
expect(viewerRes.status).toBe(403);
|
||||
expect(viewerRes.data).toMatchObject({ detail: expect.stringMatching(/admin/i) });
|
||||
|
||||
@ -41,6 +45,7 @@ test.describe('role permissions @smoke', () => {
|
||||
expectedStatus: 200,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'admin_users', BUDGET_MS.api);
|
||||
expect(adminRes.status).toBe(200);
|
||||
expect(Array.isArray(adminRes.data.items)).toBe(true);
|
||||
});
|
||||
@ -52,16 +57,20 @@ test.describe('role permissions @smoke', () => {
|
||||
test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required');
|
||||
|
||||
const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass));
|
||||
expectWithinBudget(timings, 'login_admin', BUDGET_MS.api);
|
||||
const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass));
|
||||
expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api);
|
||||
|
||||
const viewerRes = await timings.measure('viewer_role_perms', () =>
|
||||
api.withAuthBearer(viewerToken).get('/api/v1/role-permissions', { expectedStatus: 403 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'viewer_role_perms', BUDGET_MS.api);
|
||||
expect(viewerRes.status).toBe(403);
|
||||
|
||||
const adminRes = await timings.measure('admin_role_perms', () =>
|
||||
api.withAuthBearer(adminToken).get('/api/v1/role-permissions', { expectedStatus: 200 }),
|
||||
);
|
||||
expectWithinBudget(timings, 'admin_role_perms', BUDGET_MS.api);
|
||||
expect(adminRes.status).toBe(200);
|
||||
expect(adminRes.data).toHaveProperty('permissions');
|
||||
});
|
||||
@ -73,7 +82,9 @@ test.describe('role permissions @smoke', () => {
|
||||
test.skip(!ready, 'E2E_API_* and E2E_API_VIEWER_* required');
|
||||
|
||||
const adminToken = await timings.measure('login_admin', () => login(api, adminUser, adminPass));
|
||||
expectWithinBudget(timings, 'login_admin', BUDGET_MS.api);
|
||||
const viewerToken = await timings.measure('login_viewer', () => login(api, viewerUser, viewerPass));
|
||||
expectWithinBudget(timings, 'login_viewer', BUDGET_MS.api);
|
||||
|
||||
// Use a photo id that cannot exist — proves the write *gate*, not deletion.
|
||||
// The admin dependency runs before the handler body, so this never touches
|
||||
@ -86,6 +97,7 @@ test.describe('role permissions @smoke', () => {
|
||||
expectedStatus: 403,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'viewer_bulk_delete', BUDGET_MS.api);
|
||||
expect(viewerRes.status).toBe(403);
|
||||
|
||||
const adminRes = await timings.measure('admin_bulk_delete', () =>
|
||||
@ -94,6 +106,7 @@ test.describe('role permissions @smoke', () => {
|
||||
expectedStatus: 200,
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'admin_bulk_delete', BUDGET_MS.api);
|
||||
expect(adminRes.status).toBe(200);
|
||||
expect(adminRes.data).toMatchObject({ deleted_count: 0, missing_photo_ids: [nonExistentId] });
|
||||
});
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { assertPublicHost, saveStorageState, waitForUrlHost } from '@levkin/playkit';
|
||||
import { test as setup } from '../fixtures';
|
||||
import path from 'node:path';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
const authFile = path.join(__dirname, '../.auth/admin.json');
|
||||
const viewerAuthFile = path.join(__dirname, '../.auth/viewer.json');
|
||||
@ -16,6 +17,7 @@ setup('authenticate e2e admin', async ({ page, playkitConfig, loginPage, e2eCred
|
||||
await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
});
|
||||
expectWithinBudget(timings, 'setup_login', BUDGET_MS.uiLogin);
|
||||
|
||||
await saveStorageState(page, authFile);
|
||||
});
|
||||
@ -35,6 +37,7 @@ setup(
|
||||
await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
});
|
||||
expectWithinBudget(timings, 'setup_login_viewer', BUDGET_MS.uiLogin);
|
||||
|
||||
await saveStorageState(page, viewerAuthFile);
|
||||
},
|
||||
|
||||
@ -3,6 +3,7 @@ import {
|
||||
waitForUrlHost,
|
||||
} from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
test.describe('auth @smoke', () => {
|
||||
test('login page loads on public host', async ({
|
||||
@ -14,6 +15,7 @@ test.describe('auth @smoke', () => {
|
||||
assertPublicHost(playkitConfig.baseUrl);
|
||||
|
||||
await timings.measure('login_page', () => loginPage.openLogin());
|
||||
expectWithinBudget(timings, 'login_page', BUDGET_MS.uiAction);
|
||||
|
||||
await expect(page.getByRole('heading', { name: /Sign in/i })).toBeVisible();
|
||||
await expect(page.locator('#email')).toBeVisible();
|
||||
@ -42,10 +44,12 @@ test.describe('auth @smoke', () => {
|
||||
await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 });
|
||||
await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin);
|
||||
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
|
||||
await timings.measure('sign_out', () => accountMenu.signOut());
|
||||
expectWithinBudget(timings, 'sign_out', BUDGET_MS.uiAction);
|
||||
|
||||
// After sign-out, NextAuth must redirect to the public host — never 10.x.
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
type MailtrapMessage,
|
||||
} from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Forgot-password must land in the mail trap (Mailpit) with a public reset link.
|
||||
@ -40,6 +41,7 @@ test.describe('mail @smoke', () => {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'forgot_password_api', BUDGET_MS.api);
|
||||
expect(res.ok()).toBeTruthy();
|
||||
|
||||
const msg = await timings.measure<MailpitMessage | MailtrapMessage>('mail_wait', () =>
|
||||
@ -81,6 +83,7 @@ test.describe('mail @smoke', () => {
|
||||
const after = new Date(Date.now() - 5_000);
|
||||
|
||||
await timings.measure('open_home', () => page.goto(playkitConfig.baseUrl));
|
||||
expectWithinBudget(timings, 'open_home', BUDGET_MS.uiAction);
|
||||
await page.getByRole('button', { name: /Sign in/i }).click();
|
||||
await page.getByRole('button', { name: /Forgot password/i }).click();
|
||||
await page.locator('#forgot-email').fill(to);
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import path from 'node:path';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Logged-in-only gallery filters: Favorites and People are hidden from the
|
||||
@ -19,6 +20,7 @@ test.describe('gallery filters (authed) @smoke', () => {
|
||||
await timings.measure('open_search', async () => {
|
||||
await page.goto(`${playkitConfig.baseUrl}/search`);
|
||||
});
|
||||
expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction);
|
||||
|
||||
const favoritesCheckbox = page.locator('#favorites-only');
|
||||
await expect(favoritesCheckbox).toBeVisible({ timeout: 20_000 });
|
||||
@ -54,6 +56,7 @@ test.describe('gallery filters (authed) @smoke', () => {
|
||||
await timings.measure('open_search', async () => {
|
||||
await page.goto(`${playkitConfig.baseUrl}/search`);
|
||||
});
|
||||
expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction);
|
||||
|
||||
await page.getByRole('button', { name: /select people/i }).click();
|
||||
await page.getByPlaceholder('Search people...').fill(person.first_name);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { interceptNetworkCall } from '@levkin/playkit';
|
||||
import { z } from 'zod';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Public gallery search (`viewer-frontend` `/api/search`, backed by its own
|
||||
@ -43,12 +44,14 @@ test.describe('gallery search filters @smoke', () => {
|
||||
);
|
||||
const tag = tagsRes.data.items[0];
|
||||
expect(tag).toBeTruthy();
|
||||
expectWithinBudget(timings, 'api_tags', BUDGET_MS.api);
|
||||
|
||||
const res = await timings.measure('search_by_tag', () =>
|
||||
page.request.get(`${playkitConfig.baseUrl}/api/search`, {
|
||||
params: { tags: String(tag.id) },
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'search_by_tag', BUDGET_MS.api);
|
||||
expect(res.ok()).toBeTruthy();
|
||||
const body = SearchResponse.parse(await res.json());
|
||||
for (const photo of body.photos) {
|
||||
@ -70,12 +73,14 @@ test.describe('gallery search filters @smoke', () => {
|
||||
);
|
||||
const person = peopleRes.data.items[0];
|
||||
expect(person).toBeTruthy();
|
||||
expectWithinBudget(timings, 'api_people', BUDGET_MS.api);
|
||||
|
||||
const res = await timings.measure('search_by_person', () =>
|
||||
page.request.get(`${playkitConfig.baseUrl}/api/search`, {
|
||||
params: { people: String(person.id) },
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'search_by_person', BUDGET_MS.api);
|
||||
expect(res.ok()).toBeTruthy();
|
||||
const body = SearchResponse.parse(await res.json());
|
||||
for (const photo of body.photos) {
|
||||
@ -130,6 +135,7 @@ test.describe('gallery search filters @smoke', () => {
|
||||
await timings.measure('open_search', async () => {
|
||||
await page.goto(`${playkitConfig.baseUrl}/search`);
|
||||
});
|
||||
expectWithinBudget(timings, 'open_search', BUDGET_MS.uiAction);
|
||||
|
||||
// Match only the *filtered* request (query includes `tags=`) — the page
|
||||
// also fires an unfiltered `/api/search?page=1&pageSize=30` on mount.
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { assertPublicHost, waitForUrlHost } from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Idle logout must redirect to the public host (same class as #57).
|
||||
@ -22,6 +23,7 @@ test.describe('auth idle @smoke', () => {
|
||||
await page.waitForURL((url) => !url.pathname.includes('/login'), { timeout: 30_000 });
|
||||
await page.getByLabel('Account menu').waitFor({ state: 'visible', timeout: 30_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'login', BUDGET_MS.uiLogin);
|
||||
|
||||
// Short idle window — requires IdleLogoutHandler e2e_idle_ms support on DEV
|
||||
await page.goto(`${playkitConfig.baseUrl}/?e2e_idle_ms=2500`);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { startNetworkErrorMonitor } from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Silent-failure guard: these pages can *look* fine in the UI while a
|
||||
@ -24,6 +25,7 @@ test.describe('network errors (public pages) @smoke', () => {
|
||||
timeout: 20_000,
|
||||
});
|
||||
});
|
||||
expectWithinBudget(timings, 'open_gallery', BUDGET_MS.uiAction);
|
||||
} finally {
|
||||
net.assertNoErrors();
|
||||
}
|
||||
@ -45,6 +47,7 @@ test.describe('network errors (public pages) @smoke', () => {
|
||||
page.getByText(/Found \d+ photos?|No photos found matching your filters/i).first(),
|
||||
).toBeVisible({ timeout: 20_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'search_tag_filter', BUDGET_MS.uiAction);
|
||||
} finally {
|
||||
net.assertNoErrors();
|
||||
}
|
||||
@ -61,6 +64,7 @@ test.describe('network errors (authed) @smoke', () => {
|
||||
await page.goto(`${playkitConfig.baseUrl}/upload`);
|
||||
await expect(page.getByLabel('Account menu')).toBeVisible({ timeout: 20_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'open_upload', BUDGET_MS.uiAction);
|
||||
} finally {
|
||||
net.assertNoErrors();
|
||||
}
|
||||
@ -77,6 +81,7 @@ test.describe('network errors (authed) @smoke', () => {
|
||||
await expect(page.getByRole('heading', { name: /Manage Users/i })).toBeVisible();
|
||||
await accountMenu.closeManageUsers();
|
||||
});
|
||||
expectWithinBudget(timings, 'manage_users', BUDGET_MS.uiAction);
|
||||
} finally {
|
||||
net.assertNoErrors();
|
||||
}
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
type MailtrapMessage,
|
||||
} from '@levkin/playkit';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Register → confirmation mail in Mailpit → verify link on public host.
|
||||
@ -38,6 +39,7 @@ test.describe('auth register @mail', () => {
|
||||
});
|
||||
await page.waitForURL(/\/login/, { timeout: 30_000 });
|
||||
});
|
||||
expectWithinBudget(timings, 'register', BUDGET_MS.uiLogin);
|
||||
|
||||
await expect(page.getByText(/Account created|check your email|confirm/i).first()).toBeVisible({
|
||||
timeout: 15_000,
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { assertPublicHost, interceptNetworkCall, waitForUrlHost } from '@levkin/playkit';
|
||||
import path from 'node:path';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
const tinyPng = path.join(__dirname, '../fixtures/tiny.png');
|
||||
|
||||
@ -18,6 +19,7 @@ test.describe('upload @smoke', () => {
|
||||
await page.goto(`${playkitConfig.baseUrl}/upload`);
|
||||
await waitForUrlHost(page, playkitConfig.expectedHost);
|
||||
});
|
||||
expectWithinBudget(timings, 'open_upload', BUDGET_MS.uiAction);
|
||||
|
||||
expect(new URL(page.url()).pathname).toMatch(/\/upload/);
|
||||
await expect(page.getByLabel('Account menu')).toBeVisible({ timeout: 20_000 });
|
||||
@ -52,5 +54,7 @@ test.describe('upload @smoke', () => {
|
||||
.first();
|
||||
await expect(outcome).toBeVisible({ timeout: 30_000 });
|
||||
});
|
||||
// Network round trip + UI settle — same magnitude as a login flow.
|
||||
expectWithinBudget(timings, 'upload_file', BUDGET_MS.uiLogin);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { assertPublicHost, waitForUrlHost } from '@levkin/playkit';
|
||||
import path from 'node:path';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* Viewer (NextAuth) session — cookies from storageState.
|
||||
@ -20,5 +21,6 @@ test.describe('viewer session @smoke', () => {
|
||||
const body = await res.json();
|
||||
expect(body?.user?.email).toBeTruthy();
|
||||
});
|
||||
expectWithinBudget(timings, 'session', BUDGET_MS.uiAction);
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import path from 'node:path';
|
||||
import { test, expect } from '../fixtures';
|
||||
import { BUDGET_MS, expectWithinBudget } from '../timing-budgets';
|
||||
|
||||
/**
|
||||
* NextAuth (browser-session) write gates — `session.user.hasWriteAccess`
|
||||
@ -32,6 +33,7 @@ test.describe('viewer write gates (NextAuth, viewer) @smoke', () => {
|
||||
data: { firstName: 'Test', lastName: 'Viewer' },
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'viewer_identify', BUDGET_MS.api);
|
||||
expect(res.status()).toBe(403);
|
||||
const body = await res.json();
|
||||
expect(body).toMatchObject({ error: expect.stringMatching(/write access/i) });
|
||||
@ -51,6 +53,7 @@ test.describe('viewer write gates (NextAuth, admin) @smoke', () => {
|
||||
data: { firstName: 'Test', lastName: 'Admin' },
|
||||
}),
|
||||
);
|
||||
expectWithinBudget(timings, 'admin_identify', BUDGET_MS.api);
|
||||
// Admin clears the write-access gate; a nonexistent face id then 404s —
|
||||
// proves the gate didn't block a legitimate write-access user.
|
||||
expect(res.status()).toBe(404);
|
||||
|
||||
39
e2e/timing-budgets.ts
Normal file
39
e2e/timing-budgets.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import { expect } from '@playwright/test';
|
||||
import type { TimingCollector } from '@levkin/playkit';
|
||||
|
||||
/**
|
||||
* Perf-budget gate for a `timings.measure(name, ...)` step.
|
||||
*
|
||||
* `timings.measure()` on its own only *records* durations (for the
|
||||
* Pushgateway export, see ROADMAP "Timing -> Pushgateway") — nothing ever
|
||||
* failed CI for a step getting slow. This asserts the most recently
|
||||
* recorded sample for `name` against `maxMs`, so a regression shows up as a
|
||||
* red test instead of only a graph nobody's watching.
|
||||
*
|
||||
* Budgets are intentionally generous (shared CI runner + DEV LXC under
|
||||
* concurrent PR load — see docs/guides/ci-runners-and-control.md in the
|
||||
* ansible repo) — this catches order-of-magnitude regressions, not
|
||||
* micro-optimization. Pick the closest `BUDGET_MS` bucket rather than
|
||||
* inventing a new one-off number per call site.
|
||||
*/
|
||||
export function expectWithinBudget(timings: TimingCollector, name: string, maxMs: number): void {
|
||||
const samples = timings.getSamples().filter((sample) => sample.name === name);
|
||||
const sample = samples[samples.length - 1];
|
||||
expect(sample, `no timing sample recorded for "${name}" — call timings.measure() first`).toBeTruthy();
|
||||
expect(
|
||||
sample!.durationMs,
|
||||
`"${name}" took ${Math.round(sample!.durationMs)}ms, budget is ${maxMs}ms`,
|
||||
).toBeLessThanOrEqual(maxMs);
|
||||
}
|
||||
|
||||
/** Shared budget buckets — see `expectWithinBudget` for rationale. */
|
||||
export const BUDGET_MS = {
|
||||
/** Direct FastAPI/NextAuth-route calls (LAN, no browser). */
|
||||
api: 8_000,
|
||||
/** Single UI interaction: open a page/panel, apply a filter, sign out. */
|
||||
uiAction: 20_000,
|
||||
/** Full login flow (navigate + submit + redirect wait). */
|
||||
uiLogin: 45_000,
|
||||
/** Multi-step UI CRUD (create/edit/deactivate + list refetch). */
|
||||
heavyCrud: 60_000,
|
||||
} as const;
|
||||
Loading…
x
Reference in New Issue
Block a user