diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index efb9c99..b0ae1d3 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -5,6 +5,7 @@ name: CI on: push: branches: [main] + tags: ['v*'] pull_request: types: [opened, synchronize, reopened] @@ -56,3 +57,81 @@ jobs: run: | docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:latest \ detect --source /repo --no-banner --redact + + # Release: only runs on `vX.Y.Z` tag push. Gates a Gitea release behind the + # same integrity checks as CI (never trust a bare "bump + tag") plus two + # consistency checks bare tagging can't give you: tag == package.json + # version, and CHANGELOG.md actually documents this version. + release: + runs-on: [homelab, self-hosted, linux] + container: + image: node:20-bookworm + if: startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install + run: npm ci + - name: Typecheck + run: npm run typecheck + - name: Unit tests + run: npm test + - name: Build + run: npm run build + - name: Verify tag matches package.json version + run: | + TAG="${GITHUB_REF#refs/tags/v}" + PKG_VERSION="$(node -p "require('./package.json').version")" + if [ "$TAG" != "$PKG_VERSION" ]; then + echo "::error::tag v$TAG does not match package.json version $PKG_VERSION" + exit 1 + fi + echo "RELEASE_VERSION=$TAG" >> "$GITEA_ENV" + - name: Extract CHANGELOG section for this version + run: | + node -e ' + const fs = require("fs"); + const version = process.env.RELEASE_VERSION; + const text = fs.readFileSync("CHANGELOG.md", "utf8"); + const re = new RegExp(`^## ${version.replace(/\./g, "\\.")}.*$`, "m"); + const start = text.search(re); + if (start === -1) { + console.error(`::error::CHANGELOG.md has no "## ${version}" section — update it before tagging`); + process.exit(1); + } + const rest = text.slice(start); + const next = rest.slice(1).search(/^## /m); + const section = next === -1 ? rest : rest.slice(0, next + 1); + fs.writeFileSync("/tmp/release-notes.md", section.trim() + "\n"); + ' + - name: Pack npm tarball + run: npm pack --pack-destination /tmp + - name: Create Gitea release + run: | + BODY_JSON=$(node -e ' + const fs = require("fs"); + const body = fs.readFileSync("/tmp/release-notes.md", "utf8"); + process.stdout.write(JSON.stringify({ + tag_name: process.env.GITHUB_REF_NAME, + name: process.env.GITHUB_REF_NAME, + body, + draft: false, + prerelease: false, + })); + ') + RESPONSE=$(curl -sS -X POST \ + -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \ + -H "Content-Type: application/json" \ + -d "$BODY_JSON" \ + "https://git.levkin.ca/api/v1/repos/ilia/playkit/releases") + RELEASE_ID=$(node -e "console.log(JSON.parse(process.argv[1]).id)" "$RESPONSE") + if [ -z "$RELEASE_ID" ] || [ "$RELEASE_ID" = "undefined" ]; then + echo "::error::release creation failed: $RESPONSE" + exit 1 + fi + TARBALL=$(ls /tmp/levkin-playkit-*.tgz) + curl -sS -X POST \ + -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \ + -F "attachment=@${TARBALL}" \ + "https://git.levkin.ca/api/v1/repos/ilia/playkit/releases/${RELEASE_ID}/assets" diff --git a/CHANGELOG.md b/CHANGELOG.md index 4baa34f..fd5baaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- CI: add tag-triggered `release` job (`.gitea/workflows/ci.yml`) — re-runs build/test, verifies tag matches `package.json` version and `CHANGELOG.md` documents it, creates a Gitea release with an `npm pack` tarball attached. Requires a one-time `GITEA_TOKEN` Actions secret. +- Docs: bump install pin examples from `v0.1.0` to `v0.3.0` (README, CONSUMER.md) +- Docs: lead with Mailpit (homelab default) instead of Mailtrap in README email section; use `createMailInbox()` + `readMailHtml()` in the example instead of a provider-specific client +- Ops: Pushgateway + `live-playkit` Grafana board now provisioned via ansible `deploy/observability/` (pending `make deploy-observability`); removed the standalone `dashboards/playkit-overview.json` (superseded) and the `dashboards` entry from `package.json` `files` + ## 0.3.0 — 2026-07-14 - **Zod schema asserts** — `assertSchema()` + optional `schema` on `ApiClient` requests diff --git a/README.md b/README.md index 1159ef5..e8fb14e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Use it as a library from any consumer repo (punimtag, MirrorMatch, …). App-spe ```bash # git tag dependency (until a private npm registry is wired) -npm install git+https://git.levkin.ca/ilia/playkit.git#v0.1.0 +npm install git+https://git.levkin.ca/ilia/playkit.git#v0.3.0 # peer npm install -D @playwright/test @@ -72,27 +72,37 @@ test('sign-out stays on public host', async ({ page, playkitConfig, timings }) = | `createLogger` / `redactSecrets` | Structured JSON logs | | `TimingCollector` / `pushPrometheusMetrics` | Action timings → Prometheus Pushgateway → Grafana | | `createPlaykitRuntime` | One-shot config + logger + API + timings | -| `MailtrapClient` / `waitForEmail` | Assert password-reset / verify emails in Mailtrap sandbox | +| `createMailInbox` / `MailpitClient` / `MailtrapClient` | Assert password-reset / verify emails (Mailpit homelab trap or Mailtrap SaaS) | +| `assertSchema` | Zod schema asserts (standalone or via `ApiClient` `schema` option) | +| `saveStorageState` / `storageStateUse` | Auth once, reuse across specs | +| `playkitFailureArtifacts` | Trace/video/screenshot only on failure | -## Mailtrap (email testing) +## Email testing (Mailpit default, Mailtrap optional) + +`createMailInbox()` picks the provider from `PLAYKIT_MAIL_PROVIDER` (default +`mailpit`) so specs don't need to know which backend is behind it. Prefer +**Mailpit** — it's our homelab SMTP trap (`10.0.10.45`, no external +dependency); use Mailtrap only if you specifically want the SaaS sandbox. ```ts -import { MailtrapClient, firstLinkMatching, assertPublicHost } from '@levkin/playkit'; +import { createMailInbox, readMailHtml, firstLinkMatching, assertPublicHost } from '@levkin/playkit'; -const mail = MailtrapClient.fromEnv(); -if (!mail) throw new Error('set MAILTRAP_API_TOKEN + MAILTRAP_INBOX_ID'); +const mail = createMailInbox(); // reads PLAYKIT_MAIL_PROVIDER / MAILPIT_* / MAILTRAP_* +if (!mail) throw new Error('set MAILPIT_BASE_URL (or MAILTRAP_API_TOKEN + MAILTRAP_INBOX_ID)'); const after = new Date(); // … trigger forgot-password in the app … const msg = await mail.waitForEmail({ to: 'e2e@example.com', subject: /reset/i, after }); -const html = await mail.getHtml(msg.id, msg.html_path); +const html = await readMailHtml(mail, msg); // normalizes Mailpit vs Mailtrap message shape const link = firstLinkMatching(html, /reset-password/); assertPublicHost(link!); ``` -**Important:** Mailtrap only sees mail if the app’s SMTP points at the sandbox -(`sandbox.smtp.mailtrap.io` + inbox credentials). Sending via Gmail to a real -address will not appear in the sandbox. See ansible `docs/hardening/SECRETS.md`. +**Important:** the mail client only sees mail if the app's SMTP actually +points at that trap (Mailpit `10.0.10.45:1025` in DEV, or Mailtrap's +`sandbox.smtp.mailtrap.io` + inbox credentials for SaaS). Sending via Gmail to +a real address will not appear in either. See ansible `docs/hardening/SECRETS.md` +(`## Playkit / punimtag e2e secrets`). ## Develop this repo @@ -106,8 +116,12 @@ npm run build ## Release 1. Bump `version` in `package.json` -2. Update `CHANGELOG.md` -3. Tag `vX.Y.Z` and push — consumers pin the tag +2. Update `CHANGELOG.md` with a `## X.Y.Z` section (the release job extracts this verbatim as release notes) +3. Tag `vX.Y.Z` and push + +Pushing the tag triggers `.gitea/workflows/ci.yml`'s `release` job: it re-runs typecheck/test/build, verifies the tag matches `package.json`'s `version` and that `CHANGELOG.md` documents it, then creates a Gitea release (with the `npm pack` tarball attached) via the API using a repo-scoped `GITEA_TOKEN` Actions secret. If any check fails, no release is created — fix and re-tag. Consumers still pin the git tag (`#vX.Y.Z`); the Gitea release is for visibility/changelog, not an npm registry publish (see ROADMAP "private Gitea npm registry"). + +**One-time setup:** add a `GITEA_TOKEN` secret (repo `Settings → Actions → Secrets` on `ilia/playkit`) scoped to create releases on this repo — separate from the `PLAYKIT_GIT_TOKEN` consumers use to clone it. ## License diff --git a/ROADMAP.md b/ROADMAP.md index 5e5279f..ea09594 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,6 +25,10 @@ Living plan for making `@levkin/playkit` more useful across Levkin repos. - [ ] **Consumer template** — `npx @levkin/playkit init` scaffolding `e2e/` + CI snippet - [ ] **Deploy-smoke CLI** — `playkit smoke --project punimtag` post-deploy gate - [ ] **Retry policy presets** — flaky-network vs strict-CI profiles +- [ ] **Self-test against a real fake site** — today's unit tests only mock HTTP/mail; `examples/` specs are explicitly *not run* in kit CI. Stand up a tiny demo app (or point at an existing DEV LXC) and run the browser/API/mail helpers against it in CI, so a regression in `BasePage`/`ApiClient`/`assertPublicHost` is caught before a consumer pins a broken tag. +- [x] **Tag-triggered release workflow** — `.gitea/workflows/ci.yml` `release` job now runs on `vX.Y.Z` tag push: re-verifies build/test, checks tag == `package.json` version, checks `CHANGELOG.md` has that version's section, then creates a Gitea release (npm-pack tarball attached) via the API. Needs a one-time `GITEA_TOKEN` Actions secret on this repo (see README "Release"). Still open: `build-and-test`/`secret-scan` also re-run on the tag push (same `on.push` trigger) — harmless redundancy, not wired to skip. +- [ ] **Live docs** — see `docs/CONSUMER.md` decision note; likely an Outline page under the existing "QA & Dev" collection (`notes.levkin.ca`, already deployed with API automation) rather than a new static site. +- [x] **Pushgateway + dashboard wired in ansible** — `pushgateway` service, Prometheus scrape job, and a generated `live-playkit` Grafana board now live in ansible `deploy/observability/` (superseding the old standalone `dashboards/playkit-overview.json`, which is removed). Ops still needs to run `make deploy-observability` against the LXC before `PLAYKIT_METRICS_ENABLED=true` does anything in CI — check with the ansible repo owner before flipping that on. ## Later (v0.5+) — professional polish @@ -38,3 +42,14 @@ Living plan for making `@levkin/playkit` more useful across Levkin repos. - [ ] **Infisical SDK helper** — `loadSecretsFromInfisical()` for local runs (machine identity) - [ ] **JUnit + HTML report merge** — single artifact for Gitea PR checks - [ ] **Network assert helpers** — fail if request hits `10.x` / wrong host after navigation + +## Ideas pulled from similar OSS tools (2026-07 research) + +Comparing playkit against `seontechnologies/playwright-utils` (102★, functional-core/fixture-shell design), `kitium-ai/playwright-helpers` (enterprise-grade, contract/a11y/chaos), `maravexa/playwright-exporter` (scheduled synthetic monitoring), and `vitalics/playwright-prometheus-remote-write-reporter` (remote-write instead of Pushgateway). What's worth taking: + +- [ ] **Functional-core-for-everything audit** — playkit already does this for the runtime object (`createPlaykitRuntime` wraps plain functions), but individual utilities like `ApiClient` are class-only. `playwright-utils` ships every utility as *both* a standalone function (explicit deps, easy to unit test) and a fixture wrapper. Worth an audit pass on `ApiClient`/`MailpitClient` to see which could get a functional export alongside the class. +- [ ] **Network interception / network error monitor** — `playwright-utils` has dedicated helpers for asserting on intercepted requests and flagging console/network errors during a test, distinct from `ApiClient` (which is for *making* requests, not observing the page's own traffic). Genuinely missing capability, not just a naming difference. +- [ ] **Test burn-in (flake detection)** — `playwright-utils`' "burn-in" utility reruns a spec N times before merge to catch flaky tests early. This *is* the mechanism for the existing "Flake quarantine" item above — implement burn-in first, quarantine consumes its output. +- [ ] **Scheduled synthetic monitoring** — `playwright-exporter` runs suites on a cron and exposes pass/fail + duration as Prometheus metrics independent of any Pushgateway. This overlaps heavily with the "Deploy-smoke CLI" item (`playkit smoke --project punimtag`) — evaluate reusing/wrapping `playwright-exporter` on a schedule instead of building a bespoke CLI from scratch. +- [ ] **Remote-write as a Pushgateway alternative** — `playwright-prometheus-remote-write-reporter` pushes via Prometheus remote-write instead of a Pushgateway (no separate service to run/scrape). Now moot for us since Pushgateway is wired into `deploy/observability` (2026-07-14), but worth remembering if that stack ever needs simplifying. +- Confirmed sane (no action): OpenAPI contract testing, axe-core a11y, and visual regression are already on this roadmap and match what `kitium-ai/playwright-helpers` treats as "enterprise" table stakes — no new items needed, just execute what's already listed. diff --git a/dashboards/playkit-overview.json b/dashboards/playkit-overview.json deleted file mode 100644 index fea487f..0000000 --- a/dashboards/playkit-overview.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "enable": true, - "expr": "playkit_action_ok{project=~\"$project\"}", - "name": "Playkit action failed", - "type": "classic_conditions" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "panels": [ - { - "title": "Action duration (ms)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "playkit_action_duration_ms{project=~\"$project\", env=~\"$env\"}", - "legendFormat": "{{action}}" - } - ] - }, - { - "title": "Action success (1=ok)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "playkit_action_ok{project=~\"$project\", env=~\"$env\"}", - "legendFormat": "{{action}}" - } - ] - } - ], - "schemaVersion": 39, - "tags": ["playkit", "e2e"], - "templating": { - "list": [ - { - "name": "project", - "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "query": "label_values(playkit_action_duration_ms, project)", - "includeAll": true, - "current": { "text": "All", "value": "$__all" } - }, - { - "name": "env", - "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "query": "label_values(playkit_action_duration_ms, env)", - "includeAll": true, - "current": { "text": "All", "value": "$__all" } - } - ] - }, - "time": { "from": "now-6h", "to": "now" }, - "title": "Playkit E2E overview", - "uid": "playkit-overview", - "version": 1 -} diff --git a/docs/CONSUMER.md b/docs/CONSUMER.md index 91074e1..79c008f 100644 --- a/docs/CONSUMER.md +++ b/docs/CONSUMER.md @@ -3,7 +3,7 @@ ## 1. Depend on a release ```bash -npm install git+https://git.levkin.ca/ilia/playkit.git#v0.1.0 +npm install git+https://git.levkin.ca/ilia/playkit.git#v0.3.0 npm install -D @playwright/test npx playwright install chromium ``` @@ -25,7 +25,8 @@ Store in Infisical `LevkinOps` / `Development` (path e.g. `/playkit/punimtag`): - `PLAYKIT_BASE_URL=https://punimtagdev.levkin.ca` - `E2E_ADMIN_EMAIL` / `E2E_ADMIN_PASSWORD` (dedicated test user — not a human’s password) -- optional `PLAYKIT_PUSHGATEWAY_URL` +- optional `PLAYKIT_PUSHGATEWAY_URL=http://10.0.10.24:9091` (Pushgateway on the observability LXC — config lives in ansible `deploy/observability/`; scrape job + `live-playkit` Grafana board are wired, but confirm `make deploy-observability` has actually been run before turning `PLAYKIT_METRICS_ENABLED=true` on in CI) +- for mail specs: `PLAYKIT_MAIL_PROVIDER=mailpit` (default) + `MAILPIT_BASE_URL` / `MAILPIT_USER` / `MAILPIT_PASSWORD`, or `MAILTRAP_*` for SaaS Sync into Gitea Actions secrets for the consumer repo. diff --git a/package.json b/package.json index baaf1a6..cb68115 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,6 @@ }, "files": [ "dist", - "dashboards", "README.md", "ROADMAP.md", "LICENSE"