Add Node/TS automation twin sharing the extension core #1
@ -56,6 +56,10 @@ jobs:
|
|||||||
test -L automation/context_extractor/js/prompt.js
|
test -L automation/context_extractor/js/prompt.js
|
||||||
diff -q automation/context_extractor/js/dom.js extension/core/dom.js
|
diff -q automation/context_extractor/js/dom.js extension/core/dom.js
|
||||||
diff -q automation/context_extractor/js/prompt.js extension/core/prompt.js
|
diff -q automation/context_extractor/js/prompt.js extension/core/prompt.js
|
||||||
|
test -L automation-js/src/js/dom.js
|
||||||
|
test -L automation-js/src/js/prompt.js
|
||||||
|
diff -q automation-js/src/js/dom.js extension/core/dom.js
|
||||||
|
diff -q automation-js/src/js/prompt.js extension/core/prompt.js
|
||||||
|
|
||||||
- name: Install package + dev deps
|
- name: Install package + dev deps
|
||||||
run: |
|
run: |
|
||||||
@ -73,3 +77,34 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
python3 scripts/package_extension.py
|
python3 scripts/package_extension.py
|
||||||
test -f dist/context-extractor-extension.zip
|
test -f dist/context-extractor-extension.zip
|
||||||
|
|
||||||
|
quality-js:
|
||||||
|
name: automation-js (vitest + real Chromium)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: automation-js
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Node
|
||||||
|
uses: actions/setup-node@v4
|
||||||
|
with:
|
||||||
|
node-version: "20"
|
||||||
|
|
||||||
|
- name: Install deps
|
||||||
|
run: npm install
|
||||||
|
|
||||||
|
- name: Install Playwright Chromium
|
||||||
|
run: npx playwright install --with-deps chromium
|
||||||
|
|
||||||
|
- name: Typecheck
|
||||||
|
run: npm run typecheck
|
||||||
|
|
||||||
|
- name: Vitest
|
||||||
|
run: npm test
|
||||||
|
|
||||||
|
- name: Build (verifies dist/js copy step)
|
||||||
|
run: npm run build
|
||||||
|
|||||||
10
Makefile
10
Makefile
@ -1,4 +1,4 @@
|
|||||||
.PHONY: help test lint package ci install
|
.PHONY: help test test-js lint package ci install install-js
|
||||||
|
|
||||||
help: ## Show targets
|
help: ## Show targets
|
||||||
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
||||||
@ -8,9 +8,15 @@ install: ## Create venv and install automation package + Playwright Chromium
|
|||||||
cd automation && .venv/bin/pip install -U pip && .venv/bin/pip install -e ".[dev]"
|
cd automation && .venv/bin/pip install -U pip && .venv/bin/pip install -e ".[dev]"
|
||||||
cd automation && .venv/bin/playwright install chromium
|
cd automation && .venv/bin/playwright install chromium
|
||||||
|
|
||||||
|
install-js: ## npm install automation-js (uses the Chromium cache npx playwright already has)
|
||||||
|
cd automation-js && npm install
|
||||||
|
|
||||||
test: ## Run pytest
|
test: ## Run pytest
|
||||||
cd automation && .venv/bin/pytest -q
|
cd automation && .venv/bin/pytest -q
|
||||||
|
|
||||||
|
test-js: ## Run automation-js vitest suite (real headless Chromium)
|
||||||
|
cd automation-js && npm test
|
||||||
|
|
||||||
lint: ## Syntax-check extension JS + validate manifest JSON
|
lint: ## Syntax-check extension JS + validate manifest JSON
|
||||||
@node --check extension/core/dom.js
|
@node --check extension/core/dom.js
|
||||||
@node --check extension/core/prompt.js
|
@node --check extension/core/prompt.js
|
||||||
@ -24,4 +30,4 @@ lint: ## Syntax-check extension JS + validate manifest JSON
|
|||||||
package: ## Zip extension/ into dist/
|
package: ## Zip extension/ into dist/
|
||||||
python3 scripts/package_extension.py
|
python3 scripts/package_extension.py
|
||||||
|
|
||||||
ci: lint test package ## Local stand-in for GitHub Actions
|
ci: lint test test-js package ## Local stand-in for GitHub Actions
|
||||||
|
|||||||
39
README.md
39
README.md
@ -4,12 +4,16 @@
|
|||||||
**Clone:** `gitea@git.levkin.ca:ilia/context-extractor.git`
|
**Clone:** `gitea@git.levkin.ca:ilia/context-extractor.git`
|
||||||
|
|
||||||
Capture console logs, network activity, JS errors, and clean markdown content
|
Capture console logs, network activity, JS errors, and clean markdown content
|
||||||
from a webpage, formatted as an AI-ready prompt. Ships two ways from one
|
from a webpage, formatted as an AI-ready prompt. Ships three ways from one
|
||||||
shared core:
|
shared core:
|
||||||
|
|
||||||
- **`extension/`** — a browser extension (Manifest V3) for interactive use.
|
- **`extension/`** — a browser extension (Manifest V3) for interactive use.
|
||||||
- **`automation/`** — a Python package for scripted/headless use with
|
- **`automation/`** — a Python package for scripted/headless use with
|
||||||
Playwright or [Camoufox](https://camoufox.com).
|
Playwright or [Camoufox](https://camoufox.com).
|
||||||
|
- **`automation-js/`** — a Node/TS twin of `automation/` for Playwright
|
||||||
|
(`playwright-core`) or [camoufox-js](https://www.npmjs.com/package/camoufox-js)
|
||||||
|
automation that's already JS, so it can `import` the session directly
|
||||||
|
instead of shelling out to Python.
|
||||||
|
|
||||||
```
|
```
|
||||||
context-extractor/
|
context-extractor/
|
||||||
@ -24,6 +28,10 @@ context-extractor/
|
|||||||
│ ├── context_extractor/ session + CLI
|
│ ├── context_extractor/ session + CLI
|
||||||
│ │ └── js/ symlinks → extension/core/*.js
|
│ │ └── js/ symlinks → extension/core/*.js
|
||||||
│ └── tests/ pytest + fixture page
|
│ └── tests/ pytest + fixture page
|
||||||
|
├── automation-js/ Node/TS package for Playwright / camoufox-js
|
||||||
|
│ ├── src/session.ts ← ExtractorSession (async, one class)
|
||||||
|
│ │ └── js/ symlinks → extension/core/*.js
|
||||||
|
│ └── tests/ vitest + real Chromium, shared fixture
|
||||||
├── scripts/package_extension.py optional zip for distribution
|
├── scripts/package_extension.py optional zip for distribution
|
||||||
├── .gitea/workflows/ci.yml Gitea Actions: JS lint + pytest + package smoke
|
├── .gitea/workflows/ci.yml Gitea Actions: JS lint + pytest + package smoke
|
||||||
├── CURSOR_PROMPT.md paste into other Cursor chats
|
├── CURSOR_PROMPT.md paste into other Cursor chats
|
||||||
@ -32,9 +40,11 @@ context-extractor/
|
|||||||
|
|
||||||
`extension/core/dom.js` and `extension/core/prompt.js` are the single source
|
`extension/core/dom.js` and `extension/core/prompt.js` are the single source
|
||||||
of truth for DOM/markdown/prompt logic — the extension loads them directly as
|
of truth for DOM/markdown/prompt logic — the extension loads them directly as
|
||||||
content-script files, and the Python package reads the exact same files
|
content-script files, and **both** automation packages read the exact same
|
||||||
(via symlinks) and runs them with `page.evaluate()`. There's only one place
|
files (via symlinks) and run them with `page.evaluate()`. There's only one
|
||||||
to fix a markdown-formatting bug.
|
place to fix a markdown-formatting bug, and the Python/JS test suites both
|
||||||
|
run against the same fixture page (`automation/tests/fixtures/sample.html`)
|
||||||
|
so a regression can't silently diverge between the two.
|
||||||
|
|
||||||
## Use from another Cursor project
|
## Use from another Cursor project
|
||||||
|
|
||||||
@ -126,6 +136,27 @@ Headless CLI, handy in CI:
|
|||||||
context-extractor https://example.com --selector "#main" --engine camoufox --out prompt.md
|
context-extractor https://example.com --selector "#main" --engine camoufox --out prompt.md
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### JS/TS twin (`automation-js/`) — for Node/Playwright or camoufox-js automation
|
||||||
|
|
||||||
|
Same idea, no Python subprocess:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd automation-js
|
||||||
|
npm install && npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { chromium } from 'playwright-core';
|
||||||
|
import { ExtractorSession } from 'context-extractor'; // automation-js
|
||||||
|
|
||||||
|
const page = await (await chromium.launch()).newPage();
|
||||||
|
const session = new ExtractorSession(page); // attach BEFORE navigating
|
||||||
|
await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||||
|
console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||||
|
```
|
||||||
|
|
||||||
|
See `automation-js/README.md`.
|
||||||
|
|
||||||
### Why automation doesn't reuse the extension's patcher
|
### Why automation doesn't reuse the extension's patcher
|
||||||
|
|
||||||
Camoufox executes Playwright's own JS (`page.add_init_script()`,
|
Camoufox executes Playwright's own JS (`page.add_init_script()`,
|
||||||
|
|||||||
85
automation-js/README.md
Normal file
85
automation-js/README.md
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
# context-extractor (JS/TS automation)
|
||||||
|
|
||||||
|
Node/Playwright twin of the Python `automation/` package — same
|
||||||
|
`extension/core/dom.js` + `extension/core/prompt.js` (via symlinks in
|
||||||
|
`src/js/`), same output shape, same test fixture. Use this one from Node
|
||||||
|
tooling (Playwright, `playwright-core`, or [camoufox-js](https://www.npmjs.com/package/camoufox-js))
|
||||||
|
instead of shelling out to Python.
|
||||||
|
|
||||||
|
One `ExtractorSession` class covers both sync/async use cases — Node's
|
||||||
|
Playwright API is async-only to begin with, so there's no sync/async split
|
||||||
|
like the Python package's `ExtractorSession` / `AsyncExtractorSession`.
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd automation-js
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
```
|
||||||
|
|
||||||
|
Not published to a registry yet — consume it from this checkout via a
|
||||||
|
relative `file:` dependency, or copy `dist/` + `dist/js/` into a consumer.
|
||||||
|
|
||||||
|
## Use
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { chromium } from 'playwright-core'; // or 'playwright', or camoufox-js
|
||||||
|
import { ExtractorSession } from 'context-extractor'; // this package
|
||||||
|
|
||||||
|
const browser = await chromium.launch();
|
||||||
|
const page = await browser.newPage();
|
||||||
|
const session = new ExtractorSession(page); // attach BEFORE navigating
|
||||||
|
|
||||||
|
await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||||
|
|
||||||
|
console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||||
|
console.log(session.getStore()); // { console, errors, network }
|
||||||
|
```
|
||||||
|
|
||||||
|
With [camoufox-js](https://www.npmjs.com/package/camoufox-js) — same idea,
|
||||||
|
`launchServer`/`launchOptions` differ but the returned `page` is a normal
|
||||||
|
Playwright `Page`:
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { launchOptions } from 'camoufox-js';
|
||||||
|
import { firefox } from 'playwright-core';
|
||||||
|
import { ExtractorSession } from 'context-extractor';
|
||||||
|
|
||||||
|
const browser = await firefox.launch(await launchOptions({ headless: false }));
|
||||||
|
const page = await browser.newPage();
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Why this exists (vs. shelling out to the Python package)
|
||||||
|
|
||||||
|
If your automation is already Node/Playwright (Cursor-driven scripts, a
|
||||||
|
`camoufox-js` session, a Playwright test suite), the Python package meant
|
||||||
|
spawning a subprocess and shipping JSON across a process boundary just to get
|
||||||
|
a markdown dump. This package removes that boundary — same core JS, same
|
||||||
|
output, but a normal `import` + `await` in the same process/event loop as the
|
||||||
|
rest of your automation.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm test
|
||||||
|
```
|
||||||
|
|
||||||
|
Runs against a real headless Chromium (`playwright-core`) loading the shared
|
||||||
|
fixture at `../automation/tests/fixtures/sample.html` — the exact same file
|
||||||
|
the Python `automation/tests/test_session.py` suite uses, so a
|
||||||
|
markdown-formatting regression can't silently diverge between the two
|
||||||
|
automation layers.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- `extractMarkdown(selector?)` returns `{ url, title, ts, selector, markdown }`.
|
||||||
|
- `buildAiPrompt(selector?, maxChars?)` wraps that in the same
|
||||||
|
`# Page Context` / `## Page Content` / errors / console / network prompt
|
||||||
|
format as the extension's Copy button and the Python package.
|
||||||
|
- `getStore()` / `clear(which?)` expose the same bounded ring buffers
|
||||||
|
(`console`, `errors`, `network`, capped at 200 entries each) as the Python
|
||||||
|
`_Store`.
|
||||||
|
- Call `session.detach()` if you keep a page open long after you're done
|
||||||
|
capturing and want to stop accumulating entries.
|
||||||
2134
automation-js/package-lock.json
generated
Normal file
2134
automation-js/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
54
automation-js/package.json
Normal file
54
automation-js/package.json
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
{
|
||||||
|
"name": "context-extractor",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"description": "Capture console/network/errors and extract AI-ready markdown from a Playwright (Node) page — JS/TS twin of the automation/ Python package, sharing the same extension/core/*.js.",
|
||||||
|
"license": "MIT",
|
||||||
|
"type": "module",
|
||||||
|
"main": "./dist/index.js",
|
||||||
|
"module": "./dist/index.js",
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"exports": {
|
||||||
|
".": {
|
||||||
|
"types": "./dist/index.d.ts",
|
||||||
|
"import": "./dist/index.js"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"files": [
|
||||||
|
"dist",
|
||||||
|
"README.md",
|
||||||
|
"LICENSE"
|
||||||
|
],
|
||||||
|
"scripts": {
|
||||||
|
"build": "tsup && npm run copy-js",
|
||||||
|
"copy-js": "mkdir -p dist/js && cp -L src/js/dom.js src/js/prompt.js dist/js/",
|
||||||
|
"typecheck": "tsc --noEmit",
|
||||||
|
"test": "vitest run",
|
||||||
|
"test:watch": "vitest"
|
||||||
|
},
|
||||||
|
"peerDependencies": {
|
||||||
|
"playwright-core": ">=1.40.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^22.15.0",
|
||||||
|
"playwright": "^1.55.0",
|
||||||
|
"playwright-core": "^1.55.0",
|
||||||
|
"tsup": "^8.4.0",
|
||||||
|
"typescript": "^5.8.0",
|
||||||
|
"vitest": "^3.1.0"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20"
|
||||||
|
},
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://git.levkin.ca/ilia/context-extractor.git"
|
||||||
|
},
|
||||||
|
"keywords": [
|
||||||
|
"playwright",
|
||||||
|
"camoufox",
|
||||||
|
"scraping",
|
||||||
|
"llm",
|
||||||
|
"markdown",
|
||||||
|
"browser-extension"
|
||||||
|
]
|
||||||
|
}
|
||||||
9
automation-js/src/index.ts
Normal file
9
automation-js/src/index.ts
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
export {
|
||||||
|
ExtractorSession,
|
||||||
|
type CaptureStore,
|
||||||
|
type ConsoleEntry,
|
||||||
|
type ErrorEntry,
|
||||||
|
type NetworkEntry,
|
||||||
|
type ExtractedMarkdown,
|
||||||
|
} from './session.js';
|
||||||
|
export { readSharedJs } from './loadJs.js';
|
||||||
1
automation-js/src/js/dom.js
Symbolic link
1
automation-js/src/js/dom.js
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
../../../extension/core/dom.js
|
||||||
1
automation-js/src/js/prompt.js
Symbolic link
1
automation-js/src/js/prompt.js
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
../../../extension/core/prompt.js
|
||||||
26
automation-js/src/loadJs.ts
Normal file
26
automation-js/src/loadJs.ts
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
import { existsSync, readFileSync } from 'node:fs';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const here = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read one of the shared `extension/core/*.js` files from disk, the same way
|
||||||
|
* the Python `automation/` package's `_read_js()` does — so both automation
|
||||||
|
* layers (and the browser extension) run byte-identical DOM/prompt logic.
|
||||||
|
*
|
||||||
|
* Resolves `js/<name>` next to this module: `src/js/*.js` (symlinks into
|
||||||
|
* `../../extension/core/`) when running from TS source, or `dist/js/*.js`
|
||||||
|
* (copied by `npm run build`) once built.
|
||||||
|
*/
|
||||||
|
export function readSharedJs(name: 'dom.js' | 'prompt.js'): string {
|
||||||
|
const path = join(here, 'js', name);
|
||||||
|
if (!existsSync(path)) {
|
||||||
|
throw new Error(
|
||||||
|
`Missing shared JS file: ${path}. This package expects to run from a checkout of the ` +
|
||||||
|
'context-extractor repo where automation-js/src/js/*.js are symlinks into extension/core/ ' +
|
||||||
|
'(and, once built, dist/js/*.js — see the copy-js build step).',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return readFileSync(path, 'utf-8');
|
||||||
|
}
|
||||||
265
automation-js/src/session.ts
Normal file
265
automation-js/src/session.ts
Normal file
@ -0,0 +1,265 @@
|
|||||||
|
/**
|
||||||
|
* Console/network/error capture + markdown extraction for a Playwright
|
||||||
|
* (Node) page — the JS/TS twin of the Python `automation/` package's
|
||||||
|
* `ExtractorSession` / `AsyncExtractorSession`. One class here covers both,
|
||||||
|
* since Node's Playwright API is async-only to begin with.
|
||||||
|
*
|
||||||
|
* Works with anything that hands you a real Playwright `Page`: `playwright`,
|
||||||
|
* `playwright-core`, `@playwright/test`, and `camoufox-js` (which wraps
|
||||||
|
* `playwright-core` directly).
|
||||||
|
*
|
||||||
|
* Design notes (see the repo README for the full rationale — same as the
|
||||||
|
* Python package):
|
||||||
|
* - Console/network/page-error capture uses Playwright's *native* event
|
||||||
|
* hooks (`page.on('console'/'request'/'requestfinished'/'requestfailed'/'pageerror')`)
|
||||||
|
* rather than an injected JS patcher, because:
|
||||||
|
* 1. It's more robust — no reliance on `page.addInitScript()`, which is
|
||||||
|
* known to be unreliable under Camoufox's isolated-world execution
|
||||||
|
* model (https://github.com/daijro/camoufox/issues/48).
|
||||||
|
* 2. It captures more than the extension does (any resource type, not
|
||||||
|
* just fetch/XHR), and can't be blocked by a page's CSP.
|
||||||
|
* - Markdown extraction and CSS-selector helpers *do* need to run inside
|
||||||
|
* the page (DOM traversal). Those live in `extension/core/*.js` and are
|
||||||
|
* read from disk here, then run via `page.evaluate()`. That's safe under
|
||||||
|
* Camoufox's default isolated world because that code only reads the DOM
|
||||||
|
* — it never writes to the live page — so no `mainWorld`/init-script
|
||||||
|
* workaround is required.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ConsoleMessage, Page, Request } from 'playwright-core';
|
||||||
|
import { readSharedJs } from './loadJs.js';
|
||||||
|
|
||||||
|
const MAX_ENTRIES = 200;
|
||||||
|
|
||||||
|
export interface ConsoleEntry {
|
||||||
|
level: string;
|
||||||
|
ts: number;
|
||||||
|
msg: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ErrorEntry {
|
||||||
|
type: string;
|
||||||
|
ts: number;
|
||||||
|
msg: string;
|
||||||
|
source: string;
|
||||||
|
line: number;
|
||||||
|
col: number;
|
||||||
|
stack: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface NetworkEntry {
|
||||||
|
type: string;
|
||||||
|
method: string;
|
||||||
|
url: string;
|
||||||
|
status: number;
|
||||||
|
ts: number;
|
||||||
|
duration: number;
|
||||||
|
error: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureStore {
|
||||||
|
console: ConsoleEntry[];
|
||||||
|
errors: ErrorEntry[];
|
||||||
|
network: NetworkEntry[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ExtractedMarkdown {
|
||||||
|
url: string;
|
||||||
|
title: string;
|
||||||
|
ts: number;
|
||||||
|
selector: string;
|
||||||
|
markdown: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CONSOLE_LEVEL_MAP: Record<string, string> = { warning: 'warn' };
|
||||||
|
|
||||||
|
function now(): number {
|
||||||
|
return Date.now();
|
||||||
|
}
|
||||||
|
|
||||||
|
class RingStore implements CaptureStore {
|
||||||
|
console: ConsoleEntry[] = [];
|
||||||
|
errors: ErrorEntry[] = [];
|
||||||
|
network: NetworkEntry[] = [];
|
||||||
|
|
||||||
|
push<K extends keyof CaptureStore>(which: K, entry: CaptureStore[K][number]): void {
|
||||||
|
const arr = this[which];
|
||||||
|
(arr as Array<CaptureStore[K][number]>).push(entry);
|
||||||
|
while (arr.length > MAX_ENTRIES) arr.shift();
|
||||||
|
}
|
||||||
|
|
||||||
|
clear(which?: keyof CaptureStore): void {
|
||||||
|
if (which) {
|
||||||
|
this[which].length = 0;
|
||||||
|
} else {
|
||||||
|
this.console.length = 0;
|
||||||
|
this.errors.length = 0;
|
||||||
|
this.network.length = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
asDict(): CaptureStore {
|
||||||
|
return { console: this.console, errors: this.errors, network: this.network };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildExtractScript(selector: string | null | undefined): string {
|
||||||
|
const domJs = readSharedJs('dom.js');
|
||||||
|
const selJson = JSON.stringify(selector || '');
|
||||||
|
return `
|
||||||
|
(() => {
|
||||||
|
${domJs}
|
||||||
|
let el = null;
|
||||||
|
const sel = ${selJson};
|
||||||
|
if (sel) { try { el = document.querySelector(sel); } catch (_) { el = null; } }
|
||||||
|
if (!el) el = document.body;
|
||||||
|
return extractMarkdown(el);
|
||||||
|
})()
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildPromptScript(
|
||||||
|
meta: Pick<ExtractedMarkdown, 'url' | 'title' | 'ts' | 'selector'>,
|
||||||
|
markdown: string,
|
||||||
|
store: CaptureStore,
|
||||||
|
maxChars: number | undefined,
|
||||||
|
): string {
|
||||||
|
const promptJs = readSharedJs('prompt.js');
|
||||||
|
const maxCharsJs = maxChars === undefined ? 'undefined' : JSON.stringify(maxChars);
|
||||||
|
return `
|
||||||
|
(() => {
|
||||||
|
${promptJs}
|
||||||
|
return buildAIPrompt(${JSON.stringify(meta)}, ${JSON.stringify(markdown)}, ${JSON.stringify(store)}, ${maxCharsJs});
|
||||||
|
})()
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attach to a page BEFORE navigating, so no console/network/error activity
|
||||||
|
* from the initial load is missed.
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* import { chromium } from 'playwright-core';
|
||||||
|
* import { ExtractorSession } from 'context-extractor';
|
||||||
|
*
|
||||||
|
* const page = await (await chromium.launch()).newPage();
|
||||||
|
* const session = new ExtractorSession(page);
|
||||||
|
* await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||||
|
* console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export class ExtractorSession {
|
||||||
|
readonly page: Page;
|
||||||
|
private readonly store = new RingStore();
|
||||||
|
private readonly requestStarts = new WeakMap<Request, number>();
|
||||||
|
|
||||||
|
constructor(page: Page) {
|
||||||
|
this.page = page;
|
||||||
|
page.on('console', this.onConsole);
|
||||||
|
page.on('pageerror', this.onPageError);
|
||||||
|
page.on('request', this.onRequest);
|
||||||
|
page.on('requestfinished', this.onRequestFinished);
|
||||||
|
page.on('requestfailed', this.onRequestFailed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Stop listening. Call this if you keep a page open long after you're done capturing. */
|
||||||
|
detach(): void {
|
||||||
|
this.page.off('console', this.onConsole);
|
||||||
|
this.page.off('pageerror', this.onPageError);
|
||||||
|
this.page.off('request', this.onRequest);
|
||||||
|
this.page.off('requestfinished', this.onRequestFinished);
|
||||||
|
this.page.off('requestfailed', this.onRequestFailed);
|
||||||
|
}
|
||||||
|
|
||||||
|
getStore(): CaptureStore {
|
||||||
|
return this.store.asDict();
|
||||||
|
}
|
||||||
|
|
||||||
|
clear(which?: keyof CaptureStore): void {
|
||||||
|
this.store.clear(which);
|
||||||
|
}
|
||||||
|
|
||||||
|
async extractMarkdown(selector?: string | null): Promise<ExtractedMarkdown> {
|
||||||
|
const markdown = (await this.page.evaluate(buildExtractScript(selector))) as string;
|
||||||
|
return {
|
||||||
|
url: this.page.url(),
|
||||||
|
title: await this.page.title(),
|
||||||
|
ts: now(),
|
||||||
|
selector: selector || 'body',
|
||||||
|
markdown: markdown || '',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async buildAiPrompt(selector?: string | null, maxChars?: number): Promise<string> {
|
||||||
|
const extracted = await this.extractMarkdown(selector);
|
||||||
|
const meta = {
|
||||||
|
url: extracted.url,
|
||||||
|
title: extracted.title,
|
||||||
|
ts: extracted.ts,
|
||||||
|
selector: extracted.selector,
|
||||||
|
};
|
||||||
|
const script = buildPromptScript(meta, extracted.markdown, this.getStore(), maxChars);
|
||||||
|
return (await this.page.evaluate(script)) as string;
|
||||||
|
}
|
||||||
|
|
||||||
|
private onConsole = (msg: ConsoleMessage): void => {
|
||||||
|
const type = msg.type();
|
||||||
|
const level = CONSOLE_LEVEL_MAP[type] ?? type;
|
||||||
|
this.store.push('console', { level, ts: now(), msg: msg.text() });
|
||||||
|
};
|
||||||
|
|
||||||
|
private onPageError = (error: Error): void => {
|
||||||
|
this.store.push('errors', {
|
||||||
|
type: 'error',
|
||||||
|
ts: now(),
|
||||||
|
msg: error?.message ?? String(error),
|
||||||
|
source: '',
|
||||||
|
line: 0,
|
||||||
|
col: 0,
|
||||||
|
stack: error?.stack ?? '',
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
private onRequest = (request: Request): void => {
|
||||||
|
this.requestStarts.set(request, Date.now());
|
||||||
|
};
|
||||||
|
|
||||||
|
private onRequestFinished = async (request: Request): Promise<void> => {
|
||||||
|
const start = this.requestStarts.get(request);
|
||||||
|
this.requestStarts.delete(request);
|
||||||
|
const duration = start ? Date.now() - start : 0;
|
||||||
|
let status = 0;
|
||||||
|
try {
|
||||||
|
const response = await request.response();
|
||||||
|
if (response) status = response.status();
|
||||||
|
} catch {
|
||||||
|
/* response unavailable — leave status 0 */
|
||||||
|
}
|
||||||
|
this.store.push('network', {
|
||||||
|
type: request.resourceType(),
|
||||||
|
method: request.method(),
|
||||||
|
url: request.url(),
|
||||||
|
status,
|
||||||
|
ts: now(),
|
||||||
|
duration,
|
||||||
|
error: null,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
private onRequestFailed = (request: Request): void => {
|
||||||
|
const start = this.requestStarts.get(request);
|
||||||
|
this.requestStarts.delete(request);
|
||||||
|
const duration = start ? Date.now() - start : 0;
|
||||||
|
const failure = request.failure();
|
||||||
|
this.store.push('network', {
|
||||||
|
type: request.resourceType(),
|
||||||
|
method: request.method(),
|
||||||
|
url: request.url(),
|
||||||
|
status: 0,
|
||||||
|
ts: now(),
|
||||||
|
duration,
|
||||||
|
error: failure?.errorText || 'Network error',
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
117
automation-js/tests/session.test.ts
Normal file
117
automation-js/tests/session.test.ts
Normal file
@ -0,0 +1,117 @@
|
|||||||
|
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||||
|
import { chromium, type Browser, type Page } from 'playwright-core';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { dirname, join } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { ExtractorSession } from '../src/session.js';
|
||||||
|
|
||||||
|
// Same golden fixture the Python `automation/tests/` suite uses — one file,
|
||||||
|
// two automation layers, so a markdown-formatting bug can't silently diverge
|
||||||
|
// between them.
|
||||||
|
const here = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const FIXTURE_URL = pathToFileURL(
|
||||||
|
join(here, '..', '..', 'automation', 'tests', 'fixtures', 'sample.html'),
|
||||||
|
).toString();
|
||||||
|
|
||||||
|
let browser: Browser;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
browser = await chromium.launch({ headless: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await browser.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
async function withPage<T>(fn: (page: Page) => Promise<T>): Promise<T> {
|
||||||
|
const p = await browser.newPage();
|
||||||
|
try {
|
||||||
|
return await fn(p);
|
||||||
|
} finally {
|
||||||
|
await p.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('ExtractorSession', () => {
|
||||||
|
it('captures console messages, page errors, and network activity', async () => {
|
||||||
|
await withPage(async (page) => {
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||||
|
await page.waitForTimeout(150);
|
||||||
|
|
||||||
|
const store = session.getStore();
|
||||||
|
const levels = new Set(store.console.map((e) => e.level));
|
||||||
|
expect(levels.has('warn')).toBe(true);
|
||||||
|
expect(
|
||||||
|
levels.has('error') ||
|
||||||
|
store.console.some((e) => (e.msg || '').toLowerCase().includes('error')),
|
||||||
|
).toBe(true);
|
||||||
|
|
||||||
|
expect(store.errors.some((e) => (e.msg || '').includes('fixture boom'))).toBe(true);
|
||||||
|
expect(store.network.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extracts markdown from the full body and from a selector', async () => {
|
||||||
|
await withPage(async (page) => {
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
const full = await session.extractMarkdown();
|
||||||
|
expect(full.title).toBe('Context Extractor Fixture');
|
||||||
|
expect(full.markdown).toContain('Fixture Page');
|
||||||
|
expect(full.markdown).toContain('**world**');
|
||||||
|
expect(full.markdown).toContain('[docs](/docs)');
|
||||||
|
expect(full.markdown).toContain('- Alpha');
|
||||||
|
|
||||||
|
const scoped = await session.extractMarkdown('#main-content');
|
||||||
|
expect(scoped.selector).toBe('#main-content');
|
||||||
|
expect(scoped.markdown).toContain('Section');
|
||||||
|
expect(scoped.markdown).not.toContain('Skip this noise');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('excludes hidden nodes (inline style, stylesheet class, [hidden], aria-hidden) from markdown', async () => {
|
||||||
|
await withPage(async (page) => {
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||||
|
|
||||||
|
const { markdown } = await session.extractMarkdown('#main-content');
|
||||||
|
expect(markdown).not.toContain('should-not-appear');
|
||||||
|
expect(markdown).not.toContain('secretConfig');
|
||||||
|
expect(markdown).not.toContain('lixTracking');
|
||||||
|
expect(markdown).toContain('Alpha');
|
||||||
|
expect(markdown).toContain('Beta');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('builds an AI prompt with the expected sections', async () => {
|
||||||
|
await withPage(async (page) => {
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||||
|
await page.waitForTimeout(150);
|
||||||
|
|
||||||
|
const prompt = await session.buildAiPrompt('#main-content');
|
||||||
|
expect(prompt.startsWith('# Page Context')).toBe(true);
|
||||||
|
expect(prompt).toContain('## Page Content');
|
||||||
|
expect(prompt).toContain('Hello **world**');
|
||||||
|
expect(/JavaScript Errors|Console Errors/.test(prompt)).toBe(true);
|
||||||
|
expect(prompt).toContain('_Extracted by Context Extractor_');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clears the capture store, selectively and fully', async () => {
|
||||||
|
await withPage(async (page) => {
|
||||||
|
const session = new ExtractorSession(page);
|
||||||
|
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||||
|
await page.waitForTimeout(100);
|
||||||
|
|
||||||
|
expect(session.getStore().console.length).toBeGreaterThan(0);
|
||||||
|
session.clear('console');
|
||||||
|
expect(session.getStore().console).toEqual([]);
|
||||||
|
|
||||||
|
session.clear();
|
||||||
|
expect(session.getStore()).toEqual({ console: [], errors: [], network: [] });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
21
automation-js/tsconfig.json
Normal file
21
automation-js/tsconfig.json
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ES2022",
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "Bundler",
|
||||||
|
"lib": ["ES2022"],
|
||||||
|
"strict": true,
|
||||||
|
"declaration": true,
|
||||||
|
"declarationMap": true,
|
||||||
|
"sourceMap": true,
|
||||||
|
"skipLibCheck": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"outDir": "dist",
|
||||||
|
"rootDir": "src",
|
||||||
|
"types": ["node"]
|
||||||
|
},
|
||||||
|
"include": ["src/**/*.ts"],
|
||||||
|
"exclude": ["node_modules", "dist", "tests"]
|
||||||
|
}
|
||||||
12
automation-js/tsup.config.ts
Normal file
12
automation-js/tsup.config.ts
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
import { defineConfig } from 'tsup';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
entry: { index: 'src/index.ts' },
|
||||||
|
format: ['esm'],
|
||||||
|
dts: true,
|
||||||
|
sourcemap: true,
|
||||||
|
splitting: false,
|
||||||
|
treeshake: true,
|
||||||
|
clean: true,
|
||||||
|
external: ['playwright-core', '@playwright/test', 'playwright'],
|
||||||
|
});
|
||||||
8
automation-js/vitest.config.ts
Normal file
8
automation-js/vitest.config.ts
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
test: {
|
||||||
|
include: ['tests/**/*.test.ts'],
|
||||||
|
testTimeout: 20_000,
|
||||||
|
},
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user