Add Node/TS automation twin sharing the extension core.
Expose ExtractorSession for Playwright/camoufox-js so Node automations can import context dumps instead of shelling out to the Python package. Same fixture as the Python tests; wired into Makefile and Gitea CI.
This commit is contained in:
parent
f79552249e
commit
55018a6e3e
@ -56,6 +56,10 @@ jobs:
|
||||
test -L automation/context_extractor/js/prompt.js
|
||||
diff -q automation/context_extractor/js/dom.js extension/core/dom.js
|
||||
diff -q automation/context_extractor/js/prompt.js extension/core/prompt.js
|
||||
test -L automation-js/src/js/dom.js
|
||||
test -L automation-js/src/js/prompt.js
|
||||
diff -q automation-js/src/js/dom.js extension/core/dom.js
|
||||
diff -q automation-js/src/js/prompt.js extension/core/prompt.js
|
||||
|
||||
- name: Install package + dev deps
|
||||
run: |
|
||||
@ -73,3 +77,34 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/package_extension.py
|
||||
test -f dist/context-extractor-extension.zip
|
||||
|
||||
quality-js:
|
||||
name: automation-js (vitest + real Chromium)
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: automation-js
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Install deps
|
||||
run: npm install
|
||||
|
||||
- name: Install Playwright Chromium
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Typecheck
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Vitest
|
||||
run: npm test
|
||||
|
||||
- name: Build (verifies dist/js copy step)
|
||||
run: npm run build
|
||||
|
||||
10
Makefile
10
Makefile
@ -1,4 +1,4 @@
|
||||
.PHONY: help test lint package ci install
|
||||
.PHONY: help test test-js lint package ci install install-js
|
||||
|
||||
help: ## Show targets
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
||||
@ -8,9 +8,15 @@ install: ## Create venv and install automation package + Playwright Chromium
|
||||
cd automation && .venv/bin/pip install -U pip && .venv/bin/pip install -e ".[dev]"
|
||||
cd automation && .venv/bin/playwright install chromium
|
||||
|
||||
install-js: ## npm install automation-js (uses the Chromium cache npx playwright already has)
|
||||
cd automation-js && npm install
|
||||
|
||||
test: ## Run pytest
|
||||
cd automation && .venv/bin/pytest -q
|
||||
|
||||
test-js: ## Run automation-js vitest suite (real headless Chromium)
|
||||
cd automation-js && npm test
|
||||
|
||||
lint: ## Syntax-check extension JS + validate manifest JSON
|
||||
@node --check extension/core/dom.js
|
||||
@node --check extension/core/prompt.js
|
||||
@ -24,4 +30,4 @@ lint: ## Syntax-check extension JS + validate manifest JSON
|
||||
package: ## Zip extension/ into dist/
|
||||
python3 scripts/package_extension.py
|
||||
|
||||
ci: lint test package ## Local stand-in for GitHub Actions
|
||||
ci: lint test test-js package ## Local stand-in for GitHub Actions
|
||||
|
||||
39
README.md
39
README.md
@ -4,12 +4,16 @@
|
||||
**Clone:** `gitea@git.levkin.ca:ilia/context-extractor.git`
|
||||
|
||||
Capture console logs, network activity, JS errors, and clean markdown content
|
||||
from a webpage, formatted as an AI-ready prompt. Ships two ways from one
|
||||
from a webpage, formatted as an AI-ready prompt. Ships three ways from one
|
||||
shared core:
|
||||
|
||||
- **`extension/`** — a browser extension (Manifest V3) for interactive use.
|
||||
- **`automation/`** — a Python package for scripted/headless use with
|
||||
Playwright or [Camoufox](https://camoufox.com).
|
||||
- **`automation-js/`** — a Node/TS twin of `automation/` for Playwright
|
||||
(`playwright-core`) or [camoufox-js](https://www.npmjs.com/package/camoufox-js)
|
||||
automation that's already JS, so it can `import` the session directly
|
||||
instead of shelling out to Python.
|
||||
|
||||
```
|
||||
context-extractor/
|
||||
@ -24,6 +28,10 @@ context-extractor/
|
||||
│ ├── context_extractor/ session + CLI
|
||||
│ │ └── js/ symlinks → extension/core/*.js
|
||||
│ └── tests/ pytest + fixture page
|
||||
├── automation-js/ Node/TS package for Playwright / camoufox-js
|
||||
│ ├── src/session.ts ← ExtractorSession (async, one class)
|
||||
│ │ └── js/ symlinks → extension/core/*.js
|
||||
│ └── tests/ vitest + real Chromium, shared fixture
|
||||
├── scripts/package_extension.py optional zip for distribution
|
||||
├── .gitea/workflows/ci.yml Gitea Actions: JS lint + pytest + package smoke
|
||||
├── CURSOR_PROMPT.md paste into other Cursor chats
|
||||
@ -32,9 +40,11 @@ context-extractor/
|
||||
|
||||
`extension/core/dom.js` and `extension/core/prompt.js` are the single source
|
||||
of truth for DOM/markdown/prompt logic — the extension loads them directly as
|
||||
content-script files, and the Python package reads the exact same files
|
||||
(via symlinks) and runs them with `page.evaluate()`. There's only one place
|
||||
to fix a markdown-formatting bug.
|
||||
content-script files, and **both** automation packages read the exact same
|
||||
files (via symlinks) and run them with `page.evaluate()`. There's only one
|
||||
place to fix a markdown-formatting bug, and the Python/JS test suites both
|
||||
run against the same fixture page (`automation/tests/fixtures/sample.html`)
|
||||
so a regression can't silently diverge between the two.
|
||||
|
||||
## Use from another Cursor project
|
||||
|
||||
@ -126,6 +136,27 @@ Headless CLI, handy in CI:
|
||||
context-extractor https://example.com --selector "#main" --engine camoufox --out prompt.md
|
||||
```
|
||||
|
||||
### JS/TS twin (`automation-js/`) — for Node/Playwright or camoufox-js automation
|
||||
|
||||
Same idea, no Python subprocess:
|
||||
|
||||
```bash
|
||||
cd automation-js
|
||||
npm install && npm run build
|
||||
```
|
||||
|
||||
```ts
|
||||
import { chromium } from 'playwright-core';
|
||||
import { ExtractorSession } from 'context-extractor'; // automation-js
|
||||
|
||||
const page = await (await chromium.launch()).newPage();
|
||||
const session = new ExtractorSession(page); // attach BEFORE navigating
|
||||
await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||
console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||
```
|
||||
|
||||
See `automation-js/README.md`.
|
||||
|
||||
### Why automation doesn't reuse the extension's patcher
|
||||
|
||||
Camoufox executes Playwright's own JS (`page.add_init_script()`,
|
||||
|
||||
85
automation-js/README.md
Normal file
85
automation-js/README.md
Normal file
@ -0,0 +1,85 @@
|
||||
# context-extractor (JS/TS automation)
|
||||
|
||||
Node/Playwright twin of the Python `automation/` package — same
|
||||
`extension/core/dom.js` + `extension/core/prompt.js` (via symlinks in
|
||||
`src/js/`), same output shape, same test fixture. Use this one from Node
|
||||
tooling (Playwright, `playwright-core`, or [camoufox-js](https://www.npmjs.com/package/camoufox-js))
|
||||
instead of shelling out to Python.
|
||||
|
||||
One `ExtractorSession` class covers both sync/async use cases — Node's
|
||||
Playwright API is async-only to begin with, so there's no sync/async split
|
||||
like the Python package's `ExtractorSession` / `AsyncExtractorSession`.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
cd automation-js
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
|
||||
Not published to a registry yet — consume it from this checkout via a
|
||||
relative `file:` dependency, or copy `dist/` + `dist/js/` into a consumer.
|
||||
|
||||
## Use
|
||||
|
||||
```ts
|
||||
import { chromium } from 'playwright-core'; // or 'playwright', or camoufox-js
|
||||
import { ExtractorSession } from 'context-extractor'; // this package
|
||||
|
||||
const browser = await chromium.launch();
|
||||
const page = await browser.newPage();
|
||||
const session = new ExtractorSession(page); // attach BEFORE navigating
|
||||
|
||||
await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||
|
||||
console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||
console.log(session.getStore()); // { console, errors, network }
|
||||
```
|
||||
|
||||
With [camoufox-js](https://www.npmjs.com/package/camoufox-js) — same idea,
|
||||
`launchServer`/`launchOptions` differ but the returned `page` is a normal
|
||||
Playwright `Page`:
|
||||
|
||||
```ts
|
||||
import { launchOptions } from 'camoufox-js';
|
||||
import { firefox } from 'playwright-core';
|
||||
import { ExtractorSession } from 'context-extractor';
|
||||
|
||||
const browser = await firefox.launch(await launchOptions({ headless: false }));
|
||||
const page = await browser.newPage();
|
||||
const session = new ExtractorSession(page);
|
||||
```
|
||||
|
||||
## Why this exists (vs. shelling out to the Python package)
|
||||
|
||||
If your automation is already Node/Playwright (Cursor-driven scripts, a
|
||||
`camoufox-js` session, a Playwright test suite), the Python package meant
|
||||
spawning a subprocess and shipping JSON across a process boundary just to get
|
||||
a markdown dump. This package removes that boundary — same core JS, same
|
||||
output, but a normal `import` + `await` in the same process/event loop as the
|
||||
rest of your automation.
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
npm test
|
||||
```
|
||||
|
||||
Runs against a real headless Chromium (`playwright-core`) loading the shared
|
||||
fixture at `../automation/tests/fixtures/sample.html` — the exact same file
|
||||
the Python `automation/tests/test_session.py` suite uses, so a
|
||||
markdown-formatting regression can't silently diverge between the two
|
||||
automation layers.
|
||||
|
||||
## Notes
|
||||
|
||||
- `extractMarkdown(selector?)` returns `{ url, title, ts, selector, markdown }`.
|
||||
- `buildAiPrompt(selector?, maxChars?)` wraps that in the same
|
||||
`# Page Context` / `## Page Content` / errors / console / network prompt
|
||||
format as the extension's Copy button and the Python package.
|
||||
- `getStore()` / `clear(which?)` expose the same bounded ring buffers
|
||||
(`console`, `errors`, `network`, capped at 200 entries each) as the Python
|
||||
`_Store`.
|
||||
- Call `session.detach()` if you keep a page open long after you're done
|
||||
capturing and want to stop accumulating entries.
|
||||
2134
automation-js/package-lock.json
generated
Normal file
2134
automation-js/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
54
automation-js/package.json
Normal file
54
automation-js/package.json
Normal file
@ -0,0 +1,54 @@
|
||||
{
|
||||
"name": "context-extractor",
|
||||
"version": "1.0.0",
|
||||
"description": "Capture console/network/errors and extract AI-ready markdown from a Playwright (Node) page — JS/TS twin of the automation/ Python package, sharing the same extension/core/*.js.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsup && npm run copy-js",
|
||||
"copy-js": "mkdir -p dist/js && cp -L src/js/dom.js src/js/prompt.js dist/js/",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"playwright-core": ">=1.40.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.15.0",
|
||||
"playwright": "^1.55.0",
|
||||
"playwright-core": "^1.55.0",
|
||||
"tsup": "^8.4.0",
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^3.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://git.levkin.ca/ilia/context-extractor.git"
|
||||
},
|
||||
"keywords": [
|
||||
"playwright",
|
||||
"camoufox",
|
||||
"scraping",
|
||||
"llm",
|
||||
"markdown",
|
||||
"browser-extension"
|
||||
]
|
||||
}
|
||||
9
automation-js/src/index.ts
Normal file
9
automation-js/src/index.ts
Normal file
@ -0,0 +1,9 @@
|
||||
export {
|
||||
ExtractorSession,
|
||||
type CaptureStore,
|
||||
type ConsoleEntry,
|
||||
type ErrorEntry,
|
||||
type NetworkEntry,
|
||||
type ExtractedMarkdown,
|
||||
} from './session.js';
|
||||
export { readSharedJs } from './loadJs.js';
|
||||
1
automation-js/src/js/dom.js
Symbolic link
1
automation-js/src/js/dom.js
Symbolic link
@ -0,0 +1 @@
|
||||
../../../extension/core/dom.js
|
||||
1
automation-js/src/js/prompt.js
Symbolic link
1
automation-js/src/js/prompt.js
Symbolic link
@ -0,0 +1 @@
|
||||
../../../extension/core/prompt.js
|
||||
26
automation-js/src/loadJs.ts
Normal file
26
automation-js/src/loadJs.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
/**
|
||||
* Read one of the shared `extension/core/*.js` files from disk, the same way
|
||||
* the Python `automation/` package's `_read_js()` does — so both automation
|
||||
* layers (and the browser extension) run byte-identical DOM/prompt logic.
|
||||
*
|
||||
* Resolves `js/<name>` next to this module: `src/js/*.js` (symlinks into
|
||||
* `../../extension/core/`) when running from TS source, or `dist/js/*.js`
|
||||
* (copied by `npm run build`) once built.
|
||||
*/
|
||||
export function readSharedJs(name: 'dom.js' | 'prompt.js'): string {
|
||||
const path = join(here, 'js', name);
|
||||
if (!existsSync(path)) {
|
||||
throw new Error(
|
||||
`Missing shared JS file: ${path}. This package expects to run from a checkout of the ` +
|
||||
'context-extractor repo where automation-js/src/js/*.js are symlinks into extension/core/ ' +
|
||||
'(and, once built, dist/js/*.js — see the copy-js build step).',
|
||||
);
|
||||
}
|
||||
return readFileSync(path, 'utf-8');
|
||||
}
|
||||
265
automation-js/src/session.ts
Normal file
265
automation-js/src/session.ts
Normal file
@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Console/network/error capture + markdown extraction for a Playwright
|
||||
* (Node) page — the JS/TS twin of the Python `automation/` package's
|
||||
* `ExtractorSession` / `AsyncExtractorSession`. One class here covers both,
|
||||
* since Node's Playwright API is async-only to begin with.
|
||||
*
|
||||
* Works with anything that hands you a real Playwright `Page`: `playwright`,
|
||||
* `playwright-core`, `@playwright/test`, and `camoufox-js` (which wraps
|
||||
* `playwright-core` directly).
|
||||
*
|
||||
* Design notes (see the repo README for the full rationale — same as the
|
||||
* Python package):
|
||||
* - Console/network/page-error capture uses Playwright's *native* event
|
||||
* hooks (`page.on('console'/'request'/'requestfinished'/'requestfailed'/'pageerror')`)
|
||||
* rather than an injected JS patcher, because:
|
||||
* 1. It's more robust — no reliance on `page.addInitScript()`, which is
|
||||
* known to be unreliable under Camoufox's isolated-world execution
|
||||
* model (https://github.com/daijro/camoufox/issues/48).
|
||||
* 2. It captures more than the extension does (any resource type, not
|
||||
* just fetch/XHR), and can't be blocked by a page's CSP.
|
||||
* - Markdown extraction and CSS-selector helpers *do* need to run inside
|
||||
* the page (DOM traversal). Those live in `extension/core/*.js` and are
|
||||
* read from disk here, then run via `page.evaluate()`. That's safe under
|
||||
* Camoufox's default isolated world because that code only reads the DOM
|
||||
* — it never writes to the live page — so no `mainWorld`/init-script
|
||||
* workaround is required.
|
||||
*/
|
||||
|
||||
import type { ConsoleMessage, Page, Request } from 'playwright-core';
|
||||
import { readSharedJs } from './loadJs.js';
|
||||
|
||||
const MAX_ENTRIES = 200;
|
||||
|
||||
export interface ConsoleEntry {
|
||||
level: string;
|
||||
ts: number;
|
||||
msg: string;
|
||||
}
|
||||
|
||||
export interface ErrorEntry {
|
||||
type: string;
|
||||
ts: number;
|
||||
msg: string;
|
||||
source: string;
|
||||
line: number;
|
||||
col: number;
|
||||
stack: string;
|
||||
}
|
||||
|
||||
export interface NetworkEntry {
|
||||
type: string;
|
||||
method: string;
|
||||
url: string;
|
||||
status: number;
|
||||
ts: number;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface CaptureStore {
|
||||
console: ConsoleEntry[];
|
||||
errors: ErrorEntry[];
|
||||
network: NetworkEntry[];
|
||||
}
|
||||
|
||||
export interface ExtractedMarkdown {
|
||||
url: string;
|
||||
title: string;
|
||||
ts: number;
|
||||
selector: string;
|
||||
markdown: string;
|
||||
}
|
||||
|
||||
const CONSOLE_LEVEL_MAP: Record<string, string> = { warning: 'warn' };
|
||||
|
||||
function now(): number {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
class RingStore implements CaptureStore {
|
||||
console: ConsoleEntry[] = [];
|
||||
errors: ErrorEntry[] = [];
|
||||
network: NetworkEntry[] = [];
|
||||
|
||||
push<K extends keyof CaptureStore>(which: K, entry: CaptureStore[K][number]): void {
|
||||
const arr = this[which];
|
||||
(arr as Array<CaptureStore[K][number]>).push(entry);
|
||||
while (arr.length > MAX_ENTRIES) arr.shift();
|
||||
}
|
||||
|
||||
clear(which?: keyof CaptureStore): void {
|
||||
if (which) {
|
||||
this[which].length = 0;
|
||||
} else {
|
||||
this.console.length = 0;
|
||||
this.errors.length = 0;
|
||||
this.network.length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
asDict(): CaptureStore {
|
||||
return { console: this.console, errors: this.errors, network: this.network };
|
||||
}
|
||||
}
|
||||
|
||||
function buildExtractScript(selector: string | null | undefined): string {
|
||||
const domJs = readSharedJs('dom.js');
|
||||
const selJson = JSON.stringify(selector || '');
|
||||
return `
|
||||
(() => {
|
||||
${domJs}
|
||||
let el = null;
|
||||
const sel = ${selJson};
|
||||
if (sel) { try { el = document.querySelector(sel); } catch (_) { el = null; } }
|
||||
if (!el) el = document.body;
|
||||
return extractMarkdown(el);
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
function buildPromptScript(
|
||||
meta: Pick<ExtractedMarkdown, 'url' | 'title' | 'ts' | 'selector'>,
|
||||
markdown: string,
|
||||
store: CaptureStore,
|
||||
maxChars: number | undefined,
|
||||
): string {
|
||||
const promptJs = readSharedJs('prompt.js');
|
||||
const maxCharsJs = maxChars === undefined ? 'undefined' : JSON.stringify(maxChars);
|
||||
return `
|
||||
(() => {
|
||||
${promptJs}
|
||||
return buildAIPrompt(${JSON.stringify(meta)}, ${JSON.stringify(markdown)}, ${JSON.stringify(store)}, ${maxCharsJs});
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach to a page BEFORE navigating, so no console/network/error activity
|
||||
* from the initial load is missed.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* import { chromium } from 'playwright-core';
|
||||
* import { ExtractorSession } from 'context-extractor';
|
||||
*
|
||||
* const page = await (await chromium.launch()).newPage();
|
||||
* const session = new ExtractorSession(page);
|
||||
* await page.goto('https://example.com', { waitUntil: 'networkidle' });
|
||||
* console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main')
|
||||
* ```
|
||||
*/
|
||||
export class ExtractorSession {
|
||||
readonly page: Page;
|
||||
private readonly store = new RingStore();
|
||||
private readonly requestStarts = new WeakMap<Request, number>();
|
||||
|
||||
constructor(page: Page) {
|
||||
this.page = page;
|
||||
page.on('console', this.onConsole);
|
||||
page.on('pageerror', this.onPageError);
|
||||
page.on('request', this.onRequest);
|
||||
page.on('requestfinished', this.onRequestFinished);
|
||||
page.on('requestfailed', this.onRequestFailed);
|
||||
}
|
||||
|
||||
/** Stop listening. Call this if you keep a page open long after you're done capturing. */
|
||||
detach(): void {
|
||||
this.page.off('console', this.onConsole);
|
||||
this.page.off('pageerror', this.onPageError);
|
||||
this.page.off('request', this.onRequest);
|
||||
this.page.off('requestfinished', this.onRequestFinished);
|
||||
this.page.off('requestfailed', this.onRequestFailed);
|
||||
}
|
||||
|
||||
getStore(): CaptureStore {
|
||||
return this.store.asDict();
|
||||
}
|
||||
|
||||
clear(which?: keyof CaptureStore): void {
|
||||
this.store.clear(which);
|
||||
}
|
||||
|
||||
async extractMarkdown(selector?: string | null): Promise<ExtractedMarkdown> {
|
||||
const markdown = (await this.page.evaluate(buildExtractScript(selector))) as string;
|
||||
return {
|
||||
url: this.page.url(),
|
||||
title: await this.page.title(),
|
||||
ts: now(),
|
||||
selector: selector || 'body',
|
||||
markdown: markdown || '',
|
||||
};
|
||||
}
|
||||
|
||||
async buildAiPrompt(selector?: string | null, maxChars?: number): Promise<string> {
|
||||
const extracted = await this.extractMarkdown(selector);
|
||||
const meta = {
|
||||
url: extracted.url,
|
||||
title: extracted.title,
|
||||
ts: extracted.ts,
|
||||
selector: extracted.selector,
|
||||
};
|
||||
const script = buildPromptScript(meta, extracted.markdown, this.getStore(), maxChars);
|
||||
return (await this.page.evaluate(script)) as string;
|
||||
}
|
||||
|
||||
private onConsole = (msg: ConsoleMessage): void => {
|
||||
const type = msg.type();
|
||||
const level = CONSOLE_LEVEL_MAP[type] ?? type;
|
||||
this.store.push('console', { level, ts: now(), msg: msg.text() });
|
||||
};
|
||||
|
||||
private onPageError = (error: Error): void => {
|
||||
this.store.push('errors', {
|
||||
type: 'error',
|
||||
ts: now(),
|
||||
msg: error?.message ?? String(error),
|
||||
source: '',
|
||||
line: 0,
|
||||
col: 0,
|
||||
stack: error?.stack ?? '',
|
||||
});
|
||||
};
|
||||
|
||||
private onRequest = (request: Request): void => {
|
||||
this.requestStarts.set(request, Date.now());
|
||||
};
|
||||
|
||||
private onRequestFinished = async (request: Request): Promise<void> => {
|
||||
const start = this.requestStarts.get(request);
|
||||
this.requestStarts.delete(request);
|
||||
const duration = start ? Date.now() - start : 0;
|
||||
let status = 0;
|
||||
try {
|
||||
const response = await request.response();
|
||||
if (response) status = response.status();
|
||||
} catch {
|
||||
/* response unavailable — leave status 0 */
|
||||
}
|
||||
this.store.push('network', {
|
||||
type: request.resourceType(),
|
||||
method: request.method(),
|
||||
url: request.url(),
|
||||
status,
|
||||
ts: now(),
|
||||
duration,
|
||||
error: null,
|
||||
});
|
||||
};
|
||||
|
||||
private onRequestFailed = (request: Request): void => {
|
||||
const start = this.requestStarts.get(request);
|
||||
this.requestStarts.delete(request);
|
||||
const duration = start ? Date.now() - start : 0;
|
||||
const failure = request.failure();
|
||||
this.store.push('network', {
|
||||
type: request.resourceType(),
|
||||
method: request.method(),
|
||||
url: request.url(),
|
||||
status: 0,
|
||||
ts: now(),
|
||||
duration,
|
||||
error: failure?.errorText || 'Network error',
|
||||
});
|
||||
};
|
||||
}
|
||||
117
automation-js/tests/session.test.ts
Normal file
117
automation-js/tests/session.test.ts
Normal file
@ -0,0 +1,117 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { chromium, type Browser, type Page } from 'playwright-core';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { ExtractorSession } from '../src/session.js';
|
||||
|
||||
// Same golden fixture the Python `automation/tests/` suite uses — one file,
|
||||
// two automation layers, so a markdown-formatting bug can't silently diverge
|
||||
// between them.
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURE_URL = pathToFileURL(
|
||||
join(here, '..', '..', 'automation', 'tests', 'fixtures', 'sample.html'),
|
||||
).toString();
|
||||
|
||||
let browser: Browser;
|
||||
|
||||
beforeAll(async () => {
|
||||
browser = await chromium.launch({ headless: true });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await browser.close();
|
||||
});
|
||||
|
||||
async function withPage<T>(fn: (page: Page) => Promise<T>): Promise<T> {
|
||||
const p = await browser.newPage();
|
||||
try {
|
||||
return await fn(p);
|
||||
} finally {
|
||||
await p.close();
|
||||
}
|
||||
}
|
||||
|
||||
describe('ExtractorSession', () => {
|
||||
it('captures console messages, page errors, and network activity', async () => {
|
||||
await withPage(async (page) => {
|
||||
const session = new ExtractorSession(page);
|
||||
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForTimeout(150);
|
||||
|
||||
const store = session.getStore();
|
||||
const levels = new Set(store.console.map((e) => e.level));
|
||||
expect(levels.has('warn')).toBe(true);
|
||||
expect(
|
||||
levels.has('error') ||
|
||||
store.console.some((e) => (e.msg || '').toLowerCase().includes('error')),
|
||||
).toBe(true);
|
||||
|
||||
expect(store.errors.some((e) => (e.msg || '').includes('fixture boom'))).toBe(true);
|
||||
expect(store.network.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
it('extracts markdown from the full body and from a selector', async () => {
|
||||
await withPage(async (page) => {
|
||||
const session = new ExtractorSession(page);
|
||||
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||
|
||||
const full = await session.extractMarkdown();
|
||||
expect(full.title).toBe('Context Extractor Fixture');
|
||||
expect(full.markdown).toContain('Fixture Page');
|
||||
expect(full.markdown).toContain('**world**');
|
||||
expect(full.markdown).toContain('[docs](/docs)');
|
||||
expect(full.markdown).toContain('- Alpha');
|
||||
|
||||
const scoped = await session.extractMarkdown('#main-content');
|
||||
expect(scoped.selector).toBe('#main-content');
|
||||
expect(scoped.markdown).toContain('Section');
|
||||
expect(scoped.markdown).not.toContain('Skip this noise');
|
||||
});
|
||||
});
|
||||
|
||||
it('excludes hidden nodes (inline style, stylesheet class, [hidden], aria-hidden) from markdown', async () => {
|
||||
await withPage(async (page) => {
|
||||
const session = new ExtractorSession(page);
|
||||
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||
|
||||
const { markdown } = await session.extractMarkdown('#main-content');
|
||||
expect(markdown).not.toContain('should-not-appear');
|
||||
expect(markdown).not.toContain('secretConfig');
|
||||
expect(markdown).not.toContain('lixTracking');
|
||||
expect(markdown).toContain('Alpha');
|
||||
expect(markdown).toContain('Beta');
|
||||
});
|
||||
});
|
||||
|
||||
it('builds an AI prompt with the expected sections', async () => {
|
||||
await withPage(async (page) => {
|
||||
const session = new ExtractorSession(page);
|
||||
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForTimeout(150);
|
||||
|
||||
const prompt = await session.buildAiPrompt('#main-content');
|
||||
expect(prompt.startsWith('# Page Context')).toBe(true);
|
||||
expect(prompt).toContain('## Page Content');
|
||||
expect(prompt).toContain('Hello **world**');
|
||||
expect(/JavaScript Errors|Console Errors/.test(prompt)).toBe(true);
|
||||
expect(prompt).toContain('_Extracted by Context Extractor_');
|
||||
});
|
||||
});
|
||||
|
||||
it('clears the capture store, selectively and fully', async () => {
|
||||
await withPage(async (page) => {
|
||||
const session = new ExtractorSession(page);
|
||||
await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' });
|
||||
await page.waitForTimeout(100);
|
||||
|
||||
expect(session.getStore().console.length).toBeGreaterThan(0);
|
||||
session.clear('console');
|
||||
expect(session.getStore().console).toEqual([]);
|
||||
|
||||
session.clear();
|
||||
expect(session.getStore()).toEqual({ console: [], errors: [], network: [] });
|
||||
});
|
||||
});
|
||||
});
|
||||
21
automation-js/tsconfig.json
Normal file
21
automation-js/tsconfig.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["ES2022"],
|
||||
"strict": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"skipLibCheck": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "tests"]
|
||||
}
|
||||
12
automation-js/tsup.config.ts
Normal file
12
automation-js/tsup.config.ts
Normal file
@ -0,0 +1,12 @@
|
||||
import { defineConfig } from 'tsup';
|
||||
|
||||
export default defineConfig({
|
||||
entry: { index: 'src/index.ts' },
|
||||
format: ['esm'],
|
||||
dts: true,
|
||||
sourcemap: true,
|
||||
splitting: false,
|
||||
treeshake: true,
|
||||
clean: true,
|
||||
external: ['playwright-core', '@playwright/test', 'playwright'],
|
||||
});
|
||||
8
automation-js/vitest.config.ts
Normal file
8
automation-js/vitest.config.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ['tests/**/*.test.ts'],
|
||||
testTimeout: 20_000,
|
||||
},
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user