diff --git a/README.md b/README.md index c58646e..669385d 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,12 @@ so no `main_world_eval` workaround is needed there either. example, its DOM without this filter is 700k+ characters, almost all of it invisible config junk). Without this check you'd be feeding an LLM chameleon experiment payloads instead of page content. +- Long `data:` URLs (e.g. inlined images) are truncated in markdown links and + in AI-prompt network lines via `truncateDataUri`, so dumps stay usable. +- `inventoryControls()` / `inventory_controls()` lists buttons, menuitems, and + `aria-label` nodes with a **visible vs hidden** flag (capped). `buildAiPrompt` + includes an **Interesting Controls** section so agents can spot hidden Radix + twins without dumping the whole DOM. - `build_ai_prompt()` / the extension **Copy** button cap page content at **20,000 chars** by default (`maxChars` in JS, `max_chars=`/`--max-chars` in Python/CLI). A full `` extraction on a JS-heavy SPA can still be enormous even after diff --git a/automation-js/README.md b/automation-js/README.md index 66d81c5..e1b3ae0 100644 --- a/automation-js/README.md +++ b/automation-js/README.md @@ -34,6 +34,7 @@ const session = new ExtractorSession(page); // attach BEFORE navigating await page.goto('https://example.com', { waitUntil: 'networkidle' }); console.log(await session.buildAiPrompt()); // or session.extractMarkdown('#main') +console.log(await session.inventoryControls('#main')); // visible/hidden buttons & aria-labels console.log(session.getStore()); // { console, errors, network } ``` diff --git a/automation-js/src/index.ts b/automation-js/src/index.ts index 9794a98..593615d 100644 --- a/automation-js/src/index.ts +++ b/automation-js/src/index.ts @@ -5,5 +5,6 @@ export { type ErrorEntry, type NetworkEntry, type ExtractedMarkdown, + type InterestingControl, } from './session.js'; export { readSharedJs } from './loadJs.js'; diff --git a/automation-js/src/session.ts b/automation-js/src/session.ts index 379879c..6cbb081 100644 --- a/automation-js/src/session.ts +++ b/automation-js/src/session.ts @@ -118,18 +118,44 @@ return extractMarkdown(el); `; } +function buildInventoryScript(selector: string | null | undefined): string { + const domJs = readSharedJs('dom.js'); + const selJson = JSON.stringify(selector || ''); + return ` +(() => { +${domJs} +let el = null; +const sel = ${selJson}; +if (sel) { try { el = document.querySelector(sel); } catch (_) { el = null; } } +if (!el) el = document.body; +return inventoryInterestingControls(el); +})() +`; +} + +export interface InterestingControl { + tag: string; + role: string; + ariaLabel: string; + text: string; + visible: boolean; + selector: string; +} + function buildPromptScript( meta: Pick, markdown: string, store: CaptureStore, maxChars: number | undefined, + controls?: InterestingControl[], ): string { const promptJs = readSharedJs('prompt.js'); const maxCharsJs = maxChars === undefined ? 'undefined' : JSON.stringify(maxChars); + const controlsJs = JSON.stringify(controls ?? []); return ` (() => { ${promptJs} -return buildAIPrompt(${JSON.stringify(meta)}, ${JSON.stringify(markdown)}, ${JSON.stringify(store)}, ${maxCharsJs}); +return buildAIPrompt(${JSON.stringify(meta)}, ${JSON.stringify(markdown)}, ${JSON.stringify(store)}, ${maxCharsJs}, ${controlsJs}); })() `; } @@ -191,15 +217,28 @@ export class ExtractorSession { }; } + /** Inventory buttons / menuitems / aria-labels under selector (default body). */ + async inventoryControls(selector?: string | null): Promise { + const list = (await this.page.evaluate(buildInventoryScript(selector))) as InterestingControl[]; + return Array.isArray(list) ? list : []; + } + async buildAiPrompt(selector?: string | null, maxChars?: number): Promise { const extracted = await this.extractMarkdown(selector); + const controls = await this.inventoryControls(selector); const meta = { url: extracted.url, title: extracted.title, ts: extracted.ts, selector: extracted.selector, }; - const script = buildPromptScript(meta, extracted.markdown, this.getStore(), maxChars); + const script = buildPromptScript( + meta, + extracted.markdown, + this.getStore(), + maxChars, + controls, + ); return (await this.page.evaluate(script)) as string; } diff --git a/automation-js/tests/session.test.ts b/automation-js/tests/session.test.ts index 4f2ac75..970fae1 100644 --- a/automation-js/tests/session.test.ts +++ b/automation-js/tests/session.test.ts @@ -63,6 +63,9 @@ describe('ExtractorSession', () => { expect(full.markdown).toContain('**world**'); expect(full.markdown).toContain('[docs](/docs)'); expect(full.markdown).toContain('- Alpha'); + expect(full.markdown).toContain('[tiny-jpeg](data:image/jpeg;base64,…['); + expect(full.markdown).toContain('bytes truncated])'); + expect(full.markdown).not.toMatch(/4AAQSkZJRgABAQAAAQABAAD/); const scoped = await session.extractMarkdown('#main-content'); expect(scoped.selector).toBe('#main-content'); @@ -95,11 +98,28 @@ describe('ExtractorSession', () => { expect(prompt.startsWith('# Page Context')).toBe(true); expect(prompt).toContain('## Page Content'); expect(prompt).toContain('Hello **world**'); + expect(prompt).toContain('## Interesting Controls'); + expect(prompt).toContain('[visible]'); + expect(prompt).toContain('Document options'); + expect(prompt).toContain('### Hidden'); expect(/JavaScript Errors|Console Errors/.test(prompt)).toBe(true); expect(prompt).toContain('_Extracted by Context Extractor_'); }); }); + it('inventories visible and hidden controls', async () => { + await withPage(async (page) => { + const session = new ExtractorSession(page); + await page.goto(FIXTURE_URL, { waitUntil: 'domcontentloaded' }); + const controls = await session.inventoryControls('#main-content'); + const docs = controls.filter((c) => c.ariaLabel === 'Document options'); + expect(docs.length).toBe(2); + expect(docs.some((c) => c.visible)).toBe(true); + expect(docs.some((c) => !c.visible)).toBe(true); + expect(controls.some((c) => c.ariaLabel === 'Upload file' && c.visible)).toBe(true); + }); + }); + it('clears the capture store, selectively and fully', async () => { await withPage(async (page) => { const session = new ExtractorSession(page); diff --git a/automation/context_extractor/session.py b/automation/context_extractor/session.py index 8b10e2b..5f71ad3 100644 --- a/automation/context_extractor/session.py +++ b/automation/context_extractor/session.py @@ -94,15 +94,35 @@ return extractMarkdown(el); """ +def _build_inventory_script(selector: Optional[str]) -> str: + dom_js = _read_js("dom.js") + sel_json = json.dumps(selector or "") + return f""" +(() => {{ +{dom_js} +let el = null; +const sel = {sel_json}; +if (sel) {{ try {{ el = document.querySelector(sel); }} catch (_) {{ el = null; }} }} +if (!el) el = document.body; +return inventoryInterestingControls(el); +}})() +""" + + def _build_prompt_script( - meta: dict[str, Any], markdown: str, store: dict[str, Any], max_chars: Optional[int] = None + meta: dict[str, Any], + markdown: str, + store: dict[str, Any], + max_chars: Optional[int] = None, + controls: Optional[list[Any]] = None, ) -> str: prompt_js = _read_js("prompt.js") max_chars_js = json.dumps(max_chars) if max_chars is not None else "undefined" + controls_js = json.dumps(controls if controls is not None else []) return f""" (() => {{ {prompt_js} -return buildAIPrompt({json.dumps(meta)}, {json.dumps(markdown)}, {json.dumps(store)}, {max_chars_js}); +return buildAIPrompt({json.dumps(meta)}, {json.dumps(markdown)}, {json.dumps(store)}, {max_chars_js}, {controls_js}); }})() """ @@ -196,13 +216,19 @@ class ExtractorSession(_CaptureMixin): "markdown": markdown or "", } + def inventory_controls(self, selector: Optional[str] = None) -> list[Any]: + result = self.page.evaluate(_build_inventory_script(selector)) + return list(result) if result else [] + def build_ai_prompt(self, selector: Optional[str] = None, max_chars: Optional[int] = None) -> str: extracted = self.extract_markdown(selector) + controls = self.inventory_controls(selector) meta = {k: extracted[k] for k in ("url", "title", "ts", "selector")} - script = _build_prompt_script(meta, extracted["markdown"], self.get_store(), max_chars) + script = _build_prompt_script( + meta, extracted["markdown"], self.get_store(), max_chars, controls + ) return self.page.evaluate(script) - class AsyncExtractorSession(_CaptureMixin): """Async API. Use with `playwright.async_api` or `camoufox.AsyncCamoufox`. @@ -236,8 +262,15 @@ class AsyncExtractorSession(_CaptureMixin): "markdown": markdown or "", } + async def inventory_controls(self, selector: Optional[str] = None) -> list[Any]: + result = await self.page.evaluate(_build_inventory_script(selector)) + return list(result) if result else [] + async def build_ai_prompt(self, selector: Optional[str] = None, max_chars: Optional[int] = None) -> str: extracted = await self.extract_markdown(selector) + controls = await self.inventory_controls(selector) meta = {k: extracted[k] for k in ("url", "title", "ts", "selector")} - script = _build_prompt_script(meta, extracted["markdown"], self.get_store(), max_chars) + script = _build_prompt_script( + meta, extracted["markdown"], self.get_store(), max_chars, controls + ) return await self.page.evaluate(script) diff --git a/automation/tests/fixtures/sample.html b/automation/tests/fixtures/sample.html index 2685b3e..f66c109 100644 --- a/automation/tests/fixtures/sample.html +++ b/automation/tests/fixtures/sample.html @@ -12,6 +12,10 @@

Section

Hello world, visit docs.

+

Big inline art: tiny-jpeg.

+ + +
  • Alpha
  • Beta
    • diff --git a/automation/tests/test_prompt.py b/automation/tests/test_prompt.py index 376e64c..117cef9 100644 --- a/automation/tests/test_prompt.py +++ b/automation/tests/test_prompt.py @@ -14,7 +14,62 @@ def test_shared_js_files_exist_and_match_package(): assert (CORE / "prompt.js").is_file() # package-side copies/symlinks must resolve to the same source assert "function extractMarkdown" in _read_js("dom.js") + assert "function truncateDataUri" in _read_js("dom.js") + assert "function inventoryInterestingControls" in _read_js("dom.js") assert "function buildAIPrompt" in _read_js("prompt.js") + assert "function truncateDataUri" in _read_js("prompt.js") + + +def test_build_ai_prompt_truncates_data_uris_in_network(page): + page.goto("about:blank") + meta = { + "url": "https://example.test/page", + "title": "T", + "ts": 1_700_000_000_000, + "selector": "body", + } + long_data = "data:image/jpeg;base64," + ("A" * 500) + store = { + "console": [], + "errors": [], + "network": [{ + "type": "fetch", "method": "GET", "url": long_data, + "status": 200, "ts": 3, "duration": 12, "error": None, + }], + } + out = page.evaluate(_build_prompt_script(meta, "# Hello", store)) + assert "bytes truncated" in out + assert ("A" * 100) not in out + + +def test_build_ai_prompt_includes_interesting_controls(page): + page.goto("about:blank") + meta = {"url": "https://example.test/page", "title": "T", "ts": 1, "selector": "body"} + controls = [ + { + "tag": "button", + "role": "", + "ariaLabel": "Document options", + "text": "…", + "visible": True, + "selector": "#vis", + }, + { + "tag": "button", + "role": "", + "ariaLabel": "Document options", + "text": "…", + "visible": False, + "selector": "#hid", + }, + ] + out = page.evaluate( + _build_prompt_script(meta, "# Hello", {"console": [], "errors": [], "network": []}, None, controls) + ) + assert "## Interesting Controls" in out + assert "[visible]" in out + assert "### Hidden" in out + assert "Document options" in out def test_build_ai_prompt_script_formats_errors_and_failures(page): diff --git a/automation/tests/test_session.py b/automation/tests/test_session.py index f913881..f63c809 100644 --- a/automation/tests/test_session.py +++ b/automation/tests/test_session.py @@ -31,6 +31,8 @@ def test_extract_markdown_body_and_selector(page, fixture_url): assert "**world**" in full["markdown"] assert "[docs](/docs)" in full["markdown"] assert "- Alpha" in full["markdown"] + assert "bytes truncated" in full["markdown"] + assert "4AAQSkZJRgABAQAAAQABAAD" not in full["markdown"] scoped = session.extract_markdown("#main-content") assert scoped["selector"] == "#main-content" @@ -64,10 +66,23 @@ def test_build_ai_prompt_has_sections(page, fixture_url): assert prompt.startswith("# Page Context") assert "## Page Content" in prompt assert "Hello **world**" in prompt + assert "## Interesting Controls" in prompt + assert "[visible]" in prompt + assert "Document options" in prompt assert "JavaScript Errors" in prompt or "Console Errors" in prompt assert "_Extracted by Context Extractor_" in prompt +def test_inventory_controls_visible_and_hidden(page, fixture_url): + session = ExtractorSession(page) + page.goto(fixture_url, wait_until="domcontentloaded") + controls = session.inventory_controls("#main-content") + docs = [c for c in controls if c.get("ariaLabel") == "Document options"] + assert len(docs) == 2 + assert any(c.get("visible") for c in docs) + assert any(not c.get("visible") for c in docs) + + def test_clear_store(page, fixture_url): session = ExtractorSession(page) page.goto(fixture_url, wait_until="domcontentloaded") diff --git a/extension/core/dom.js b/extension/core/dom.js index ef906d0..ea69c08 100644 --- a/extension/core/dom.js +++ b/extension/core/dom.js @@ -62,6 +62,24 @@ function getSelector(el) { // --------------------------------------------------------------------------- const NEVER_RENDERED_TAGS = new Set(["script", "style", "noscript", "template", "svg", "iframe", "canvas"]); +/** Max length before a data: URL is collapsed in dumps (keeps prompts usable). */ +const DATA_URI_SOFT_MAX = 120; + +/** + * Collapse long data: URIs so markdown / network dumps don't blow up (e.g. Outline + * inlining base64 images). Non-data URLs are returned unchanged. + * Keep in sync with truncateDataUri in core/prompt.js. + */ +function truncateDataUri(url, softMax) { + if (typeof url !== "string" || !url.startsWith("data:")) return url || ""; + const limit = softMax || DATA_URI_SOFT_MAX; + if (url.length <= limit) return url; + const comma = url.indexOf(","); + const header = comma >= 0 ? url.slice(0, Math.min(comma, 64)) : url.slice(0, 64); + const payloadLen = comma >= 0 ? url.length - comma - 1 : url.length; + return header + ",…[" + payloadLen + " bytes truncated]"; +} + function isHidden(el) { if (!el || el.nodeType !== 1) return false; if (el.hidden) return true; @@ -95,7 +113,7 @@ function extractMarkdown(node) { const kids = walkInline(el); switch (tag) { case "a": { - const href = el.getAttribute("href") || ""; + const href = truncateDataUri(el.getAttribute("href") || ""); return "[" + kids + "](" + href + ")"; } case "strong": @@ -184,3 +202,47 @@ function extractMarkdown(node) { md = md.replace(/\n{3,}/g, "\n\n"); return md.trim(); } + +// --------------------------------------------------------------------------- +// Interesting controls inventory (agent / SPA debugging) +// --------------------------------------------------------------------------- +const CONTROLS_CAP = 80; +const CONTROL_SELECTOR = + 'button, [role="button"], [role="menuitem"], [role="option"], [aria-label]'; + +/** + * List interactive / labeled controls under root with visibility flags. + * Helps agents find hidden Radix twins ("Document options") without dumping + * the whole DOM. + */ +function inventoryInterestingControls(root, cap) { + const base = root && root.nodeType === 1 ? root : document.body; + if (!base || !base.querySelectorAll) return []; + const limit = cap || CONTROLS_CAP; + const nodes = base.querySelectorAll(CONTROL_SELECTOR); + const seen = new Set(); + const out = []; + for (let i = 0; i < nodes.length && out.length < limit; i++) { + const el = nodes[i]; + if (seen.has(el)) continue; + seen.add(el); + const tag = el.tagName ? el.tagName.toLowerCase() : ""; + const role = (el.getAttribute && el.getAttribute("role")) || ""; + const ariaLabel = (el.getAttribute && el.getAttribute("aria-label")) || ""; + let text = ""; + try { + text = (el.textContent || "").replace(/\s+/g, " ").trim().slice(0, 80); + } catch (_) { + text = ""; + } + out.push({ + tag: tag, + role: role, + ariaLabel: ariaLabel, + text: text, + visible: !isHidden(el), + selector: getSelector(el), + }); + } + return out; +} diff --git a/extension/core/prompt.js b/extension/core/prompt.js index 37b5d08..2d318b0 100644 --- a/extension/core/prompt.js +++ b/extension/core/prompt.js @@ -16,8 +16,33 @@ // Pass a tighter selector (or a larger maxChars) instead of relying on this // cap for real content; it exists as a safety net, not a summarizer. const DEFAULT_MAX_MARKDOWN_CHARS = 20000; +const DATA_URI_SOFT_MAX = 120; -function buildAIPrompt(meta, markdown, store, maxChars) { +/** + * Collapse long data: URIs in network/prompt lines. + * Keep in sync with truncateDataUri in core/dom.js. + */ +function truncateDataUri(url, softMax) { + if (typeof url !== "string" || !url.startsWith("data:")) return url || ""; + const limit = softMax || DATA_URI_SOFT_MAX; + if (url.length <= limit) return url; + const comma = url.indexOf(","); + const header = comma >= 0 ? url.slice(0, Math.min(comma, 64)) : url.slice(0, 64); + const payloadLen = comma >= 0 ? url.length - comma - 1 : url.length; + return header + ",…[" + payloadLen + " bytes truncated]"; +} + +function formatControlLine(c) { + const bits = []; + if (c.tag) bits.push(c.tag); + if (c.role) bits.push("role=" + c.role); + if (c.ariaLabel) bits.push('aria-label="' + c.ariaLabel + '"'); + if (c.text) bits.push('"' + c.text + '"'); + if (c.selector) bits.push("`" + c.selector + "`"); + return (c.visible ? "[visible] " : "[hidden] ") + bits.join(" "); +} + +function buildAIPrompt(meta, markdown, store, maxChars, controls) { const limit = maxChars || DEFAULT_MAX_MARKDOWN_CHARS; const parts = []; @@ -42,6 +67,28 @@ function buildAIPrompt(meta, markdown, store, maxChars) { parts.push(""); } + const controlList = Array.isArray(controls) ? controls : []; + if (controlList.length) { + const visible = controlList.filter((c) => c && c.visible); + const hiddenCount = controlList.length - visible.length; + parts.push("## Interesting Controls"); + parts.push(""); + parts.push( + `_Showing ${visible.length} visible` + + (hiddenCount ? `, ${hiddenCount} hidden` : "") + + ` (of ${controlList.length} inventoried)._`, + ); + parts.push(""); + visible.forEach((c) => parts.push("- " + formatControlLine(c))); + if (hiddenCount) { + parts.push(""); + parts.push("### Hidden"); + parts.push(""); + controlList.filter((c) => c && !c.visible).forEach((c) => parts.push("- " + formatControlLine(c))); + } + parts.push(""); + } + const errors = (store && store.errors) || []; if (errors.length) { parts.push("## JavaScript Errors"); @@ -80,14 +127,16 @@ function buildAIPrompt(meta, markdown, store, maxChars) { parts.push(""); failed.forEach((n) => { const status = n.error ? "ERR" : n.status; - parts.push(`- ${n.method || "GET"} ${status} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`); + const url = truncateDataUri(n.url || ""); + parts.push(`- ${n.method || "GET"} ${status} ${url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`); }); parts.push(""); } else if (network.length) { parts.push("## Recent Requests"); parts.push(""); network.slice(-15).forEach((n) => { - parts.push(`- ${n.method || "GET"} ${n.status || "—"} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}`); + const url = truncateDataUri(n.url || ""); + parts.push(`- ${n.method || "GET"} ${n.status || "—"} ${url}${n.duration ? " (" + n.duration + "ms)" : ""}`); }); parts.push(""); } diff --git a/extension/popup.js b/extension/popup.js index 62ae41c..6451799 100644 --- a/extension/popup.js +++ b/extension/popup.js @@ -412,7 +412,7 @@ function wire() { out.push(""); state.store.network.forEach((n) => { const status = n.error ? "ERR" : (n.status || "—"); - out.push(`- ${n.method || "GET"} ${status} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`); + out.push(`- ${n.method || "GET"} ${status} ${truncateDataUri(n.url || "")}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`); }); out.push(""); out.push("---");