Initial Context Extractor: extension + Playwright/Camoufox package

Ship a shared markdown/prompt core used by a Brave/Chrome MV3 extension and a Python automation API, with pytest coverage, CI, and packaging smoke.

automation/tests/conftest.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from playwright.sync_api import sync_playwright
+
+FIXTURE = Path(__file__).parent / "fixtures" / "sample.html"
+
+
+@pytest.fixture(scope="session")
+def playwright_browser():
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        yield browser
+        browser.close()
+
+
+@pytest.fixture
+def page(playwright_browser):
+    page = playwright_browser.new_page()
+    yield page
+    page.close()
+
+
+@pytest.fixture
+def fixture_url() -> str:
+    return FIXTURE.resolve().as_uri()

automation/tests/fixtures/sample.html

@@ -0,0 +1,33 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>Context Extractor Fixture</title>
+</head>
+<body>
+  <header>
+    <h1>Fixture Page</h1>
+    <nav>Skip this noise</nav>
+  </header>
+  <main id="main-content">
+    <h2>Section</h2>
+    <p>Hello <strong>world</strong>, visit <a href="/docs">docs</a>.</p>
+    <ul>
+      <li>Alpha</li>
+      <li>Beta</li>
+    </ul>
+    <div style="display:none">hidden-inline-style-blob {"secretConfig": "should-not-appear"}</div>
+    <div class="css-hidden">hidden-via-stylesheet-blob {"lixTracking": "should-not-appear"}</div>
+    <div hidden>hidden-attribute-blob should-not-appear</div>
+    <div aria-hidden="true">aria-hidden-blob should-not-appear</div>
+  </main>
+  <style>.css-hidden { display: none; }</style>
+  <script>
+    console.log("fixture boot");
+    console.warn("fixture warn");
+    console.error("fixture error log");
+    setTimeout(() => { throw new Error("fixture boom"); }, 20);
+    fetch("/missing-endpoint").catch(() => {});
+  </script>
+</body>
+</html>

automation/tests/test_prompt.py

@@ -0,0 +1,71 @@
+"""Unit-style tests for the shared prompt.js formatter (via page.evaluate)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from context_extractor.session import _build_prompt_script, _read_js
+
+CORE = Path(__file__).resolve().parents[2] / "extension" / "core"
+
+
+def test_shared_js_files_exist_and_match_package():
+    assert (CORE / "dom.js").is_file()
+    assert (CORE / "prompt.js").is_file()
+    # package-side copies/symlinks must resolve to the same source
+    assert "function extractMarkdown" in _read_js("dom.js")
+    assert "function buildAIPrompt" in _read_js("prompt.js")
+
+
+def test_build_ai_prompt_script_formats_errors_and_failures(page):
+    page.goto("about:blank")
+    meta = {
+        "url": "https://example.test/page",
+        "title": "T",
+        "ts": 1_700_000_000_000,
+        "selector": "body",
+    }
+    store = {
+        "console": [{"level": "warn", "ts": 1, "msg": "careful"}],
+        "errors": [{
+            "type": "error", "ts": 2, "msg": "boom",
+            "source": "a.js", "line": 3, "col": 4, "stack": "Error: boom\n  at x",
+        }],
+        "network": [{
+            "type": "fetch", "method": "GET", "url": "https://example.test/api",
+            "status": 500, "ts": 3, "duration": 12, "error": None,
+        }],
+    }
+    script = _build_prompt_script(meta, "# Hello", store)
+    out = page.evaluate(script)
+    assert "# Page Context" in out
+    assert "https://example.test/page" in out
+    assert "## Page Content" in out
+    assert "# Hello" in out
+    assert "## JavaScript Errors" in out
+    assert "boom" in out
+    assert "## Console Errors & Warnings" in out
+    assert "[WARN] careful" in out
+    assert "## Failed Requests" in out
+    assert "GET 500 https://example.test/api" in out
+
+
+def test_build_ai_prompt_truncates_huge_markdown_by_default(page):
+    """Regression test: a JS-heavy SPA (e.g. LinkedIn) can extract 700k+ chars
+    of body content. The prompt must not blindly include all of it."""
+    page.goto("about:blank")
+    meta = {"url": "https://example.test/huge", "title": "T", "ts": 1, "selector": "body"}
+    huge_markdown = "x" * 50_000
+    script = _build_prompt_script(meta, huge_markdown, {"console": [], "errors": [], "network": []})
+    out = page.evaluate(script)
+    assert "truncated to 20,000 of 50,000 chars" in out
+    assert len(out) < 21_000  # header/footer overhead + capped body, nowhere near 50k
+
+
+def test_build_ai_prompt_respects_custom_max_chars(page):
+    page.goto("about:blank")
+    meta = {"url": "https://example.test/huge", "title": "T", "ts": 1, "selector": "body"}
+    markdown = "y" * 1000
+    script = _build_prompt_script(meta, markdown, {"console": [], "errors": [], "network": []}, max_chars=200)
+    out = page.evaluate(script)
+    assert "truncated to 200 of 1,000 chars" in out

automation/tests/test_session.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from context_extractor import AsyncExtractorSession, ExtractorSession, __version__
+
+
+def test_version_semverish():
+    assert __version__.count(".") >= 1
+
+
+def test_capture_console_errors_and_network(page, fixture_url):
+    session = ExtractorSession(page)
+    page.goto(fixture_url, wait_until="domcontentloaded")
+    page.wait_for_timeout(150)
+
+    store = session.get_store()
+    levels = {e["level"] for e in store["console"]}
+    assert "warn" in levels
+    assert "error" in levels or any("error" in (e.get("msg") or "").lower() for e in store["console"])
+
+    assert any(e["msg"] == "fixture boom" or "fixture boom" in (e.get("msg") or "") for e in store["errors"])
+    assert store["network"], "expected at least the document request"
+
+
+def test_extract_markdown_body_and_selector(page, fixture_url):
+    session = ExtractorSession(page)
+    page.goto(fixture_url, wait_until="domcontentloaded")
+
+    full = session.extract_markdown()
+    assert full["title"] == "Context Extractor Fixture"
+    assert "Fixture Page" in full["markdown"]
+    assert "**world**" in full["markdown"]
+    assert "[docs](/docs)" in full["markdown"]
+    assert "- Alpha" in full["markdown"]
+
+    scoped = session.extract_markdown("#main-content")
+    assert scoped["selector"] == "#main-content"
+    assert "Section" in scoped["markdown"]
+    assert "Skip this noise" not in scoped["markdown"]
+
+
+def test_hidden_nodes_are_excluded_from_markdown(page, fixture_url):
+    """Regression test: SPAs (e.g. LinkedIn) stash hydration/experiment JSON in
+    hidden DOM nodes, not just <script> tags. Any hidden node must be excluded
+    regardless of *how* it's hidden (inline style, stylesheet class, [hidden],
+    aria-hidden)."""
+    session = ExtractorSession(page)
+    page.goto(fixture_url, wait_until="domcontentloaded")
+
+    markdown = session.extract_markdown("#main-content")["markdown"]
+    assert "should-not-appear" not in markdown
+    assert "secretConfig" not in markdown
+    assert "lixTracking" not in markdown
+    # Sanity: visible content in the same container still comes through.
+    assert "Alpha" in markdown
+    assert "Beta" in markdown
+
+
+def test_build_ai_prompt_has_sections(page, fixture_url):
+    session = ExtractorSession(page)
+    page.goto(fixture_url, wait_until="domcontentloaded")
+    page.wait_for_timeout(150)
+
+    prompt = session.build_ai_prompt("#main-content")
+    assert prompt.startswith("# Page Context")
+    assert "## Page Content" in prompt
+    assert "Hello **world**" in prompt
+    assert "JavaScript Errors" in prompt or "Console Errors" in prompt
+    assert "_Extracted by Context Extractor_" in prompt
+
+
+def test_clear_store(page, fixture_url):
+    session = ExtractorSession(page)
+    page.goto(fixture_url, wait_until="domcontentloaded")
+    page.wait_for_timeout(100)
+    assert session.get_store()["console"]
+    session.clear("console")
+    assert session.get_store()["console"] == []
+    session.clear()
+    store = session.get_store()
+    assert store == {"console": [], "errors": [], "network": []}
+
+
+def test_async_session_exported():
+    assert AsyncExtractorSession is not None