Initial Context Extractor: extension + Playwright/Camoufox package
Some checks failed
CI / Lint + tests (push) Has been cancelled
Some checks failed
CI / Lint + tests (push) Has been cancelled
Ship a shared markdown/prompt core used by a Brave/Chrome MV3 extension and a Python automation API, with pytest coverage, CI, and packaging smoke.
This commit is contained in:
commit
c91cd221d1
75
.github/workflows/ci.yml
vendored
Normal file
75
.github/workflows/ci.yml
vendored
Normal file
@ -0,0 +1,75 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master, main]
|
||||
pull_request:
|
||||
branches: [master, main]
|
||||
|
||||
jobs:
|
||||
quality:
|
||||
name: Lint + tests
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: automation
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
cache-dependency-path: automation/pyproject.toml
|
||||
|
||||
- name: Set up Node (JS syntax check only)
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Syntax-check extension JS
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for f in \
|
||||
extension/core/dom.js \
|
||||
extension/core/prompt.js \
|
||||
extension/page-patcher.js \
|
||||
extension/content.js \
|
||||
extension/background.js \
|
||||
extension/popup.js
|
||||
do
|
||||
node --check "$f"
|
||||
echo "OK $f"
|
||||
done
|
||||
python3 -m json.tool extension/manifest.json >/dev/null
|
||||
echo "OK extension/manifest.json"
|
||||
|
||||
- name: Verify shared-core symlinks
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -L automation/context_extractor/js/dom.js
|
||||
test -L automation/context_extractor/js/prompt.js
|
||||
diff -q automation/context_extractor/js/dom.js extension/core/dom.js
|
||||
diff -q automation/context_extractor/js/prompt.js extension/core/prompt.js
|
||||
|
||||
- name: Install package + dev deps
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -e ".[dev]"
|
||||
|
||||
- name: Install Playwright Chromium
|
||||
run: playwright install --with-deps chromium
|
||||
|
||||
- name: Pytest
|
||||
run: pytest -q
|
||||
|
||||
- name: Package extension zip (smoke)
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: |
|
||||
python3 scripts/package_extension.py
|
||||
test -f dist/context-extractor-extension.zip
|
||||
12
.gitignore
vendored
Normal file
12
.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
dist/
|
||||
*.zip
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
.venv/
|
||||
venv/
|
||||
*.egg-info/
|
||||
node_modules/
|
||||
.DS_Store
|
||||
.coverage
|
||||
htmlcov/
|
||||
15
.gitleaks.toml
Normal file
15
.gitleaks.toml
Normal file
@ -0,0 +1,15 @@
|
||||
# Minimal gitleaks config for this repo.
|
||||
# Extends the default ruleset; no secrets are expected in source.
|
||||
title = "context-extractor"
|
||||
|
||||
[extend]
|
||||
useDefault = true
|
||||
|
||||
# Example/fixture HTML and markdown prompt docs can look secret-like;
|
||||
# they are intentional fixtures, not credentials.
|
||||
[allowlist]
|
||||
description = "Known false positives"
|
||||
paths = [
|
||||
'''automation/tests/fixtures/''',
|
||||
'''CURSOR_PROMPT\.md''',
|
||||
]
|
||||
40
CURSOR_PROMPT.md
Normal file
40
CURSOR_PROMPT.md
Normal file
@ -0,0 +1,40 @@
|
||||
# Cursor consumer prompt
|
||||
|
||||
Paste this into another Cursor chat when that project should use Context Extractor
|
||||
(path below — adjust if you moved the repo).
|
||||
|
||||
---
|
||||
|
||||
```text
|
||||
Use Context Extractor from ~/Documents/code/context-extractor for browser capture / scrape debugging.
|
||||
|
||||
Setup (once):
|
||||
cd ~/Documents/code/context-extractor/automation && python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install -e ".[dev]"
|
||||
playwright install chromium
|
||||
# optional anti-detect: pip install -e ".[camoufox]"
|
||||
|
||||
Python scrape / CI loop:
|
||||
from playwright.sync_api import sync_playwright
|
||||
# or: from camoufox import Camoufox
|
||||
from context_extractor import ExtractorSession
|
||||
|
||||
with sync_playwright() as p:
|
||||
page = p.chromium.launch(headless=True).new_page()
|
||||
session = ExtractorSession(page) # attach BEFORE goto
|
||||
page.goto(URL, wait_until="domcontentloaded")
|
||||
prompt = session.build_ai_prompt(SELECTOR) # None = whole body
|
||||
# also available: session.extract_markdown(), session.get_store(), page.content() for raw HTML
|
||||
|
||||
CLI: context-extractor URL --selector CSS --engine chromium|camoufox --out prompt.md
|
||||
|
||||
Manual / interactive (Brave or Chrome):
|
||||
Load unpacked: ~/Documents/code/context-extractor/extension
|
||||
Then use "Copy" on the broken page.
|
||||
|
||||
When debugging scrapers or flaky pages in THIS project:
|
||||
1. Capture with ExtractorSession / the extension.
|
||||
2. Paste the prompt into chat and ask for selector fixes, parse logic, or why the page failed.
|
||||
3. Prefer selectors from #main / data-* / stable IDs over brittle chrome-copy paths.
|
||||
Do not reinvent markdown extraction or console/network capture — reuse this package.
|
||||
```
|
||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Context Extractor contributors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
27
Makefile
Normal file
27
Makefile
Normal file
@ -0,0 +1,27 @@
|
||||
.PHONY: help test lint package ci install
|
||||
|
||||
help: ## Show targets
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-12s %s\n", $$1, $$2}'
|
||||
|
||||
install: ## Create venv and install automation package + Playwright Chromium
|
||||
cd automation && python3 -m venv .venv
|
||||
cd automation && .venv/bin/pip install -U pip && .venv/bin/pip install -e ".[dev]"
|
||||
cd automation && .venv/bin/playwright install chromium
|
||||
|
||||
test: ## Run pytest
|
||||
cd automation && .venv/bin/pytest -q
|
||||
|
||||
lint: ## Syntax-check extension JS + validate manifest JSON
|
||||
@node --check extension/core/dom.js
|
||||
@node --check extension/core/prompt.js
|
||||
@node --check extension/page-patcher.js
|
||||
@node --check extension/content.js
|
||||
@node --check extension/background.js
|
||||
@node --check extension/popup.js
|
||||
@python3 -m json.tool extension/manifest.json >/dev/null
|
||||
@echo "lint OK"
|
||||
|
||||
package: ## Zip extension/ into dist/
|
||||
python3 scripts/package_extension.py
|
||||
|
||||
ci: lint test package ## Local stand-in for GitHub Actions
|
||||
179
README.md
Normal file
179
README.md
Normal file
@ -0,0 +1,179 @@
|
||||
# Context Extractor
|
||||
|
||||
Capture console logs, network activity, JS errors, and clean markdown content
|
||||
from a webpage, formatted as an AI-ready prompt. Ships two ways from one
|
||||
shared core:
|
||||
|
||||
- **`extension/`** — a browser extension (Manifest V3) for interactive use.
|
||||
- **`automation/`** — a Python package for scripted/headless use with
|
||||
Playwright or [Camoufox](https://camoufox.com).
|
||||
|
||||
```
|
||||
context-extractor/
|
||||
├── extension/ browser extension (Chrome + Brave, unpacked)
|
||||
│ ├── core/dom.js ← shared: selector + markdown extraction
|
||||
│ ├── core/prompt.js ← shared: AI-prompt formatter
|
||||
│ ├── page-patcher.js MAIN-world console/fetch/XHR/error capture
|
||||
│ ├── content.js ISOLATED-world store, picker, messaging
|
||||
│ ├── background.js
|
||||
│ └── popup.html / popup.js
|
||||
├── automation/ Python package for Playwright / Camoufox
|
||||
│ ├── context_extractor/ session + CLI
|
||||
│ │ └── js/ symlinks → extension/core/*.js
|
||||
│ └── tests/ pytest + fixture page
|
||||
├── scripts/package_extension.py optional zip for distribution
|
||||
├── .github/workflows/ci.yml JS lint + pytest + package smoke
|
||||
├── CURSOR_PROMPT.md paste into other Cursor chats
|
||||
└── Makefile make test / make lint / make package
|
||||
```
|
||||
|
||||
`extension/core/dom.js` and `extension/core/prompt.js` are the single source
|
||||
of truth for DOM/markdown/prompt logic — the extension loads them directly as
|
||||
content-script files, and the Python package reads the exact same files
|
||||
(via symlinks) and runs them with `page.evaluate()`. There's only one place
|
||||
to fix a markdown-formatting bug.
|
||||
|
||||
## Use from another Cursor project
|
||||
|
||||
Copy the block in [`CURSOR_PROMPT.md`](CURSOR_PROMPT.md) into that project's chat
|
||||
(or drop it in `.cursor/rules/`).
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
make install # once
|
||||
make ci # lint + pytest + package smoke (same checks as GitHub Actions)
|
||||
```
|
||||
|
||||
Or manually:
|
||||
|
||||
```bash
|
||||
cd automation
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install -e ".[dev]"
|
||||
playwright install chromium
|
||||
pytest -q
|
||||
```
|
||||
|
||||
## 1. Browser extension (Chrome and Brave)
|
||||
|
||||
Brave is Chromium under the hood and loads unpacked Manifest V3 extensions
|
||||
exactly like Chrome does — there is no separate "Brave build." Same steps in
|
||||
both:
|
||||
|
||||
1. Go to `chrome://extensions` (or `brave://extensions`).
|
||||
2. Enable **Developer Mode**.
|
||||
3. **Load unpacked** → select the `extension/` folder.
|
||||
4. Reload any tab you want to capture from (content scripts only attach to
|
||||
pages loaded/reloaded after install).
|
||||
|
||||
Click the toolbar icon to see captured console/network/errors, extract
|
||||
markdown from the whole page or a CSS selector (use **Pick** to click an
|
||||
element), and **Copy** to put a ready-to-paste prompt on the clipboard.
|
||||
|
||||
### What changed vs. a typical extension implementation
|
||||
|
||||
The original console/fetch/XHR patcher was injected as an inline `<script>`
|
||||
tag, which strict-CSP sites (`script-src` without `'unsafe-inline'`) silently
|
||||
block. This version declares `page-patcher.js` as a second content script
|
||||
with `"world": "MAIN"` (Chrome/Brave 111+) instead — it runs directly in the
|
||||
page's own JS realm without ever going through the page's HTML parser, so
|
||||
it isn't subject to that CSP restriction. It talks back to the isolated-world
|
||||
`content.js` (which owns the store, the element picker, and
|
||||
`chrome.runtime` messaging) via a `CustomEvent` bridge, since MAIN and
|
||||
ISOLATED worlds don't share JS globals.
|
||||
|
||||
### Packaging a zip (optional)
|
||||
|
||||
```bash
|
||||
python3 scripts/package_extension.py # -> dist/context-extractor-extension.zip
|
||||
```
|
||||
|
||||
Not needed for local use — "Load unpacked" against `extension/` is enough.
|
||||
|
||||
## 2. Playwright / Camoufox automation
|
||||
|
||||
```bash
|
||||
cd automation
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install -e . # plain Playwright
|
||||
pip install -e ".[camoufox]" # + Camoufox extra
|
||||
playwright install chromium # only needed for the plain-Playwright path
|
||||
```
|
||||
|
||||
```python
|
||||
from playwright.sync_api import sync_playwright
|
||||
from context_extractor import ExtractorSession
|
||||
|
||||
with sync_playwright() as p:
|
||||
page = p.chromium.launch().new_page()
|
||||
session = ExtractorSession(page) # attach BEFORE navigating
|
||||
page.goto("https://example.com", wait_until="networkidle")
|
||||
print(session.build_ai_prompt()) # or session.extract_markdown("#main")
|
||||
```
|
||||
|
||||
Camoufox (sync or async) works the same way — see
|
||||
`automation/examples/scrape_camoufox.py` and
|
||||
`automation/examples/scrape_camoufox_async.py`. Use `AsyncExtractorSession`
|
||||
for `camoufox.async_api.AsyncCamoufox` / `playwright.async_api`.
|
||||
|
||||
Headless CLI, handy in CI:
|
||||
|
||||
```bash
|
||||
context-extractor https://example.com --selector "#main" --engine camoufox --out prompt.md
|
||||
```
|
||||
|
||||
### Why automation doesn't reuse the extension's patcher
|
||||
|
||||
Camoufox executes Playwright's own JS (`page.add_init_script()`,
|
||||
`page.evaluate()` writes) in an isolated context by design, as an
|
||||
anti-fingerprinting measure — this is a
|
||||
[known, documented limitation](https://github.com/daijro/camoufox/issues/48)
|
||||
that breaks naive init-script injection. Rather than fight that, the
|
||||
automation layer captures console/network/errors via Playwright's *native*
|
||||
event hooks (`page.on("console"/"request"/"requestfinished"/"requestfailed"/"pageerror")`)
|
||||
instead of injecting JS into the page at all. This is more robust than the
|
||||
extension's approach (it can't be blocked by page CSP, and it's not limited
|
||||
to fetch/XHR — it sees every resource type), and it works identically
|
||||
whether the page is Chromium, Firefox, WebKit, or Camoufox.
|
||||
|
||||
Markdown extraction and CSS-selector logic *do* run inside the page (they
|
||||
need real DOM traversal), via `page.evaluate()` against `core/dom.js`. That's
|
||||
safe under Camoufox's default isolated world because that code only *reads*
|
||||
the DOM and mutates a detached clone — it never writes to the live page —
|
||||
so no `main_world_eval` workaround is needed there either.
|
||||
|
||||
## Extraction behavior worth knowing
|
||||
|
||||
- `extractMarkdown()` walks the *rendered* page and skips anything actually
|
||||
invisible: `display:none`, `visibility:hidden`, `[hidden]`,
|
||||
`aria-hidden="true"` — however it's hidden (inline style, stylesheet class,
|
||||
or attribute). SPAs routinely stash large hydration/experiment JSON
|
||||
payloads in hidden DOM nodes, not just `<script>` tags (LinkedIn is a good
|
||||
example — its DOM without this filter is 700k+ characters, almost all of
|
||||
it invisible config junk). Without this check you'd be feeding an LLM
|
||||
chameleon experiment payloads instead of page content.
|
||||
- `build_ai_prompt()` / the extension **Copy** button cap page content at **20,000 chars**
|
||||
by default (`maxChars` in JS, `max_chars=`/`--max-chars` in Python/CLI). A
|
||||
full `<body>` extraction on a JS-heavy SPA can still be enormous even after
|
||||
hidden-node filtering — the cap is a safety net, not a summarizer. Prefer
|
||||
a narrower selector (**Pick** in the extension, or a known selector in
|
||||
automation) over relying on the cap for real content.
|
||||
|
||||
## Known limitations
|
||||
|
||||
- The element-picker UI (click-to-grab-a-selector) is extension-only; it's a
|
||||
visual affordance that doesn't make sense in headless automation. For
|
||||
scripted use, pass the selector you already know to `extract_markdown()` /
|
||||
`build_ai_prompt()`.
|
||||
- The extension's network capture only sees `fetch` + `XMLHttpRequest` (it's
|
||||
patched in JS). The automation layer sees every resource type since it
|
||||
uses Playwright's own request events.
|
||||
- `page-patcher.js` still can't see activity from a page's Web Workers or
|
||||
Service Workers; neither can the automation layer without additional
|
||||
`page.on("worker")` wiring (not implemented — open an issue/extend
|
||||
`session.py` if you need it).
|
||||
- Hidden-node filtering catches `display:none`/`visibility:hidden`/`[hidden]`/
|
||||
`aria-hidden`, but not "visually clipped but AT-readable" sr-only patterns
|
||||
(e.g. `position:absolute;clip:rect(0,0,0,0)`) — those are left in on
|
||||
purpose since they're usually real text, not hydration data.
|
||||
12
automation/README.md
Normal file
12
automation/README.md
Normal file
@ -0,0 +1,12 @@
|
||||
# Automation package
|
||||
|
||||
Install from this directory:
|
||||
|
||||
```bash
|
||||
python3 -m venv .venv && source .venv/bin/activate
|
||||
pip install -e ".[dev]"
|
||||
playwright install chromium
|
||||
pytest
|
||||
```
|
||||
|
||||
Project docs and consumer Cursor prompt live in the repo root (`../README.md`, `../CURSOR_PROMPT.md`).
|
||||
4
automation/context_extractor/__init__.py
Normal file
4
automation/context_extractor/__init__.py
Normal file
@ -0,0 +1,4 @@
|
||||
from .session import AsyncExtractorSession, ExtractorSession
|
||||
|
||||
__all__ = ["AsyncExtractorSession", "ExtractorSession"]
|
||||
__version__ = "1.3.0"
|
||||
74
automation/context_extractor/cli.py
Normal file
74
automation/context_extractor/cli.py
Normal file
@ -0,0 +1,74 @@
|
||||
"""Headless CLI: fetch a URL, capture console/network/errors, print an AI-ready prompt.
|
||||
|
||||
Usage:
|
||||
context-extractor https://example.com
|
||||
context-extractor https://example.com --selector "#main" --engine camoufox
|
||||
context-extractor https://example.com --wait 2000 --out prompt.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .session import ExtractorSession
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(prog="context-extractor", description=__doc__)
|
||||
parser.add_argument("url", help="URL to load")
|
||||
parser.add_argument("--selector", default=None, help="CSS selector to extract (default: body)")
|
||||
parser.add_argument("--wait", type=int, default=0, help="Extra milliseconds to wait after load")
|
||||
parser.add_argument(
|
||||
"--max-chars", type=int, default=None,
|
||||
help="Cap page content in the prompt to this many chars (default: 20000). "
|
||||
"SPA pages can extract hundreds of thousands of chars; this keeps output LLM-sized.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--engine", choices=["chromium", "firefox", "webkit", "camoufox"], default="chromium",
|
||||
help="Which browser engine to drive (default: chromium)",
|
||||
)
|
||||
parser.add_argument("--headed", action="store_true", help="Show the browser window")
|
||||
parser.add_argument("--out", default=None, help="Write the prompt to a file instead of stdout")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.engine == "camoufox":
|
||||
try:
|
||||
from camoufox import Camoufox
|
||||
except ImportError:
|
||||
print(
|
||||
"camoufox is not installed. Install with: pip install 'context-extractor[camoufox]'",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
with Camoufox(headless=not args.headed) as browser:
|
||||
page = browser.new_page()
|
||||
prompt = _run(page, args)
|
||||
else:
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = getattr(p, args.engine).launch(headless=not args.headed)
|
||||
page = browser.new_page()
|
||||
prompt = _run(page, args)
|
||||
browser.close()
|
||||
|
||||
if args.out:
|
||||
with open(args.out, "w", encoding="utf-8") as f:
|
||||
f.write(prompt)
|
||||
print(f"Wrote {len(prompt):,} chars to {args.out}", file=sys.stderr)
|
||||
else:
|
||||
print(prompt)
|
||||
return 0
|
||||
|
||||
|
||||
def _run(page, args) -> str:
|
||||
session = ExtractorSession(page)
|
||||
page.goto(args.url, wait_until="domcontentloaded")
|
||||
if args.wait:
|
||||
page.wait_for_timeout(args.wait)
|
||||
return session.build_ai_prompt(args.selector, max_chars=args.max_chars)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1
automation/context_extractor/js/dom.js
Symbolic link
1
automation/context_extractor/js/dom.js
Symbolic link
@ -0,0 +1 @@
|
||||
../../../extension/core/dom.js
|
||||
1
automation/context_extractor/js/prompt.js
Symbolic link
1
automation/context_extractor/js/prompt.js
Symbolic link
@ -0,0 +1 @@
|
||||
../../../extension/core/prompt.js
|
||||
243
automation/context_extractor/session.py
Normal file
243
automation/context_extractor/session.py
Normal file
@ -0,0 +1,243 @@
|
||||
"""Console/network/error capture + markdown extraction for a Playwright page.
|
||||
|
||||
Works with:
|
||||
- Plain Playwright (any browser: chromium, firefox, webkit)
|
||||
- Camoufox (daijro/camoufox), sync or async — Camoufox hands you a normal
|
||||
Playwright Page/BrowserContext, so everything here works unmodified.
|
||||
|
||||
Design notes (see project README for the full rationale):
|
||||
- Console/network/page-error capture uses Playwright's *native* event hooks
|
||||
(page.on("console"/"request"/"requestfinished"/"requestfailed"/"pageerror")).
|
||||
This deliberately avoids injecting a JS patcher into the page (the trick
|
||||
the browser extension has to use), because:
|
||||
1. It's more robust — no reliance on page.add_init_script(), which is
|
||||
known to be unreliable under Camoufox's isolated-world execution
|
||||
model (see https://github.com/daijro/camoufox/issues/48).
|
||||
2. It captures more than the extension does (any resource type, not
|
||||
just fetch/XHR), and can't be blocked by a page's CSP.
|
||||
- Markdown extraction and CSS-selector helpers *do* need to run inside the
|
||||
page (DOM traversal). Those live in extension/core/*.js and are read from
|
||||
disk here, then run via page.evaluate(). This is safe under Camoufox's
|
||||
default *isolated* world because that code only reads the DOM / mutates a
|
||||
detached clone — it never writes to the live page — so no
|
||||
`main_world_eval` workaround is required.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
_JS_DIR = Path(__file__).parent / "js"
|
||||
MAX_ENTRIES = 200
|
||||
|
||||
|
||||
def _read_js(name: str) -> str:
|
||||
path = _JS_DIR / name
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Missing shared JS file: {path}. This package expects to run from "
|
||||
"a checkout of the context-extractor repo where automation/context_extractor/js/*.js "
|
||||
"are symlinks into extension/core/."
|
||||
)
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _now_ms() -> int:
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
class _Store:
|
||||
"""Bounded ring-buffer store matching the extension's shape exactly, so
|
||||
build_ai_prompt() output is identical between the extension and here."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.console: list[dict[str, Any]] = []
|
||||
self.errors: list[dict[str, Any]] = []
|
||||
self.network: list[dict[str, Any]] = []
|
||||
|
||||
def push(self, which: str, entry: dict[str, Any]) -> None:
|
||||
arr = getattr(self, which)
|
||||
arr.append(entry)
|
||||
while len(arr) > MAX_ENTRIES:
|
||||
arr.pop(0)
|
||||
|
||||
def clear(self, which: Optional[str] = None) -> None:
|
||||
if which:
|
||||
getattr(self, which).clear()
|
||||
else:
|
||||
self.console.clear()
|
||||
self.errors.clear()
|
||||
self.network.clear()
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"console": self.console, "errors": self.errors, "network": self.network}
|
||||
|
||||
|
||||
_CONSOLE_LEVEL_MAP = {"warning": "warn"}
|
||||
|
||||
|
||||
def _build_extract_script(selector: Optional[str]) -> str:
|
||||
dom_js = _read_js("dom.js")
|
||||
sel_json = json.dumps(selector or "")
|
||||
return f"""
|
||||
(() => {{
|
||||
{dom_js}
|
||||
let el = null;
|
||||
const sel = {sel_json};
|
||||
if (sel) {{ try {{ el = document.querySelector(sel); }} catch (_) {{ el = null; }} }}
|
||||
if (!el) el = document.body;
|
||||
return extractMarkdown(el);
|
||||
}})()
|
||||
"""
|
||||
|
||||
|
||||
def _build_prompt_script(
|
||||
meta: dict[str, Any], markdown: str, store: dict[str, Any], max_chars: Optional[int] = None
|
||||
) -> str:
|
||||
prompt_js = _read_js("prompt.js")
|
||||
max_chars_js = json.dumps(max_chars) if max_chars is not None else "undefined"
|
||||
return f"""
|
||||
(() => {{
|
||||
{prompt_js}
|
||||
return buildAIPrompt({json.dumps(meta)}, {json.dumps(markdown)}, {json.dumps(store)}, {max_chars_js});
|
||||
}})()
|
||||
"""
|
||||
|
||||
|
||||
class _CaptureMixin:
|
||||
"""Shared event-handling logic. Playwright event callbacks are plain
|
||||
synchronous functions in both the sync and async APIs (Playwright invokes
|
||||
them itself; you never await them), so this is safe to share as-is."""
|
||||
|
||||
def _init_capture_state(self) -> None:
|
||||
self.store = _Store()
|
||||
self._request_starts: dict[int, float] = {}
|
||||
|
||||
def _on_console(self, msg) -> None:
|
||||
level = _CONSOLE_LEVEL_MAP.get(msg.type, msg.type)
|
||||
self.store.push("console", {"level": level, "ts": _now_ms(), "msg": msg.text})
|
||||
|
||||
def _on_pageerror(self, error) -> None:
|
||||
message = getattr(error, "message", None) or str(error)
|
||||
stack = getattr(error, "stack", "") or ""
|
||||
self.store.push("errors", {
|
||||
"type": "error", "ts": _now_ms(), "msg": message,
|
||||
"source": "", "line": 0, "col": 0, "stack": stack,
|
||||
})
|
||||
|
||||
def _on_request(self, request) -> None:
|
||||
self._request_starts[id(request)] = time.monotonic()
|
||||
|
||||
def _on_requestfinished(self, request) -> None:
|
||||
start = self._request_starts.pop(id(request), None)
|
||||
duration = round((time.monotonic() - start) * 1000) if start else 0
|
||||
status = 0
|
||||
try:
|
||||
response = request.response()
|
||||
if response:
|
||||
status = response.status
|
||||
except Exception:
|
||||
pass
|
||||
self.store.push("network", {
|
||||
"type": request.resource_type, "method": request.method, "url": request.url,
|
||||
"status": status, "ts": _now_ms(), "duration": duration, "error": None,
|
||||
})
|
||||
|
||||
def _on_requestfailed(self, request) -> None:
|
||||
start = self._request_starts.pop(id(request), None)
|
||||
duration = round((time.monotonic() - start) * 1000) if start else 0
|
||||
failure = getattr(request, "failure", None)
|
||||
error_text = failure.get("errorText") if isinstance(failure, dict) else str(failure or "Network error")
|
||||
self.store.push("network", {
|
||||
"type": request.resource_type, "method": request.method, "url": request.url,
|
||||
"status": 0, "ts": _now_ms(), "duration": duration, "error": error_text or "Network error",
|
||||
})
|
||||
|
||||
def get_store(self) -> dict[str, Any]:
|
||||
return self.store.as_dict()
|
||||
|
||||
def clear(self, which: Optional[str] = None) -> None:
|
||||
self.store.clear(which)
|
||||
|
||||
|
||||
class ExtractorSession(_CaptureMixin):
|
||||
"""Sync API. Use with `playwright.sync_api` or sync `camoufox.Camoufox`.
|
||||
|
||||
Example:
|
||||
from playwright.sync_api import sync_playwright
|
||||
from context_extractor import ExtractorSession
|
||||
|
||||
with sync_playwright() as p:
|
||||
page = p.chromium.launch().new_page()
|
||||
session = ExtractorSession(page)
|
||||
page.goto("https://example.com")
|
||||
print(session.build_ai_prompt())
|
||||
"""
|
||||
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
self._init_capture_state()
|
||||
page.on("console", self._on_console)
|
||||
page.on("pageerror", self._on_pageerror)
|
||||
page.on("request", self._on_request)
|
||||
page.on("requestfinished", self._on_requestfinished)
|
||||
page.on("requestfailed", self._on_requestfailed)
|
||||
|
||||
def extract_markdown(self, selector: Optional[str] = None) -> dict[str, Any]:
|
||||
markdown = self.page.evaluate(_build_extract_script(selector))
|
||||
return {
|
||||
"url": self.page.url,
|
||||
"title": self.page.title(),
|
||||
"ts": _now_ms(),
|
||||
"selector": selector or "body",
|
||||
"markdown": markdown or "",
|
||||
}
|
||||
|
||||
def build_ai_prompt(self, selector: Optional[str] = None, max_chars: Optional[int] = None) -> str:
|
||||
extracted = self.extract_markdown(selector)
|
||||
meta = {k: extracted[k] for k in ("url", "title", "ts", "selector")}
|
||||
script = _build_prompt_script(meta, extracted["markdown"], self.get_store(), max_chars)
|
||||
return self.page.evaluate(script)
|
||||
|
||||
|
||||
class AsyncExtractorSession(_CaptureMixin):
|
||||
"""Async API. Use with `playwright.async_api` or `camoufox.AsyncCamoufox`.
|
||||
|
||||
Example:
|
||||
from camoufox.async_api import AsyncCamoufox
|
||||
from context_extractor import AsyncExtractorSession
|
||||
|
||||
async with AsyncCamoufox(headless=True) as browser:
|
||||
page = await browser.new_page()
|
||||
session = AsyncExtractorSession(page)
|
||||
await page.goto("https://example.com")
|
||||
print(await session.build_ai_prompt())
|
||||
"""
|
||||
|
||||
def __init__(self, page) -> None:
|
||||
self.page = page
|
||||
self._init_capture_state()
|
||||
page.on("console", self._on_console)
|
||||
page.on("pageerror", self._on_pageerror)
|
||||
page.on("request", self._on_request)
|
||||
page.on("requestfinished", self._on_requestfinished)
|
||||
page.on("requestfailed", self._on_requestfailed)
|
||||
|
||||
async def extract_markdown(self, selector: Optional[str] = None) -> dict[str, Any]:
|
||||
markdown = await self.page.evaluate(_build_extract_script(selector))
|
||||
return {
|
||||
"url": self.page.url,
|
||||
"title": await self.page.title(),
|
||||
"ts": _now_ms(),
|
||||
"selector": selector or "body",
|
||||
"markdown": markdown or "",
|
||||
}
|
||||
|
||||
async def build_ai_prompt(self, selector: Optional[str] = None, max_chars: Optional[int] = None) -> str:
|
||||
extracted = await self.extract_markdown(selector)
|
||||
meta = {k: extracted[k] for k in ("url", "title", "ts", "selector")}
|
||||
script = _build_prompt_script(meta, extracted["markdown"], self.get_store(), max_chars)
|
||||
return await self.page.evaluate(script)
|
||||
40
automation/examples/scrape_camoufox.py
Normal file
40
automation/examples/scrape_camoufox.py
Normal file
@ -0,0 +1,40 @@
|
||||
"""Minimal example: capture context from a page using Camoufox (anti-detect Firefox).
|
||||
|
||||
Camoufox hands you a normal Playwright Page, so ExtractorSession works
|
||||
unmodified. No `main_world_eval` or init-script workarounds are needed here:
|
||||
capture uses Playwright's native page.on(...) hooks (not JS injection), and
|
||||
markdown extraction only *reads* the DOM, which works fine in Camoufox's
|
||||
default isolated world.
|
||||
|
||||
Run:
|
||||
pip install -e ./automation[camoufox]
|
||||
python automation/examples/scrape_camoufox.py https://example.com
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
from camoufox import Camoufox
|
||||
|
||||
from context_extractor import ExtractorSession
|
||||
|
||||
|
||||
def main() -> None:
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else "https://example.com"
|
||||
|
||||
with Camoufox(headless=True, geoip=True) as browser:
|
||||
page = browser.new_page()
|
||||
|
||||
session = ExtractorSession(page)
|
||||
|
||||
page.goto(url, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(1000) # let SPA fetches settle
|
||||
|
||||
print(session.build_ai_prompt())
|
||||
|
||||
# Or grab pieces individually:
|
||||
# extracted = session.extract_markdown("#main-content")
|
||||
# store = session.get_store() # {"console": [...], "errors": [...], "network": [...]}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
31
automation/examples/scrape_camoufox_async.py
Normal file
31
automation/examples/scrape_camoufox_async.py
Normal file
@ -0,0 +1,31 @@
|
||||
"""Async variant, using AsyncCamoufox + AsyncExtractorSession.
|
||||
|
||||
Run:
|
||||
pip install -e ./automation[camoufox]
|
||||
python automation/examples/scrape_camoufox_async.py https://example.com
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from camoufox.async_api import AsyncCamoufox
|
||||
|
||||
from context_extractor import AsyncExtractorSession
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else "https://example.com"
|
||||
|
||||
async with AsyncCamoufox(headless=True, geoip=True) as browser:
|
||||
page = await browser.new_page()
|
||||
|
||||
session = AsyncExtractorSession(page)
|
||||
|
||||
await page.goto(url, wait_until="domcontentloaded")
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
print(await session.build_ai_prompt())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
35
automation/examples/scrape_chromium.py
Normal file
35
automation/examples/scrape_chromium.py
Normal file
@ -0,0 +1,35 @@
|
||||
"""Minimal example: capture context from a page using plain Playwright + Chromium.
|
||||
|
||||
Run:
|
||||
pip install -e ./automation
|
||||
playwright install chromium
|
||||
python automation/examples/scrape_chromium.py https://example.com
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
from context_extractor import ExtractorSession
|
||||
|
||||
|
||||
def main() -> None:
|
||||
url = sys.argv[1] if len(sys.argv) > 1 else "https://example.com"
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
page = browser.new_page()
|
||||
|
||||
# Attach capture *before* navigating so console/network from the very
|
||||
# first load are seen.
|
||||
session = ExtractorSession(page)
|
||||
|
||||
page.goto(url, wait_until="networkidle")
|
||||
|
||||
print(session.build_ai_prompt()) # selector=None -> whole <body>
|
||||
|
||||
browser.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
34
automation/pyproject.toml
Normal file
34
automation/pyproject.toml
Normal file
@ -0,0 +1,34 @@
|
||||
[project]
|
||||
name = "context-extractor"
|
||||
version = "1.3.0"
|
||||
description = "Capture console/network/errors and extract AI-ready markdown from a Playwright or Camoufox page."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
license = { text = "MIT" }
|
||||
authors = [{ name = "Context Extractor contributors" }]
|
||||
keywords = ["playwright", "camoufox", "scraping", "llm", "markdown", "browser-extension"]
|
||||
dependencies = [
|
||||
"playwright>=1.44",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
camoufox = ["camoufox[geoip]>=0.4"]
|
||||
dev = ["pytest>=8.0", "pytest-asyncio>=0.23"]
|
||||
|
||||
[project.scripts]
|
||||
context-extractor = "context_extractor.cli:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["context_extractor*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
context_extractor = ["js/*.js"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
asyncio_mode = "auto"
|
||||
filterwarnings = ["ignore::DeprecationWarning"]
|
||||
28
automation/tests/conftest.py
Normal file
28
automation/tests/conftest.py
Normal file
@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "sample.html"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def playwright_browser():
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
yield browser
|
||||
browser.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def page(playwright_browser):
|
||||
page = playwright_browser.new_page()
|
||||
yield page
|
||||
page.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fixture_url() -> str:
|
||||
return FIXTURE.resolve().as_uri()
|
||||
33
automation/tests/fixtures/sample.html
vendored
Normal file
33
automation/tests/fixtures/sample.html
vendored
Normal file
@ -0,0 +1,33 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Context Extractor Fixture</title>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Fixture Page</h1>
|
||||
<nav>Skip this noise</nav>
|
||||
</header>
|
||||
<main id="main-content">
|
||||
<h2>Section</h2>
|
||||
<p>Hello <strong>world</strong>, visit <a href="/docs">docs</a>.</p>
|
||||
<ul>
|
||||
<li>Alpha</li>
|
||||
<li>Beta</li>
|
||||
</ul>
|
||||
<div style="display:none">hidden-inline-style-blob {"secretConfig": "should-not-appear"}</div>
|
||||
<div class="css-hidden">hidden-via-stylesheet-blob {"lixTracking": "should-not-appear"}</div>
|
||||
<div hidden>hidden-attribute-blob should-not-appear</div>
|
||||
<div aria-hidden="true">aria-hidden-blob should-not-appear</div>
|
||||
</main>
|
||||
<style>.css-hidden { display: none; }</style>
|
||||
<script>
|
||||
console.log("fixture boot");
|
||||
console.warn("fixture warn");
|
||||
console.error("fixture error log");
|
||||
setTimeout(() => { throw new Error("fixture boom"); }, 20);
|
||||
fetch("/missing-endpoint").catch(() => {});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
71
automation/tests/test_prompt.py
Normal file
71
automation/tests/test_prompt.py
Normal file
@ -0,0 +1,71 @@
|
||||
"""Unit-style tests for the shared prompt.js formatter (via page.evaluate)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from context_extractor.session import _build_prompt_script, _read_js
|
||||
|
||||
CORE = Path(__file__).resolve().parents[2] / "extension" / "core"
|
||||
|
||||
|
||||
def test_shared_js_files_exist_and_match_package():
|
||||
assert (CORE / "dom.js").is_file()
|
||||
assert (CORE / "prompt.js").is_file()
|
||||
# package-side copies/symlinks must resolve to the same source
|
||||
assert "function extractMarkdown" in _read_js("dom.js")
|
||||
assert "function buildAIPrompt" in _read_js("prompt.js")
|
||||
|
||||
|
||||
def test_build_ai_prompt_script_formats_errors_and_failures(page):
|
||||
page.goto("about:blank")
|
||||
meta = {
|
||||
"url": "https://example.test/page",
|
||||
"title": "T",
|
||||
"ts": 1_700_000_000_000,
|
||||
"selector": "body",
|
||||
}
|
||||
store = {
|
||||
"console": [{"level": "warn", "ts": 1, "msg": "careful"}],
|
||||
"errors": [{
|
||||
"type": "error", "ts": 2, "msg": "boom",
|
||||
"source": "a.js", "line": 3, "col": 4, "stack": "Error: boom\n at x",
|
||||
}],
|
||||
"network": [{
|
||||
"type": "fetch", "method": "GET", "url": "https://example.test/api",
|
||||
"status": 500, "ts": 3, "duration": 12, "error": None,
|
||||
}],
|
||||
}
|
||||
script = _build_prompt_script(meta, "# Hello", store)
|
||||
out = page.evaluate(script)
|
||||
assert "# Page Context" in out
|
||||
assert "https://example.test/page" in out
|
||||
assert "## Page Content" in out
|
||||
assert "# Hello" in out
|
||||
assert "## JavaScript Errors" in out
|
||||
assert "boom" in out
|
||||
assert "## Console Errors & Warnings" in out
|
||||
assert "[WARN] careful" in out
|
||||
assert "## Failed Requests" in out
|
||||
assert "GET 500 https://example.test/api" in out
|
||||
|
||||
|
||||
def test_build_ai_prompt_truncates_huge_markdown_by_default(page):
|
||||
"""Regression test: a JS-heavy SPA (e.g. LinkedIn) can extract 700k+ chars
|
||||
of body content. The prompt must not blindly include all of it."""
|
||||
page.goto("about:blank")
|
||||
meta = {"url": "https://example.test/huge", "title": "T", "ts": 1, "selector": "body"}
|
||||
huge_markdown = "x" * 50_000
|
||||
script = _build_prompt_script(meta, huge_markdown, {"console": [], "errors": [], "network": []})
|
||||
out = page.evaluate(script)
|
||||
assert "truncated to 20,000 of 50,000 chars" in out
|
||||
assert len(out) < 21_000 # header/footer overhead + capped body, nowhere near 50k
|
||||
|
||||
|
||||
def test_build_ai_prompt_respects_custom_max_chars(page):
|
||||
page.goto("about:blank")
|
||||
meta = {"url": "https://example.test/huge", "title": "T", "ts": 1, "selector": "body"}
|
||||
markdown = "y" * 1000
|
||||
script = _build_prompt_script(meta, markdown, {"console": [], "errors": [], "network": []}, max_chars=200)
|
||||
out = page.evaluate(script)
|
||||
assert "truncated to 200 of 1,000 chars" in out
|
||||
84
automation/tests/test_session.py
Normal file
84
automation/tests/test_session.py
Normal file
@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from context_extractor import AsyncExtractorSession, ExtractorSession, __version__
|
||||
|
||||
|
||||
def test_version_semverish():
|
||||
assert __version__.count(".") >= 1
|
||||
|
||||
|
||||
def test_capture_console_errors_and_network(page, fixture_url):
|
||||
session = ExtractorSession(page)
|
||||
page.goto(fixture_url, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(150)
|
||||
|
||||
store = session.get_store()
|
||||
levels = {e["level"] for e in store["console"]}
|
||||
assert "warn" in levels
|
||||
assert "error" in levels or any("error" in (e.get("msg") or "").lower() for e in store["console"])
|
||||
|
||||
assert any(e["msg"] == "fixture boom" or "fixture boom" in (e.get("msg") or "") for e in store["errors"])
|
||||
assert store["network"], "expected at least the document request"
|
||||
|
||||
|
||||
def test_extract_markdown_body_and_selector(page, fixture_url):
|
||||
session = ExtractorSession(page)
|
||||
page.goto(fixture_url, wait_until="domcontentloaded")
|
||||
|
||||
full = session.extract_markdown()
|
||||
assert full["title"] == "Context Extractor Fixture"
|
||||
assert "Fixture Page" in full["markdown"]
|
||||
assert "**world**" in full["markdown"]
|
||||
assert "[docs](/docs)" in full["markdown"]
|
||||
assert "- Alpha" in full["markdown"]
|
||||
|
||||
scoped = session.extract_markdown("#main-content")
|
||||
assert scoped["selector"] == "#main-content"
|
||||
assert "Section" in scoped["markdown"]
|
||||
assert "Skip this noise" not in scoped["markdown"]
|
||||
|
||||
|
||||
def test_hidden_nodes_are_excluded_from_markdown(page, fixture_url):
|
||||
"""Regression test: SPAs (e.g. LinkedIn) stash hydration/experiment JSON in
|
||||
hidden DOM nodes, not just <script> tags. Any hidden node must be excluded
|
||||
regardless of *how* it's hidden (inline style, stylesheet class, [hidden],
|
||||
aria-hidden)."""
|
||||
session = ExtractorSession(page)
|
||||
page.goto(fixture_url, wait_until="domcontentloaded")
|
||||
|
||||
markdown = session.extract_markdown("#main-content")["markdown"]
|
||||
assert "should-not-appear" not in markdown
|
||||
assert "secretConfig" not in markdown
|
||||
assert "lixTracking" not in markdown
|
||||
# Sanity: visible content in the same container still comes through.
|
||||
assert "Alpha" in markdown
|
||||
assert "Beta" in markdown
|
||||
|
||||
|
||||
def test_build_ai_prompt_has_sections(page, fixture_url):
|
||||
session = ExtractorSession(page)
|
||||
page.goto(fixture_url, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(150)
|
||||
|
||||
prompt = session.build_ai_prompt("#main-content")
|
||||
assert prompt.startswith("# Page Context")
|
||||
assert "## Page Content" in prompt
|
||||
assert "Hello **world**" in prompt
|
||||
assert "JavaScript Errors" in prompt or "Console Errors" in prompt
|
||||
assert "_Extracted by Context Extractor_" in prompt
|
||||
|
||||
|
||||
def test_clear_store(page, fixture_url):
|
||||
session = ExtractorSession(page)
|
||||
page.goto(fixture_url, wait_until="domcontentloaded")
|
||||
page.wait_for_timeout(100)
|
||||
assert session.get_store()["console"]
|
||||
session.clear("console")
|
||||
assert session.get_store()["console"] == []
|
||||
session.clear()
|
||||
store = session.get_store()
|
||||
assert store == {"console": [], "errors": [], "network": []}
|
||||
|
||||
|
||||
def test_async_session_exported():
|
||||
assert AsyncExtractorSession is not None
|
||||
18
extension/background.js
Normal file
18
extension/background.js
Normal file
@ -0,0 +1,18 @@
|
||||
// background.js — Context Extractor service worker
|
||||
// Persists the selector chosen via the in-page element picker so the popup
|
||||
// can pre-fill it the next time it opens.
|
||||
|
||||
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||||
if (!msg || !msg.type) return false;
|
||||
|
||||
if (msg.type === "PICKER_SELECTED") {
|
||||
const selector = msg.selector || "";
|
||||
chrome.storage.session
|
||||
.set({ pickerSelector: selector })
|
||||
.then(() => sendResponse({ ok: true }))
|
||||
.catch((err) => sendResponse({ ok: false, error: String(err) }));
|
||||
return true; // async response
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
219
extension/content.js
Normal file
219
extension/content.js
Normal file
@ -0,0 +1,219 @@
|
||||
// content.js — Context Extractor (isolated world)
|
||||
//
|
||||
// Loaded together with core/dom.js in the same content_scripts entry (see
|
||||
// manifest.json), so getSelector() and extractMarkdown() are already in
|
||||
// scope here — no import needed. Also injected on-demand by the popup via
|
||||
// chrome.scripting.executeScript when a tab was open before the extension
|
||||
// loaded (or after a reload of the extension without a page reload).
|
||||
//
|
||||
// Capture (console/fetch/XHR/errors) happens in page-patcher.js, which runs
|
||||
// in the MAIN world and forwards entries here via a CustomEvent, since the
|
||||
// isolated and main worlds don't share JS globals.
|
||||
(() => {
|
||||
// Guard against double injection (manifest auto-inject + popup inject).
|
||||
if (window.__ctxExtractorIsolated) return;
|
||||
window.__ctxExtractorIsolated = true;
|
||||
|
||||
const MAX_ENTRIES = 200;
|
||||
const BRIDGE_EVENT = "__ctxExtractorEvent";
|
||||
|
||||
const store = {
|
||||
console: [], // { level, ts, msg }
|
||||
errors: [], // { type, ts, msg, source, line, col, stack }
|
||||
network: [] // { type, method, url, status, ts, duration, error }
|
||||
};
|
||||
|
||||
function push(arr, entry) {
|
||||
arr.push(entry);
|
||||
while (arr.length > MAX_ENTRIES) arr.shift();
|
||||
}
|
||||
|
||||
document.addEventListener(BRIDGE_EVENT, (ev) => {
|
||||
const detail = ev && ev.detail;
|
||||
if (!detail) return;
|
||||
const { kind, payload } = detail;
|
||||
if (!payload) return;
|
||||
if (kind === "console") push(store.console, payload);
|
||||
else if (kind === "error") push(store.errors, payload);
|
||||
else if (kind === "network") push(store.network, payload);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Element picker
|
||||
// ---------------------------------------------------------------------------
|
||||
let pickerActive = false;
|
||||
let pickerNodes = null;
|
||||
|
||||
function startPicker() {
|
||||
if (pickerActive) return;
|
||||
pickerActive = true;
|
||||
|
||||
const overlay = document.createElement("div");
|
||||
overlay.style.cssText = [
|
||||
"position:fixed", "inset:0", "z-index:2147483646",
|
||||
"background:transparent", "cursor:crosshair", "pointer-events:auto"
|
||||
].join(";");
|
||||
|
||||
const highlight = document.createElement("div");
|
||||
highlight.style.cssText = [
|
||||
"position:fixed", "z-index:2147483647",
|
||||
"border:2px solid #4f98a3",
|
||||
"background:rgba(79,152,163,0.18)",
|
||||
"pointer-events:none",
|
||||
"transition:all 40ms linear",
|
||||
"box-sizing:border-box",
|
||||
"border-radius:2px"
|
||||
].join(";");
|
||||
|
||||
const label = document.createElement("div");
|
||||
label.style.cssText = [
|
||||
"position:fixed", "z-index:2147483647",
|
||||
"background:#1c1b19", "color:#e8e6e3",
|
||||
"font:12px/1.4 ui-monospace,Menlo,Consolas,monospace",
|
||||
"padding:4px 8px", "border-radius:4px",
|
||||
"border:1px solid #4f98a3",
|
||||
"pointer-events:none",
|
||||
"max-width:60vw", "overflow:hidden",
|
||||
"text-overflow:ellipsis", "white-space:nowrap"
|
||||
].join(";");
|
||||
label.textContent = "Pick an element — ESC to cancel";
|
||||
|
||||
document.documentElement.appendChild(overlay);
|
||||
document.documentElement.appendChild(highlight);
|
||||
document.documentElement.appendChild(label);
|
||||
|
||||
let currentEl = null;
|
||||
|
||||
function positionHighlight(el) {
|
||||
if (!el) return;
|
||||
const r = el.getBoundingClientRect();
|
||||
highlight.style.left = r.left + "px";
|
||||
highlight.style.top = r.top + "px";
|
||||
highlight.style.width = r.width + "px";
|
||||
highlight.style.height = r.height + "px";
|
||||
|
||||
const sel = getSelector(el);
|
||||
label.textContent = sel || el.tagName.toLowerCase();
|
||||
const lx = Math.min(r.left, window.innerWidth - 320);
|
||||
const ly = r.top > 28 ? r.top - 24 : r.bottom + 4;
|
||||
label.style.left = Math.max(4, lx) + "px";
|
||||
label.style.top = Math.max(4, ly) + "px";
|
||||
}
|
||||
|
||||
function onMove(e) {
|
||||
overlay.style.pointerEvents = "none";
|
||||
const el = document.elementFromPoint(e.clientX, e.clientY);
|
||||
overlay.style.pointerEvents = "auto";
|
||||
if (el && el !== highlight && el !== label && el !== overlay) {
|
||||
currentEl = el;
|
||||
positionHighlight(el);
|
||||
}
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
pickerActive = false;
|
||||
overlay.removeEventListener("mousemove", onMove, true);
|
||||
overlay.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("keydown", onKey, true);
|
||||
overlay.remove();
|
||||
highlight.remove();
|
||||
label.remove();
|
||||
pickerNodes = null;
|
||||
}
|
||||
|
||||
function onClick(e) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
overlay.style.pointerEvents = "none";
|
||||
const el = currentEl || document.elementFromPoint(e.clientX, e.clientY);
|
||||
const sel = el ? getSelector(el) : "";
|
||||
try {
|
||||
chrome.runtime.sendMessage({ type: "PICKER_SELECTED", selector: sel });
|
||||
} catch (_) {}
|
||||
cleanup();
|
||||
}
|
||||
|
||||
function onKey(e) {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
overlay.addEventListener("mousemove", onMove, true);
|
||||
overlay.addEventListener("click", onClick, true);
|
||||
document.addEventListener("keydown", onKey, true);
|
||||
|
||||
pickerNodes = { overlay, highlight, label, cleanup };
|
||||
}
|
||||
|
||||
function stopPicker() {
|
||||
if (pickerNodes && pickerNodes.cleanup) pickerNodes.cleanup();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Message handling
|
||||
// ---------------------------------------------------------------------------
|
||||
chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
|
||||
if (!msg || !msg.type) {
|
||||
sendResponse({ ok: false, error: "No message type" });
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
switch (msg.type) {
|
||||
case "GET_STORE": {
|
||||
sendResponse({ ok: true, store });
|
||||
break;
|
||||
}
|
||||
case "EXTRACT_CONTENT": {
|
||||
const selector = (msg.selector || "").trim();
|
||||
let target = null;
|
||||
if (selector) {
|
||||
try { target = document.querySelector(selector); } catch (_) { target = null; }
|
||||
}
|
||||
if (!target) target = document.body;
|
||||
const markdown = extractMarkdown(target);
|
||||
sendResponse({
|
||||
ok: true,
|
||||
meta: {
|
||||
url: location.href,
|
||||
title: document.title || "",
|
||||
ts: Date.now(),
|
||||
selector: selector || "body"
|
||||
},
|
||||
markdown
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "START_PICKER": {
|
||||
startPicker();
|
||||
sendResponse({ ok: true });
|
||||
break;
|
||||
}
|
||||
case "STOP_PICKER": {
|
||||
stopPicker();
|
||||
sendResponse({ ok: true });
|
||||
break;
|
||||
}
|
||||
case "CLEAR_STORE": {
|
||||
const which = msg.which;
|
||||
if (which && store[which]) store[which].length = 0;
|
||||
else {
|
||||
store.console.length = 0;
|
||||
store.errors.length = 0;
|
||||
store.network.length = 0;
|
||||
}
|
||||
sendResponse({ ok: true });
|
||||
break;
|
||||
}
|
||||
default:
|
||||
sendResponse({ ok: false, error: "Unknown type: " + msg.type });
|
||||
}
|
||||
} catch (err) {
|
||||
sendResponse({ ok: false, error: String(err && err.message || err) });
|
||||
}
|
||||
return true; // keep channel open for async
|
||||
});
|
||||
})();
|
||||
186
extension/core/dom.js
Normal file
186
extension/core/dom.js
Normal file
@ -0,0 +1,186 @@
|
||||
// core/dom.js — shared DOM helpers (Context Extractor)
|
||||
//
|
||||
// Pure, read-only DOM logic with no chrome.* / node dependency. Loaded two ways:
|
||||
// 1. Extension: listed before content.js in manifest.json's content_scripts,
|
||||
// so these top-level declarations land in the same isolated-world scope.
|
||||
// 2. Automation (Playwright/Camoufox): file contents are read from disk and
|
||||
// evaluated in-page via page.evaluate(). Safe under Camoufox's default
|
||||
// isolated world since it only *reads* the DOM (getComputedStyle included)
|
||||
// — it never writes to the live page, and no longer even clones it.
|
||||
//
|
||||
// Do not add chrome.*, window.close(), or any write to the live document here.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CSS selector synthesis
|
||||
// ---------------------------------------------------------------------------
|
||||
function getSelector(el) {
|
||||
if (!(el instanceof Element)) return "";
|
||||
if (el.id && /^[A-Za-z][\w-]*$/.test(el.id)) return "#" + el.id;
|
||||
|
||||
const parts = [];
|
||||
let node = el;
|
||||
while (node && node.nodeType === 1 && node !== document.documentElement) {
|
||||
let part = node.tagName.toLowerCase();
|
||||
if (node.id && /^[A-Za-z][\w-]*$/.test(node.id)) {
|
||||
parts.unshift("#" + node.id);
|
||||
break;
|
||||
}
|
||||
if (node.classList && node.classList.length) {
|
||||
const cls = Array.from(node.classList)
|
||||
.filter((c) => /^[A-Za-z_][\w-]*$/.test(c))
|
||||
.slice(0, 2)
|
||||
.map((c) => "." + c)
|
||||
.join("");
|
||||
part += cls;
|
||||
}
|
||||
const parent = node.parentElement;
|
||||
if (parent) {
|
||||
const sameTag = Array.from(parent.children).filter(
|
||||
(c) => c.tagName === node.tagName
|
||||
);
|
||||
if (sameTag.length > 1) {
|
||||
const idx = sameTag.indexOf(node) + 1;
|
||||
part += ":nth-of-type(" + idx + ")";
|
||||
}
|
||||
}
|
||||
parts.unshift(part);
|
||||
node = node.parentElement;
|
||||
}
|
||||
return parts.join(" > ");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Markdown extraction
|
||||
//
|
||||
// Walks the *live* node directly — no cloning needed. Only ever reads
|
||||
// (textContent, attributes, getComputedStyle); never mutates. Skips tags that
|
||||
// never carry visible content, plus anything actually invisible to a human
|
||||
// (display:none / visibility:hidden / [hidden] / [aria-hidden="true"]).
|
||||
// SPAs routinely stash large hydration/experiment JSON payloads in hidden DOM
|
||||
// nodes (not just <script> tags) — without this check that JSON gets read as
|
||||
// if it were visible page text.
|
||||
// ---------------------------------------------------------------------------
|
||||
const NEVER_RENDERED_TAGS = new Set(["script", "style", "noscript", "template", "svg", "iframe", "canvas"]);
|
||||
|
||||
function isHidden(el) {
|
||||
if (!el || el.nodeType !== 1) return false;
|
||||
if (el.hidden) return true;
|
||||
const ariaHidden = el.getAttribute && el.getAttribute("aria-hidden");
|
||||
if (ariaHidden === "true") return true;
|
||||
try {
|
||||
const cs = window.getComputedStyle(el);
|
||||
if (cs && (cs.display === "none" || cs.visibility === "hidden" || cs.visibility === "collapse")) {
|
||||
return true;
|
||||
}
|
||||
} catch (_) {
|
||||
// getComputedStyle can throw on detached/foreign nodes; treat as visible.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldSkip(el) {
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
return NEVER_RENDERED_TAGS.has(tag) || isHidden(el);
|
||||
}
|
||||
|
||||
function extractMarkdown(node) {
|
||||
if (!node) return "";
|
||||
|
||||
const out = [];
|
||||
|
||||
function emit(s) { out.push(s); }
|
||||
|
||||
function inline(el) {
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
const kids = walkInline(el);
|
||||
switch (tag) {
|
||||
case "a": {
|
||||
const href = el.getAttribute("href") || "";
|
||||
return "[" + kids + "](" + href + ")";
|
||||
}
|
||||
case "strong":
|
||||
case "b":
|
||||
return "**" + kids + "**";
|
||||
case "em":
|
||||
case "i":
|
||||
return "_" + kids + "_";
|
||||
case "code":
|
||||
return "`" + kids + "`";
|
||||
case "br":
|
||||
return "\n";
|
||||
default:
|
||||
return kids;
|
||||
}
|
||||
}
|
||||
|
||||
function walkInline(el) {
|
||||
let s = "";
|
||||
el.childNodes.forEach((c) => {
|
||||
if (c.nodeType === 3) s += c.nodeValue;
|
||||
else if (c.nodeType === 1 && !shouldSkip(c)) s += inline(c);
|
||||
});
|
||||
return s;
|
||||
}
|
||||
|
||||
function block(el) {
|
||||
if (shouldSkip(el)) return;
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
|
||||
if (/^h[1-6]$/.test(tag)) {
|
||||
const level = Number(tag[1]);
|
||||
emit("\n" + "#".repeat(level) + " " + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (tag) {
|
||||
case "p":
|
||||
emit("\n" + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
case "br":
|
||||
emit("\n");
|
||||
return;
|
||||
case "hr":
|
||||
emit("\n---\n");
|
||||
return;
|
||||
case "li":
|
||||
emit("- " + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
case "pre":
|
||||
emit("\n```\n" + (el.textContent || "").replace(/\n+$/, "") + "\n```\n");
|
||||
return;
|
||||
case "blockquote":
|
||||
emit("\n> " + walkInline(el).trim().replace(/\n/g, "\n> ") + "\n");
|
||||
return;
|
||||
case "ul":
|
||||
case "ol":
|
||||
case "div":
|
||||
case "section":
|
||||
case "article":
|
||||
case "main":
|
||||
case "header":
|
||||
case "footer":
|
||||
case "nav":
|
||||
case "aside":
|
||||
case "body":
|
||||
el.childNodes.forEach((c) => {
|
||||
if (c.nodeType === 1) block(c);
|
||||
else if (c.nodeType === 3) {
|
||||
const t = c.nodeValue;
|
||||
if (t && t.trim()) emit(t);
|
||||
}
|
||||
});
|
||||
return;
|
||||
default: {
|
||||
const text = walkInline(el);
|
||||
if (text) emit(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
block(node);
|
||||
|
||||
let md = out.join("");
|
||||
md = md.replace(/[ \t]+\n/g, "\n");
|
||||
md = md.replace(/\n{3,}/g, "\n\n");
|
||||
return md.trim();
|
||||
}
|
||||
99
extension/core/prompt.js
Normal file
99
extension/core/prompt.js
Normal file
@ -0,0 +1,99 @@
|
||||
// core/prompt.js — shared AI-prompt formatter (Context Extractor)
|
||||
//
|
||||
// Pure string formatting, no DOM/chrome.* dependency. Shared by:
|
||||
// 1. Extension popup (popup.html loads this before popup.js).
|
||||
// 2. Automation (evaluated in-page via page.evaluate so both surfaces
|
||||
// produce byte-identical output for the same inputs).
|
||||
//
|
||||
// store shape: { console: [{level,ts,msg}], errors: [{type,ts,msg,source,line,col,stack}],
|
||||
// network: [{type,method,url,status,ts,duration,error}] }
|
||||
// meta shape: { url, title, ts, selector }
|
||||
//
|
||||
// markdown is capped at maxChars (default 20,000) before it goes in the
|
||||
// prompt. Extracting the whole <body> of a JS-heavy SPA easily produces
|
||||
// hundreds of thousands of characters (LinkedIn: 700k+) — most LLM context
|
||||
// windows can't take that, and it's rarely what you want to paste anyway.
|
||||
// Pass a tighter selector (or a larger maxChars) instead of relying on this
|
||||
// cap for real content; it exists as a safety net, not a summarizer.
|
||||
const DEFAULT_MAX_MARKDOWN_CHARS = 20000;
|
||||
|
||||
function buildAIPrompt(meta, markdown, store, maxChars) {
|
||||
const limit = maxChars || DEFAULT_MAX_MARKDOWN_CHARS;
|
||||
const parts = [];
|
||||
|
||||
parts.push("# Page Context");
|
||||
if (meta) {
|
||||
if (meta.url) parts.push(`- URL: ${meta.url}`);
|
||||
if (meta.title) parts.push(`- Title: ${meta.title}`);
|
||||
if (meta.ts) parts.push(`- Captured: ${new Date(meta.ts).toISOString()}`);
|
||||
if (meta.selector) parts.push(`- Selector: \`${meta.selector}\``);
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
const trimmed = markdown && markdown.trim();
|
||||
if (trimmed) {
|
||||
const truncated = trimmed.length > limit;
|
||||
const heading = truncated
|
||||
? `## Page Content (truncated to ${limit.toLocaleString()} of ${trimmed.length.toLocaleString()} chars — use a narrower selector to see more)`
|
||||
: "## Page Content";
|
||||
parts.push(heading);
|
||||
parts.push("");
|
||||
parts.push(truncated ? trimmed.slice(0, limit) + "\n…" : trimmed);
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
const errors = (store && store.errors) || [];
|
||||
if (errors.length) {
|
||||
parts.push("## JavaScript Errors");
|
||||
parts.push("");
|
||||
errors.forEach((e, i) => {
|
||||
const where = [e.source, e.line ? "line " + e.line : "", e.col ? "col " + e.col : ""]
|
||||
.filter(Boolean).join(" ");
|
||||
parts.push(`### ${i + 1}. ${e.type || "error"}: ${e.msg || ""}`);
|
||||
if (where) parts.push(`- Location: ${where}`);
|
||||
if (e.ts) parts.push(`- Time: ${new Date(e.ts).toISOString()}`);
|
||||
if (e.stack) {
|
||||
parts.push("");
|
||||
parts.push("```");
|
||||
parts.push(e.stack);
|
||||
parts.push("```");
|
||||
}
|
||||
parts.push("");
|
||||
});
|
||||
}
|
||||
|
||||
const consoleEntries = (store && store.console) || [];
|
||||
const consoleIssues = consoleEntries.filter((c) => c.level === "error" || c.level === "warn");
|
||||
if (consoleIssues.length) {
|
||||
parts.push("## Console Errors & Warnings");
|
||||
parts.push("");
|
||||
consoleIssues.forEach((c) => {
|
||||
parts.push(`- [${(c.level || "log").toUpperCase()}] ${c.msg || ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
const network = (store && store.network) || [];
|
||||
const failed = network.filter((n) => (n.status >= 400) || n.error);
|
||||
if (failed.length) {
|
||||
parts.push("## Failed Requests");
|
||||
parts.push("");
|
||||
failed.forEach((n) => {
|
||||
const status = n.error ? "ERR" : n.status;
|
||||
parts.push(`- ${n.method || "GET"} ${status} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
} else if (network.length) {
|
||||
parts.push("## Recent Requests");
|
||||
parts.push("");
|
||||
network.slice(-15).forEach((n) => {
|
||||
parts.push(`- ${n.method || "GET"} ${n.status || "—"} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
parts.push("---");
|
||||
parts.push("_Extracted by Context Extractor_");
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
30
extension/manifest.json
Normal file
30
extension/manifest.json
Normal file
@ -0,0 +1,30 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "Context Extractor",
|
||||
"version": "1.3.2",
|
||||
"description": "Extract clean, AI-ready context from any webpage — page content, console logs, network requests, and JS errors. Works unpacked in Chrome and Brave.",
|
||||
"permissions": ["activeTab", "scripting", "storage"],
|
||||
"host_permissions": ["<all_urls>"],
|
||||
"background": {
|
||||
"service_worker": "background.js"
|
||||
},
|
||||
"action": {
|
||||
"default_popup": "popup.html",
|
||||
"default_title": "Context Extractor"
|
||||
},
|
||||
"content_scripts": [
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["core/dom.js", "content.js"],
|
||||
"run_at": "document_start",
|
||||
"all_frames": false
|
||||
},
|
||||
{
|
||||
"matches": ["<all_urls>"],
|
||||
"js": ["page-patcher.js"],
|
||||
"world": "MAIN",
|
||||
"run_at": "document_start",
|
||||
"all_frames": false
|
||||
}
|
||||
]
|
||||
}
|
||||
176
extension/page-patcher.js
Normal file
176
extension/page-patcher.js
Normal file
@ -0,0 +1,176 @@
|
||||
// page-patcher.js — Context Extractor
|
||||
//
|
||||
// Declared in manifest.json with "world": "MAIN" (Chrome/Brave 111+), so this
|
||||
// file executes directly in the page's own JS realm — no inline <script>
|
||||
// element is created, which means it is NOT subject to the page's script-src
|
||||
// CSP the way a page-inserted inline script would be. This replaces the old
|
||||
// approach (content.js injecting a <script> tag), which silently failed to
|
||||
// capture page-script activity on strict-CSP sites.
|
||||
//
|
||||
// MAIN world scripts cannot call chrome.runtime.* directly, so results are
|
||||
// handed to the isolated-world content.js via a CustomEvent bridge.
|
||||
(() => {
|
||||
if (window.__ctxExtractorMain) return;
|
||||
window.__ctxExtractorMain = true;
|
||||
|
||||
const BRIDGE_EVENT = "__ctxExtractorEvent";
|
||||
|
||||
const SEND = (kind, payload) => {
|
||||
try {
|
||||
document.dispatchEvent(new CustomEvent(BRIDGE_EVENT, { detail: { kind, payload } }));
|
||||
} catch (_) {}
|
||||
};
|
||||
|
||||
const safeStringify = (v) => {
|
||||
if (v === null) return "null";
|
||||
if (v === undefined) return "undefined";
|
||||
const t = typeof v;
|
||||
if (t === "string") return v;
|
||||
if (t === "number" || t === "boolean" || t === "bigint") return String(v);
|
||||
if (t === "function") return "[Function" + (v.name ? " " + v.name : "") + "]";
|
||||
if (t === "symbol") return v.toString();
|
||||
if (v instanceof Error) return v.stack || (v.name + ": " + v.message);
|
||||
try {
|
||||
const seen = new WeakSet();
|
||||
return JSON.stringify(v, (k, val) => {
|
||||
if (typeof val === "object" && val !== null) {
|
||||
if (seen.has(val)) return "[Circular]";
|
||||
seen.add(val);
|
||||
}
|
||||
if (typeof val === "function") return "[Function]";
|
||||
if (typeof val === "bigint") return val.toString() + "n";
|
||||
return val;
|
||||
});
|
||||
} catch (_) {
|
||||
try { return String(v); } catch (__) { return "[Unserializable]"; }
|
||||
}
|
||||
};
|
||||
|
||||
const fmtArgs = (args) => Array.from(args).map(safeStringify).join(" ");
|
||||
|
||||
// --- console ---
|
||||
const levels = ["log", "warn", "error", "info", "debug"];
|
||||
for (const lvl of levels) {
|
||||
const orig = console[lvl];
|
||||
if (typeof orig !== "function") continue;
|
||||
console[lvl] = function (...args) {
|
||||
try {
|
||||
SEND("console", { level: lvl, ts: Date.now(), msg: fmtArgs(args) });
|
||||
} catch (_) {}
|
||||
return orig.apply(this, args);
|
||||
};
|
||||
}
|
||||
|
||||
// --- window errors ---
|
||||
window.addEventListener("error", (e) => {
|
||||
SEND("error", {
|
||||
type: "error",
|
||||
ts: Date.now(),
|
||||
msg: (e && e.message) || "Unknown error",
|
||||
source: (e && e.filename) || "",
|
||||
line: (e && e.lineno) || 0,
|
||||
col: (e && e.colno) || 0,
|
||||
stack: (e && e.error && e.error.stack) || ""
|
||||
});
|
||||
});
|
||||
|
||||
window.addEventListener("unhandledrejection", (e) => {
|
||||
let msg = "Unhandled promise rejection";
|
||||
let stack = "";
|
||||
const r = e && e.reason;
|
||||
if (r instanceof Error) { msg = r.message; stack = r.stack || ""; }
|
||||
else if (r !== undefined) { msg = safeStringify(r); }
|
||||
SEND("error", { type: "unhandledrejection", ts: Date.now(), msg, source: "", line: 0, col: 0, stack });
|
||||
});
|
||||
|
||||
// --- XHR ---
|
||||
const XHRProto = XMLHttpRequest.prototype;
|
||||
const origOpen = XHRProto.open;
|
||||
const origSend = XHRProto.send;
|
||||
XHRProto.open = function (method, url) {
|
||||
try {
|
||||
this.__ctx = { method: String(method || "GET").toUpperCase(), url: String(url || ""), start: 0 };
|
||||
} catch (_) {}
|
||||
return origOpen.apply(this, arguments);
|
||||
};
|
||||
XHRProto.send = function () {
|
||||
try {
|
||||
if (this.__ctx) this.__ctx.start = performance.now();
|
||||
this.addEventListener("loadend", () => {
|
||||
try {
|
||||
const ctx = this.__ctx || {};
|
||||
SEND("network", {
|
||||
type: "xhr",
|
||||
method: ctx.method || "GET",
|
||||
url: ctx.url || "",
|
||||
status: this.status || 0,
|
||||
ts: Date.now(),
|
||||
duration: ctx.start ? Math.round(performance.now() - ctx.start) : 0,
|
||||
error: null
|
||||
});
|
||||
} catch (_) {}
|
||||
});
|
||||
this.addEventListener("error", () => {
|
||||
try {
|
||||
const ctx = this.__ctx || {};
|
||||
SEND("network", {
|
||||
type: "xhr",
|
||||
method: ctx.method || "GET",
|
||||
url: ctx.url || "",
|
||||
status: 0,
|
||||
ts: Date.now(),
|
||||
duration: ctx.start ? Math.round(performance.now() - ctx.start) : 0,
|
||||
error: "Network error"
|
||||
});
|
||||
} catch (_) {}
|
||||
});
|
||||
} catch (_) {}
|
||||
return origSend.apply(this, arguments);
|
||||
};
|
||||
|
||||
// --- fetch ---
|
||||
const origFetch = window.fetch;
|
||||
if (typeof origFetch === "function") {
|
||||
window.fetch = function (input, init) {
|
||||
const start = performance.now();
|
||||
let method = "GET";
|
||||
let url = "";
|
||||
try {
|
||||
if (typeof input === "string") { url = input; }
|
||||
else if (input && typeof input.url === "string") { url = input.url; method = input.method || method; }
|
||||
if (init && init.method) method = init.method;
|
||||
method = String(method).toUpperCase();
|
||||
} catch (_) {}
|
||||
return origFetch.apply(this, arguments).then(
|
||||
(res) => {
|
||||
try {
|
||||
SEND("network", {
|
||||
type: "fetch",
|
||||
method,
|
||||
url,
|
||||
status: res.status || 0,
|
||||
ts: Date.now(),
|
||||
duration: Math.round(performance.now() - start),
|
||||
error: null
|
||||
});
|
||||
} catch (_) {}
|
||||
return res;
|
||||
},
|
||||
(err) => {
|
||||
try {
|
||||
SEND("network", {
|
||||
type: "fetch",
|
||||
method,
|
||||
url,
|
||||
status: 0,
|
||||
ts: Date.now(),
|
||||
duration: Math.round(performance.now() - start),
|
||||
error: (err && err.message) || String(err)
|
||||
});
|
||||
} catch (_) {}
|
||||
throw err;
|
||||
}
|
||||
);
|
||||
};
|
||||
}
|
||||
})();
|
||||
283
extension/popup.html
Normal file
283
extension/popup.html
Normal file
@ -0,0 +1,283 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>Context Extractor</title>
|
||||
<style>
|
||||
:root {
|
||||
--bg: #171614;
|
||||
--surface: #1c1b19;
|
||||
--surface-2: #232220;
|
||||
--border: #2e2c29;
|
||||
--text: #e8e6e3;
|
||||
--muted: #8a8580;
|
||||
--primary: #4f98a3;
|
||||
--primary-hover: #5fb0bd;
|
||||
--ok: #6ecf7a;
|
||||
--warn: #e7c46a;
|
||||
--err: #e96b8a;
|
||||
--redir: #e7c46a;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html, body {
|
||||
margin: 0; padding: 0;
|
||||
width: 440px;
|
||||
background: var(--bg); color: var(--text);
|
||||
font: 13px/1.45 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
}
|
||||
.header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 10px 12px; border-bottom: 1px solid var(--border);
|
||||
background: var(--surface);
|
||||
}
|
||||
.header h1 {
|
||||
margin: 0; font-size: 13px; font-weight: 600; color: var(--text);
|
||||
letter-spacing: 0.2px;
|
||||
}
|
||||
.header .sub { font-size: 11px; color: var(--muted); }
|
||||
|
||||
.tabs {
|
||||
display: grid; grid-template-columns: repeat(4, 1fr);
|
||||
background: var(--surface);
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.tab {
|
||||
position: relative;
|
||||
background: transparent; color: var(--muted);
|
||||
border: 0; border-bottom: 2px solid transparent;
|
||||
padding: 9px 4px; font-size: 12px; font-weight: 500;
|
||||
cursor: pointer; text-align: center;
|
||||
font-family: inherit;
|
||||
}
|
||||
.tab:hover { color: var(--text); }
|
||||
.tab.active { color: var(--text); border-bottom-color: var(--primary); }
|
||||
.tab .count {
|
||||
color: var(--muted); font-size: 11px; margin-left: 4px;
|
||||
}
|
||||
.tab .dot {
|
||||
display: none;
|
||||
position: absolute; top: 6px; right: 8px;
|
||||
width: 6px; height: 6px; border-radius: 50%;
|
||||
background: var(--err);
|
||||
}
|
||||
.tab.has-issue .dot { display: block; }
|
||||
|
||||
.panel { display: none; padding: 12px; }
|
||||
.panel.active { display: block; }
|
||||
|
||||
.row { display: flex; gap: 6px; align-items: center; }
|
||||
.row + .row { margin-top: 8px; }
|
||||
|
||||
input[type="text"] {
|
||||
flex: 1; min-width: 0;
|
||||
background: var(--surface-2); color: var(--text);
|
||||
border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 6px 8px; font: 12px ui-monospace, Menlo, Consolas, monospace;
|
||||
outline: none;
|
||||
}
|
||||
input[type="text"]:focus { border-color: var(--primary); }
|
||||
|
||||
button {
|
||||
background: var(--surface-2); color: var(--text);
|
||||
border: 1px solid var(--border); border-radius: 4px;
|
||||
padding: 6px 10px; font-size: 12px; cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
button:hover { background: #2a2825; border-color: #3a3733; }
|
||||
button.primary {
|
||||
background: var(--primary); border-color: var(--primary); color: #0f1f22;
|
||||
font-weight: 600;
|
||||
}
|
||||
button.primary:hover { background: var(--primary-hover); border-color: var(--primary-hover); }
|
||||
button.ghost { background: transparent; }
|
||||
button.danger { color: var(--err); border-color: #3a2a2f; }
|
||||
button.danger:hover { background: #2a1f23; }
|
||||
|
||||
.actions {
|
||||
display: flex; gap: 6px; margin-top: 10px;
|
||||
padding-top: 10px; border-top: 1px solid var(--border);
|
||||
}
|
||||
.actions .spacer { flex: 1; }
|
||||
|
||||
.preview {
|
||||
margin-top: 10px;
|
||||
background: var(--surface-2); border: 1px solid var(--border);
|
||||
border-radius: 4px; padding: 8px;
|
||||
max-height: 280px; overflow: auto;
|
||||
font: 11.5px/1.45 ui-monospace, Menlo, Consolas, monospace;
|
||||
white-space: pre-wrap; word-break: break-word;
|
||||
color: #d6d3cf;
|
||||
}
|
||||
.preview.empty { color: var(--muted); }
|
||||
|
||||
.meta {
|
||||
font-size: 11px; color: var(--muted);
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.list {
|
||||
background: var(--surface-2); border: 1px solid var(--border);
|
||||
border-radius: 4px;
|
||||
max-height: 320px; overflow: auto;
|
||||
}
|
||||
.list .empty {
|
||||
padding: 18px; text-align: center; color: var(--muted); font-size: 12px;
|
||||
}
|
||||
|
||||
.log-row {
|
||||
padding: 6px 8px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font: 11.5px ui-monospace, Menlo, Consolas, monospace;
|
||||
display: flex; gap: 8px;
|
||||
}
|
||||
.log-row:last-child { border-bottom: 0; }
|
||||
.log-row .lvl {
|
||||
flex: 0 0 44px; text-transform: uppercase;
|
||||
font-size: 10px; font-weight: 700; padding-top: 1px;
|
||||
}
|
||||
.log-row .msg {
|
||||
flex: 1; min-width: 0; word-break: break-word; white-space: pre-wrap;
|
||||
color: #d6d3cf;
|
||||
}
|
||||
.log-row.error .lvl { color: var(--err); }
|
||||
.log-row.warn .lvl { color: var(--warn); }
|
||||
.log-row.info .lvl { color: var(--primary); }
|
||||
.log-row.log .lvl { color: var(--muted); }
|
||||
.log-row.debug .lvl { color: var(--muted); }
|
||||
|
||||
.net-row {
|
||||
display: grid;
|
||||
grid-template-columns: 50px 50px 1fr 60px;
|
||||
gap: 8px; align-items: center;
|
||||
padding: 6px 8px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font: 11px ui-monospace, Menlo, Consolas, monospace;
|
||||
}
|
||||
.net-row:last-child { border-bottom: 0; }
|
||||
.net-row .method { color: var(--muted); font-weight: 700; }
|
||||
.net-row .status { font-weight: 700; }
|
||||
.net-row .url {
|
||||
color: #d6d3cf; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.net-row .dur { color: var(--muted); text-align: right; }
|
||||
.net-row.ok .status { color: var(--ok); }
|
||||
.net-row.redir .status { color: var(--redir); }
|
||||
.net-row.err .status { color: var(--err); }
|
||||
|
||||
.err-row {
|
||||
padding: 8px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
font: 11.5px ui-monospace, Menlo, Consolas, monospace;
|
||||
}
|
||||
.err-row:last-child { border-bottom: 0; }
|
||||
.err-row .head {
|
||||
color: var(--err); font-weight: 700; margin-bottom: 4px; word-break: break-word;
|
||||
}
|
||||
.err-row .sub { color: var(--muted); font-size: 10.5px; margin-bottom: 4px; }
|
||||
.err-row .stack {
|
||||
color: #c4c1bd; white-space: pre-wrap; word-break: break-word;
|
||||
max-height: 120px; overflow: auto;
|
||||
}
|
||||
|
||||
.summary {
|
||||
font-size: 11px; color: var(--muted);
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
.summary b { color: var(--text); font-weight: 600; }
|
||||
|
||||
.toast {
|
||||
position: fixed; left: 50%; bottom: 14px; transform: translateX(-50%);
|
||||
background: #2a3a3d; color: var(--text);
|
||||
border: 1px solid var(--primary);
|
||||
padding: 6px 12px; border-radius: 4px;
|
||||
font-size: 12px;
|
||||
opacity: 0; pointer-events: none;
|
||||
transition: opacity 140ms ease;
|
||||
z-index: 10;
|
||||
}
|
||||
.toast.show { opacity: 1; }
|
||||
|
||||
::-webkit-scrollbar { width: 8px; height: 8px; }
|
||||
::-webkit-scrollbar-thumb { background: #2e2c29; border-radius: 4px; }
|
||||
::-webkit-scrollbar-thumb:hover { background: #3a3733; }
|
||||
::-webkit-scrollbar-track { background: transparent; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="header">
|
||||
<h1>Context Extractor</h1>
|
||||
<span class="sub" id="pageHint">—</span>
|
||||
</div>
|
||||
|
||||
<div class="tabs" role="tablist">
|
||||
<button class="tab active" data-tab="content" role="tab">
|
||||
Content<span class="count" id="count-content"></span><span class="dot"></span>
|
||||
</button>
|
||||
<button class="tab" data-tab="console" role="tab">
|
||||
Console<span class="count" id="count-console"></span><span class="dot"></span>
|
||||
</button>
|
||||
<button class="tab" data-tab="network" role="tab">
|
||||
Network<span class="count" id="count-network"></span><span class="dot"></span>
|
||||
</button>
|
||||
<button class="tab" data-tab="errors" role="tab">
|
||||
Errors<span class="count" id="count-errors"></span><span class="dot"></span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Content tab -->
|
||||
<div class="panel active" data-panel="content">
|
||||
<div class="row">
|
||||
<input type="text" id="selectorInput" placeholder="CSS selector (blank = body)" />
|
||||
<button id="pickBtn" title="Pick an element on the page">Pick</button>
|
||||
</div>
|
||||
<div class="row">
|
||||
<button id="extractBtn" class="primary" style="flex:1">Extract content</button>
|
||||
</div>
|
||||
<div class="meta" id="extractMeta"></div>
|
||||
<div class="preview empty" id="contentPreview">No content extracted yet.</div>
|
||||
<div class="actions">
|
||||
<button id="copyContent" class="primary">Copy</button>
|
||||
<div class="spacer"></div>
|
||||
<button id="clearContent" class="danger ghost">Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Console tab -->
|
||||
<div class="panel" data-panel="console">
|
||||
<div class="summary" id="consoleSummary">No console activity captured.</div>
|
||||
<div class="list" id="consoleList"></div>
|
||||
<div class="actions">
|
||||
<button id="copyConsole" class="primary">Copy</button>
|
||||
<div class="spacer"></div>
|
||||
<button id="clearConsole" class="danger ghost">Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Network tab -->
|
||||
<div class="panel" data-panel="network">
|
||||
<div class="summary" id="networkSummary">No requests captured.</div>
|
||||
<div class="list" id="networkList"></div>
|
||||
<div class="actions">
|
||||
<button id="copyNetwork" class="primary">Copy</button>
|
||||
<div class="spacer"></div>
|
||||
<button id="clearNetwork" class="danger ghost">Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Errors tab -->
|
||||
<div class="panel" data-panel="errors">
|
||||
<div class="summary" id="errorsSummary">No JavaScript errors captured.</div>
|
||||
<div class="list" id="errorsList"></div>
|
||||
<div class="actions">
|
||||
<button id="copyErrors" class="primary">Copy</button>
|
||||
<div class="spacer"></div>
|
||||
<button id="clearErrors" class="danger ghost">Clear</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="toast" id="toast">Copied ✓</div>
|
||||
|
||||
<script src="core/prompt.js"></script>
|
||||
<script src="popup.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
512
extension/popup.js
Normal file
512
extension/popup.js
Normal file
@ -0,0 +1,512 @@
|
||||
// popup.js — Context Extractor
|
||||
//
|
||||
// buildAIPrompt() comes from core/prompt.js, loaded via a <script> tag in
|
||||
// popup.html before this file, so it's already in scope here.
|
||||
|
||||
const state = {
|
||||
tabId: null,
|
||||
store: { console: [], network: [], errors: [] },
|
||||
lastExtract: null // { meta, markdown }
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Messaging helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
async function getActiveTab() {
|
||||
const [tab] = await chrome.tabs.query({ active: true, currentWindow: true });
|
||||
return tab || null;
|
||||
}
|
||||
|
||||
function isRestrictedUrl(url) {
|
||||
if (!url) return true;
|
||||
return /^(chrome|brave|edge|about|devtools|chrome-extension|brave-extension|moz-extension):/i.test(url)
|
||||
|| url.startsWith("https://chrome.google.com/webstore")
|
||||
|| url.startsWith("https://chromewebstore.google.com");
|
||||
}
|
||||
|
||||
function sendRaw(message) {
|
||||
return new Promise((resolve) => {
|
||||
if (state.tabId == null) {
|
||||
return resolve({ ok: false, error: "No active tab." });
|
||||
}
|
||||
try {
|
||||
chrome.tabs.sendMessage(state.tabId, message, (response) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
return resolve({
|
||||
ok: false,
|
||||
error: chrome.runtime.lastError.message || "No content script on this tab."
|
||||
});
|
||||
}
|
||||
resolve(response || { ok: false, error: "Empty response from content script." });
|
||||
});
|
||||
} catch (err) {
|
||||
resolve({ ok: false, error: String(err && err.message || err) });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function injectContentScripts() {
|
||||
if (state.tabId == null) throw new Error("No active tab.");
|
||||
// Isolated world: DOM helpers + messaging/store/picker.
|
||||
await chrome.scripting.executeScript({
|
||||
target: { tabId: state.tabId },
|
||||
files: ["core/dom.js", "content.js"]
|
||||
});
|
||||
// MAIN world: console/fetch/XHR patches (may miss early page activity if
|
||||
// injected late, but still useful going forward).
|
||||
try {
|
||||
await chrome.scripting.executeScript({
|
||||
target: { tabId: state.tabId },
|
||||
files: ["page-patcher.js"],
|
||||
world: "MAIN"
|
||||
});
|
||||
} catch (_) {
|
||||
// MAIN-world inject can fail on some pages; extraction still works.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message to the content script. If it isn't present yet (tab was open
|
||||
* before the extension loaded/reloaded), inject it and retry once.
|
||||
*/
|
||||
async function sendToContent(message) {
|
||||
const first = await sendRaw(message);
|
||||
if (first && first.ok) return first;
|
||||
|
||||
const err = (first && first.error) || "";
|
||||
const needsInject = /Receiving end does not exist|Could not establish connection|No content script/i.test(err)
|
||||
|| first == null;
|
||||
|
||||
if (!needsInject) return first;
|
||||
|
||||
try {
|
||||
await injectContentScripts();
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false,
|
||||
error: "Could not inject into this tab: " + String(e && e.message || e)
|
||||
};
|
||||
}
|
||||
|
||||
// Tiny pause so the listener is registered before we message again.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
return sendRaw(message);
|
||||
}
|
||||
|
||||
function showExtractError(preview, meta, message) {
|
||||
preview.classList.add("empty");
|
||||
preview.textContent = message || "Could not extract content from this page.";
|
||||
meta.textContent = "";
|
||||
state.lastExtract = null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tabs
|
||||
// ---------------------------------------------------------------------------
|
||||
function activateTab(name) {
|
||||
document.querySelectorAll(".tab").forEach((t) => {
|
||||
t.classList.toggle("active", t.dataset.tab === name);
|
||||
});
|
||||
document.querySelectorAll(".panel").forEach((p) => {
|
||||
p.classList.toggle("active", p.dataset.panel === name);
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll(".tab").forEach((t) => {
|
||||
t.addEventListener("click", () => activateTab(t.dataset.tab));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Toast
|
||||
// ---------------------------------------------------------------------------
|
||||
let toastTimer = null;
|
||||
function toast(msg) {
|
||||
const el = document.getElementById("toast");
|
||||
el.textContent = msg || "Copied ✓";
|
||||
el.classList.add("show");
|
||||
clearTimeout(toastTimer);
|
||||
toastTimer = setTimeout(() => el.classList.remove("show"), 1800);
|
||||
}
|
||||
|
||||
async function copyText(text) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
toast("Copied ✓");
|
||||
} catch (e) {
|
||||
// Fallback
|
||||
const ta = document.createElement("textarea");
|
||||
ta.value = text;
|
||||
document.body.appendChild(ta);
|
||||
ta.select();
|
||||
try { document.execCommand("copy"); toast("Copied ✓"); }
|
||||
catch (_) { toast("Copy failed"); }
|
||||
ta.remove();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rendering
|
||||
// ---------------------------------------------------------------------------
|
||||
function fmtTime(ts) {
|
||||
try {
|
||||
const d = new Date(ts);
|
||||
return d.toLocaleTimeString([], { hour12: false });
|
||||
} catch (_) { return ""; }
|
||||
}
|
||||
|
||||
function truncate(s, n) {
|
||||
if (s == null) return "";
|
||||
s = String(s);
|
||||
return s.length > n ? s.slice(0, n) + "…" : s;
|
||||
}
|
||||
|
||||
function statusClass(status, error) {
|
||||
if (error || !status) return "err";
|
||||
if (status >= 400) return "err";
|
||||
if (status >= 300) return "redir";
|
||||
if (status >= 200) return "ok";
|
||||
return "err";
|
||||
}
|
||||
|
||||
function renderConsole() {
|
||||
const list = document.getElementById("consoleList");
|
||||
const summary = document.getElementById("consoleSummary");
|
||||
const entries = state.store.console || [];
|
||||
|
||||
const counts = { error: 0, warn: 0, info: 0, log: 0, debug: 0 };
|
||||
entries.forEach((e) => { if (counts[e.level] != null) counts[e.level]++; });
|
||||
summary.innerHTML = `<b>${entries.length}</b> entries · ` +
|
||||
`<span style="color:var(--err)">err ${counts.error}</span> · ` +
|
||||
`<span style="color:var(--warn)">warn ${counts.warn}</span> · ` +
|
||||
`<span style="color:var(--primary)">info ${counts.info}</span> · ` +
|
||||
`log ${counts.log} · debug ${counts.debug}`;
|
||||
|
||||
if (!entries.length) {
|
||||
list.innerHTML = `<div class="empty">No console activity captured.</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = entries.slice().reverse().map((e) => {
|
||||
const lvl = (e.level || "log").toLowerCase();
|
||||
return `<div class="log-row ${lvl}">
|
||||
<div class="lvl">${lvl}</div>
|
||||
<div class="msg">${escapeHtml(truncate(e.msg, 800))}</div>
|
||||
</div>`;
|
||||
});
|
||||
list.innerHTML = rows.join("");
|
||||
}
|
||||
|
||||
function renderNetwork() {
|
||||
const list = document.getElementById("networkList");
|
||||
const summary = document.getElementById("networkSummary");
|
||||
const entries = state.store.network || [];
|
||||
|
||||
const fails = entries.filter((n) => (n.status >= 400) || n.error).length;
|
||||
summary.innerHTML = `<b>${entries.length}</b> requests · ` +
|
||||
`<span style="color:var(--err)">${fails} failed</span>`;
|
||||
|
||||
if (!entries.length) {
|
||||
list.innerHTML = `<div class="empty">No requests captured.</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = entries.slice().reverse().map((n) => {
|
||||
const cls = statusClass(n.status, n.error);
|
||||
const statusText = n.error ? "ERR" : (n.status || "—");
|
||||
return `<div class="net-row ${cls}">
|
||||
<div class="method">${escapeHtml(n.method || "")}</div>
|
||||
<div class="status">${escapeHtml(String(statusText))}</div>
|
||||
<div class="url" title="${escapeHtml(n.url || "")}">${escapeHtml(truncate(n.url, 200))}</div>
|
||||
<div class="dur">${n.duration ? n.duration + "ms" : ""}</div>
|
||||
</div>`;
|
||||
});
|
||||
list.innerHTML = rows.join("");
|
||||
}
|
||||
|
||||
function renderErrors() {
|
||||
const list = document.getElementById("errorsList");
|
||||
const summary = document.getElementById("errorsSummary");
|
||||
const entries = state.store.errors || [];
|
||||
|
||||
summary.innerHTML = `<b>${entries.length}</b> JavaScript errors captured.`;
|
||||
|
||||
if (!entries.length) {
|
||||
list.innerHTML = `<div class="empty">No JavaScript errors captured.</div>`;
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = entries.slice().reverse().map((e) => {
|
||||
const where = [e.source, e.line ? "line " + e.line : "", e.col ? "col " + e.col : ""]
|
||||
.filter(Boolean).join(" · ");
|
||||
return `<div class="err-row">
|
||||
<div class="head">${escapeHtml(e.type || "error")}: ${escapeHtml(truncate(e.msg, 300))}</div>
|
||||
<div class="sub">${escapeHtml(where || fmtTime(e.ts))}</div>
|
||||
${e.stack ? `<div class="stack">${escapeHtml(truncate(e.stack, 1500))}</div>` : ""}
|
||||
</div>`;
|
||||
});
|
||||
list.innerHTML = rows.join("");
|
||||
}
|
||||
|
||||
function updateBadges() {
|
||||
const cs = state.store.console || [];
|
||||
const ns = state.store.network || [];
|
||||
const es = state.store.errors || [];
|
||||
|
||||
document.getElementById("count-console").textContent = cs.length ? " " + cs.length : "";
|
||||
document.getElementById("count-network").textContent = ns.length ? " " + ns.length : "";
|
||||
document.getElementById("count-errors").textContent = es.length ? " " + es.length : "";
|
||||
|
||||
const consoleHasIssue = cs.some((e) => e.level === "error" || e.level === "warn");
|
||||
const networkHasIssue = ns.some((n) => (n.status >= 400) || n.error);
|
||||
const errorsHasIssue = es.length > 0;
|
||||
|
||||
setIssue("console", consoleHasIssue);
|
||||
setIssue("network", networkHasIssue);
|
||||
setIssue("errors", errorsHasIssue);
|
||||
}
|
||||
|
||||
function setIssue(tab, on) {
|
||||
const el = document.querySelector(`.tab[data-tab="${tab}"]`);
|
||||
if (el) el.classList.toggle("has-issue", !!on);
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
return String(s == null ? "" : s)
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Store fetch
|
||||
// ---------------------------------------------------------------------------
|
||||
async function refreshStore() {
|
||||
const res = await sendToContent({ type: "GET_STORE" });
|
||||
if (res && res.ok && res.store) {
|
||||
state.store = {
|
||||
console: res.store.console || [],
|
||||
network: res.store.network || [],
|
||||
errors: res.store.errors || []
|
||||
};
|
||||
} else {
|
||||
state.store = { console: [], network: [], errors: [] };
|
||||
}
|
||||
renderConsole();
|
||||
renderNetwork();
|
||||
renderErrors();
|
||||
updateBadges();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Content extraction
|
||||
// ---------------------------------------------------------------------------
|
||||
async function doExtract() {
|
||||
const selector = document.getElementById("selectorInput").value.trim();
|
||||
const preview = document.getElementById("contentPreview");
|
||||
const meta = document.getElementById("extractMeta");
|
||||
|
||||
const tab = await getActiveTab();
|
||||
state.tabId = tab ? tab.id : null;
|
||||
if (!tab) {
|
||||
showExtractError(preview, meta, "No active tab.");
|
||||
return;
|
||||
}
|
||||
if (isRestrictedUrl(tab.url)) {
|
||||
showExtractError(
|
||||
preview,
|
||||
meta,
|
||||
"Can't run on this page (" + (tab.url || "unknown") + "). Open a normal http(s) tab and try again."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
preview.classList.add("empty");
|
||||
preview.textContent = "Extracting…";
|
||||
|
||||
const res = await sendToContent({ type: "EXTRACT_CONTENT", selector });
|
||||
|
||||
if (!res || !res.ok) {
|
||||
showExtractError(
|
||||
preview,
|
||||
meta,
|
||||
(res && res.error)
|
||||
? "Could not extract: " + res.error + "\n\nTip: reload the extension on brave://extensions, then click Extract again (injection is automatic now)."
|
||||
: "Could not extract content from this page."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
state.lastExtract = { meta: res.meta, markdown: res.markdown || "" };
|
||||
meta.textContent = `${res.meta.selector} · ${res.markdown.length.toLocaleString()} chars · ${fmtTime(res.meta.ts)}`;
|
||||
if (res.markdown && res.markdown.length) {
|
||||
preview.classList.remove("empty");
|
||||
preview.textContent = res.markdown.length > 5000
|
||||
? res.markdown.slice(0, 5000) + "\n\n… [truncated — full content included when you click Copy]"
|
||||
: res.markdown;
|
||||
} else {
|
||||
preview.classList.add("empty");
|
||||
preview.textContent = "No content found for that selector.";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire up buttons
|
||||
// ---------------------------------------------------------------------------
|
||||
function wire() {
|
||||
document.getElementById("extractBtn").addEventListener("click", doExtract);
|
||||
|
||||
document.getElementById("pickBtn").addEventListener("click", async () => {
|
||||
const tab = await getActiveTab();
|
||||
state.tabId = tab ? tab.id : null;
|
||||
if (!tab || isRestrictedUrl(tab.url)) {
|
||||
toast("Can't pick on this page");
|
||||
return;
|
||||
}
|
||||
const res = await sendToContent({ type: "START_PICKER" });
|
||||
if (!res || !res.ok) {
|
||||
toast((res && res.error) || "Picker failed");
|
||||
return;
|
||||
}
|
||||
window.close();
|
||||
});
|
||||
|
||||
document.getElementById("copyContent").addEventListener("click", async () => {
|
||||
// Make sure we have an extraction
|
||||
if (!state.lastExtract) await doExtract();
|
||||
const ex = state.lastExtract || { meta: null, markdown: "" };
|
||||
await refreshStore(); // ensure store fresh for prompt
|
||||
const text = buildAIPrompt(ex.meta, ex.markdown, state.store);
|
||||
copyText(text);
|
||||
});
|
||||
|
||||
document.getElementById("copyConsole").addEventListener("click", async () => {
|
||||
await refreshStore();
|
||||
const tab = await getActiveTab();
|
||||
const url = tab && tab.url ? tab.url : "";
|
||||
const out = [];
|
||||
out.push("# Console Output");
|
||||
if (url) out.push(`- URL: ${url}`);
|
||||
out.push(`- Captured: ${new Date().toISOString()}`);
|
||||
out.push(`- Entries: ${state.store.console.length}`);
|
||||
out.push("");
|
||||
state.store.console.forEach((c) => {
|
||||
out.push(`[${new Date(c.ts).toISOString()}] [${(c.level || "log").toUpperCase()}] ${c.msg || ""}`);
|
||||
});
|
||||
out.push("");
|
||||
out.push("---");
|
||||
out.push("_Extracted by Context Extractor_");
|
||||
copyText(out.join("\n"));
|
||||
});
|
||||
|
||||
document.getElementById("copyNetwork").addEventListener("click", async () => {
|
||||
await refreshStore();
|
||||
const tab = await getActiveTab();
|
||||
const url = tab && tab.url ? tab.url : "";
|
||||
const out = [];
|
||||
out.push("# Network Activity");
|
||||
if (url) out.push(`- URL: ${url}`);
|
||||
out.push(`- Captured: ${new Date().toISOString()}`);
|
||||
out.push(`- Requests: ${state.store.network.length}`);
|
||||
out.push("");
|
||||
state.store.network.forEach((n) => {
|
||||
const status = n.error ? "ERR" : (n.status || "—");
|
||||
out.push(`- ${n.method || "GET"} ${status} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`);
|
||||
});
|
||||
out.push("");
|
||||
out.push("---");
|
||||
out.push("_Extracted by Context Extractor_");
|
||||
copyText(out.join("\n"));
|
||||
});
|
||||
|
||||
document.getElementById("copyErrors").addEventListener("click", async () => {
|
||||
await refreshStore();
|
||||
const tab = await getActiveTab();
|
||||
const url = tab && tab.url ? tab.url : "";
|
||||
const out = [];
|
||||
out.push("# JavaScript Errors");
|
||||
if (url) out.push(`- URL: ${url}`);
|
||||
out.push(`- Captured: ${new Date().toISOString()}`);
|
||||
out.push(`- Errors: ${state.store.errors.length}`);
|
||||
out.push("");
|
||||
state.store.errors.forEach((e, i) => {
|
||||
const where = [e.source, e.line ? "line " + e.line : "", e.col ? "col " + e.col : ""]
|
||||
.filter(Boolean).join(" ");
|
||||
out.push(`## ${i + 1}. ${e.type || "error"}: ${e.msg || ""}`);
|
||||
if (where) out.push(`- Location: ${where}`);
|
||||
out.push(`- Time: ${new Date(e.ts).toISOString()}`);
|
||||
if (e.stack) {
|
||||
out.push("");
|
||||
out.push("```");
|
||||
out.push(e.stack);
|
||||
out.push("```");
|
||||
}
|
||||
out.push("");
|
||||
});
|
||||
out.push("---");
|
||||
out.push("_Extracted by Context Extractor_");
|
||||
copyText(out.join("\n"));
|
||||
});
|
||||
|
||||
// Clear buttons
|
||||
document.getElementById("clearContent").addEventListener("click", async () => {
|
||||
state.lastExtract = null;
|
||||
document.getElementById("contentPreview").classList.add("empty");
|
||||
document.getElementById("contentPreview").textContent = "No content extracted yet.";
|
||||
document.getElementById("extractMeta").textContent = "";
|
||||
toast("Cleared");
|
||||
});
|
||||
|
||||
document.getElementById("clearConsole").addEventListener("click", async () => {
|
||||
await sendToContent({ type: "CLEAR_STORE", which: "console" });
|
||||
await refreshStore();
|
||||
toast("Cleared");
|
||||
});
|
||||
|
||||
document.getElementById("clearNetwork").addEventListener("click", async () => {
|
||||
await sendToContent({ type: "CLEAR_STORE", which: "network" });
|
||||
await refreshStore();
|
||||
toast("Cleared");
|
||||
});
|
||||
|
||||
document.getElementById("clearErrors").addEventListener("click", async () => {
|
||||
await sendToContent({ type: "CLEAR_STORE", which: "errors" });
|
||||
await refreshStore();
|
||||
toast("Cleared");
|
||||
});
|
||||
|
||||
document.getElementById("selectorInput").addEventListener("keydown", (e) => {
|
||||
if (e.key === "Enter") doExtract();
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Init
|
||||
// ---------------------------------------------------------------------------
|
||||
(async function init() {
|
||||
wire();
|
||||
|
||||
const tab = await getActiveTab();
|
||||
state.tabId = tab ? tab.id : null;
|
||||
|
||||
// Set page hint
|
||||
if (tab && tab.url) {
|
||||
try {
|
||||
const u = new URL(tab.url);
|
||||
document.getElementById("pageHint").textContent = u.hostname;
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
// Pre-fill selector from session storage
|
||||
try {
|
||||
const data = await chrome.storage.session.get("pickerSelector");
|
||||
if (data && data.pickerSelector) {
|
||||
document.getElementById("selectorInput").value = data.pickerSelector;
|
||||
// Clear it after consuming so it doesn't stick forever
|
||||
await chrome.storage.session.remove("pickerSelector");
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
await refreshStore();
|
||||
})();
|
||||
28
scripts/package_extension.py
Normal file
28
scripts/package_extension.py
Normal file
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Zip up extension/ into dist/context-extractor-extension.zip for distribution.
|
||||
|
||||
For local dev, you don't need this — just "Load unpacked" and point Chrome or
|
||||
Brave straight at the extension/ folder. This script is only useful if you
|
||||
want a shareable zip (e.g. to hand to someone else, or side-load elsewhere).
|
||||
"""
|
||||
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SRC = ROOT / "extension"
|
||||
DIST = ROOT / "dist"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
DIST.mkdir(exist_ok=True)
|
||||
out_path = DIST / "context-extractor-extension.zip"
|
||||
with zipfile.ZipFile(out_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for path in sorted(SRC.rglob("*")):
|
||||
if path.is_file():
|
||||
zf.write(path, path.relative_to(SRC))
|
||||
print(f"Wrote {out_path} ({out_path.stat().st_size:,} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user