Initial Context Extractor: extension + Playwright/Camoufox package
CI / Lint + tests (push) Has been cancelled
CI / Lint + tests (push) Has been cancelled
Ship a shared markdown/prompt core used by a Brave/Chrome MV3 extension and a Python automation API, with pytest coverage, CI, and packaging smoke.
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
// core/dom.js — shared DOM helpers (Context Extractor)
|
||||
//
|
||||
// Pure, read-only DOM logic with no chrome.* / node dependency. Loaded two ways:
|
||||
// 1. Extension: listed before content.js in manifest.json's content_scripts,
|
||||
// so these top-level declarations land in the same isolated-world scope.
|
||||
// 2. Automation (Playwright/Camoufox): file contents are read from disk and
|
||||
// evaluated in-page via page.evaluate(). Safe under Camoufox's default
|
||||
// isolated world since it only *reads* the DOM (getComputedStyle included)
|
||||
// — it never writes to the live page, and no longer even clones it.
|
||||
//
|
||||
// Do not add chrome.*, window.close(), or any write to the live document here.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CSS selector synthesis
|
||||
// ---------------------------------------------------------------------------
|
||||
function getSelector(el) {
|
||||
if (!(el instanceof Element)) return "";
|
||||
if (el.id && /^[A-Za-z][\w-]*$/.test(el.id)) return "#" + el.id;
|
||||
|
||||
const parts = [];
|
||||
let node = el;
|
||||
while (node && node.nodeType === 1 && node !== document.documentElement) {
|
||||
let part = node.tagName.toLowerCase();
|
||||
if (node.id && /^[A-Za-z][\w-]*$/.test(node.id)) {
|
||||
parts.unshift("#" + node.id);
|
||||
break;
|
||||
}
|
||||
if (node.classList && node.classList.length) {
|
||||
const cls = Array.from(node.classList)
|
||||
.filter((c) => /^[A-Za-z_][\w-]*$/.test(c))
|
||||
.slice(0, 2)
|
||||
.map((c) => "." + c)
|
||||
.join("");
|
||||
part += cls;
|
||||
}
|
||||
const parent = node.parentElement;
|
||||
if (parent) {
|
||||
const sameTag = Array.from(parent.children).filter(
|
||||
(c) => c.tagName === node.tagName
|
||||
);
|
||||
if (sameTag.length > 1) {
|
||||
const idx = sameTag.indexOf(node) + 1;
|
||||
part += ":nth-of-type(" + idx + ")";
|
||||
}
|
||||
}
|
||||
parts.unshift(part);
|
||||
node = node.parentElement;
|
||||
}
|
||||
return parts.join(" > ");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Markdown extraction
|
||||
//
|
||||
// Walks the *live* node directly — no cloning needed. Only ever reads
|
||||
// (textContent, attributes, getComputedStyle); never mutates. Skips tags that
|
||||
// never carry visible content, plus anything actually invisible to a human
|
||||
// (display:none / visibility:hidden / [hidden] / [aria-hidden="true"]).
|
||||
// SPAs routinely stash large hydration/experiment JSON payloads in hidden DOM
|
||||
// nodes (not just <script> tags) — without this check that JSON gets read as
|
||||
// if it were visible page text.
|
||||
// ---------------------------------------------------------------------------
|
||||
const NEVER_RENDERED_TAGS = new Set(["script", "style", "noscript", "template", "svg", "iframe", "canvas"]);
|
||||
|
||||
function isHidden(el) {
|
||||
if (!el || el.nodeType !== 1) return false;
|
||||
if (el.hidden) return true;
|
||||
const ariaHidden = el.getAttribute && el.getAttribute("aria-hidden");
|
||||
if (ariaHidden === "true") return true;
|
||||
try {
|
||||
const cs = window.getComputedStyle(el);
|
||||
if (cs && (cs.display === "none" || cs.visibility === "hidden" || cs.visibility === "collapse")) {
|
||||
return true;
|
||||
}
|
||||
} catch (_) {
|
||||
// getComputedStyle can throw on detached/foreign nodes; treat as visible.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldSkip(el) {
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
return NEVER_RENDERED_TAGS.has(tag) || isHidden(el);
|
||||
}
|
||||
|
||||
function extractMarkdown(node) {
|
||||
if (!node) return "";
|
||||
|
||||
const out = [];
|
||||
|
||||
function emit(s) { out.push(s); }
|
||||
|
||||
function inline(el) {
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
const kids = walkInline(el);
|
||||
switch (tag) {
|
||||
case "a": {
|
||||
const href = el.getAttribute("href") || "";
|
||||
return "[" + kids + "](" + href + ")";
|
||||
}
|
||||
case "strong":
|
||||
case "b":
|
||||
return "**" + kids + "**";
|
||||
case "em":
|
||||
case "i":
|
||||
return "_" + kids + "_";
|
||||
case "code":
|
||||
return "`" + kids + "`";
|
||||
case "br":
|
||||
return "\n";
|
||||
default:
|
||||
return kids;
|
||||
}
|
||||
}
|
||||
|
||||
function walkInline(el) {
|
||||
let s = "";
|
||||
el.childNodes.forEach((c) => {
|
||||
if (c.nodeType === 3) s += c.nodeValue;
|
||||
else if (c.nodeType === 1 && !shouldSkip(c)) s += inline(c);
|
||||
});
|
||||
return s;
|
||||
}
|
||||
|
||||
function block(el) {
|
||||
if (shouldSkip(el)) return;
|
||||
const tag = el.tagName ? el.tagName.toLowerCase() : "";
|
||||
|
||||
if (/^h[1-6]$/.test(tag)) {
|
||||
const level = Number(tag[1]);
|
||||
emit("\n" + "#".repeat(level) + " " + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (tag) {
|
||||
case "p":
|
||||
emit("\n" + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
case "br":
|
||||
emit("\n");
|
||||
return;
|
||||
case "hr":
|
||||
emit("\n---\n");
|
||||
return;
|
||||
case "li":
|
||||
emit("- " + walkInline(el).trim() + "\n");
|
||||
return;
|
||||
case "pre":
|
||||
emit("\n```\n" + (el.textContent || "").replace(/\n+$/, "") + "\n```\n");
|
||||
return;
|
||||
case "blockquote":
|
||||
emit("\n> " + walkInline(el).trim().replace(/\n/g, "\n> ") + "\n");
|
||||
return;
|
||||
case "ul":
|
||||
case "ol":
|
||||
case "div":
|
||||
case "section":
|
||||
case "article":
|
||||
case "main":
|
||||
case "header":
|
||||
case "footer":
|
||||
case "nav":
|
||||
case "aside":
|
||||
case "body":
|
||||
el.childNodes.forEach((c) => {
|
||||
if (c.nodeType === 1) block(c);
|
||||
else if (c.nodeType === 3) {
|
||||
const t = c.nodeValue;
|
||||
if (t && t.trim()) emit(t);
|
||||
}
|
||||
});
|
||||
return;
|
||||
default: {
|
||||
const text = walkInline(el);
|
||||
if (text) emit(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
block(node);
|
||||
|
||||
let md = out.join("");
|
||||
md = md.replace(/[ \t]+\n/g, "\n");
|
||||
md = md.replace(/\n{3,}/g, "\n\n");
|
||||
return md.trim();
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
// core/prompt.js — shared AI-prompt formatter (Context Extractor)
|
||||
//
|
||||
// Pure string formatting, no DOM/chrome.* dependency. Shared by:
|
||||
// 1. Extension popup (popup.html loads this before popup.js).
|
||||
// 2. Automation (evaluated in-page via page.evaluate so both surfaces
|
||||
// produce byte-identical output for the same inputs).
|
||||
//
|
||||
// store shape: { console: [{level,ts,msg}], errors: [{type,ts,msg,source,line,col,stack}],
|
||||
// network: [{type,method,url,status,ts,duration,error}] }
|
||||
// meta shape: { url, title, ts, selector }
|
||||
//
|
||||
// markdown is capped at maxChars (default 20,000) before it goes in the
|
||||
// prompt. Extracting the whole <body> of a JS-heavy SPA easily produces
|
||||
// hundreds of thousands of characters (LinkedIn: 700k+) — most LLM context
|
||||
// windows can't take that, and it's rarely what you want to paste anyway.
|
||||
// Pass a tighter selector (or a larger maxChars) instead of relying on this
|
||||
// cap for real content; it exists as a safety net, not a summarizer.
|
||||
const DEFAULT_MAX_MARKDOWN_CHARS = 20000;
|
||||
|
||||
function buildAIPrompt(meta, markdown, store, maxChars) {
|
||||
const limit = maxChars || DEFAULT_MAX_MARKDOWN_CHARS;
|
||||
const parts = [];
|
||||
|
||||
parts.push("# Page Context");
|
||||
if (meta) {
|
||||
if (meta.url) parts.push(`- URL: ${meta.url}`);
|
||||
if (meta.title) parts.push(`- Title: ${meta.title}`);
|
||||
if (meta.ts) parts.push(`- Captured: ${new Date(meta.ts).toISOString()}`);
|
||||
if (meta.selector) parts.push(`- Selector: \`${meta.selector}\``);
|
||||
}
|
||||
parts.push("");
|
||||
|
||||
const trimmed = markdown && markdown.trim();
|
||||
if (trimmed) {
|
||||
const truncated = trimmed.length > limit;
|
||||
const heading = truncated
|
||||
? `## Page Content (truncated to ${limit.toLocaleString()} of ${trimmed.length.toLocaleString()} chars — use a narrower selector to see more)`
|
||||
: "## Page Content";
|
||||
parts.push(heading);
|
||||
parts.push("");
|
||||
parts.push(truncated ? trimmed.slice(0, limit) + "\n…" : trimmed);
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
const errors = (store && store.errors) || [];
|
||||
if (errors.length) {
|
||||
parts.push("## JavaScript Errors");
|
||||
parts.push("");
|
||||
errors.forEach((e, i) => {
|
||||
const where = [e.source, e.line ? "line " + e.line : "", e.col ? "col " + e.col : ""]
|
||||
.filter(Boolean).join(" ");
|
||||
parts.push(`### ${i + 1}. ${e.type || "error"}: ${e.msg || ""}`);
|
||||
if (where) parts.push(`- Location: ${where}`);
|
||||
if (e.ts) parts.push(`- Time: ${new Date(e.ts).toISOString()}`);
|
||||
if (e.stack) {
|
||||
parts.push("");
|
||||
parts.push("```");
|
||||
parts.push(e.stack);
|
||||
parts.push("```");
|
||||
}
|
||||
parts.push("");
|
||||
});
|
||||
}
|
||||
|
||||
const consoleEntries = (store && store.console) || [];
|
||||
const consoleIssues = consoleEntries.filter((c) => c.level === "error" || c.level === "warn");
|
||||
if (consoleIssues.length) {
|
||||
parts.push("## Console Errors & Warnings");
|
||||
parts.push("");
|
||||
consoleIssues.forEach((c) => {
|
||||
parts.push(`- [${(c.level || "log").toUpperCase()}] ${c.msg || ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
const network = (store && store.network) || [];
|
||||
const failed = network.filter((n) => (n.status >= 400) || n.error);
|
||||
if (failed.length) {
|
||||
parts.push("## Failed Requests");
|
||||
parts.push("");
|
||||
failed.forEach((n) => {
|
||||
const status = n.error ? "ERR" : n.status;
|
||||
parts.push(`- ${n.method || "GET"} ${status} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}${n.error ? " — " + n.error : ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
} else if (network.length) {
|
||||
parts.push("## Recent Requests");
|
||||
parts.push("");
|
||||
network.slice(-15).forEach((n) => {
|
||||
parts.push(`- ${n.method || "GET"} ${n.status || "—"} ${n.url}${n.duration ? " (" + n.duration + "ms)" : ""}`);
|
||||
});
|
||||
parts.push("");
|
||||
}
|
||||
|
||||
parts.push("---");
|
||||
parts.push("_Extracted by Context Extractor_");
|
||||
|
||||
return parts.join("\n");
|
||||
}
|
||||
Reference in New Issue
Block a user