feat: add JS Remotely (javascript.jobs) extractor #7

Merged
ilia merged 2 commits from feat/javascriptjobs-extractor into main 2026-08-10 17:20:32 -05:00
12 changed files with 466 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
# javascriptjobs-extractor
Pulls remote listings from [javascript.jobs](https://javascript.jobs) (the
board formerly promoted under the `jsremotely.com` alias — that domain now
301s straight here). JS/TS-focused: React, Vue, Node, Angular, TypeScript.
- No authentication, no public API/RSS — this is a server-rendered HTML
scrape of `/remote`, which supports a `keyword` query param for server-side
term filtering (`?keyword=react&page=2`), so pipeline search terms are
iterated as separate paginated fetches rather than filtered client-side.
- Listing cards give title/employer/job-type badge/salary text/logo. Detail
pages (`/job/<slug>`) additionally embed a `JobPosting` (schema.org)
`<script type="application/ld+json">` block with description, exact
`datePosted`, `employmentType`, and structured `baseSalary`.
- That embedded block is **not valid JSON** (unescaped newlines inside
`description`, and at least one observed missing comma between sibling
keys), so we extract fields with targeted regexes instead of
`JSON.parse`. If a detail fetch fails or a field is missing, we fall back
to the listing-card values.
- `baseSalary` in that block is unreliable on the source side: postings
with no salary entered still emit `minValue`/`maxValue: 0` (we treat 0 as
"not specified"), and at least one observed posting had `minValue` equal
to `maxValue` (both set to the range's upper bound) despite the
human-readable listing-card text showing a real range. We keep the raw
listing-card salary string in `salary` as a trustworthy fallback alongside
the (possibly wrong) structured `salaryMinAmount`/`salaryMaxAmount`.
- Caps via `javascriptjobsMaxJobsPerTerm` (default 50) and
`javascriptjobsMaxPages` (default 3, pages per term).
- Every matched job triggers one extra HTTP GET (detail page) for
enrichment — keep the caps conservative given this is a low-volume niche
board.
+333
View File
@@ -0,0 +1,333 @@
/**
* JS Remotely / javascript.jobs — JS/TS-focused remote board (server-rendered
* HTML, no public API or RSS).
*
* https://javascript.jobs/remote?keyword=<term>&page=<n>
*
* `jsremotely.com` now 301s straight to `javascript.jobs`, so we hit the
* canonical domain directly. The `/remote` listing supports a `keyword`
* query param that filters server-side, so we iterate pipeline search terms
* as separate paginated fetches rather than filtering client-side.
*
* Listing cards give us title/employer/badges/salary text; detail pages
* additionally embed a `JobPosting` (schema.org) block for description,
* exact date posted, employment type, and structured salary. That block is
* emitted with unescaped newlines and occasional missing commas, so it is
* not valid JSON — we pull fields out with targeted regexes instead of
* `JSON.parse`.
*/
import type {
ExtractorManifest,
ExtractorRunResult,
} from "@shared/types/extractors";
import type { CreateJobInput } from "@shared/types/jobs";
const ORIGIN = "https://javascript.jobs";
const USER_AGENT = "JobOps/1.0 (+https://github.com/) job-search pipeline";
/** One job card's markup is well under this; keeps field regexes from
* bleeding into the next card (or page footer for the last card). */
const CARD_WINDOW = 2500;
interface ListingJob {
slug: string;
title: string;
employer: string;
jobType?: string;
isRemoteBadge: boolean;
salaryText?: string;
companyLogo?: string;
}
function decodeHtmlEntities(value: string): string {
return value
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'");
}
function collapseWhitespace(value: string): string {
return value.replace(/\s+/g, " ").trim();
}
function parseListingJobs(html: string): ListingJob[] {
const chunks = html.split('<a href="https://javascript.jobs/job/').slice(1);
const jobs: ListingJob[] = [];
for (const chunk of chunks) {
const slug = chunk.split('"')[0]?.trim();
if (!slug) continue;
const card = chunk.slice(0, CARD_WINDOW);
const titleMatch = card.match(
/tw-text-lg tw-font-medium">\s*([^<]+?)\s*<\/div>/,
);
const employerMatch = card.match(/tw-card-title">\s*([^<]+?)\s*<\/span>/);
if (!titleMatch || !employerMatch) continue;
const badges = [
...card.matchAll(/tw-rounded-\[3px\]">\s*([^<]+?)\s*<\/span>/g),
].map((m) => collapseWhitespace(m[1]));
const isRemoteBadge = badges.some((b) => /^remote$/i.test(b));
const jobType = badges.find((b) => !/^remote$/i.test(b));
const salaryMatch = card.match(/Salary:\s*([\s\S]*?)<\/span>/);
const logoMatch = card.match(/<img[^>]*src="([^"]+)"/);
jobs.push({
slug,
title: decodeHtmlEntities(titleMatch[1].trim()),
employer: decodeHtmlEntities(employerMatch[1].trim()),
jobType,
isRemoteBadge,
salaryText: salaryMatch
? collapseWhitespace(decodeHtmlEntities(salaryMatch[1]))
: undefined,
companyLogo: logoMatch?.[1],
});
}
return jobs;
}
interface DetailFields {
description?: string;
datePosted?: string;
employmentType?: string;
organizationName?: string;
organizationLogo?: string;
salaryCurrency?: string;
salaryMin?: number;
salaryMax?: number;
salaryUnit?: string;
}
/** Best-effort string field extraction over the (invalid-JSON) JobPosting
* block: stops at the next quoted key or a closing brace rather than
* assuming a specific neighboring key or consistent `key: value` spacing. */
function extractStringField(raw: string, key: string): string | undefined {
const pattern = new RegExp(
`"${key}"\\s*:\\s*"([\\s\\S]*?)"\\s*(?=,\\s*\\n?\\s*"[A-Za-z]+"\\s*:|\\n?\\s*})`,
);
const match = raw.match(pattern);
const value = match?.[1]?.trim();
return value ? value : undefined;
}
/** javascript.jobs emits `minValue`/`maxValue: 0` placeholders even when no
* salary was entered, so treat 0 the same as "not specified". */
function extractNumberField(raw: string, key: string): number | undefined {
const match = raw.match(new RegExp(`"${key}"\\s*:\\s*([0-9.]+)`));
const value = match ? Number.parseFloat(match[1]) : undefined;
return value != null && Number.isFinite(value) && value > 0
? value
: undefined;
}
function extractObjectBlock(raw: string, key: string): string | undefined {
const match = raw.match(new RegExp(`"${key}"\\s*:\\s*{([\\s\\S]*?)}`));
return match?.[1];
}
function parseJobPosting(html: string): DetailFields | null {
const match = html.match(
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/,
);
const raw = match?.[1];
if (!raw) return null;
const orgBlock = extractObjectBlock(raw, "hiringOrganization");
// baseSalary nests a `value` object inside it; matching keys over the
// whole ld+json block is simpler than balancing the extra brace pair and
// these key names don't collide with anything else in this schema.
const salaryUnit = extractStringField(raw, "unitText");
return {
description: extractStringField(raw, "description"),
datePosted: extractStringField(raw, "datePosted"),
employmentType: extractStringField(raw, "employmentType"),
organizationName: orgBlock
? extractStringField(orgBlock, "name")
: undefined,
organizationLogo: orgBlock
? extractStringField(orgBlock, "logo")
: undefined,
salaryCurrency: extractStringField(raw, "currency"),
salaryMin: extractNumberField(raw, "minValue"),
salaryMax: extractNumberField(raw, "maxValue"),
salaryUnit,
};
}
function normalizeJobType(raw: string | undefined): string | undefined {
if (!raw) return undefined;
return raw.replace(/_/g, " ").trim() || undefined;
}
function salaryIntervalFromUnit(unit: string | undefined): string | undefined {
if (!unit) return undefined;
const lower = unit.toLowerCase();
if (lower.startsWith("year")) return "yearly";
if (lower.startsWith("month")) return "monthly";
if (lower.startsWith("week")) return "weekly";
if (lower.startsWith("day")) return "daily";
if (lower.startsWith("hour")) return "hourly";
return undefined;
}
async function fetchText(url: string): Promise<string> {
const response = await fetch(url, {
headers: { Accept: "text/html", "User-Agent": USER_AGENT },
});
if (!response.ok) {
throw new Error(
`javascript.jobs request failed (${response.status}) for ${url}`,
);
}
return response.text();
}
function listingUrl(term: string, page: number): string {
const url = new URL(`${ORIGIN}/remote`);
if (term) url.searchParams.set("keyword", term);
if (page > 1) url.searchParams.set("page", String(page));
return url.toString();
}
function readCappedInt(
raw: string | undefined,
fallback: number,
min: number,
max: number,
): number {
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
if (!Number.isFinite(parsed)) return fallback;
return Math.min(Math.max(parsed, min), max);
}
export const manifest: ExtractorManifest = {
id: "javascriptjobs",
displayName: "JS Remotely (javascript.jobs)",
providesSources: ["javascriptjobs"],
async run(context): Promise<ExtractorRunResult> {
if (context.shouldCancel?.()) return { success: true, jobs: [] };
const cap = readCappedInt(
context.settings.javascriptjobsMaxJobsPerTerm,
50,
1,
500,
);
const maxPages = readCappedInt(
context.settings.javascriptjobsMaxPages,
3,
1,
20,
);
const terms = context.searchTerms.length > 0 ? context.searchTerms : [""];
const seen = new Set<string>();
const out: CreateJobInput[] = [];
try {
for (let t = 0; t < terms.length; t += 1) {
if (context.shouldCancel?.()) break;
const term = terms[t].trim();
let collectedForTerm = 0;
for (let page = 1; page <= maxPages; page += 1) {
if (context.shouldCancel?.()) break;
if (collectedForTerm >= cap) break;
const url = listingUrl(term, page);
context.onProgress?.({
phase: "list",
termsProcessed: t,
termsTotal: terms.length,
currentUrl: url,
detail: `JS Remotely: term ${t + 1}/${terms.length}, page ${page}/${maxPages}`,
});
const html = await fetchText(url);
const listings = parseListingJobs(html);
if (listings.length === 0) break;
for (const listing of listings) {
if (collectedForTerm >= cap) break;
if (seen.has(listing.slug)) continue;
const jobUrl = `${ORIGIN}/job/${listing.slug}`;
let jobDescription: string | undefined;
let datePosted: string | undefined;
let jobType = listing.jobType;
let employer = listing.employer;
let companyLogo = listing.companyLogo;
let salaryCurrency: string | undefined;
let salaryMinAmount: number | undefined;
let salaryMaxAmount: number | undefined;
let salaryInterval: string | undefined;
try {
const detailHtml = await fetchText(jobUrl);
const detail = parseJobPosting(detailHtml);
if (detail) {
jobDescription = detail.description;
datePosted = detail.datePosted;
jobType = normalizeJobType(detail.employmentType) ?? jobType;
employer = detail.organizationName ?? employer;
companyLogo = detail.organizationLogo ?? companyLogo;
if (detail.salaryMin != null || detail.salaryMax != null) {
salaryCurrency = detail.salaryCurrency;
salaryMinAmount = detail.salaryMin;
salaryMaxAmount = detail.salaryMax;
salaryInterval = salaryIntervalFromUnit(detail.salaryUnit);
}
}
} catch {
// Fall back to listing-card fields when the detail fetch fails.
}
seen.add(listing.slug);
collectedForTerm += 1;
out.push({
source: "javascriptjobs",
sourceJobId: listing.slug,
title: listing.title,
employer,
jobUrl,
applicationLink: jobUrl,
location: "Remote",
isRemote: true,
jobType,
datePosted,
jobDescription,
companyLogo,
salary: listing.salaryText,
salaryCurrency,
salaryMinAmount,
salaryMaxAmount,
salaryInterval,
});
}
}
context.onProgress?.({
phase: "list",
termsProcessed: t + 1,
termsTotal: terms.length,
jobPagesProcessed: out.length,
detail: `JS Remotely: completed term ${t + 1}/${terms.length} (${collectedForTerm} found)`,
});
}
return { success: true, jobs: out };
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
return { success: false, jobs: out, error: message };
}
},
};
export default manifest;
+17
View File
@@ -0,0 +1,17 @@
{
"name": "javascriptjobs-extractor",
"version": "0.0.1",
"type": "module",
"description": "JS Remotely / javascript.jobs remote job-board extractor",
"main": "manifest.ts",
"dependencies": {
"job-ops-shared": "^1.0.0"
},
"devDependencies": {
"@types/node": "^24.0.0",
"typescript": "~5.9.0"
},
"scripts": {
"check:types": "tsc --noEmit"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"compilerOptions": {
"module": "ESNext",
"moduleResolution": "bundler",
"target": "ES2022",
"outDir": "dist",
"strict": true,
"noUnusedLocals": false,
"lib": ["ES2022", "DOM"],
"types": ["node"],
"baseUrl": ".",
"paths": {
"@shared/*": ["../../shared/src/*"]
}
},
"include": ["./manifest.ts", "./src/**/*"]
}
@@ -193,6 +193,8 @@ export function buildMaxCoverageDiscoveryLimits(args: {
himalayasMaxJobsPerTerm: boardCap,
weworkremotelyMaxJobsPerTerm: boardCap,
workingnomadsMaxJobsPerTerm: boardCap,
javascriptjobsMaxJobsPerTerm: Math.min(500, boardCap),
javascriptjobsMaxPages: 10,
fourdayweekMaxJobsPerTerm: boardCap,
testdevjobsMaxJobsPerTerm: boardCap,
testdevjobsMaxPages: 10,
@@ -268,6 +268,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
himalayas: "https://himalayas.app",
weworkremotely: "https://weworkremotely.com",
workingnomads: "https://www.workingnomads.com",
javascriptjobs: "https://javascript.jobs",
testdevjobs: "https://testdevjobs.com",
wellfound: "https://wellfound.com",
builtin: "https://builtin.com",
+32
View File
@@ -536,6 +536,34 @@
"undici-types": "~7.16.0"
}
},
"extractors/javascriptjobs": {
"name": "javascriptjobs-extractor",
"version": "0.0.1",
"dependencies": {
"job-ops-shared": "^1.0.0"
},
"devDependencies": {
"@types/node": "^24.0.0",
"typescript": "~5.9.0"
}
},
"extractors/javascriptjobs/node_modules/@types/node": {
"version": "24.13.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz",
"integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.18.0"
}
},
"extractors/javascriptjobs/node_modules/undici-types": {
"version": "7.18.2",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz",
"integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==",
"dev": true,
"license": "MIT"
},
"extractors/jobicy": {
"name": "jobicy-extractor",
"version": "0.0.1",
@@ -15979,6 +16007,10 @@
"node": ">=0.10.0"
}
},
"node_modules/javascriptjobs-extractor": {
"resolved": "extractors/javascriptjobs",
"link": true
},
"node_modules/jest-util": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz",
+8
View File
@@ -186,6 +186,14 @@ const ALL_TARGETS: Target[] = [
icimsMaxPagesPerSearch: "2",
},
},
{
id: "javascriptjobs",
importPath: "../extractors/javascriptjobs/manifest",
settings: {
javascriptjobsMaxJobsPerTerm: "5",
javascriptjobsMaxPages: "1",
},
},
{
id: "jobicy",
importPath: "../extractors/jobicy/manifest",
+7
View File
@@ -22,6 +22,7 @@ export const EXTRACTOR_SOURCE_IDS = [
"himalayas",
"weworkremotely",
"workingnomads",
"javascriptjobs",
"fourdayweek",
"testdevjobs",
"wellfound",
@@ -194,6 +195,12 @@ export const EXTRACTOR_SOURCE_METADATA: Record<
category: "pipeline",
region: "remote",
},
javascriptjobs: {
label: "JS Remotely (javascript.jobs)",
order: 192,
category: "pipeline",
region: "remote",
},
fourdayweek: {
label: "4 Day Week",
order: 195,
+14
View File
@@ -445,6 +445,20 @@ export const settingsRegistry = {
parse: parseIntOrNull,
serialize: serializeNullableNumber,
},
javascriptjobsMaxJobsPerTerm: {
kind: "typed" as const,
schema: z.number().int().min(1).max(500),
default: (): number => 50,
parse: parseIntOrNull,
serialize: serializeNullableNumber,
},
javascriptjobsMaxPages: {
kind: "typed" as const,
schema: z.number().int().min(1).max(20),
default: (): number => 3,
parse: parseIntOrNull,
serialize: serializeNullableNumber,
},
testdevjobsMaxJobsPerTerm: {
kind: "typed" as const,
schema: z.number().int().min(1).max(1000),
+2
View File
@@ -202,6 +202,8 @@ export const createAppSettings = (
himalayasMaxJobsPerTerm: { value: 50, default: 50, override: null },
weworkremotelyMaxJobsPerTerm: { value: 50, default: 50, override: null },
workingnomadsMaxJobsPerTerm: { value: 50, default: 50, override: null },
javascriptjobsMaxJobsPerTerm: { value: 50, default: 50, override: null },
javascriptjobsMaxPages: { value: 3, default: 3, override: null },
testdevjobsMaxJobsPerTerm: { value: 100, default: 100, override: null },
testdevjobsMaxPages: { value: 3, default: 3, override: null },
builtinMaxJobsPerTerm: { value: 100, default: 100, override: null },
+2
View File
@@ -213,6 +213,8 @@ export interface AppSettings {
himalayasMaxJobsPerTerm: Resolved<number>;
weworkremotelyMaxJobsPerTerm: Resolved<number>;
workingnomadsMaxJobsPerTerm: Resolved<number>;
javascriptjobsMaxJobsPerTerm: Resolved<number>;
javascriptjobsMaxPages: Resolved<number>;
testdevjobsMaxJobsPerTerm: Resolved<number>;
testdevjobsMaxPages: Resolved<number>;
builtinMaxJobsPerTerm: Resolved<number>;