feat: add JS Remotely (javascript.jobs) extractor
Adds a pipeline extractor for javascript.jobs (the board now promoted under the jsremotely.com alias, which 301s straight here). No public API/RSS, so it scrapes the /remote listing (server-side keyword+page filtering) and enriches each match from the job detail page's JobPosting schema.org block, which is emitted as near-JSON (unescaped newlines, an occasional missing comma) and is parsed with targeted regexes instead of JSON.parse. Also guards against the source's own baseSalary bug (0 placeholders, and at least one posting with minValue == maxValue) by keeping the listing-card salary text as a trustworthy fallback. Wires the new "javascriptjobs" source through the shared extractor catalog, settings registry/types/factories, the max-coverage automatic-run preset, demo seed data, and the smoke-test target.
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
# javascriptjobs-extractor
|
||||
|
||||
Pulls remote listings from [javascript.jobs](https://javascript.jobs) (the
|
||||
board formerly promoted under the `jsremotely.com` alias — that domain now
|
||||
301s straight here). JS/TS-focused: React, Vue, Node, Angular, TypeScript.
|
||||
|
||||
- No authentication, no public API/RSS — this is a server-rendered HTML
|
||||
scrape of `/remote`, which supports a `keyword` query param for server-side
|
||||
term filtering (`?keyword=react&page=2`), so pipeline search terms are
|
||||
iterated as separate paginated fetches rather than filtered client-side.
|
||||
- Listing cards give title/employer/job-type badge/salary text/logo. Detail
|
||||
pages (`/job/<slug>`) additionally embed a `JobPosting` (schema.org)
|
||||
`<script type="application/ld+json">` block with description, exact
|
||||
`datePosted`, `employmentType`, and structured `baseSalary`.
|
||||
- That embedded block is **not valid JSON** (unescaped newlines inside
|
||||
`description`, and at least one observed missing comma between sibling
|
||||
keys), so we extract fields with targeted regexes instead of
|
||||
`JSON.parse`. If a detail fetch fails or a field is missing, we fall back
|
||||
to the listing-card values.
|
||||
- `baseSalary` in that block is unreliable on the source side: postings
|
||||
with no salary entered still emit `minValue`/`maxValue: 0` (we treat 0 as
|
||||
"not specified"), and at least one observed posting had `minValue` equal
|
||||
to `maxValue` (both set to the range's upper bound) despite the
|
||||
human-readable listing-card text showing a real range. We keep the raw
|
||||
listing-card salary string in `salary` as a trustworthy fallback alongside
|
||||
the (possibly wrong) structured `salaryMinAmount`/`salaryMaxAmount`.
|
||||
- Caps via `javascriptjobsMaxJobsPerTerm` (default 50) and
|
||||
`javascriptjobsMaxPages` (default 3, pages per term).
|
||||
- Every matched job triggers one extra HTTP GET (detail page) for
|
||||
enrichment — keep the caps conservative given this is a low-volume niche
|
||||
board.
|
||||
@@ -0,0 +1,333 @@
|
||||
/**
|
||||
* JS Remotely / javascript.jobs — JS/TS-focused remote board (server-rendered
|
||||
* HTML, no public API or RSS).
|
||||
*
|
||||
* https://javascript.jobs/remote?keyword=<term>&page=<n>
|
||||
*
|
||||
* `jsremotely.com` now 301s straight to `javascript.jobs`, so we hit the
|
||||
* canonical domain directly. The `/remote` listing supports a `keyword`
|
||||
* query param that filters server-side, so we iterate pipeline search terms
|
||||
* as separate paginated fetches rather than filtering client-side.
|
||||
*
|
||||
* Listing cards give us title/employer/badges/salary text; detail pages
|
||||
* additionally embed a `JobPosting` (schema.org) block for description,
|
||||
* exact date posted, employment type, and structured salary. That block is
|
||||
* emitted with unescaped newlines and occasional missing commas, so it is
|
||||
* not valid JSON — we pull fields out with targeted regexes instead of
|
||||
* `JSON.parse`.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorRunResult,
|
||||
} from "@shared/types/extractors";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
|
||||
const ORIGIN = "https://javascript.jobs";
|
||||
const USER_AGENT = "JobOps/1.0 (+https://github.com/) job-search pipeline";
|
||||
/** One job card's markup is well under this; keeps field regexes from
|
||||
* bleeding into the next card (or page footer for the last card). */
|
||||
const CARD_WINDOW = 2500;
|
||||
|
||||
interface ListingJob {
|
||||
slug: string;
|
||||
title: string;
|
||||
employer: string;
|
||||
jobType?: string;
|
||||
isRemoteBadge: boolean;
|
||||
salaryText?: string;
|
||||
companyLogo?: string;
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
|
||||
function collapseWhitespace(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function parseListingJobs(html: string): ListingJob[] {
|
||||
const chunks = html.split('<a href="https://javascript.jobs/job/').slice(1);
|
||||
const jobs: ListingJob[] = [];
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const slug = chunk.split('"')[0]?.trim();
|
||||
if (!slug) continue;
|
||||
const card = chunk.slice(0, CARD_WINDOW);
|
||||
|
||||
const titleMatch = card.match(
|
||||
/tw-text-lg tw-font-medium">\s*([^<]+?)\s*<\/div>/,
|
||||
);
|
||||
const employerMatch = card.match(/tw-card-title">\s*([^<]+?)\s*<\/span>/);
|
||||
if (!titleMatch || !employerMatch) continue;
|
||||
|
||||
const badges = [
|
||||
...card.matchAll(/tw-rounded-\[3px\]">\s*([^<]+?)\s*<\/span>/g),
|
||||
].map((m) => collapseWhitespace(m[1]));
|
||||
const isRemoteBadge = badges.some((b) => /^remote$/i.test(b));
|
||||
const jobType = badges.find((b) => !/^remote$/i.test(b));
|
||||
|
||||
const salaryMatch = card.match(/Salary:\s*([\s\S]*?)<\/span>/);
|
||||
const logoMatch = card.match(/<img[^>]*src="([^"]+)"/);
|
||||
|
||||
jobs.push({
|
||||
slug,
|
||||
title: decodeHtmlEntities(titleMatch[1].trim()),
|
||||
employer: decodeHtmlEntities(employerMatch[1].trim()),
|
||||
jobType,
|
||||
isRemoteBadge,
|
||||
salaryText: salaryMatch
|
||||
? collapseWhitespace(decodeHtmlEntities(salaryMatch[1]))
|
||||
: undefined,
|
||||
companyLogo: logoMatch?.[1],
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
|
||||
interface DetailFields {
|
||||
description?: string;
|
||||
datePosted?: string;
|
||||
employmentType?: string;
|
||||
organizationName?: string;
|
||||
organizationLogo?: string;
|
||||
salaryCurrency?: string;
|
||||
salaryMin?: number;
|
||||
salaryMax?: number;
|
||||
salaryUnit?: string;
|
||||
}
|
||||
|
||||
/** Best-effort string field extraction over the (invalid-JSON) JobPosting
|
||||
* block: stops at the next quoted key or a closing brace rather than
|
||||
* assuming a specific neighboring key or consistent `key: value` spacing. */
|
||||
function extractStringField(raw: string, key: string): string | undefined {
|
||||
const pattern = new RegExp(
|
||||
`"${key}"\\s*:\\s*"([\\s\\S]*?)"\\s*(?=,\\s*\\n?\\s*"[A-Za-z]+"\\s*:|\\n?\\s*})`,
|
||||
);
|
||||
const match = raw.match(pattern);
|
||||
const value = match?.[1]?.trim();
|
||||
return value ? value : undefined;
|
||||
}
|
||||
|
||||
/** javascript.jobs emits `minValue`/`maxValue: 0` placeholders even when no
|
||||
* salary was entered, so treat 0 the same as "not specified". */
|
||||
function extractNumberField(raw: string, key: string): number | undefined {
|
||||
const match = raw.match(new RegExp(`"${key}"\\s*:\\s*([0-9.]+)`));
|
||||
const value = match ? Number.parseFloat(match[1]) : undefined;
|
||||
return value != null && Number.isFinite(value) && value > 0
|
||||
? value
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function extractObjectBlock(raw: string, key: string): string | undefined {
|
||||
const match = raw.match(new RegExp(`"${key}"\\s*:\\s*{([\\s\\S]*?)}`));
|
||||
return match?.[1];
|
||||
}
|
||||
|
||||
function parseJobPosting(html: string): DetailFields | null {
|
||||
const match = html.match(
|
||||
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/,
|
||||
);
|
||||
const raw = match?.[1];
|
||||
if (!raw) return null;
|
||||
|
||||
const orgBlock = extractObjectBlock(raw, "hiringOrganization");
|
||||
// baseSalary nests a `value` object inside it; matching keys over the
|
||||
// whole ld+json block is simpler than balancing the extra brace pair and
|
||||
// these key names don't collide with anything else in this schema.
|
||||
const salaryUnit = extractStringField(raw, "unitText");
|
||||
|
||||
return {
|
||||
description: extractStringField(raw, "description"),
|
||||
datePosted: extractStringField(raw, "datePosted"),
|
||||
employmentType: extractStringField(raw, "employmentType"),
|
||||
organizationName: orgBlock
|
||||
? extractStringField(orgBlock, "name")
|
||||
: undefined,
|
||||
organizationLogo: orgBlock
|
||||
? extractStringField(orgBlock, "logo")
|
||||
: undefined,
|
||||
salaryCurrency: extractStringField(raw, "currency"),
|
||||
salaryMin: extractNumberField(raw, "minValue"),
|
||||
salaryMax: extractNumberField(raw, "maxValue"),
|
||||
salaryUnit,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeJobType(raw: string | undefined): string | undefined {
|
||||
if (!raw) return undefined;
|
||||
return raw.replace(/_/g, " ").trim() || undefined;
|
||||
}
|
||||
|
||||
function salaryIntervalFromUnit(unit: string | undefined): string | undefined {
|
||||
if (!unit) return undefined;
|
||||
const lower = unit.toLowerCase();
|
||||
if (lower.startsWith("year")) return "yearly";
|
||||
if (lower.startsWith("month")) return "monthly";
|
||||
if (lower.startsWith("week")) return "weekly";
|
||||
if (lower.startsWith("day")) return "daily";
|
||||
if (lower.startsWith("hour")) return "hourly";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
const response = await fetch(url, {
|
||||
headers: { Accept: "text/html", "User-Agent": USER_AGENT },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`javascript.jobs request failed (${response.status}) for ${url}`,
|
||||
);
|
||||
}
|
||||
return response.text();
|
||||
}
|
||||
|
||||
function listingUrl(term: string, page: number): string {
|
||||
const url = new URL(`${ORIGIN}/remote`);
|
||||
if (term) url.searchParams.set("keyword", term);
|
||||
if (page > 1) url.searchParams.set("page", String(page));
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function readCappedInt(
|
||||
raw: string | undefined,
|
||||
fallback: number,
|
||||
min: number,
|
||||
max: number,
|
||||
): number {
|
||||
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.min(Math.max(parsed, min), max);
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "javascriptjobs",
|
||||
displayName: "JS Remotely (javascript.jobs)",
|
||||
providesSources: ["javascriptjobs"],
|
||||
async run(context): Promise<ExtractorRunResult> {
|
||||
if (context.shouldCancel?.()) return { success: true, jobs: [] };
|
||||
|
||||
const cap = readCappedInt(
|
||||
context.settings.javascriptjobsMaxJobsPerTerm,
|
||||
50,
|
||||
1,
|
||||
500,
|
||||
);
|
||||
const maxPages = readCappedInt(
|
||||
context.settings.javascriptjobsMaxPages,
|
||||
3,
|
||||
1,
|
||||
20,
|
||||
);
|
||||
const terms = context.searchTerms.length > 0 ? context.searchTerms : [""];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const out: CreateJobInput[] = [];
|
||||
|
||||
try {
|
||||
for (let t = 0; t < terms.length; t += 1) {
|
||||
if (context.shouldCancel?.()) break;
|
||||
const term = terms[t].trim();
|
||||
let collectedForTerm = 0;
|
||||
|
||||
for (let page = 1; page <= maxPages; page += 1) {
|
||||
if (context.shouldCancel?.()) break;
|
||||
if (collectedForTerm >= cap) break;
|
||||
|
||||
const url = listingUrl(term, page);
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: t,
|
||||
termsTotal: terms.length,
|
||||
currentUrl: url,
|
||||
detail: `JS Remotely: term ${t + 1}/${terms.length}, page ${page}/${maxPages}`,
|
||||
});
|
||||
|
||||
const html = await fetchText(url);
|
||||
const listings = parseListingJobs(html);
|
||||
if (listings.length === 0) break;
|
||||
|
||||
for (const listing of listings) {
|
||||
if (collectedForTerm >= cap) break;
|
||||
if (seen.has(listing.slug)) continue;
|
||||
|
||||
const jobUrl = `${ORIGIN}/job/${listing.slug}`;
|
||||
let jobDescription: string | undefined;
|
||||
let datePosted: string | undefined;
|
||||
let jobType = listing.jobType;
|
||||
let employer = listing.employer;
|
||||
let companyLogo = listing.companyLogo;
|
||||
let salaryCurrency: string | undefined;
|
||||
let salaryMinAmount: number | undefined;
|
||||
let salaryMaxAmount: number | undefined;
|
||||
let salaryInterval: string | undefined;
|
||||
|
||||
try {
|
||||
const detailHtml = await fetchText(jobUrl);
|
||||
const detail = parseJobPosting(detailHtml);
|
||||
if (detail) {
|
||||
jobDescription = detail.description;
|
||||
datePosted = detail.datePosted;
|
||||
jobType = normalizeJobType(detail.employmentType) ?? jobType;
|
||||
employer = detail.organizationName ?? employer;
|
||||
companyLogo = detail.organizationLogo ?? companyLogo;
|
||||
if (detail.salaryMin != null || detail.salaryMax != null) {
|
||||
salaryCurrency = detail.salaryCurrency;
|
||||
salaryMinAmount = detail.salaryMin;
|
||||
salaryMaxAmount = detail.salaryMax;
|
||||
salaryInterval = salaryIntervalFromUnit(detail.salaryUnit);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fall back to listing-card fields when the detail fetch fails.
|
||||
}
|
||||
|
||||
seen.add(listing.slug);
|
||||
collectedForTerm += 1;
|
||||
out.push({
|
||||
source: "javascriptjobs",
|
||||
sourceJobId: listing.slug,
|
||||
title: listing.title,
|
||||
employer,
|
||||
jobUrl,
|
||||
applicationLink: jobUrl,
|
||||
location: "Remote",
|
||||
isRemote: true,
|
||||
jobType,
|
||||
datePosted,
|
||||
jobDescription,
|
||||
companyLogo,
|
||||
salary: listing.salaryText,
|
||||
salaryCurrency,
|
||||
salaryMinAmount,
|
||||
salaryMaxAmount,
|
||||
salaryInterval,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: t + 1,
|
||||
termsTotal: terms.length,
|
||||
jobPagesProcessed: out.length,
|
||||
detail: `JS Remotely: completed term ${t + 1}/${terms.length} (${collectedForTerm} found)`,
|
||||
});
|
||||
}
|
||||
|
||||
return { success: true, jobs: out };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: out, error: message };
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "javascriptjobs-extractor",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"description": "JS Remotely / javascript.jobs remote job-board extractor",
|
||||
"main": "manifest.ts",
|
||||
"dependencies": {
|
||||
"job-ops-shared": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
"typescript": "~5.9.0"
|
||||
},
|
||||
"scripts": {
|
||||
"check:types": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./manifest.ts", "./src/**/*"]
|
||||
}
|
||||
@@ -193,6 +193,8 @@ export function buildMaxCoverageDiscoveryLimits(args: {
|
||||
himalayasMaxJobsPerTerm: boardCap,
|
||||
weworkremotelyMaxJobsPerTerm: boardCap,
|
||||
workingnomadsMaxJobsPerTerm: boardCap,
|
||||
javascriptjobsMaxJobsPerTerm: Math.min(500, boardCap),
|
||||
javascriptjobsMaxPages: 10,
|
||||
fourdayweekMaxJobsPerTerm: boardCap,
|
||||
testdevjobsMaxJobsPerTerm: boardCap,
|
||||
testdevjobsMaxPages: 10,
|
||||
|
||||
@@ -268,6 +268,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
|
||||
himalayas: "https://himalayas.app",
|
||||
weworkremotely: "https://weworkremotely.com",
|
||||
workingnomads: "https://www.workingnomads.com",
|
||||
javascriptjobs: "https://javascript.jobs",
|
||||
testdevjobs: "https://testdevjobs.com",
|
||||
wellfound: "https://wellfound.com",
|
||||
builtin: "https://builtin.com",
|
||||
|
||||
@@ -186,6 +186,14 @@ const ALL_TARGETS: Target[] = [
|
||||
icimsMaxPagesPerSearch: "2",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "javascriptjobs",
|
||||
importPath: "../extractors/javascriptjobs/manifest",
|
||||
settings: {
|
||||
javascriptjobsMaxJobsPerTerm: "5",
|
||||
javascriptjobsMaxPages: "1",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "jobicy",
|
||||
importPath: "../extractors/jobicy/manifest",
|
||||
|
||||
@@ -22,6 +22,7 @@ export const EXTRACTOR_SOURCE_IDS = [
|
||||
"himalayas",
|
||||
"weworkremotely",
|
||||
"workingnomads",
|
||||
"javascriptjobs",
|
||||
"fourdayweek",
|
||||
"testdevjobs",
|
||||
"wellfound",
|
||||
@@ -194,6 +195,12 @@ export const EXTRACTOR_SOURCE_METADATA: Record<
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
javascriptjobs: {
|
||||
label: "JS Remotely (javascript.jobs)",
|
||||
order: 192,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
fourdayweek: {
|
||||
label: "4 Day Week",
|
||||
order: 195,
|
||||
|
||||
@@ -445,6 +445,20 @@ export const settingsRegistry = {
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
javascriptjobsMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(500),
|
||||
default: (): number => 50,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
javascriptjobsMaxPages: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(20),
|
||||
default: (): number => 3,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
testdevjobsMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
|
||||
@@ -202,6 +202,8 @@ export const createAppSettings = (
|
||||
himalayasMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
weworkremotelyMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
workingnomadsMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
javascriptjobsMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
javascriptjobsMaxPages: { value: 3, default: 3, override: null },
|
||||
testdevjobsMaxJobsPerTerm: { value: 100, default: 100, override: null },
|
||||
testdevjobsMaxPages: { value: 3, default: 3, override: null },
|
||||
builtinMaxJobsPerTerm: { value: 100, default: 100, override: null },
|
||||
|
||||
@@ -213,6 +213,8 @@ export interface AppSettings {
|
||||
himalayasMaxJobsPerTerm: Resolved<number>;
|
||||
weworkremotelyMaxJobsPerTerm: Resolved<number>;
|
||||
workingnomadsMaxJobsPerTerm: Resolved<number>;
|
||||
javascriptjobsMaxJobsPerTerm: Resolved<number>;
|
||||
javascriptjobsMaxPages: Resolved<number>;
|
||||
testdevjobsMaxJobsPerTerm: Resolved<number>;
|
||||
testdevjobsMaxPages: Resolved<number>;
|
||||
builtinMaxJobsPerTerm: Resolved<number>;
|
||||
|
||||
Reference in New Issue
Block a user