feat(extractors): add 17 job source extractors and cross-source dedup
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
Adds extractor packages: arbeitnow, ashby, careerjet, fourdayweek,
greenhouse, himalayas, jobicy, jooble, lever, reed, remoteok, remotive,
themuse, usajobs, weworkremotely, workday — each with manifest, package
metadata and README.
Pipeline / shared:
- shared/job-fingerprint: stable hash for cross-source dedup, with tests
- discover-jobs: dedup via fingerprint and richer per-source merging
- jobs repository: fingerprint-aware upsert / lookup
- settings-registry, settings types/routes, demo-defaults: knobs for the
new sources
- shared extractors index: register the new manifests
- location-support, profiles route: small fixes for the new sources
Tooling:
- scripts/smoke-extractors.ts to sanity-check each source locally
- scripts/jobber-cron-{cherepaha,dobkin}.env.example: per-host cron
templates (CHANGEME placeholders only)
- .env.example: documented env vars for the new extractors
- .gitignore: ignore extractors/*/storage/ runtime caches (was ukvisajobs only)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
# remotive-extractor
|
||||
|
||||
Pulls listings from the public [Remotive API](https://remotive.com/api/remote-jobs).
|
||||
|
||||
- No authentication required.
|
||||
- Each pipeline `searchTerm` is passed as the `search` query parameter;
|
||||
without terms we fetch the generic remote feed.
|
||||
- Caps results per term via the `remotiveMaxJobsPerTerm` setting (default 100).
|
||||
- All listings are flagged `isRemote: true`.
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Remotive public remote-jobs API.
|
||||
*
|
||||
* https://remotive.com/api/remote-jobs?limit=N&search=term
|
||||
*
|
||||
* No auth. Returns up to `limit` results per call with a `search` keyword
|
||||
* filter. We iterate pipeline search terms as the `search` parameter.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorRunResult,
|
||||
} from "@shared/types/extractors";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
|
||||
const API_URL = "https://remotive.com/api/remote-jobs";
|
||||
|
||||
interface RemotiveJob {
|
||||
id?: number;
|
||||
url?: string;
|
||||
title?: string;
|
||||
company_name?: string;
|
||||
company_logo?: string;
|
||||
category?: string;
|
||||
tags?: string[];
|
||||
job_type?: string;
|
||||
publication_date?: string;
|
||||
candidate_required_location?: string;
|
||||
salary?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface RemotiveResponse {
|
||||
jobs?: RemotiveJob[];
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const trimmed = value.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function normalizeJobType(raw: string | undefined): string | undefined {
|
||||
if (!raw) return undefined;
|
||||
return raw.replace(/_/g, " ").trim() || undefined;
|
||||
}
|
||||
|
||||
function mapJob(raw: RemotiveJob): CreateJobInput | null {
|
||||
const jobUrl = asString(raw.url);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
const tags = Array.isArray(raw.tags)
|
||||
? raw.tags.filter((t): t is string => typeof t === "string" && t.length > 0)
|
||||
: [];
|
||||
|
||||
return {
|
||||
source: "remotive",
|
||||
sourceJobId: raw.id != null ? String(raw.id) : undefined,
|
||||
title: asString(raw.title) ?? "Unknown Title",
|
||||
employer: asString(raw.company_name) ?? "Unknown Employer",
|
||||
jobUrl,
|
||||
applicationLink: jobUrl,
|
||||
location: asString(raw.candidate_required_location) ?? "Remote",
|
||||
isRemote: true,
|
||||
jobType: normalizeJobType(raw.job_type),
|
||||
companyIndustry: asString(raw.category),
|
||||
companyLogo: asString(raw.company_logo),
|
||||
datePosted: asString(raw.publication_date),
|
||||
salary: asString(raw.salary),
|
||||
jobDescription: asString(raw.description),
|
||||
disciplines: tags.length > 0 ? tags.join(", ") : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchJobs(
|
||||
search: string | null,
|
||||
limit: number,
|
||||
): Promise<RemotiveJob[]> {
|
||||
const url = new URL(API_URL);
|
||||
url.searchParams.set("limit", String(Math.min(Math.max(limit, 1), 100)));
|
||||
if (search) url.searchParams.set("search", search);
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Remotive request failed with status ${response.status}`);
|
||||
}
|
||||
const body = (await response.json()) as RemotiveResponse;
|
||||
return Array.isArray(body.jobs) ? body.jobs : [];
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "remotive",
|
||||
displayName: "Remotive",
|
||||
providesSources: ["remotive"],
|
||||
async run(context): Promise<ExtractorRunResult> {
|
||||
if (context.shouldCancel?.()) return { success: true, jobs: [] };
|
||||
|
||||
const maxJobsPerTerm = context.settings.remotiveMaxJobsPerTerm
|
||||
? Number.parseInt(context.settings.remotiveMaxJobsPerTerm, 10)
|
||||
: 100;
|
||||
|
||||
const terms = context.searchTerms.length > 0 ? context.searchTerms : [null];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const out: CreateJobInput[] = [];
|
||||
|
||||
try {
|
||||
for (let i = 0; i < terms.length; i += 1) {
|
||||
if (context.shouldCancel?.()) break;
|
||||
const term = terms[i];
|
||||
const search = term ? term.trim() : null;
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: i,
|
||||
termsTotal: terms.length,
|
||||
currentUrl: search ?? "(all remote)",
|
||||
detail: `Remotive: term ${i + 1}/${terms.length}`,
|
||||
});
|
||||
|
||||
const raw = await fetchJobs(search, maxJobsPerTerm);
|
||||
let collected = 0;
|
||||
for (const item of raw) {
|
||||
if (collected >= maxJobsPerTerm) break;
|
||||
const mapped = mapJob(item);
|
||||
if (!mapped) continue;
|
||||
const key = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(mapped);
|
||||
collected += 1;
|
||||
}
|
||||
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: i + 1,
|
||||
termsTotal: terms.length,
|
||||
currentUrl: search ?? "(all remote)",
|
||||
jobPagesProcessed: out.length,
|
||||
detail: `Remotive: completed term ${i + 1}/${terms.length} (${collected} found)`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: out, error: message };
|
||||
}
|
||||
|
||||
return { success: true, jobs: out };
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "remotive-extractor",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"description": "Remotive public remote-jobs API extractor",
|
||||
"main": "manifest.ts",
|
||||
"dependencies": {
|
||||
"job-ops-shared": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
"typescript": "~5.9.0"
|
||||
},
|
||||
"scripts": {
|
||||
"check:types": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./manifest.ts", "./src/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user