feat(extractors): add 17 job source extractors and cross-source dedup
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
Adds extractor packages: arbeitnow, ashby, careerjet, fourdayweek,
greenhouse, himalayas, jobicy, jooble, lever, reed, remoteok, remotive,
themuse, usajobs, weworkremotely, workday — each with manifest, package
metadata and README.
Pipeline / shared:
- shared/job-fingerprint: stable hash for cross-source dedup, with tests
- discover-jobs: dedup via fingerprint and richer per-source merging
- jobs repository: fingerprint-aware upsert / lookup
- settings-registry, settings types/routes, demo-defaults: knobs for the
new sources
- shared extractors index: register the new manifests
- location-support, profiles route: small fixes for the new sources
Tooling:
- scripts/smoke-extractors.ts to sanity-check each source locally
- scripts/jobber-cron-{cherepaha,dobkin}.env.example: per-host cron
templates (CHANGEME placeholders only)
- .env.example: documented env vars for the new extractors
- .gitignore: ignore extractors/*/storage/ runtime caches (was ukvisajobs only)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
# ashby-extractor
|
||||
|
||||
Public Ashby job-board feeds via
|
||||
`GET https://api.ashbyhq.com/posting-api/job-board/{company}`.
|
||||
|
||||
- No auth.
|
||||
- Configure target slugs via `ashbyCompanies` (comma/newline) or
|
||||
`ASHBY_COMPANIES` env.
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* Ashby public job board API.
|
||||
*
|
||||
* https://developers.ashbyhq.com/reference/posting-api-job-board
|
||||
* GET https://api.ashbyhq.com/posting-api/job-board/{company}
|
||||
*
|
||||
* No auth. Each entry in `ashbyCompanies` is fetched independently.
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorRunResult,
|
||||
} from "@shared/types/extractors";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
|
||||
interface AshbyAddress {
|
||||
postalAddress?: {
|
||||
addressLocality?: string;
|
||||
addressRegion?: string;
|
||||
addressCountry?: string;
|
||||
};
|
||||
}
|
||||
interface AshbyJob {
|
||||
id?: string;
|
||||
title?: string;
|
||||
jobUrl?: string;
|
||||
applyUrl?: string;
|
||||
publishedAt?: string;
|
||||
employmentType?: string;
|
||||
isRemote?: boolean;
|
||||
team?: string;
|
||||
department?: string;
|
||||
location?: string;
|
||||
locationName?: string;
|
||||
secondaryLocations?: Array<{ location?: string; locationName?: string }>;
|
||||
address?: AshbyAddress;
|
||||
descriptionPlain?: string;
|
||||
descriptionHtml?: string;
|
||||
}
|
||||
interface AshbyResponse {
|
||||
jobs?: AshbyJob[];
|
||||
apiVersion?: string;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | undefined {
|
||||
if (typeof value !== "string") return undefined;
|
||||
const trimmed = value.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function readCompanies(raw: string | undefined): string[] {
|
||||
if (!raw) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
return parsed
|
||||
.map((entry) => (typeof entry === "string" ? entry.trim() : ""))
|
||||
.filter(Boolean);
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
return raw
|
||||
.split(/[\n,;|]+/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function locationFor(job: AshbyJob): string | undefined {
|
||||
const primary =
|
||||
asString(job.locationName) ?? asString(job.location) ?? undefined;
|
||||
const secondary =
|
||||
job.secondaryLocations
|
||||
?.map((entry) => asString(entry.locationName) ?? asString(entry.location))
|
||||
.filter((value): value is string => Boolean(value)) ?? [];
|
||||
const all = [primary, ...secondary].filter((value): value is string =>
|
||||
Boolean(value),
|
||||
);
|
||||
return all.length > 0 ? all.join("; ") : undefined;
|
||||
}
|
||||
|
||||
function mapJob(job: AshbyJob, company: string): CreateJobInput | null {
|
||||
const jobUrl = asString(job.jobUrl) ?? asString(job.applyUrl);
|
||||
if (!jobUrl) return null;
|
||||
const employer = company
|
||||
.split(/[-_]/)
|
||||
.filter(Boolean)
|
||||
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
|
||||
.join(" ");
|
||||
return {
|
||||
source: "ashby",
|
||||
sourceJobId: asString(job.id),
|
||||
title: asString(job.title) ?? "Unknown Title",
|
||||
employer: employer || company,
|
||||
jobUrl,
|
||||
applicationLink: asString(job.applyUrl) ?? jobUrl,
|
||||
location: locationFor(job),
|
||||
isRemote: typeof job.isRemote === "boolean" ? job.isRemote : undefined,
|
||||
jobType: asString(job.employmentType),
|
||||
jobFunction: asString(job.team),
|
||||
companyIndustry: asString(job.department),
|
||||
datePosted: asString(job.publishedAt),
|
||||
jobDescription:
|
||||
asString(job.descriptionPlain) ?? asString(job.descriptionHtml),
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchCompany(company: string): Promise<AshbyJob[]> {
|
||||
const url = `https://api.ashbyhq.com/posting-api/job-board/${encodeURIComponent(company)}`;
|
||||
const response = await fetch(url, {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (response.status === 404) return [];
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Ashby request for "${company}" failed with status ${response.status}`,
|
||||
);
|
||||
}
|
||||
const body = (await response.json()) as AshbyResponse;
|
||||
return Array.isArray(body.jobs) ? body.jobs : [];
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "ashby",
|
||||
displayName: "Ashby (ATS)",
|
||||
providesSources: ["ashby"],
|
||||
async run(context): Promise<ExtractorRunResult> {
|
||||
if (context.shouldCancel?.()) return { success: true, jobs: [] };
|
||||
|
||||
const companies = readCompanies(context.settings.ashbyCompanies);
|
||||
if (companies.length === 0) {
|
||||
return {
|
||||
success: true,
|
||||
jobs: [],
|
||||
error:
|
||||
"No Ashby companies configured. Set ASHBY_COMPANIES or the ashbyCompanies setting (comma- or newline-separated slugs).",
|
||||
};
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const out: CreateJobInput[] = [];
|
||||
|
||||
try {
|
||||
for (let i = 0; i < companies.length; i += 1) {
|
||||
if (context.shouldCancel?.()) break;
|
||||
const company = companies[i];
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: i,
|
||||
termsTotal: companies.length,
|
||||
currentUrl: company,
|
||||
detail: `Ashby: ${company} (${i + 1}/${companies.length})`,
|
||||
});
|
||||
|
||||
let added = 0;
|
||||
const jobs = await fetchCompany(company);
|
||||
for (const job of jobs) {
|
||||
const mapped = mapJob(job, company);
|
||||
if (!mapped) continue;
|
||||
const key = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(mapped);
|
||||
added += 1;
|
||||
}
|
||||
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: i + 1,
|
||||
termsTotal: companies.length,
|
||||
currentUrl: company,
|
||||
jobPagesProcessed: out.length,
|
||||
detail: `Ashby: ${company} → ${added} jobs (${out.length} total)`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: out, error: message };
|
||||
}
|
||||
|
||||
return { success: true, jobs: out };
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "ashby-extractor",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"description": "Ashby public ATS extractor",
|
||||
"main": "manifest.ts",
|
||||
"dependencies": {
|
||||
"job-ops-shared": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
"typescript": "~5.9.0"
|
||||
},
|
||||
"scripts": {
|
||||
"check:types": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./manifest.ts", "./src/**/*"]
|
||||
}
|
||||
Reference in New Issue
Block a user