Hiring cafe extractor (#192)

* feat(hiringcafe): register new source across shared/server/client enums

* feat(hiringcafe-extractor): add browser-backed Hiring Cafe dataset extractor

* feat(orchestrator): integrate Hiring Cafe discovery service into pipeline

* feat(orchestrator-ui): add Hiring Cafe to source availability and run estimates

* chore(hiringcafe): wire CI/docker and add extractor documentation

* chore(format): apply biome formatting for Hiring Cafe integration

* add original websites

* coomints

* number or null
This commit is contained in:
Shaheer Sarfaraz
2026-02-19 12:51:55 +00:00
committed by GitHub
parent 16dd17ebea
commit d34a9f041b
31 changed files with 1363 additions and 5 deletions
@@ -24,7 +24,13 @@ interface PipelineProgress {
| "failed";
message: string;
detail?: string;
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna" | null;
crawlingSource:
| "gradcracker"
| "jobspy"
| "ukvisajobs"
| "adzuna"
| "hiringcafe"
| null;
crawlingSourcesCompleted: number;
crawlingSourcesTotal: number;
crawlingTermsProcessed: number;
@@ -85,6 +91,7 @@ const sourceLabel: Record<
jobspy: "JobSpy",
ukvisajobs: "UKVisaJobs",
adzuna: "Adzuna",
hiringcafe: "Hiring Cafe",
};
const clamp = (value: number, min: number, max: number) =>
@@ -92,4 +92,20 @@ describe("automatic-run utilities", () => {
expect(estimate.discovered.cap).toBeGreaterThan(0);
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
});
it("includes hiringcafe in estimate caps using the shared term budget", () => {
const estimate = calculateAutomaticEstimate({
values: {
topN: 10,
minSuitabilityScore: 50,
searchTerms: ["backend", "platform"],
runBudget: 120,
country: "united kingdom",
},
sources: ["hiringcafe"],
});
expect(estimate.discovered.cap).toBeGreaterThan(0);
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
});
});
@@ -77,6 +77,7 @@ export function deriveExtractorLimits(args: {
const includesGradcracker = args.sources.includes("gradcracker");
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
const includesAdzuna = args.sources.includes("adzuna");
const includesHiringCafe = args.sources.includes("hiringcafe");
const weightedContributors =
(includesIndeed ? termCount : 0) +
@@ -84,7 +85,8 @@ export function deriveExtractorLimits(args: {
(includesGlassdoor ? termCount : 0) +
(includesGradcracker ? termCount : 0) +
(includesUkVisaJobs ? 1 : 0) +
(includesAdzuna ? termCount : 0);
(includesAdzuna ? termCount : 0) +
(includesHiringCafe ? termCount : 0);
if (weightedContributors <= 0) {
return {
@@ -143,6 +145,7 @@ export function calculateAutomaticEstimate(args: {
const hasLinkedIn = sources.includes("linkedin");
const hasGlassdoor = sources.includes("glassdoor");
const hasAdzuna = sources.includes("adzuna");
const hasHiringCafe = sources.includes("hiringcafe");
const limits = deriveExtractorLimits({
budget: values.runBudget,
searchTerms: values.searchTerms,
@@ -158,8 +161,12 @@ export function calculateAutomaticEstimate(args: {
: 0;
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
const adzunaCap = hasAdzuna ? limits.adzunaMaxJobsPerTerm * termCount : 0;
const hiringCafeCap = hasHiringCafe
? limits.jobspyResultsWanted * termCount
: 0;
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap;
const discoveredCap =
jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap + hiringCafeCap;
const discoveredMin = Math.round(discoveredCap * 0.35);
const discoveredMax = Math.round(discoveredCap * 0.75);
const processedMin = Math.min(values.topN, discoveredMin);
@@ -14,6 +14,7 @@ export const orderedSources: JobSource[] = [
"linkedin",
"glassdoor",
"adzuna",
"hiringcafe",
"ukvisajobs",
];
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];
@@ -168,7 +168,8 @@ export const getSourcesWithJobs = (jobs: JobListItem[]): JobSource[] => {
export const getEnabledSources = (
settings: AppSettings | null,
): JobSource[] => {
if (!settings) return [...DEFAULT_PIPELINE_SOURCES, "glassdoor"];
if (!settings)
return [...DEFAULT_PIPELINE_SOURCES, "glassdoor", "hiringcafe"];
const enabled: JobSource[] = [];
const hasUkVisaJobsAuth = Boolean(
@@ -191,6 +192,10 @@ export const getEnabledSources = (
if (hasAdzunaAuth) enabled.push(source);
continue;
}
if (source === "hiringcafe") {
enabled.push(source);
continue;
}
if (
source === "indeed" ||
source === "linkedin" ||
+1
View File
@@ -144,5 +144,6 @@ export const sourceLabel: Record<Job["source"], string> = {
glassdoor: "Glassdoor",
ukvisajobs: "UK Visa Jobs",
adzuna: "Adzuna",
hiringcafe: "Hiring Cafe",
manual: "Manual",
};
@@ -101,6 +101,7 @@ const runPipelineSchema = z.object({
"glassdoor",
"ukvisajobs",
"adzuna",
"hiringcafe",
]),
)
.min(1)
@@ -253,6 +253,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
gradcracker: "https://www.gradcracker.com",
ukvisajobs: "https://www.ukvisajobs.com",
adzuna: "https://www.adzuna.com",
hiringcafe: "https://hiring.cafe",
manual: "https://example.com",
};
+1
View File
@@ -40,6 +40,7 @@ export const jobs = sqliteTable("jobs", {
"glassdoor",
"ukvisajobs",
"adzuna",
"hiringcafe",
"manual",
],
})
+6 -1
View File
@@ -14,7 +14,12 @@ export type PipelineStep =
| "cancelled"
| "failed";
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna";
export type CrawlSource =
| "gradcracker"
| "jobspy"
| "ukvisajobs"
| "adzuna"
| "hiringcafe";
export interface PipelineProgress {
step: PipelineStep;
@@ -23,6 +23,10 @@ vi.mock("../../services/adzuna", () => ({
runAdzuna: vi.fn(),
}));
vi.mock("../../services/hiring-cafe", () => ({
runHiringCafe: vi.fn(),
}));
vi.mock("../../services/ukvisajobs", () => ({
runUkVisaJobs: vi.fn(),
}));
@@ -218,6 +222,126 @@ describe("discoverJobsStep", () => {
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
});
it("runs hiringcafe when selected and passes country/terms/cap", async () => {
const settingsRepo = await import("../../repositories/settings");
const hiringCafe = await import("../../services/hiring-cafe");
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
searchTerms: JSON.stringify(["engineer"]),
jobspyCountryIndeed: "united states",
jobspyResultsWanted: "25",
} as any);
vi.mocked(hiringCafe.runHiringCafe).mockResolvedValue({
success: true,
jobs: [
{
source: "hiringcafe",
sourceJobId: "hc-1",
title: "Engineer",
employer: "ACME",
jobUrl: "https://example.com/hc",
applicationLink: "https://example.com/hc",
},
],
} as any);
const result = await discoverJobsStep({
mergedConfig: {
...config,
sources: ["hiringcafe"],
},
});
expect(result.discoveredJobs).toHaveLength(1);
expect(vi.mocked(hiringCafe.runHiringCafe)).toHaveBeenCalledWith(
expect.objectContaining({
country: "united states",
searchTerms: ["engineer"],
maxJobsPerTerm: 25,
}),
);
});
it("updates Hiring Cafe terms and pages via progress callbacks", async () => {
const settingsRepo = await import("../../repositories/settings");
const hiringCafe = await import("../../services/hiring-cafe");
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
searchTerms: JSON.stringify(["engineer", "frontend"]),
jobspyCountryIndeed: "united kingdom",
jobspyResultsWanted: "50",
} as any);
vi.mocked(hiringCafe.runHiringCafe).mockImplementation(
async (options: any) => {
options?.onProgress?.({
type: "term_start",
termIndex: 1,
termTotal: 2,
searchTerm: "engineer",
});
options?.onProgress?.({
type: "page_fetched",
termIndex: 1,
termTotal: 2,
searchTerm: "engineer",
pageNo: 0,
resultsOnPage: 10,
totalCollected: 10,
});
options?.onProgress?.({
type: "term_complete",
termIndex: 1,
termTotal: 2,
searchTerm: "engineer",
jobsFoundTerm: 10,
});
return { success: true, jobs: [] } as any;
},
);
await discoverJobsStep({
mergedConfig: {
...config,
sources: ["hiringcafe"],
},
});
const progress = getProgress();
expect(progress.crawlingTermsProcessed).toBe(1);
expect(progress.crawlingTermsTotal).toBe(2);
expect(progress.crawlingListPagesProcessed).toBe(1);
expect(progress.crawlingJobPagesEnqueued).toBe(10);
expect(progress.crawlingJobPagesProcessed).toBe(10);
});
it("returns Hiring Cafe source error when extractor fails", async () => {
const settingsRepo = await import("../../repositories/settings");
const hiringCafe = await import("../../services/hiring-cafe");
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
searchTerms: JSON.stringify(["engineer"]),
jobspyCountryIndeed: "united kingdom",
jobspyResultsWanted: "50",
} as any);
vi.mocked(hiringCafe.runHiringCafe).mockResolvedValue({
success: false,
jobs: [],
error: "blocked upstream",
} as any);
await expect(
discoverJobsStep({
mergedConfig: {
...config,
sources: ["hiringcafe"],
},
}),
).rejects.toThrow("All sources failed: hiringcafe: blocked upstream");
});
it("maps Gradcracker progress callback into live crawling counters", async () => {
const settingsRepo = await import("../../repositories/settings");
const crawler = await import("../../services/crawler");
@@ -402,6 +526,7 @@ describe("discoverJobsStep", () => {
it("does not throw when no sources are requested", async () => {
const settingsRepo = await import("../../repositories/settings");
const adzuna = await import("../../services/adzuna");
const hiringCafe = await import("../../services/hiring-cafe");
const jobSpy = await import("../../services/jobspy");
const crawler = await import("../../services/crawler");
const ukVisa = await import("../../services/ukvisajobs");
@@ -422,6 +547,7 @@ describe("discoverJobsStep", () => {
expect(result.sourceErrors).toEqual([]);
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
expect(vi.mocked(hiringCafe.runHiringCafe)).not.toHaveBeenCalled();
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
});
@@ -10,6 +10,7 @@ import * as jobsRepo from "../../repositories/jobs";
import * as settingsRepo from "../../repositories/settings";
import { runAdzuna } from "../../services/adzuna";
import { runCrawler } from "../../services/crawler";
import { runHiringCafe } from "../../services/hiring-cafe";
import { runJobSpy } from "../../services/jobspy";
import { runUkVisaJobs } from "../../services/ukvisajobs";
import { progressHelpers, updateProgress } from "../progress";
@@ -75,12 +76,14 @@ export async function discoverJobsStep(args: {
const shouldRunJobSpy = jobSpySites.length > 0;
const shouldRunAdzuna = compatibleSources.includes("adzuna");
const shouldRunHiringCafe = compatibleSources.includes("hiringcafe");
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
const totalSources =
Number(shouldRunJobSpy) +
Number(shouldRunAdzuna) +
Number(shouldRunHiringCafe) +
Number(shouldRunGradcracker) +
Number(shouldRunUkVisaJobs);
let completedSources = 0;
@@ -236,6 +239,84 @@ export async function discoverJobsStep(args: {
return { discoveredJobs, sourceErrors };
}
if (shouldRunHiringCafe) {
progressHelpers.startSource("hiringcafe", completedSources, totalSources, {
termsTotal: searchTerms.length,
detail: "Hiring Cafe: fetching jobs...",
});
const hiringCafeMaxJobsPerTerm = settings.jobspyResultsWanted
? parseInt(settings.jobspyResultsWanted, 10)
: 200;
const hiringCafeResult = await runHiringCafe({
country: selectedCountry,
searchTerms,
maxJobsPerTerm: hiringCafeMaxJobsPerTerm,
onProgress: (event) => {
if (event.type === "term_start") {
progressHelpers.crawlingUpdate({
source: "hiringcafe",
termsProcessed: Math.max(event.termIndex - 1, 0),
termsTotal: event.termTotal,
phase: "list",
currentUrl: event.searchTerm,
});
updateProgress({
step: "crawling",
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
});
return;
}
if (event.type === "page_fetched") {
const displayPageNo = event.pageNo + 1;
progressHelpers.crawlingUpdate({
source: "hiringcafe",
termsProcessed: Math.max(event.termIndex - 1, 0),
termsTotal: event.termTotal,
listPagesProcessed: displayPageNo,
jobPagesEnqueued: event.totalCollected,
jobPagesProcessed: event.totalCollected,
phase: "list",
currentUrl: `page ${displayPageNo}`,
});
updateProgress({
step: "crawling",
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal}, page ${displayPageNo} (${event.totalCollected} collected)`,
});
return;
}
progressHelpers.crawlingUpdate({
source: "hiringcafe",
termsProcessed: event.termIndex,
termsTotal: event.termTotal,
phase: "list",
currentUrl: event.searchTerm,
});
updateProgress({
step: "crawling",
detail: `Hiring Cafe: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
});
},
});
if (!hiringCafeResult.success) {
sourceErrors.push(
`hiringcafe: ${hiringCafeResult.error ?? "unknown error"}`,
);
} else {
discoveredJobs.push(...hiringCafeResult.jobs);
}
markSourceComplete();
}
if (args.shouldCancel?.()) {
return { discoveredJobs, sourceErrors };
}
if (shouldRunGradcracker) {
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
detail: "Gradcracker: scraping...",
@@ -0,0 +1,270 @@
import { spawn, spawnSync } from "node:child_process";
import { mkdir, readFile, rm } from "node:fs/promises";
import { createRequire } from "node:module";
import { dirname, join } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import { logger } from "@infra/logger";
import { sanitizeUnknown } from "@infra/sanitize";
import type { CreateJobInput } from "@shared/types";
import { toNumberOrNull, toStringOrNull } from "@shared/utils/type-conversion";
const __dirname = dirname(fileURLToPath(import.meta.url));
const HIRING_CAFE_DIR = join(__dirname, "../../../../extractors/hiringcafe");
const DATASET_PATH = join(
HIRING_CAFE_DIR,
"storage/datasets/default/jobs.json",
);
const STORAGE_DATASET_DIR = join(HIRING_CAFE_DIR, "storage/datasets/default");
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
const require = createRequire(import.meta.url);
const TSX_CLI_PATH = resolveTsxCliPath();
type HiringCafeRawJob = Record<string, unknown>;
export type HiringCafeProgressEvent =
| {
type: "term_start";
termIndex: number;
termTotal: number;
searchTerm: string;
}
| {
type: "page_fetched";
termIndex: number;
termTotal: number;
searchTerm: string;
pageNo: number;
resultsOnPage: number;
totalCollected: number;
}
| {
type: "term_complete";
termIndex: number;
termTotal: number;
searchTerm: string;
jobsFoundTerm: number;
};
export interface RunHiringCafeOptions {
searchTerms?: string[];
country?: string;
maxJobsPerTerm?: number;
onProgress?: (event: HiringCafeProgressEvent) => void;
}
export interface HiringCafeResult {
success: boolean;
jobs: CreateJobInput[];
error?: string;
}
function resolveTsxCliPath(): string | null {
try {
return require.resolve("tsx/dist/cli.mjs");
} catch {
return null;
}
}
function canRunNpmCommand(): boolean {
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
return !result.error && result.status === 0;
}
function parseProgressLine(line: string): HiringCafeProgressEvent | null {
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(raw) as Record<string, unknown>;
} catch {
return null;
}
const event = toStringOrNull(parsed.event);
const termIndex = toNumberOrNull(parsed.termIndex);
const termTotal = toNumberOrNull(parsed.termTotal);
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
if (!event || termIndex === null || termTotal === null) {
return null;
}
if (event === "term_start") {
return { type: "term_start", termIndex, termTotal, searchTerm };
}
if (event === "page_fetched") {
const pageNo = toNumberOrNull(parsed.pageNo);
if (pageNo === null) return null;
return {
type: "page_fetched",
termIndex,
termTotal,
searchTerm,
pageNo,
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
};
}
if (event === "term_complete") {
return {
type: "term_complete",
termIndex,
termTotal,
searchTerm,
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
};
}
return null;
}
function mapHiringCafeRow(row: HiringCafeRawJob): CreateJobInput | null {
const jobUrl = toStringOrNull(row.jobUrl);
if (!jobUrl) return null;
return {
source: "hiringcafe",
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
title: toStringOrNull(row.title) ?? "Unknown Title",
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
jobUrl,
applicationLink: toStringOrNull(row.applicationLink) ?? jobUrl,
location: toStringOrNull(row.location) ?? undefined,
salary: toStringOrNull(row.salary) ?? undefined,
datePosted: toStringOrNull(row.datePosted) ?? undefined,
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
jobType: toStringOrNull(row.jobType) ?? undefined,
};
}
async function readDataset(): Promise<CreateJobInput[]> {
const content = await readFile(DATASET_PATH, "utf-8");
const parsed = JSON.parse(content) as unknown;
if (!Array.isArray(parsed)) return [];
const jobs: CreateJobInput[] = [];
const seen = new Set<string>();
for (const value of parsed) {
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
const mapped = mapHiringCafeRow(value as HiringCafeRawJob);
if (!mapped) continue;
const dedupeKey = mapped.sourceJobId || mapped.jobUrl;
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
jobs.push(mapped);
}
return jobs;
}
async function clearStorageDataset(): Promise<void> {
await rm(STORAGE_DATASET_DIR, { recursive: true, force: true });
await mkdir(STORAGE_DATASET_DIR, { recursive: true });
}
export async function runHiringCafe(
options: RunHiringCafeOptions = {},
): Promise<HiringCafeResult> {
const searchTerms =
options.searchTerms && options.searchTerms.length > 0
? options.searchTerms
: ["web developer"];
const country = (options.country || "united kingdom").trim().toLowerCase();
const maxJobsPerTerm = options.maxJobsPerTerm ?? 200;
const useNpmCommand = canRunNpmCommand();
if (!useNpmCommand && !TSX_CLI_PATH) {
return {
success: false,
jobs: [],
error: "Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
};
}
try {
await clearStorageDataset();
await new Promise<void>((resolve, reject) => {
const extractorEnv = {
...process.env,
JOBOPS_EMIT_PROGRESS: "1",
HIRING_CAFE_SEARCH_TERMS: JSON.stringify(searchTerms),
HIRING_CAFE_COUNTRY: country,
HIRING_CAFE_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
HIRING_CAFE_OUTPUT_JSON: DATASET_PATH,
};
const child = useNpmCommand
? spawn("npm", ["run", "start"], {
cwd: HIRING_CAFE_DIR,
stdio: ["ignore", "pipe", "pipe"],
env: extractorEnv,
})
: (() => {
const tsxCliPath = TSX_CLI_PATH;
if (!tsxCliPath) {
throw new Error(
"Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
);
}
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
cwd: HIRING_CAFE_DIR,
stdio: ["ignore", "pipe", "pipe"],
env: extractorEnv,
});
})();
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
const progressEvent = parseProgressLine(line);
if (progressEvent) {
options.onProgress?.(progressEvent);
return;
}
stream.write(`${line}\n`);
};
const stdoutRl = child.stdout
? createInterface({ input: child.stdout })
: null;
const stderrRl = child.stderr
? createInterface({ input: child.stderr })
: null;
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
child.on("close", (code) => {
stdoutRl?.close();
stderrRl?.close();
if (code === 0) resolve();
else
reject(new Error(`Hiring Cafe extractor exited with code ${code}`));
});
child.on("error", reject);
});
const jobs = await readDataset();
return { success: true, jobs };
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
logger.warn("Hiring Cafe extractor run failed", {
error: message,
details: sanitizeUnknown(error),
});
return { success: false, jobs: [], error: message };
}
}