Hiring cafe extractor (#192)
* feat(hiringcafe): register new source across shared/server/client enums * feat(hiringcafe-extractor): add browser-backed Hiring Cafe dataset extractor * feat(orchestrator): integrate Hiring Cafe discovery service into pipeline * feat(orchestrator-ui): add Hiring Cafe to source availability and run estimates * chore(hiringcafe): wire CI/docker and add extractor documentation * chore(format): apply biome formatting for Hiring Cafe integration * add original websites * coomints * number or null
This commit is contained in:
@@ -24,7 +24,13 @@ interface PipelineProgress {
|
||||
| "failed";
|
||||
message: string;
|
||||
detail?: string;
|
||||
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna" | null;
|
||||
crawlingSource:
|
||||
| "gradcracker"
|
||||
| "jobspy"
|
||||
| "ukvisajobs"
|
||||
| "adzuna"
|
||||
| "hiringcafe"
|
||||
| null;
|
||||
crawlingSourcesCompleted: number;
|
||||
crawlingSourcesTotal: number;
|
||||
crawlingTermsProcessed: number;
|
||||
@@ -85,6 +91,7 @@ const sourceLabel: Record<
|
||||
jobspy: "JobSpy",
|
||||
ukvisajobs: "UKVisaJobs",
|
||||
adzuna: "Adzuna",
|
||||
hiringcafe: "Hiring Cafe",
|
||||
};
|
||||
|
||||
const clamp = (value: number, min: number, max: number) =>
|
||||
|
||||
@@ -92,4 +92,20 @@ describe("automatic-run utilities", () => {
|
||||
expect(estimate.discovered.cap).toBeGreaterThan(0);
|
||||
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
|
||||
});
|
||||
|
||||
it("includes hiringcafe in estimate caps using the shared term budget", () => {
|
||||
const estimate = calculateAutomaticEstimate({
|
||||
values: {
|
||||
topN: 10,
|
||||
minSuitabilityScore: 50,
|
||||
searchTerms: ["backend", "platform"],
|
||||
runBudget: 120,
|
||||
country: "united kingdom",
|
||||
},
|
||||
sources: ["hiringcafe"],
|
||||
});
|
||||
|
||||
expect(estimate.discovered.cap).toBeGreaterThan(0);
|
||||
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -77,6 +77,7 @@ export function deriveExtractorLimits(args: {
|
||||
const includesGradcracker = args.sources.includes("gradcracker");
|
||||
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
|
||||
const includesAdzuna = args.sources.includes("adzuna");
|
||||
const includesHiringCafe = args.sources.includes("hiringcafe");
|
||||
|
||||
const weightedContributors =
|
||||
(includesIndeed ? termCount : 0) +
|
||||
@@ -84,7 +85,8 @@ export function deriveExtractorLimits(args: {
|
||||
(includesGlassdoor ? termCount : 0) +
|
||||
(includesGradcracker ? termCount : 0) +
|
||||
(includesUkVisaJobs ? 1 : 0) +
|
||||
(includesAdzuna ? termCount : 0);
|
||||
(includesAdzuna ? termCount : 0) +
|
||||
(includesHiringCafe ? termCount : 0);
|
||||
|
||||
if (weightedContributors <= 0) {
|
||||
return {
|
||||
@@ -143,6 +145,7 @@ export function calculateAutomaticEstimate(args: {
|
||||
const hasLinkedIn = sources.includes("linkedin");
|
||||
const hasGlassdoor = sources.includes("glassdoor");
|
||||
const hasAdzuna = sources.includes("adzuna");
|
||||
const hasHiringCafe = sources.includes("hiringcafe");
|
||||
const limits = deriveExtractorLimits({
|
||||
budget: values.runBudget,
|
||||
searchTerms: values.searchTerms,
|
||||
@@ -158,8 +161,12 @@ export function calculateAutomaticEstimate(args: {
|
||||
: 0;
|
||||
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
|
||||
const adzunaCap = hasAdzuna ? limits.adzunaMaxJobsPerTerm * termCount : 0;
|
||||
const hiringCafeCap = hasHiringCafe
|
||||
? limits.jobspyResultsWanted * termCount
|
||||
: 0;
|
||||
|
||||
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap;
|
||||
const discoveredCap =
|
||||
jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap + hiringCafeCap;
|
||||
const discoveredMin = Math.round(discoveredCap * 0.35);
|
||||
const discoveredMax = Math.round(discoveredCap * 0.75);
|
||||
const processedMin = Math.min(values.topN, discoveredMin);
|
||||
|
||||
@@ -14,6 +14,7 @@ export const orderedSources: JobSource[] = [
|
||||
"linkedin",
|
||||
"glassdoor",
|
||||
"adzuna",
|
||||
"hiringcafe",
|
||||
"ukvisajobs",
|
||||
];
|
||||
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];
|
||||
|
||||
@@ -168,7 +168,8 @@ export const getSourcesWithJobs = (jobs: JobListItem[]): JobSource[] => {
|
||||
export const getEnabledSources = (
|
||||
settings: AppSettings | null,
|
||||
): JobSource[] => {
|
||||
if (!settings) return [...DEFAULT_PIPELINE_SOURCES, "glassdoor"];
|
||||
if (!settings)
|
||||
return [...DEFAULT_PIPELINE_SOURCES, "glassdoor", "hiringcafe"];
|
||||
|
||||
const enabled: JobSource[] = [];
|
||||
const hasUkVisaJobsAuth = Boolean(
|
||||
@@ -191,6 +192,10 @@ export const getEnabledSources = (
|
||||
if (hasAdzunaAuth) enabled.push(source);
|
||||
continue;
|
||||
}
|
||||
if (source === "hiringcafe") {
|
||||
enabled.push(source);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
source === "indeed" ||
|
||||
source === "linkedin" ||
|
||||
|
||||
@@ -144,5 +144,6 @@ export const sourceLabel: Record<Job["source"], string> = {
|
||||
glassdoor: "Glassdoor",
|
||||
ukvisajobs: "UK Visa Jobs",
|
||||
adzuna: "Adzuna",
|
||||
hiringcafe: "Hiring Cafe",
|
||||
manual: "Manual",
|
||||
};
|
||||
|
||||
@@ -101,6 +101,7 @@ const runPipelineSchema = z.object({
|
||||
"glassdoor",
|
||||
"ukvisajobs",
|
||||
"adzuna",
|
||||
"hiringcafe",
|
||||
]),
|
||||
)
|
||||
.min(1)
|
||||
|
||||
@@ -253,6 +253,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
|
||||
gradcracker: "https://www.gradcracker.com",
|
||||
ukvisajobs: "https://www.ukvisajobs.com",
|
||||
adzuna: "https://www.adzuna.com",
|
||||
hiringcafe: "https://hiring.cafe",
|
||||
manual: "https://example.com",
|
||||
};
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ export const jobs = sqliteTable("jobs", {
|
||||
"glassdoor",
|
||||
"ukvisajobs",
|
||||
"adzuna",
|
||||
"hiringcafe",
|
||||
"manual",
|
||||
],
|
||||
})
|
||||
|
||||
@@ -14,7 +14,12 @@ export type PipelineStep =
|
||||
| "cancelled"
|
||||
| "failed";
|
||||
|
||||
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna";
|
||||
export type CrawlSource =
|
||||
| "gradcracker"
|
||||
| "jobspy"
|
||||
| "ukvisajobs"
|
||||
| "adzuna"
|
||||
| "hiringcafe";
|
||||
|
||||
export interface PipelineProgress {
|
||||
step: PipelineStep;
|
||||
|
||||
@@ -23,6 +23,10 @@ vi.mock("../../services/adzuna", () => ({
|
||||
runAdzuna: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../services/hiring-cafe", () => ({
|
||||
runHiringCafe: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../services/ukvisajobs", () => ({
|
||||
runUkVisaJobs: vi.fn(),
|
||||
}));
|
||||
@@ -218,6 +222,126 @@ describe("discoverJobsStep", () => {
|
||||
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("runs hiringcafe when selected and passes country/terms/cap", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const hiringCafe = await import("../../services/hiring-cafe");
|
||||
|
||||
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||
searchTerms: JSON.stringify(["engineer"]),
|
||||
jobspyCountryIndeed: "united states",
|
||||
jobspyResultsWanted: "25",
|
||||
} as any);
|
||||
|
||||
vi.mocked(hiringCafe.runHiringCafe).mockResolvedValue({
|
||||
success: true,
|
||||
jobs: [
|
||||
{
|
||||
source: "hiringcafe",
|
||||
sourceJobId: "hc-1",
|
||||
title: "Engineer",
|
||||
employer: "ACME",
|
||||
jobUrl: "https://example.com/hc",
|
||||
applicationLink: "https://example.com/hc",
|
||||
},
|
||||
],
|
||||
} as any);
|
||||
|
||||
const result = await discoverJobsStep({
|
||||
mergedConfig: {
|
||||
...config,
|
||||
sources: ["hiringcafe"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.discoveredJobs).toHaveLength(1);
|
||||
expect(vi.mocked(hiringCafe.runHiringCafe)).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
country: "united states",
|
||||
searchTerms: ["engineer"],
|
||||
maxJobsPerTerm: 25,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("updates Hiring Cafe terms and pages via progress callbacks", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const hiringCafe = await import("../../services/hiring-cafe");
|
||||
|
||||
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||
searchTerms: JSON.stringify(["engineer", "frontend"]),
|
||||
jobspyCountryIndeed: "united kingdom",
|
||||
jobspyResultsWanted: "50",
|
||||
} as any);
|
||||
|
||||
vi.mocked(hiringCafe.runHiringCafe).mockImplementation(
|
||||
async (options: any) => {
|
||||
options?.onProgress?.({
|
||||
type: "term_start",
|
||||
termIndex: 1,
|
||||
termTotal: 2,
|
||||
searchTerm: "engineer",
|
||||
});
|
||||
options?.onProgress?.({
|
||||
type: "page_fetched",
|
||||
termIndex: 1,
|
||||
termTotal: 2,
|
||||
searchTerm: "engineer",
|
||||
pageNo: 0,
|
||||
resultsOnPage: 10,
|
||||
totalCollected: 10,
|
||||
});
|
||||
options?.onProgress?.({
|
||||
type: "term_complete",
|
||||
termIndex: 1,
|
||||
termTotal: 2,
|
||||
searchTerm: "engineer",
|
||||
jobsFoundTerm: 10,
|
||||
});
|
||||
return { success: true, jobs: [] } as any;
|
||||
},
|
||||
);
|
||||
|
||||
await discoverJobsStep({
|
||||
mergedConfig: {
|
||||
...config,
|
||||
sources: ["hiringcafe"],
|
||||
},
|
||||
});
|
||||
|
||||
const progress = getProgress();
|
||||
expect(progress.crawlingTermsProcessed).toBe(1);
|
||||
expect(progress.crawlingTermsTotal).toBe(2);
|
||||
expect(progress.crawlingListPagesProcessed).toBe(1);
|
||||
expect(progress.crawlingJobPagesEnqueued).toBe(10);
|
||||
expect(progress.crawlingJobPagesProcessed).toBe(10);
|
||||
});
|
||||
|
||||
it("returns Hiring Cafe source error when extractor fails", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const hiringCafe = await import("../../services/hiring-cafe");
|
||||
|
||||
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||
searchTerms: JSON.stringify(["engineer"]),
|
||||
jobspyCountryIndeed: "united kingdom",
|
||||
jobspyResultsWanted: "50",
|
||||
} as any);
|
||||
|
||||
vi.mocked(hiringCafe.runHiringCafe).mockResolvedValue({
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "blocked upstream",
|
||||
} as any);
|
||||
|
||||
await expect(
|
||||
discoverJobsStep({
|
||||
mergedConfig: {
|
||||
...config,
|
||||
sources: ["hiringcafe"],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("All sources failed: hiringcafe: blocked upstream");
|
||||
});
|
||||
|
||||
it("maps Gradcracker progress callback into live crawling counters", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const crawler = await import("../../services/crawler");
|
||||
@@ -402,6 +526,7 @@ describe("discoverJobsStep", () => {
|
||||
it("does not throw when no sources are requested", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const adzuna = await import("../../services/adzuna");
|
||||
const hiringCafe = await import("../../services/hiring-cafe");
|
||||
const jobSpy = await import("../../services/jobspy");
|
||||
const crawler = await import("../../services/crawler");
|
||||
const ukVisa = await import("../../services/ukvisajobs");
|
||||
@@ -422,6 +547,7 @@ describe("discoverJobsStep", () => {
|
||||
expect(result.sourceErrors).toEqual([]);
|
||||
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(hiringCafe.runHiringCafe)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ import * as jobsRepo from "../../repositories/jobs";
|
||||
import * as settingsRepo from "../../repositories/settings";
|
||||
import { runAdzuna } from "../../services/adzuna";
|
||||
import { runCrawler } from "../../services/crawler";
|
||||
import { runHiringCafe } from "../../services/hiring-cafe";
|
||||
import { runJobSpy } from "../../services/jobspy";
|
||||
import { runUkVisaJobs } from "../../services/ukvisajobs";
|
||||
import { progressHelpers, updateProgress } from "../progress";
|
||||
@@ -75,12 +76,14 @@ export async function discoverJobsStep(args: {
|
||||
|
||||
const shouldRunJobSpy = jobSpySites.length > 0;
|
||||
const shouldRunAdzuna = compatibleSources.includes("adzuna");
|
||||
const shouldRunHiringCafe = compatibleSources.includes("hiringcafe");
|
||||
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
|
||||
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
|
||||
|
||||
const totalSources =
|
||||
Number(shouldRunJobSpy) +
|
||||
Number(shouldRunAdzuna) +
|
||||
Number(shouldRunHiringCafe) +
|
||||
Number(shouldRunGradcracker) +
|
||||
Number(shouldRunUkVisaJobs);
|
||||
let completedSources = 0;
|
||||
@@ -236,6 +239,84 @@ export async function discoverJobsStep(args: {
|
||||
return { discoveredJobs, sourceErrors };
|
||||
}
|
||||
|
||||
if (shouldRunHiringCafe) {
|
||||
progressHelpers.startSource("hiringcafe", completedSources, totalSources, {
|
||||
termsTotal: searchTerms.length,
|
||||
detail: "Hiring Cafe: fetching jobs...",
|
||||
});
|
||||
|
||||
const hiringCafeMaxJobsPerTerm = settings.jobspyResultsWanted
|
||||
? parseInt(settings.jobspyResultsWanted, 10)
|
||||
: 200;
|
||||
|
||||
const hiringCafeResult = await runHiringCafe({
|
||||
country: selectedCountry,
|
||||
searchTerms,
|
||||
maxJobsPerTerm: hiringCafeMaxJobsPerTerm,
|
||||
onProgress: (event) => {
|
||||
if (event.type === "term_start") {
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "hiringcafe",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
phase: "list",
|
||||
currentUrl: event.searchTerm,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "page_fetched") {
|
||||
const displayPageNo = event.pageNo + 1;
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "hiringcafe",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: displayPageNo,
|
||||
jobPagesEnqueued: event.totalCollected,
|
||||
jobPagesProcessed: event.totalCollected,
|
||||
phase: "list",
|
||||
currentUrl: `page ${displayPageNo}`,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal}, page ${displayPageNo} (${event.totalCollected} collected)`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "hiringcafe",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
phase: "list",
|
||||
currentUrl: event.searchTerm,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Hiring Cafe: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
if (!hiringCafeResult.success) {
|
||||
sourceErrors.push(
|
||||
`hiringcafe: ${hiringCafeResult.error ?? "unknown error"}`,
|
||||
);
|
||||
} else {
|
||||
discoveredJobs.push(...hiringCafeResult.jobs);
|
||||
}
|
||||
|
||||
markSourceComplete();
|
||||
}
|
||||
|
||||
if (args.shouldCancel?.()) {
|
||||
return { discoveredJobs, sourceErrors };
|
||||
}
|
||||
|
||||
if (shouldRunGradcracker) {
|
||||
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
|
||||
detail: "Gradcracker: scraping...",
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { mkdir, readFile, rm } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { logger } from "@infra/logger";
|
||||
import { sanitizeUnknown } from "@infra/sanitize";
|
||||
import type { CreateJobInput } from "@shared/types";
|
||||
import { toNumberOrNull, toStringOrNull } from "@shared/utils/type-conversion";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const HIRING_CAFE_DIR = join(__dirname, "../../../../extractors/hiringcafe");
|
||||
const DATASET_PATH = join(
|
||||
HIRING_CAFE_DIR,
|
||||
"storage/datasets/default/jobs.json",
|
||||
);
|
||||
const STORAGE_DATASET_DIR = join(HIRING_CAFE_DIR, "storage/datasets/default");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const TSX_CLI_PATH = resolveTsxCliPath();
|
||||
|
||||
type HiringCafeRawJob = Record<string, unknown>;
|
||||
|
||||
export type HiringCafeProgressEvent =
|
||||
| {
|
||||
type: "term_start";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
}
|
||||
| {
|
||||
type: "page_fetched";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo: number;
|
||||
resultsOnPage: number;
|
||||
totalCollected: number;
|
||||
}
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
};
|
||||
|
||||
export interface RunHiringCafeOptions {
|
||||
searchTerms?: string[];
|
||||
country?: string;
|
||||
maxJobsPerTerm?: number;
|
||||
onProgress?: (event: HiringCafeProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface HiringCafeResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function resolveTsxCliPath(): string | null {
|
||||
try {
|
||||
return require.resolve("tsx/dist/cli.mjs");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canRunNpmCommand(): boolean {
|
||||
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
|
||||
return !result.error && result.status === 0;
|
||||
}
|
||||
|
||||
function parseProgressLine(line: string): HiringCafeProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const event = toStringOrNull(parsed.event);
|
||||
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||
|
||||
if (!event || termIndex === null || termTotal === null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (event === "term_start") {
|
||||
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||
}
|
||||
|
||||
if (event === "page_fetched") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
if (pageNo === null) return null;
|
||||
|
||||
return {
|
||||
type: "page_fetched",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
pageNo,
|
||||
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "term_complete") {
|
||||
return {
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapHiringCafeRow(row: HiringCafeRawJob): CreateJobInput | null {
|
||||
const jobUrl = toStringOrNull(row.jobUrl);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
return {
|
||||
source: "hiringcafe",
|
||||
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
|
||||
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
|
||||
jobUrl,
|
||||
applicationLink: toStringOrNull(row.applicationLink) ?? jobUrl,
|
||||
location: toStringOrNull(row.location) ?? undefined,
|
||||
salary: toStringOrNull(row.salary) ?? undefined,
|
||||
datePosted: toStringOrNull(row.datePosted) ?? undefined,
|
||||
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
|
||||
jobType: toStringOrNull(row.jobType) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function readDataset(): Promise<CreateJobInput[]> {
|
||||
const content = await readFile(DATASET_PATH, "utf-8");
|
||||
const parsed = JSON.parse(content) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const value of parsed) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
|
||||
|
||||
const mapped = mapHiringCafeRow(value as HiringCafeRawJob);
|
||||
if (!mapped) continue;
|
||||
|
||||
const dedupeKey = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(dedupeKey)) continue;
|
||||
|
||||
seen.add(dedupeKey);
|
||||
jobs.push(mapped);
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
|
||||
async function clearStorageDataset(): Promise<void> {
|
||||
await rm(STORAGE_DATASET_DIR, { recursive: true, force: true });
|
||||
await mkdir(STORAGE_DATASET_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
export async function runHiringCafe(
|
||||
options: RunHiringCafeOptions = {},
|
||||
): Promise<HiringCafeResult> {
|
||||
const searchTerms =
|
||||
options.searchTerms && options.searchTerms.length > 0
|
||||
? options.searchTerms
|
||||
: ["web developer"];
|
||||
const country = (options.country || "united kingdom").trim().toLowerCase();
|
||||
const maxJobsPerTerm = options.maxJobsPerTerm ?? 200;
|
||||
|
||||
const useNpmCommand = canRunNpmCommand();
|
||||
if (!useNpmCommand && !TSX_CLI_PATH) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await clearStorageDataset();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const extractorEnv = {
|
||||
...process.env,
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
HIRING_CAFE_SEARCH_TERMS: JSON.stringify(searchTerms),
|
||||
HIRING_CAFE_COUNTRY: country,
|
||||
HIRING_CAFE_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
|
||||
HIRING_CAFE_OUTPUT_JSON: DATASET_PATH,
|
||||
};
|
||||
|
||||
const child = useNpmCommand
|
||||
? spawn("npm", ["run", "start"], {
|
||||
cwd: HIRING_CAFE_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
})
|
||||
: (() => {
|
||||
const tsxCliPath = TSX_CLI_PATH;
|
||||
if (!tsxCliPath) {
|
||||
throw new Error(
|
||||
"Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
|
||||
);
|
||||
}
|
||||
|
||||
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
|
||||
cwd: HIRING_CAFE_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
});
|
||||
})();
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const progressEvent = parseProgressLine(line);
|
||||
if (progressEvent) {
|
||||
options.onProgress?.(progressEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else
|
||||
reject(new Error(`Hiring Cafe extractor exited with code ${code}`));
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const jobs = await readDataset();
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
logger.warn("Hiring Cafe extractor run failed", {
|
||||
error: message,
|
||||
details: sanitizeUnknown(error),
|
||||
});
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user