Auto-Registering Extractor System (#223)
* initial commit? * Address PR feedback on extractor discovery and startup resilience * Address latest PR review comments * fix city resolution fallback when input parses empty * address PR feedback on extractor registry and pipeline validation * address copilot comments on manifests and registry startup * fix extractor discovery export handling and env isolation in tests * enforce duplicate manifest id failures in strict mode * Fix remaining extractor registry and runtime review comments * docs * docs * test all, logic remains in extractors * Address PR review feedback on extractor registry and validation * Revert extractor moduleResolution to bundler * Enforce shared city filtering across all discovery sources * Deduplicate extractor strict city post-filtering
This commit is contained in:
Shaheer Sarfaraz
@@ -0,0 +1,120 @@
|
||||
import { getAdzunaCountryCode } from "@shared/location-support.js";
|
||||
import { resolveSearchCities } from "@shared/search-cities.js";
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorProgressEvent,
|
||||
} from "@shared/types/extractors";
|
||||
import { runAdzuna } from "./src/run";
|
||||
|
||||
function toProgress(event: {
|
||||
type: string;
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo?: number;
|
||||
totalCollected?: number;
|
||||
}): ExtractorProgressEvent {
|
||||
if (event.type === "term_start") {
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `Adzuna: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.type === "page_fetched") {
|
||||
const pageNo = event.pageNo ?? 0;
|
||||
const totalCollected = event.totalCollected ?? 0;
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: pageNo,
|
||||
jobPagesEnqueued: totalCollected,
|
||||
jobPagesProcessed: totalCollected,
|
||||
currentUrl: `page ${pageNo}`,
|
||||
detail: `Adzuna: term ${event.termIndex}/${event.termTotal}, page ${pageNo} (${totalCollected} collected)`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `Adzuna: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
};
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "adzuna",
|
||||
displayName: "Adzuna",
|
||||
providesSources: ["adzuna"],
|
||||
requiredEnvVars: ["ADZUNA_APP_ID", "ADZUNA_APP_KEY"],
|
||||
async run(context) {
|
||||
if (context.shouldCancel?.()) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
const countryCode = getAdzunaCountryCode(context.selectedCountry);
|
||||
if (!countryCode) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: `unsupported country ${context.selectedCountry}`,
|
||||
};
|
||||
}
|
||||
|
||||
const maxJobsPerTerm = context.settings.adzunaMaxJobsPerTerm
|
||||
? parseInt(context.settings.adzunaMaxJobsPerTerm, 10)
|
||||
: 50;
|
||||
|
||||
let result: Awaited<ReturnType<typeof runAdzuna>>;
|
||||
try {
|
||||
result = await runAdzuna({
|
||||
country: countryCode,
|
||||
countryKey: context.selectedCountry,
|
||||
searchTerms: context.searchTerms,
|
||||
locations: resolveSearchCities({
|
||||
single:
|
||||
context.settings.searchCities ?? context.settings.jobspyLocation,
|
||||
}),
|
||||
maxJobsPerTerm,
|
||||
onProgress: (event) => {
|
||||
if (context.shouldCancel?.()) return;
|
||||
|
||||
context.onProgress?.(toProgress(event));
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: typeof error === "string"
|
||||
? error
|
||||
: "Unexpected error while running Adzuna extractor.";
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: message,
|
||||
};
|
||||
}
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobs: result.jobs,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,301 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeCountryKey } from "@shared/location-support.js";
|
||||
import {
|
||||
resolveSearchCities,
|
||||
shouldApplyStrictCityFilter,
|
||||
} from "@shared/search-cities.js";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
import {
|
||||
toNumberOrNull,
|
||||
toStringOrNull,
|
||||
} from "@shared/utils/type-conversion.js";
|
||||
|
||||
const srcDir = dirname(fileURLToPath(import.meta.url));
|
||||
const EXTRACTOR_DIR = join(srcDir, "..");
|
||||
const DATASET_PATH = join(EXTRACTOR_DIR, "storage/datasets/default/jobs.json");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
const require = createRequire(import.meta.url);
|
||||
const TSX_CLI_PATH = resolveTsxCliPath();
|
||||
|
||||
type AdzunaRawJob = Record<string, unknown>;
|
||||
|
||||
export type AdzunaProgressEvent =
|
||||
| {
|
||||
type: "term_start";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
}
|
||||
| {
|
||||
type: "page_fetched";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo: number;
|
||||
resultsOnPage: number;
|
||||
totalCollected: number;
|
||||
}
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
};
|
||||
|
||||
export interface RunAdzunaOptions {
|
||||
searchTerms?: string[];
|
||||
country?: string;
|
||||
countryKey?: string;
|
||||
locations?: string[];
|
||||
maxJobsPerTerm?: number;
|
||||
onProgress?: (event: AdzunaProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface AdzunaResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function shouldApplyStrictLocationFilter(
|
||||
location: string,
|
||||
countryKey: string,
|
||||
): boolean {
|
||||
return shouldApplyStrictCityFilter(location, countryKey);
|
||||
}
|
||||
|
||||
function resolveTsxCliPath(): string | null {
|
||||
try {
|
||||
return require.resolve("tsx/dist/cli.mjs");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canRunNpmCommand(): boolean {
|
||||
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
|
||||
return !result.error && result.status === 0;
|
||||
}
|
||||
|
||||
function parseAdzunaProgressLine(line: string): AdzunaProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const event = toStringOrNull(parsed.event);
|
||||
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||
if (!event || termIndex === null || termTotal === null) return null;
|
||||
|
||||
if (event === "term_start") {
|
||||
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||
}
|
||||
|
||||
if (event === "page_fetched") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
if (pageNo === null) return null;
|
||||
return {
|
||||
type: "page_fetched",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
pageNo,
|
||||
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "term_complete") {
|
||||
return {
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapAdzunaRow(row: AdzunaRawJob): CreateJobInput | null {
|
||||
const jobUrl = toStringOrNull(row.jobUrl);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
return {
|
||||
source: "adzuna",
|
||||
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
|
||||
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
|
||||
jobUrl,
|
||||
applicationLink:
|
||||
toStringOrNull(row.applicationLink) ??
|
||||
toStringOrNull(row.jobUrl) ??
|
||||
undefined,
|
||||
location: toStringOrNull(row.location) ?? undefined,
|
||||
salary: toStringOrNull(row.salary) ?? undefined,
|
||||
datePosted: toStringOrNull(row.datePosted) ?? undefined,
|
||||
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
|
||||
jobType: toStringOrNull(row.jobType) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function readDataset(): Promise<CreateJobInput[]> {
|
||||
const content = await readFile(DATASET_PATH, "utf-8");
|
||||
const parsed = JSON.parse(content) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const value of parsed) {
|
||||
if (!value || typeof value !== "object") continue;
|
||||
const mapped = mapAdzunaRow(value as AdzunaRawJob);
|
||||
if (!mapped) continue;
|
||||
const key = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
jobs.push(mapped);
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
export async function runAdzuna(
|
||||
options: RunAdzunaOptions = {},
|
||||
): Promise<AdzunaResult> {
|
||||
const appId = process.env.ADZUNA_APP_ID?.trim();
|
||||
const appKey = process.env.ADZUNA_APP_KEY?.trim();
|
||||
if (!appId || !appKey) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Missing Adzuna credentials (ADZUNA_APP_ID / ADZUNA_APP_KEY)",
|
||||
};
|
||||
}
|
||||
|
||||
const country = (options.country || "gb").trim().toLowerCase();
|
||||
const countryKey = normalizeCountryKey(options.countryKey ?? "");
|
||||
const maxJobsPerTerm = options.maxJobsPerTerm ?? 50;
|
||||
const searchTerms =
|
||||
options.searchTerms && options.searchTerms.length > 0
|
||||
? options.searchTerms
|
||||
: ["web developer"];
|
||||
const locations = resolveSearchCities({
|
||||
list: options.locations,
|
||||
env: process.env.ADZUNA_LOCATION_QUERY,
|
||||
});
|
||||
const runLocations = locations.length > 0 ? locations : [null];
|
||||
const termTotal = searchTerms.length * runLocations.length;
|
||||
const useNpmCommand = canRunNpmCommand();
|
||||
if (!useNpmCommand && !TSX_CLI_PATH) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (let runIndex = 0; runIndex < runLocations.length; runIndex += 1) {
|
||||
const location = runLocations[runIndex];
|
||||
const strictLocationFilter =
|
||||
location !== null &&
|
||||
shouldApplyStrictLocationFilter(location, countryKey);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const extractorEnv = {
|
||||
...process.env,
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
ADZUNA_APP_ID: appId,
|
||||
ADZUNA_APP_KEY: appKey,
|
||||
ADZUNA_COUNTRY: country,
|
||||
ADZUNA_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
|
||||
ADZUNA_SEARCH_TERMS: JSON.stringify(searchTerms),
|
||||
ADZUNA_OUTPUT_JSON: DATASET_PATH,
|
||||
ADZUNA_LOCATION_QUERY: strictLocationFilter ? location : "",
|
||||
};
|
||||
const child = useNpmCommand
|
||||
? spawn("npm", ["run", "start"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
})
|
||||
: (() => {
|
||||
const tsxCliPath = TSX_CLI_PATH;
|
||||
if (!tsxCliPath) {
|
||||
throw new Error(
|
||||
"Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||
);
|
||||
}
|
||||
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
});
|
||||
})();
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const progressEvent = parseAdzunaProgressLine(line);
|
||||
if (progressEvent) {
|
||||
const termOffset = runIndex * searchTerms.length;
|
||||
options.onProgress?.({
|
||||
...progressEvent,
|
||||
termIndex: termOffset + progressEvent.termIndex,
|
||||
termTotal,
|
||||
});
|
||||
return;
|
||||
}
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`Adzuna extractor exited with code ${code}`));
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const runJobs = await readDataset();
|
||||
const filtered = runJobs;
|
||||
|
||||
for (const job of filtered) {
|
||||
const key = job.sourceJobId || job.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
jobs.push(job);
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { shouldApplyStrictLocationFilter } from "../src/run";
|
||||
|
||||
describe("adzuna location query strictness", () => {
|
||||
it("enables strict filtering when city differs from country", () => {
|
||||
expect(shouldApplyStrictLocationFilter("Leeds", "united kingdom")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("disables strict filtering when location is country-level", () => {
|
||||
expect(shouldApplyStrictLocationFilter("UK", "united kingdom")).toBe(false);
|
||||
expect(shouldApplyStrictLocationFilter("United States", "us")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorRuntimeContext,
|
||||
} from "@shared/types/extractors";
|
||||
import { runCrawler } from "./src/run";
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "gradcracker",
|
||||
displayName: "Gradcracker",
|
||||
providesSources: ["gradcracker"],
|
||||
async run(context: ExtractorRuntimeContext) {
|
||||
if (context.shouldCancel?.()) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
const existingJobUrls = await context.getExistingJobUrls?.();
|
||||
const maxJobsPerTerm = context.settings.gradcrackerMaxJobsPerTerm
|
||||
? parseInt(context.settings.gradcrackerMaxJobsPerTerm, 10)
|
||||
: 50;
|
||||
|
||||
const result = await runCrawler({
|
||||
existingJobUrls,
|
||||
searchTerms: context.searchTerms,
|
||||
maxJobsPerTerm,
|
||||
onProgress: (progress) => {
|
||||
if (context.shouldCancel?.()) return;
|
||||
|
||||
context.onProgress?.({
|
||||
phase: progress.phase,
|
||||
currentUrl: progress.currentUrl,
|
||||
listPagesProcessed: progress.listPagesProcessed,
|
||||
listPagesTotal: progress.listPagesTotal,
|
||||
jobCardsFound: progress.jobCardsFound,
|
||||
jobPagesEnqueued: progress.jobPagesEnqueued,
|
||||
jobPagesSkipped: progress.jobPagesSkipped,
|
||||
jobPagesProcessed: progress.jobPagesProcessed,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobs: result.jobs,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,185 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type CreateJobInput = {
|
||||
source: "gradcracker";
|
||||
title: string;
|
||||
employer: string;
|
||||
jobUrl: string;
|
||||
employerUrl?: string;
|
||||
applicationLink?: string;
|
||||
disciplines?: string;
|
||||
deadline?: string;
|
||||
salary?: string;
|
||||
location?: string;
|
||||
degreeRequired?: string;
|
||||
starting?: string;
|
||||
jobDescription?: string;
|
||||
};
|
||||
|
||||
const srcDir = dirname(fileURLToPath(import.meta.url));
|
||||
const EXTRACTOR_DIR = join(srcDir, "..");
|
||||
const STORAGE_DIR = join(EXTRACTOR_DIR, "storage/datasets/default");
|
||||
const JOBOPS_STORAGE_DIR = join(EXTRACTOR_DIR, "storage/jobops");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
|
||||
export interface CrawlerResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface RunCrawlerOptions {
|
||||
existingJobUrls?: string[];
|
||||
onProgress?: (update: JobExtractorProgress) => void;
|
||||
searchTerms?: string[];
|
||||
maxJobsPerTerm?: number;
|
||||
}
|
||||
|
||||
interface JobExtractorProgress {
|
||||
phase?: "list" | "job";
|
||||
currentUrl?: string;
|
||||
listPagesProcessed?: number;
|
||||
listPagesTotal?: number;
|
||||
jobCardsFound?: number;
|
||||
jobPagesEnqueued?: number;
|
||||
jobPagesSkipped?: number;
|
||||
jobPagesProcessed?: number;
|
||||
ts?: string;
|
||||
}
|
||||
|
||||
async function writeExistingJobUrlsFile(
|
||||
existingJobUrls: string[] | undefined,
|
||||
): Promise<string | null> {
|
||||
if (!existingJobUrls || existingJobUrls.length === 0) return null;
|
||||
await mkdir(JOBOPS_STORAGE_DIR, { recursive: true });
|
||||
const filePath = join(JOBOPS_STORAGE_DIR, "existing-job-urls.json");
|
||||
await writeFile(filePath, JSON.stringify(existingJobUrls), "utf-8");
|
||||
return filePath;
|
||||
}
|
||||
|
||||
export async function runCrawler(
|
||||
options: RunCrawlerOptions = {},
|
||||
): Promise<CrawlerResult> {
|
||||
try {
|
||||
await clearStorageDataset();
|
||||
const existingJobUrlsFile = await writeExistingJobUrlsFile(
|
||||
options.existingJobUrls,
|
||||
);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn("npm", ["run", "start"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
shell: true,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: {
|
||||
...process.env,
|
||||
JOBOPS_SKIP_APPLY_FOR_EXISTING: "1",
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
GRADCRACKER_SEARCH_TERMS: options.searchTerms
|
||||
? JSON.stringify(options.searchTerms)
|
||||
: "",
|
||||
GRADCRACKER_MAX_JOBS_PER_TERM: options.maxJobsPerTerm
|
||||
? String(options.maxJobsPerTerm)
|
||||
: "",
|
||||
...(existingJobUrlsFile
|
||||
? { JOBOPS_EXISTING_JOB_URLS_FILE: existingJobUrlsFile }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
if (line.startsWith(JOBOPS_PROGRESS_PREFIX)) {
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as JobExtractorProgress;
|
||||
options.onProgress?.(parsed);
|
||||
} catch {
|
||||
// ignore malformed progress lines
|
||||
}
|
||||
return;
|
||||
}
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`Crawler exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const jobs = await readCrawledJobs();
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
|
||||
async function readCrawledJobs(): Promise<CreateJobInput[]> {
|
||||
try {
|
||||
const files = await readdir(STORAGE_DIR);
|
||||
const jsonFiles = files.filter((file) => file.endsWith(".json"));
|
||||
const jobs: CreateJobInput[] = [];
|
||||
|
||||
for (const file of jsonFiles) {
|
||||
const content = await readFile(join(STORAGE_DIR, file), "utf-8");
|
||||
const data = JSON.parse(content) as Record<string, unknown>;
|
||||
|
||||
jobs.push({
|
||||
source: "gradcracker",
|
||||
title: (data.title as string) || "Unknown Title",
|
||||
employer: (data.employer as string) || "Unknown Employer",
|
||||
employerUrl: data.employerUrl as string | undefined,
|
||||
jobUrl: (data.url as string) || (data.jobUrl as string),
|
||||
applicationLink: data.applicationLink as string | undefined,
|
||||
disciplines:
|
||||
typeof data.disciplines === "string"
|
||||
? data.disciplines
|
||||
: Array.isArray(data.disciplines)
|
||||
? data.disciplines
|
||||
.filter((value): value is string => typeof value === "string")
|
||||
.join(", ")
|
||||
: undefined,
|
||||
deadline: data.deadline as string | undefined,
|
||||
salary: data.salary as string | undefined,
|
||||
location: data.location as string | undefined,
|
||||
degreeRequired: data.degreeRequired as string | undefined,
|
||||
starting: data.starting as string | undefined,
|
||||
jobDescription: data.jobDescription as string | undefined,
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function clearStorageDataset(): Promise<void> {
|
||||
try {
|
||||
await rm(STORAGE_DIR, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,11 @@
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["DOM"]
|
||||
"lib": ["DOM"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { resolveSearchCities } from "@shared/search-cities.js";
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorProgressEvent,
|
||||
} from "@shared/types/extractors";
|
||||
import { runHiringCafe } from "./src/run";
|
||||
|
||||
function toProgress(event: {
|
||||
type: string;
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo?: number;
|
||||
totalCollected?: number;
|
||||
}): ExtractorProgressEvent {
|
||||
if (event.type === "term_start") {
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.type === "page_fetched") {
|
||||
const pageNo = (event.pageNo ?? 0) + 1;
|
||||
const totalCollected = event.totalCollected ?? 0;
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: pageNo,
|
||||
jobPagesEnqueued: totalCollected,
|
||||
jobPagesProcessed: totalCollected,
|
||||
currentUrl: `page ${pageNo}`,
|
||||
detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal}, page ${pageNo} (${totalCollected} collected)`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `Hiring Cafe: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
};
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "hiringcafe",
|
||||
displayName: "Hiring Cafe",
|
||||
providesSources: ["hiringcafe"],
|
||||
async run(context) {
|
||||
if (context.shouldCancel?.()) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
const maxJobsPerTerm = context.settings.jobspyResultsWanted
|
||||
? parseInt(context.settings.jobspyResultsWanted, 10)
|
||||
: 200;
|
||||
|
||||
const result = await runHiringCafe({
|
||||
country: context.selectedCountry,
|
||||
countryKey: context.selectedCountry,
|
||||
searchTerms: context.searchTerms,
|
||||
locations: resolveSearchCities({
|
||||
single:
|
||||
context.settings.searchCities ?? context.settings.jobspyLocation,
|
||||
}),
|
||||
maxJobsPerTerm,
|
||||
onProgress: (event) => {
|
||||
if (context.shouldCancel?.()) return;
|
||||
|
||||
context.onProgress?.(toProgress(event));
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobs: result.jobs,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,307 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { mkdir, readFile, rm } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeCountryKey } from "@shared/location-support.js";
|
||||
import {
|
||||
resolveSearchCities,
|
||||
shouldApplyStrictCityFilter,
|
||||
} from "@shared/search-cities.js";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
import {
|
||||
toNumberOrNull,
|
||||
toStringOrNull,
|
||||
} from "@shared/utils/type-conversion.js";
|
||||
|
||||
const srcDir = dirname(fileURLToPath(import.meta.url));
|
||||
const EXTRACTOR_DIR = join(srcDir, "..");
|
||||
const DATASET_PATH = join(EXTRACTOR_DIR, "storage/datasets/default/jobs.json");
|
||||
const STORAGE_DATASET_DIR = join(EXTRACTOR_DIR, "storage/datasets/default");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
const require = createRequire(import.meta.url);
|
||||
const TSX_CLI_PATH = resolveTsxCliPath();
|
||||
|
||||
type HiringCafeRawJob = Record<string, unknown>;
|
||||
|
||||
export type HiringCafeProgressEvent =
|
||||
| {
|
||||
type: "term_start";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
}
|
||||
| {
|
||||
type: "page_fetched";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo: number;
|
||||
resultsOnPage: number;
|
||||
totalCollected: number;
|
||||
}
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
};
|
||||
|
||||
export interface RunHiringCafeOptions {
|
||||
searchTerms?: string[];
|
||||
country?: string;
|
||||
countryKey?: string;
|
||||
locations?: string[];
|
||||
locationRadiusMiles?: number;
|
||||
maxJobsPerTerm?: number;
|
||||
onProgress?: (event: HiringCafeProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface HiringCafeResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export function shouldApplyStrictLocationFilter(
|
||||
location: string,
|
||||
countryKey: string,
|
||||
): boolean {
|
||||
return shouldApplyStrictCityFilter(location, countryKey);
|
||||
}
|
||||
|
||||
function resolveTsxCliPath(): string | null {
|
||||
try {
|
||||
return require.resolve("tsx/dist/cli.mjs");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canRunNpmCommand(): boolean {
|
||||
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
|
||||
return !result.error && result.status === 0;
|
||||
}
|
||||
|
||||
function parseProgressLine(line: string): HiringCafeProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const event = toStringOrNull(parsed.event);
|
||||
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||
if (!event || termIndex === null || termTotal === null) return null;
|
||||
|
||||
if (event === "term_start") {
|
||||
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||
}
|
||||
|
||||
if (event === "page_fetched") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
if (pageNo === null) return null;
|
||||
return {
|
||||
type: "page_fetched",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
pageNo,
|
||||
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "term_complete") {
|
||||
return {
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapHiringCafeRow(row: HiringCafeRawJob): CreateJobInput | null {
|
||||
const jobUrl = toStringOrNull(row.jobUrl);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
return {
|
||||
source: "hiringcafe",
|
||||
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
|
||||
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
|
||||
jobUrl,
|
||||
applicationLink: toStringOrNull(row.applicationLink) ?? jobUrl,
|
||||
location: toStringOrNull(row.location) ?? undefined,
|
||||
salary: toStringOrNull(row.salary) ?? undefined,
|
||||
datePosted: toStringOrNull(row.datePosted) ?? undefined,
|
||||
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
|
||||
jobType: toStringOrNull(row.jobType) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function readDataset(): Promise<CreateJobInput[]> {
|
||||
const content = await readFile(DATASET_PATH, "utf-8");
|
||||
const parsed = JSON.parse(content) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const value of parsed) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) continue;
|
||||
const mapped = mapHiringCafeRow(value as HiringCafeRawJob);
|
||||
if (!mapped) continue;
|
||||
const dedupeKey = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(dedupeKey)) continue;
|
||||
seen.add(dedupeKey);
|
||||
jobs.push(mapped);
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
async function clearStorageDataset(): Promise<void> {
|
||||
await rm(STORAGE_DATASET_DIR, { recursive: true, force: true });
|
||||
await mkdir(STORAGE_DATASET_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
export async function runHiringCafe(
|
||||
options: RunHiringCafeOptions = {},
|
||||
): Promise<HiringCafeResult> {
|
||||
const searchTerms =
|
||||
options.searchTerms && options.searchTerms.length > 0
|
||||
? options.searchTerms
|
||||
: ["web developer"];
|
||||
const country = (options.country || "united kingdom").trim().toLowerCase();
|
||||
const countryKey = normalizeCountryKey(options.countryKey ?? "");
|
||||
const maxJobsPerTerm = options.maxJobsPerTerm ?? 200;
|
||||
const locationRadiusMiles = Math.max(
|
||||
1,
|
||||
Math.floor(options.locationRadiusMiles ?? 1),
|
||||
);
|
||||
const locations = resolveSearchCities({
|
||||
list: options.locations,
|
||||
env: process.env.HIRING_CAFE_LOCATION_QUERY,
|
||||
});
|
||||
const runLocations = locations.length > 0 ? locations : [null];
|
||||
const termTotal = searchTerms.length * runLocations.length;
|
||||
|
||||
const useNpmCommand = canRunNpmCommand();
|
||||
if (!useNpmCommand && !TSX_CLI_PATH) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (let runIndex = 0; runIndex < runLocations.length; runIndex += 1) {
|
||||
const location = runLocations[runIndex];
|
||||
const strictLocationFilter =
|
||||
location !== null &&
|
||||
shouldApplyStrictLocationFilter(location, countryKey);
|
||||
|
||||
await clearStorageDataset();
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const extractorEnv = {
|
||||
...process.env,
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
HIRING_CAFE_SEARCH_TERMS: JSON.stringify(searchTerms),
|
||||
HIRING_CAFE_COUNTRY: country,
|
||||
HIRING_CAFE_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
|
||||
HIRING_CAFE_OUTPUT_JSON: DATASET_PATH,
|
||||
HIRING_CAFE_LOCATION_QUERY: strictLocationFilter ? location : "",
|
||||
HIRING_CAFE_LOCATION_RADIUS_MILES: strictLocationFilter
|
||||
? String(locationRadiusMiles)
|
||||
: "",
|
||||
};
|
||||
|
||||
const child = useNpmCommand
|
||||
? spawn("npm", ["run", "start"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
})
|
||||
: (() => {
|
||||
const tsxCliPath = TSX_CLI_PATH;
|
||||
if (!tsxCliPath) {
|
||||
throw new Error(
|
||||
"Unable to execute Hiring Cafe extractor (npm/tsx unavailable)",
|
||||
);
|
||||
}
|
||||
|
||||
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
});
|
||||
})();
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const progressEvent = parseProgressLine(line);
|
||||
if (progressEvent) {
|
||||
const termOffset = runIndex * searchTerms.length;
|
||||
options.onProgress?.({
|
||||
...progressEvent,
|
||||
termIndex: termOffset + progressEvent.termIndex,
|
||||
termTotal,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else
|
||||
reject(new Error(`Hiring Cafe extractor exited with code ${code}`));
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const runJobs = await readDataset();
|
||||
const filtered = runJobs;
|
||||
|
||||
for (const job of filtered) {
|
||||
const key = job.sourceJobId || job.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
jobs.push(job);
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { shouldApplyStrictLocationFilter } from "../src/run";
|
||||
|
||||
describe("hiringcafe location query strictness", () => {
|
||||
it("enables strict filtering when city differs from country", () => {
|
||||
expect(shouldApplyStrictLocationFilter("Leeds", "united kingdom")).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("disables strict filtering when location is country-level", () => {
|
||||
expect(shouldApplyStrictLocationFilter("UK", "united kingdom")).toBe(false);
|
||||
expect(shouldApplyStrictLocationFilter("United States", "us")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorRuntimeContext,
|
||||
} from "@shared/types/extractors";
|
||||
import { runJobSpy } from "./src/run";
|
||||
|
||||
type JobSpySite = NonNullable<Parameters<typeof runJobSpy>[0]["sites"]>[number];
|
||||
|
||||
const JOBSPY_SOURCES = new Set<JobSpySite>(["indeed", "linkedin", "glassdoor"]);
|
||||
|
||||
function isJobSpySite(source: string): source is JobSpySite {
|
||||
return JOBSPY_SOURCES.has(source as JobSpySite);
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "jobspy",
|
||||
displayName: "JobSpy",
|
||||
providesSources: ["indeed", "linkedin", "glassdoor"],
|
||||
async run(context: ExtractorRuntimeContext) {
|
||||
if (context.shouldCancel?.()) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
const sites = context.selectedSources.filter(isJobSpySite);
|
||||
|
||||
const result = await runJobSpy({
|
||||
sites,
|
||||
searchTerms: context.searchTerms,
|
||||
location:
|
||||
context.settings.searchCities ?? context.settings.jobspyLocation,
|
||||
resultsWanted: context.settings.jobspyResultsWanted
|
||||
? parseInt(context.settings.jobspyResultsWanted, 10)
|
||||
: undefined,
|
||||
countryIndeed: context.settings.jobspyCountryIndeed,
|
||||
onProgress: (event) => {
|
||||
if (context.shouldCancel?.()) return;
|
||||
|
||||
if (event.type === "term_start") {
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `JobSpy: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
context.onProgress?.({
|
||||
phase: "list",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm,
|
||||
detail: `JobSpy: completed ${event.termIndex}/${event.termTotal} (${event.searchTerm}) with ${event.jobsFoundTerm} jobs`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobs: result.jobs,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,397 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readFile, unlink } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { resolveSearchCities } from "@shared/search-cities.js";
|
||||
import type { CreateJobInput, JobSource } from "@shared/types/jobs";
|
||||
import {
|
||||
toNumberOrNull,
|
||||
toStringOrNull,
|
||||
} from "@shared/utils/type-conversion.js";
|
||||
|
||||
const srcDir = dirname(fileURLToPath(import.meta.url));
|
||||
const EXTRACTOR_DIR = join(srcDir, "..");
|
||||
const JOBSPY_SCRIPT = join(EXTRACTOR_DIR, "scrape_jobs.py");
|
||||
const OUTPUT_DIR = join(EXTRACTOR_DIR, "storage/imports");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
|
||||
export type JobSpyProgressEvent =
|
||||
| {
|
||||
type: "term_start";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
}
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
};
|
||||
|
||||
export function parseJobSpyProgressLine(
|
||||
line: string,
|
||||
): JobSpyProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const eventName = toStringOrNull(parsed.event);
|
||||
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||
|
||||
if (!eventName || termIndex === null || termTotal === null) return null;
|
||||
if (eventName === "term_start") {
|
||||
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||
}
|
||||
if (eventName === "term_complete") {
|
||||
return {
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function toBooleanOrNull(value: unknown): boolean | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (typeof value === "boolean") return value;
|
||||
if (typeof value === "number") return value !== 0;
|
||||
if (typeof value === "string") {
|
||||
const normalized = value.trim().toLowerCase();
|
||||
if (!normalized) return null;
|
||||
if (["1", "true", "yes", "y", "on"].includes(normalized)) return true;
|
||||
if (["0", "false", "no", "n", "off"].includes(normalized)) return false;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function toJsonStringOrNull(value: unknown): string | null {
|
||||
if (value === null || value === undefined) return null;
|
||||
if (typeof value === "string") return toStringOrNull(value);
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function toJobSource(site: unknown): JobSource | null {
|
||||
const raw = toStringOrNull(site)?.toLowerCase();
|
||||
if (raw === "gradcracker") return "gradcracker";
|
||||
if (raw === "indeed") return "indeed";
|
||||
if (raw === "linkedin") return "linkedin";
|
||||
if (raw === "glassdoor") return "glassdoor";
|
||||
return null;
|
||||
}
|
||||
|
||||
function formatSalary(params: {
|
||||
minAmount: number | null;
|
||||
maxAmount: number | null;
|
||||
currency: string | null;
|
||||
interval: string | null;
|
||||
}): string | null {
|
||||
const { minAmount, maxAmount, currency, interval } = params;
|
||||
if (minAmount === null && maxAmount === null) return null;
|
||||
|
||||
const fmt = (n: number) => `${Math.round(n)}`;
|
||||
let range: string;
|
||||
if (minAmount !== null && maxAmount !== null) {
|
||||
range = `${fmt(minAmount)}-${fmt(maxAmount)}`;
|
||||
} else if (minAmount !== null) {
|
||||
range = `${fmt(minAmount)}+`;
|
||||
} else if (maxAmount !== null) {
|
||||
range = fmt(maxAmount);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
const currencyPart = currency ? `${currency} ` : "";
|
||||
const intervalPart = interval ? ` / ${interval}` : "";
|
||||
return `${currencyPart}${range}${intervalPart}`.trim();
|
||||
}
|
||||
|
||||
export interface RunJobSpyOptions {
|
||||
sites?: Array<JobSource>;
|
||||
searchTerms?: string[];
|
||||
location?: string;
|
||||
locations?: string[];
|
||||
resultsWanted?: number;
|
||||
hoursOld?: number;
|
||||
countryIndeed?: string;
|
||||
linkedinFetchDescription?: boolean;
|
||||
isRemote?: boolean;
|
||||
onProgress?: (event: JobSpyProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface JobSpyResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export async function runJobSpy(
|
||||
options: RunJobSpyOptions = {},
|
||||
): Promise<JobSpyResult> {
|
||||
await mkdir(OUTPUT_DIR, { recursive: true });
|
||||
|
||||
const sites = (options.sites ?? ["indeed", "linkedin", "glassdoor"])
|
||||
.filter(
|
||||
(site) =>
|
||||
site === "indeed" || site === "linkedin" || site === "glassdoor",
|
||||
)
|
||||
.join(",");
|
||||
|
||||
const searchTerms = resolveSearchTerms(options);
|
||||
const locations = resolveSearchCities({
|
||||
list: options.locations,
|
||||
single: options.location,
|
||||
env: process.env.JOBSPY_LOCATION,
|
||||
fallback: "UK",
|
||||
});
|
||||
const countryIndeed =
|
||||
options.countryIndeed ?? process.env.JOBSPY_COUNTRY_INDEED ?? "UK";
|
||||
if (searchTerms.length === 0) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seenJobUrls = new Set<string>();
|
||||
const totalRuns = searchTerms.length * locations.length;
|
||||
let runIndex = 0;
|
||||
|
||||
for (const searchTerm of searchTerms) {
|
||||
for (const location of locations) {
|
||||
runIndex += 1;
|
||||
const suffix = `${runIndex}_${slugForFilename(searchTerm)}_${slugForFilename(location)}`;
|
||||
const outputCsv = join(OUTPUT_DIR, `jobspy_jobs_${suffix}.csv`);
|
||||
const outputJson = join(OUTPUT_DIR, `jobspy_jobs_${suffix}.json`);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const pythonPath = process.env.PYTHON_PATH
|
||||
? process.env.PYTHON_PATH
|
||||
: process.platform === "win32"
|
||||
? "python"
|
||||
: "python3";
|
||||
|
||||
const child = spawn(pythonPath, [JOBSPY_SCRIPT], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
shell: false,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: {
|
||||
...process.env,
|
||||
JOBSPY_SITES: sites || "indeed,linkedin,glassdoor",
|
||||
JOBSPY_SEARCH_TERM: searchTerm,
|
||||
JOBSPY_TERM_INDEX: String(runIndex),
|
||||
JOBSPY_TERM_TOTAL: String(totalRuns),
|
||||
JOBSPY_LOCATION: location,
|
||||
JOBSPY_RESULTS_WANTED: String(
|
||||
options.resultsWanted ??
|
||||
process.env.JOBSPY_RESULTS_WANTED ??
|
||||
200,
|
||||
),
|
||||
JOBSPY_HOURS_OLD: String(
|
||||
options.hoursOld ?? process.env.JOBSPY_HOURS_OLD ?? 72,
|
||||
),
|
||||
JOBSPY_COUNTRY_INDEED: countryIndeed,
|
||||
JOBSPY_LINKEDIN_FETCH_DESCRIPTION: String(
|
||||
options.linkedinFetchDescription ??
|
||||
process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION ??
|
||||
"1",
|
||||
),
|
||||
JOBSPY_IS_REMOTE: String(
|
||||
options.isRemote ?? process.env.JOBSPY_IS_REMOTE ?? "0",
|
||||
),
|
||||
JOBSPY_OUTPUT_CSV: outputCsv,
|
||||
JOBSPY_OUTPUT_JSON: outputJson,
|
||||
},
|
||||
});
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const event = parseJobSpyProgressLine(line);
|
||||
if (event) {
|
||||
options.onProgress?.(event);
|
||||
return;
|
||||
}
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`JobSpy exited with code ${code}`));
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const raw = await readFile(outputJson, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Array<Record<string, unknown>>;
|
||||
const filtered = mapJobSpyRows(parsed);
|
||||
|
||||
for (const job of filtered) {
|
||||
if (seenJobUrls.has(job.jobUrl)) continue;
|
||||
seenJobUrls.add(job.jobUrl);
|
||||
jobs.push(job);
|
||||
}
|
||||
|
||||
try {
|
||||
await unlink(outputJson);
|
||||
await unlink(outputCsv);
|
||||
} catch {
|
||||
// ignore cleanup errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSearchTerms(options: RunJobSpyOptions): string[] {
|
||||
const fromOptions = options.searchTerms?.length ? options.searchTerms : null;
|
||||
const fromEnv = parseSearchTermsEnv(process.env.JOBSPY_SEARCH_TERMS);
|
||||
const raw = fromOptions ?? fromEnv ?? ["web developer"];
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const term of raw) {
|
||||
const normalized = term.trim();
|
||||
if (!normalized) continue;
|
||||
const key = normalized.toLowerCase();
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(normalized);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseSearchTermsEnv(raw: string | undefined): string[] | null {
|
||||
if (!raw) return null;
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return null;
|
||||
|
||||
if (trimmed.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
if (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.every((value) => typeof value === "string")
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
|
||||
const delimiter = trimmed.includes("|")
|
||||
? "|"
|
||||
: trimmed.includes("\n")
|
||||
? "\n"
|
||||
: ",";
|
||||
const split = trimmed
|
||||
.split(delimiter)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean);
|
||||
return split.length > 0 ? split : null;
|
||||
}
|
||||
|
||||
function slugForFilename(input: string): string {
|
||||
const slug = input
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "_")
|
||||
.replace(/^_+|_+$/g, "")
|
||||
.slice(0, 40);
|
||||
return slug || "term";
|
||||
}
|
||||
|
||||
function mapJobSpyRows(
|
||||
parsed: Array<Record<string, unknown>>,
|
||||
): CreateJobInput[] {
|
||||
const jobs: CreateJobInput[] = [];
|
||||
|
||||
for (const row of parsed) {
|
||||
const source = toJobSource(row.site);
|
||||
if (!source) continue;
|
||||
|
||||
const jobUrl = toStringOrNull(row.job_url);
|
||||
if (!jobUrl) continue;
|
||||
|
||||
const minAmount = toNumberOrNull(row.min_amount);
|
||||
const maxAmount = toNumberOrNull(row.max_amount);
|
||||
const currency = toStringOrNull(row.currency);
|
||||
const interval = toStringOrNull(row.interval);
|
||||
const salary = formatSalary({ minAmount, maxAmount, currency, interval });
|
||||
|
||||
const jobUrlDirect = toStringOrNull(row.job_url_direct);
|
||||
|
||||
jobs.push({
|
||||
source,
|
||||
sourceJobId: toStringOrNull(row.id) ?? undefined,
|
||||
jobUrlDirect: jobUrlDirect ?? undefined,
|
||||
datePosted: toStringOrNull(row.date_posted) ?? undefined,
|
||||
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||
employer: toStringOrNull(row.company) ?? "Unknown Employer",
|
||||
employerUrl: toStringOrNull(row.company_url) ?? undefined,
|
||||
jobUrl,
|
||||
applicationLink: jobUrlDirect ?? jobUrl,
|
||||
location: toStringOrNull(row.location) ?? undefined,
|
||||
jobDescription: toStringOrNull(row.description) ?? undefined,
|
||||
salary: salary ?? undefined,
|
||||
jobType: toStringOrNull(row.job_type) ?? undefined,
|
||||
salarySource: toStringOrNull(row.salary_source) ?? undefined,
|
||||
salaryInterval: interval ?? undefined,
|
||||
salaryMinAmount: minAmount ?? undefined,
|
||||
salaryMaxAmount: maxAmount ?? undefined,
|
||||
salaryCurrency: currency ?? undefined,
|
||||
isRemote: toBooleanOrNull(row.is_remote) ?? undefined,
|
||||
jobLevel: toStringOrNull(row.job_level) ?? undefined,
|
||||
jobFunction: toStringOrNull(row.job_function) ?? undefined,
|
||||
listingType: toStringOrNull(row.listing_type) ?? undefined,
|
||||
emails: toJsonStringOrNull(row.emails) ?? undefined,
|
||||
companyIndustry: toStringOrNull(row.company_industry) ?? undefined,
|
||||
companyLogo: toStringOrNull(row.company_logo) ?? undefined,
|
||||
companyUrlDirect: toStringOrNull(row.company_url_direct) ?? undefined,
|
||||
companyAddresses: toJsonStringOrNull(row.company_addresses) ?? undefined,
|
||||
companyNumEmployees:
|
||||
toStringOrNull(row.company_num_employees) ?? undefined,
|
||||
companyRevenue: toStringOrNull(row.company_revenue) ?? undefined,
|
||||
companyDescription: toStringOrNull(row.company_description) ?? undefined,
|
||||
skills: toJsonStringOrNull(row.skills) ?? undefined,
|
||||
experienceRange: toJsonStringOrNull(row.experience_range) ?? undefined,
|
||||
companyRating: toNumberOrNull(row.company_rating) ?? undefined,
|
||||
companyReviewsCount:
|
||||
toNumberOrNull(row.company_reviews_count) ?? undefined,
|
||||
vacancyCount: toNumberOrNull(row.vacancy_count) ?? undefined,
|
||||
workFromHomeType: toStringOrNull(row.work_from_home_type) ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseJobSpyProgressLine } from "../src/run";
|
||||
|
||||
describe("parseJobSpyProgressLine", () => {
|
||||
it("parses term_start progress lines", () => {
|
||||
const event = parseJobSpyProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"term_start","termIndex":1,"termTotal":3,"searchTerm":"engineer"}',
|
||||
);
|
||||
|
||||
expect(event).toEqual({
|
||||
type: "term_start",
|
||||
termIndex: 1,
|
||||
termTotal: 3,
|
||||
searchTerm: "engineer",
|
||||
});
|
||||
});
|
||||
|
||||
it("parses term_complete progress lines", () => {
|
||||
const event = parseJobSpyProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"term_complete","termIndex":2,"termTotal":3,"searchTerm":"frontend","jobsFoundTerm":17}',
|
||||
);
|
||||
|
||||
expect(event).toEqual({
|
||||
type: "term_complete",
|
||||
termIndex: 2,
|
||||
termTotal: 3,
|
||||
searchTerm: "frontend",
|
||||
jobsFoundTerm: 17,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null for malformed payloads", () => {
|
||||
expect(parseJobSpyProgressLine("JOBOPS_PROGRESS {bad json")).toBeNull();
|
||||
expect(parseJobSpyProgressLine("JOBOPS_PROGRESS {}")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for non-progress lines", () => {
|
||||
expect(parseJobSpyProgressLine("Found 20 jobs")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ES2022",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"],
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@shared/*": ["../../shared/src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import type {
|
||||
ExtractorManifest,
|
||||
ExtractorProgressEvent,
|
||||
ExtractorRuntimeContext,
|
||||
} from "@shared/types/extractors";
|
||||
import { runUkVisaJobs } from "./src/run";
|
||||
|
||||
function toProgress(event: {
|
||||
type: string;
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo?: number;
|
||||
maxPages?: number;
|
||||
totalCollected?: number;
|
||||
message?: string;
|
||||
}): ExtractorProgressEvent {
|
||||
if (event.type === "init") {
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: 0,
|
||||
listPagesTotal: event.maxPages ?? 0,
|
||||
currentUrl: event.searchTerm || "all jobs",
|
||||
detail: `UKVisaJobs: term ${event.termIndex}/${event.termTotal} (${event.searchTerm || "all jobs"})`,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.type === "page_fetched") {
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: event.pageNo ?? 0,
|
||||
listPagesTotal: event.maxPages ?? 0,
|
||||
jobPagesEnqueued: event.totalCollected ?? 0,
|
||||
jobPagesProcessed: event.totalCollected ?? 0,
|
||||
currentUrl: `page ${event.pageNo ?? 0}/${event.maxPages ?? 0}`,
|
||||
detail: `UKVisaJobs: term ${event.termIndex}/${event.termTotal}, page ${event.pageNo ?? 0}/${event.maxPages ?? 0} (${event.totalCollected ?? 0} collected)`,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.type === "term_complete") {
|
||||
return {
|
||||
phase: "list",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
currentUrl: event.searchTerm || "all jobs",
|
||||
detail: `UKVisaJobs: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm || "all jobs"})`,
|
||||
};
|
||||
}
|
||||
|
||||
if (event.type === "empty_page") {
|
||||
return {
|
||||
detail: `UKVisaJobs: page ${event.pageNo ?? 0} returned no jobs`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
detail: `UKVisaJobs: ${event.message ?? "unknown event"}`,
|
||||
};
|
||||
}
|
||||
|
||||
export const manifest: ExtractorManifest = {
|
||||
id: "ukvisajobs",
|
||||
displayName: "UK Visa Jobs",
|
||||
providesSources: ["ukvisajobs"],
|
||||
requiredEnvVars: ["UKVISAJOBS_EMAIL", "UKVISAJOBS_PASSWORD"],
|
||||
async run(context: ExtractorRuntimeContext) {
|
||||
if (context.shouldCancel?.()) {
|
||||
return { success: true, jobs: [] };
|
||||
}
|
||||
|
||||
const maxJobs = context.settings.ukvisajobsMaxJobs
|
||||
? parseInt(context.settings.ukvisajobsMaxJobs, 10)
|
||||
: 50;
|
||||
|
||||
const result = await runUkVisaJobs({
|
||||
maxJobs,
|
||||
searchTerms: context.searchTerms,
|
||||
onProgress: (event) => {
|
||||
if (context.shouldCancel?.()) return;
|
||||
|
||||
context.onProgress?.(toProgress(event));
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: result.error,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobs: result.jobs,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
export default manifest;
|
||||
@@ -0,0 +1,445 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { mkdir, readdir, readFile, rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type CreateJobInput = {
|
||||
source: "ukvisajobs";
|
||||
sourceJobId?: string;
|
||||
title: string;
|
||||
employer: string;
|
||||
employerUrl?: string;
|
||||
jobUrl: string;
|
||||
applicationLink?: string;
|
||||
location?: string;
|
||||
deadline?: string;
|
||||
salary?: string;
|
||||
jobDescription?: string;
|
||||
datePosted?: string;
|
||||
degreeRequired?: string;
|
||||
jobType?: string;
|
||||
jobLevel?: string;
|
||||
};
|
||||
|
||||
import {
|
||||
toNumberOrNull,
|
||||
toStringOrNull,
|
||||
} from "@shared/utils/type-conversion.js";
|
||||
|
||||
const srcDir = dirname(fileURLToPath(import.meta.url));
|
||||
const EXTRACTOR_DIR = join(srcDir, "..");
|
||||
const STORAGE_DIR = join(EXTRACTOR_DIR, "storage/datasets/default");
|
||||
const AUTH_CACHE_PATH = join(EXTRACTOR_DIR, "storage/ukvisajobs-auth.json");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
let isUkVisaJobsRunning = false;
|
||||
|
||||
interface UkVisaJobsAuthSession {
|
||||
token?: string;
|
||||
authToken?: string;
|
||||
csrfToken?: string;
|
||||
ciSession?: string;
|
||||
}
|
||||
|
||||
export interface RunUkVisaJobsOptions {
|
||||
maxJobs?: number;
|
||||
searchKeyword?: string;
|
||||
searchTerms?: string[];
|
||||
onProgress?: (event: UkVisaJobsProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface UkVisaJobsResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
type UkVisaJobsExtractorProgressEvent =
|
||||
| {
|
||||
type: "init";
|
||||
maxPages: number;
|
||||
maxJobs: number;
|
||||
searchKeyword: string;
|
||||
}
|
||||
| {
|
||||
type: "page_fetched";
|
||||
pageNo: number;
|
||||
maxPages: number;
|
||||
jobsOnPage: number;
|
||||
totalCollected: number;
|
||||
totalAvailable: number;
|
||||
}
|
||||
| {
|
||||
type: "done";
|
||||
maxPages: number;
|
||||
totalCollected: number;
|
||||
totalAvailable: number;
|
||||
}
|
||||
| {
|
||||
type: "empty_page";
|
||||
pageNo: number;
|
||||
maxPages: number;
|
||||
totalCollected: number;
|
||||
}
|
||||
| {
|
||||
type: "error";
|
||||
message: string;
|
||||
pageNo?: number;
|
||||
status?: number;
|
||||
};
|
||||
|
||||
type UkVisaJobsExtractorEventWithTerm = UkVisaJobsExtractorProgressEvent & {
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
};
|
||||
|
||||
export type UkVisaJobsProgressEvent =
|
||||
| UkVisaJobsExtractorEventWithTerm
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
totalCollected: number;
|
||||
};
|
||||
|
||||
export function parseUkVisaJobsProgressLine(
|
||||
line: string,
|
||||
): UkVisaJobsExtractorProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const event = toStringOrNull(parsed.event);
|
||||
if (!event) return null;
|
||||
|
||||
if (event === "init") {
|
||||
const maxPages = toNumberOrNull(parsed.maxPages);
|
||||
const maxJobs = toNumberOrNull(parsed.maxJobs);
|
||||
if (maxPages === null || maxJobs === null) return null;
|
||||
return {
|
||||
type: "init",
|
||||
maxPages,
|
||||
maxJobs,
|
||||
searchKeyword: toStringOrNull(parsed.searchKeyword) ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "page_fetched") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
const maxPages = toNumberOrNull(parsed.maxPages);
|
||||
if (pageNo === null || maxPages === null) return null;
|
||||
return {
|
||||
type: "page_fetched",
|
||||
pageNo,
|
||||
maxPages,
|
||||
jobsOnPage: toNumberOrNull(parsed.jobsOnPage) ?? 0,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
totalAvailable: toNumberOrNull(parsed.totalAvailable) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "done") {
|
||||
const maxPages = toNumberOrNull(parsed.maxPages);
|
||||
if (maxPages === null) return null;
|
||||
return {
|
||||
type: "done",
|
||||
maxPages,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
totalAvailable: toNumberOrNull(parsed.totalAvailable) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "empty_page") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
const maxPages = toNumberOrNull(parsed.maxPages);
|
||||
if (pageNo === null || maxPages === null) return null;
|
||||
return {
|
||||
type: "empty_page",
|
||||
pageNo,
|
||||
maxPages,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "error") {
|
||||
return {
|
||||
type: "error",
|
||||
message: toStringOrNull(parsed.message) ?? "unknown error",
|
||||
pageNo: toNumberOrNull(parsed.pageNo) ?? undefined,
|
||||
status: toNumberOrNull(parsed.status) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function cleanHtml(html: string): string {
|
||||
let text = html.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, "");
|
||||
|
||||
const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/i);
|
||||
const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
|
||||
if (mainMatch) {
|
||||
text = mainMatch[1];
|
||||
} else if (bodyMatch) {
|
||||
text = bodyMatch[1];
|
||||
}
|
||||
|
||||
text = text.replace(/<[^>]+>/g, " ");
|
||||
text = text
|
||||
.replace(/ /g, " ")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"');
|
||||
text = text.replace(/\s+/g, " ").trim();
|
||||
|
||||
if (text.length > 8000) {
|
||||
text = `${text.substring(0, 8000)}...`;
|
||||
}
|
||||
|
||||
return text;
|
||||
}
|
||||
|
||||
async function fetchJobDescription(url: string): Promise<string | null> {
|
||||
try {
|
||||
const authSession = await loadCachedAuthSession();
|
||||
const cookieParts: string[] = [];
|
||||
if (authSession?.csrfToken) {
|
||||
cookieParts.push(`csrf_token=${authSession.csrfToken}`);
|
||||
}
|
||||
if (authSession?.ciSession) {
|
||||
cookieParts.push(`ci_session=${authSession.ciSession}`);
|
||||
}
|
||||
const token = authSession?.authToken || authSession?.token;
|
||||
if (token) cookieParts.push(`authToken=${token}`);
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
};
|
||||
|
||||
if (cookieParts.length > 0) {
|
||||
headers.Cookie = cookieParts.join("; ");
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!response.ok) return null;
|
||||
|
||||
const html = await response.text();
|
||||
const cleaned = cleanHtml(html);
|
||||
return cleaned.length > 100 ? cleaned : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadCachedAuthSession(): Promise<UkVisaJobsAuthSession | null> {
|
||||
try {
|
||||
const data = await readFile(AUTH_CACHE_PATH, "utf-8");
|
||||
return JSON.parse(data) as UkVisaJobsAuthSession;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function clearStorageDataset(): Promise<void> {
|
||||
try {
|
||||
await rm(STORAGE_DIR, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
export async function runUkVisaJobs(
|
||||
options: RunUkVisaJobsOptions = {},
|
||||
): Promise<UkVisaJobsResult> {
|
||||
if (isUkVisaJobsRunning) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "UK Visa Jobs extractor is already running",
|
||||
};
|
||||
}
|
||||
|
||||
isUkVisaJobsRunning = true;
|
||||
try {
|
||||
const terms: string[] = [];
|
||||
if (options.searchTerms && options.searchTerms.length > 0) {
|
||||
terms.push(...options.searchTerms);
|
||||
} else if (options.searchKeyword) {
|
||||
terms.push(options.searchKeyword);
|
||||
} else {
|
||||
terms.push("");
|
||||
}
|
||||
|
||||
const allJobs: CreateJobInput[] = [];
|
||||
const seenIds = new Set<string>();
|
||||
const termTotal = terms.length;
|
||||
|
||||
for (let i = 0; i < terms.length; i += 1) {
|
||||
const term = terms[i];
|
||||
const termIndex = i + 1;
|
||||
|
||||
try {
|
||||
await clearStorageDataset();
|
||||
await mkdir(STORAGE_DIR, { recursive: true });
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn("npx", ["tsx", "src/main.ts"], {
|
||||
cwd: EXTRACTOR_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: {
|
||||
...process.env,
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
UKVISAJOBS_MAX_JOBS: String(options.maxJobs ?? 50),
|
||||
UKVISAJOBS_SEARCH_KEYWORD: term,
|
||||
},
|
||||
});
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const progressEvent = parseUkVisaJobsProgressLine(line);
|
||||
if (progressEvent) {
|
||||
options.onProgress?.({
|
||||
...progressEvent,
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm: term,
|
||||
});
|
||||
return;
|
||||
}
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else {
|
||||
reject(
|
||||
new Error(`UK Visa Jobs extractor exited with code ${code}`),
|
||||
);
|
||||
}
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const runJobs = await readDataset();
|
||||
let jobsFoundTerm = 0;
|
||||
|
||||
for (const job of runJobs) {
|
||||
const id = job.sourceJobId || job.jobUrl;
|
||||
if (seenIds.has(id)) continue;
|
||||
seenIds.add(id);
|
||||
|
||||
const isPoorDescription =
|
||||
!job.jobDescription ||
|
||||
job.jobDescription.length < 100 ||
|
||||
job.jobDescription.startsWith("Visa sponsorship info:");
|
||||
|
||||
if (isPoorDescription && job.jobUrl) {
|
||||
const enriched = await fetchJobDescription(job.jobUrl);
|
||||
if (enriched) {
|
||||
job.jobDescription = enriched;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
allJobs.push(job);
|
||||
jobsFoundTerm += 1;
|
||||
}
|
||||
|
||||
options.onProgress?.({
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm: term,
|
||||
jobsFoundTerm,
|
||||
totalCollected: allJobs.length,
|
||||
});
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Unknown error";
|
||||
options.onProgress?.({
|
||||
type: "error",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm: term,
|
||||
message,
|
||||
});
|
||||
}
|
||||
|
||||
if (i < terms.length - 1) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
}
|
||||
}
|
||||
|
||||
return { success: true, jobs: allJobs };
|
||||
} finally {
|
||||
isUkVisaJobsRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function readDataset(): Promise<CreateJobInput[]> {
|
||||
const jobs: CreateJobInput[] = [];
|
||||
|
||||
try {
|
||||
const files = await readdir(STORAGE_DIR);
|
||||
const jsonFiles = files.filter(
|
||||
(file) => file.endsWith(".json") && file !== "jobs.json",
|
||||
);
|
||||
|
||||
for (const file of jsonFiles.sort()) {
|
||||
try {
|
||||
const content = await readFile(join(STORAGE_DIR, file), "utf-8");
|
||||
const job = JSON.parse(content) as Record<string, unknown>;
|
||||
|
||||
jobs.push({
|
||||
source: "ukvisajobs",
|
||||
sourceJobId: job.sourceJobId as string | undefined,
|
||||
title: (job.title as string) || "Unknown Title",
|
||||
employer: (job.employer as string) || "Unknown Employer",
|
||||
employerUrl: job.employerUrl as string | undefined,
|
||||
jobUrl: job.jobUrl as string,
|
||||
applicationLink:
|
||||
(job.applicationLink as string | undefined) ||
|
||||
(job.jobUrl as string),
|
||||
location: job.location as string | undefined,
|
||||
deadline: job.deadline as string | undefined,
|
||||
salary: job.salary as string | undefined,
|
||||
jobDescription: job.jobDescription as string | undefined,
|
||||
datePosted: job.datePosted as string | undefined,
|
||||
degreeRequired: job.degreeRequired as string | undefined,
|
||||
jobType: job.jobType as string | undefined,
|
||||
jobLevel: job.jobLevel as string | undefined,
|
||||
});
|
||||
} catch {
|
||||
// ignore invalid file
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore missing dir
|
||||
}
|
||||
|
||||
return jobs;
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseUkVisaJobsProgressLine } from "../src/run";
|
||||
|
||||
describe("parseUkVisaJobsProgressLine", () => {
|
||||
it("parses init events", () => {
|
||||
const event = parseUkVisaJobsProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"init","maxPages":4,"maxJobs":50,"searchKeyword":"engineer"}',
|
||||
);
|
||||
|
||||
expect(event).toEqual({
|
||||
type: "init",
|
||||
maxPages: 4,
|
||||
maxJobs: 50,
|
||||
searchKeyword: "engineer",
|
||||
});
|
||||
});
|
||||
|
||||
it("parses page_fetched events", () => {
|
||||
const event = parseUkVisaJobsProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"page_fetched","pageNo":2,"maxPages":4,"jobsOnPage":15,"totalCollected":28,"totalAvailable":105}',
|
||||
);
|
||||
|
||||
expect(event).toEqual({
|
||||
type: "page_fetched",
|
||||
pageNo: 2,
|
||||
maxPages: 4,
|
||||
jobsOnPage: 15,
|
||||
totalCollected: 28,
|
||||
totalAvailable: 105,
|
||||
});
|
||||
});
|
||||
|
||||
it("parses terminal and error events", () => {
|
||||
expect(
|
||||
parseUkVisaJobsProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"empty_page","pageNo":3,"maxPages":4,"totalCollected":28}',
|
||||
),
|
||||
).toEqual({
|
||||
type: "empty_page",
|
||||
pageNo: 3,
|
||||
maxPages: 4,
|
||||
totalCollected: 28,
|
||||
});
|
||||
|
||||
expect(
|
||||
parseUkVisaJobsProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"done","maxPages":4,"totalCollected":42,"totalAvailable":105}',
|
||||
),
|
||||
).toEqual({
|
||||
type: "done",
|
||||
maxPages: 4,
|
||||
totalCollected: 42,
|
||||
totalAvailable: 105,
|
||||
});
|
||||
|
||||
expect(
|
||||
parseUkVisaJobsProgressLine(
|
||||
'JOBOPS_PROGRESS {"event":"error","message":"boom","pageNo":2,"status":500}',
|
||||
),
|
||||
).toEqual({
|
||||
type: "error",
|
||||
message: "boom",
|
||||
pageNo: 2,
|
||||
status: 500,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores malformed or unrelated lines", () => {
|
||||
expect(parseUkVisaJobsProgressLine("JOBOPS_PROGRESS {bad")).toBeNull();
|
||||
expect(parseUkVisaJobsProgressLine("normal log line")).toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user