feat: add Adzuna extractor with orchestrator integration (#177)

* feat(settings): add adzuna source fields and country compatibility

* feat(discovery): integrate adzuna extractor into pipeline

* feat(client): wire adzuna in source selection and run budgeting

* docs(extractors): add adzuna guide and configuration notes

* chore(workspaces): register adzuna extractor in lockfile

* fix(adzuna): run extractor via npm script instead of npx

* fix(adzuna): execute extractor via node+tsx without shell

* fix(adzuna): prefer npm run start without shell, fallback to tsx

* fix(docker): include adzuna extractor workspace in image

* chore(adzuna): reuse shared type-conversion utilities

* type-check adzuna

* formatting

* deeedooop

* better instructions
This commit is contained in:
Shaheer Sarfaraz 2026-02-17 16:49:42 +00:00 committed by GitHub
parent 4da264eb48
commit c5c6675f04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 1092 additions and 11 deletions

View File

@ -39,6 +39,15 @@ UKVISAJOBS_EMAIL=
UKVISAJOBS_PASSWORD=
UKVISAJOBS_HEADLESS=true
# =============================================================================
# Adzuna (multi-country API source) - optional
# =============================================================================
# App credentials from Adzuna developer account.
ADZUNA_APP_ID=
ADZUNA_APP_KEY=
# Optional default per-term cap (can be overridden by UI run budget logic).
# ADZUNA_MAX_JOBS_PER_TERM=50
# =============================================================================
# JobSpy - Job search configuration
# =============================================================================

View File

@ -49,7 +49,11 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
project: [orchestrator, gradcracker-extractor, ukvisajobs-extractor]
project:
- orchestrator
- adzuna-extractor
- gradcracker-extractor
- ukvisajobs-extractor
steps:
- uses: actions/checkout@v4
- name: Setup Node

View File

@ -35,6 +35,7 @@ COPY package*.json ./
COPY docs-site/package*.json ./docs-site/
COPY shared/package*.json ./shared/
COPY orchestrator/package*.json ./orchestrator/
COPY extractors/adzuna/package*.json ./extractors/adzuna/
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
@ -52,6 +53,7 @@ WORKDIR /app
COPY shared ./shared
COPY docs-site ./docs-site
COPY orchestrator ./orchestrator
COPY extractors/adzuna ./extractors/adzuna
COPY extractors/gradcracker ./extractors/gradcracker
COPY extractors/jobspy ./extractors/jobspy
COPY extractors/ukvisajobs ./extractors/ukvisajobs
@ -97,6 +99,7 @@ COPY package*.json ./
COPY docs-site/package*.json ./docs-site/
COPY shared/package*.json ./shared/
COPY orchestrator/package*.json ./orchestrator/
COPY extractors/adzuna/package*.json ./extractors/adzuna/
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
@ -110,6 +113,7 @@ COPY --from=builder /app/orchestrator/dist ./orchestrator/dist
COPY --from=builder /app/docs-site/build ./orchestrator/dist/docs
COPY shared ./shared
COPY orchestrator ./orchestrator
COPY extractors/adzuna ./extractors/adzuna
COPY extractors/gradcracker ./extractors/gradcracker
COPY extractors/jobspy ./extractors/jobspy
COPY extractors/ukvisajobs ./extractors/ukvisajobs

View File

@ -60,7 +60,7 @@ docker compose up -d
## Why JobOps?
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor** + specialized boards (Gradcracker, UK Visa Jobs).
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor, Adzuna** + specialized boards (Gradcracker, UK Visa Jobs).
* **AI Scoring**: Ranks jobs by fit against *your* profile using your preferred LLM (OpenRouter/OpenAI/Gemini).
* **Auto-Tailoring**: Generates custom resumes (PDFs) for every application using RxResume v4.
* **Email Tracking**: Connect Gmail to auto-detect interviews, offers, and rejections.
@ -81,6 +81,7 @@ docker compose up -d
| **LinkedIn** | Global / General |
| **Indeed** | Global / General |
| **Glassdoor** | Global / General |
| **Adzuna** | Multi-country API source |
| **Gradcracker** | STEM / Grads (UK) |
| **UK Visa Jobs** | Sponsorship (UK) |

View File

@ -0,0 +1,58 @@
---
id: adzuna
title: Adzuna Extractor
description: API-based Adzuna extraction with orchestrator ingestion and progress updates.
sidebar_position: 6
---
## What it is
Adzuna is an API-backed extractor implemented in two lean pieces:
1. `extractors/adzuna/src/main.ts` fetches paginated Adzuna search results and writes `jobs.json`.
2. `orchestrator/src/server/services/adzuna.ts` runs the extractor, parses progress lines, and maps rows into `CreateJobInput`.
It de-duplicates in the existing repository path using `sourceJobId` fallback to `jobUrl`.
## Why it exists
Adzuna provides stable API discovery for countries that are not covered by UK-only sources. It adds a lower-maintenance source without introducing new API routes or UI sections.
## How to use it
1. Create an Adzuna developer account.
2. Open [Adzuna Access Details](https://developer.adzuna.com/admin/access_details).
3. Copy your **App ID** and **App Key**.
4. In Job Ops, open **Settings** and paste them into `Adzuna App ID` and `Adzuna App Key` under **Environment & Accounts**.
5. In **Pipeline Run** (Automatic tab), select a compatible country and enable **Adzuna** in Sources.
6. Start the run; Adzuna progress appears in the existing crawl progress stream.
Default controls:
- `ADZUNA_APP_ID`
- `ADZUNA_APP_KEY`
- `ADZUNA_MAX_JOBS_PER_TERM` (default `50`)
Supported countries in this integration:
- United Kingdom, United States, Austria, Australia, Belgium, Brazil, Canada, Switzerland, Germany, Spain, France, India, Italy, Mexico, Netherlands, New Zealand, Poland, Singapore, South Africa.
## Common problems
### Adzuna is disabled in source selection
- `Adzuna App ID` and `Adzuna App Key` are missing from Settings (or env).
### Adzuna is skipped for my selected country
- The selected country is not in the supported list above.
### Adzuna fails with authorization errors
- Verify `ADZUNA_APP_ID` and `ADZUNA_APP_KEY` are valid and active in your Adzuna account.
## Related pages
- [Extractors Overview](/docs/next/extractors/overview)
- [Pipeline Run](/docs/next/features/pipeline-run)
- [Settings](/docs/next/features/settings)

View File

@ -13,12 +13,14 @@ This page helps you choose the right extractor for your run, understand key cons
| --- | --- | --- | --- | --- |
| [Gradcracker](/docs/next/extractors/gradcracker) | UK graduate roles from Gradcracker | Crawling stability depends on page structure and anti-bot behavior; tuned for low concurrency | `GRADCRACKER_SEARCH_TERMS`, `GRADCRACKER_MAX_JOBS_PER_TERM`, `JOBOPS_SKIP_APPLY_FOR_EXISTING` | Scrapes listing metadata, then detail pages and apply URL resolution |
| [JobSpy](/docs/next/extractors/jobspy) | Multi-source discovery (Indeed, LinkedIn, Glassdoor) | Requires Python wrapper execution per term; source availability and quality vary by site/location | `JOBSPY_SITES`, `JOBSPY_SEARCH_TERMS`, `JOBSPY_RESULTS_WANTED`, `JOBSPY_HOURS_OLD`, `JOBSPY_LINKEDIN_FETCH_DESCRIPTION` | Produces JSON per term, then orchestrator normalizes and de-duplicates by `jobUrl` |
| [Adzuna](/docs/next/extractors/adzuna) | API-based multi-country discovery with low scraping overhead | Requires valid App ID/App Key; country must be in Adzuna-supported list | `ADZUNA_APP_ID`, `ADZUNA_APP_KEY`, `ADZUNA_MAX_JOBS_PER_TERM` | API pagination to dataset output; orchestrator maps progress and de-duplicates by `sourceJobId`/`jobUrl` |
| [UKVisaJobs](/docs/next/extractors/ukvisajobs) | UK visa sponsorship-focused roles | Requires authenticated session and periodic token/cookie refresh | `UKVISAJOBS_EMAIL`, `UKVISAJOBS_PASSWORD`, `UKVISAJOBS_MAX_JOBS`, `UKVISAJOBS_SEARCH_KEYWORD` | API pagination + dataset output; orchestrator de-dupes and may fetch missing descriptions |
| [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review |
## Which extractor should I use?
- Use **JobSpy** for broad first-pass sourcing across common boards.
- Use **Adzuna** when you want API-first discovery in supported non-UK markets.
- Use **Gradcracker** when targeting graduate pipelines in the UK.
- Use **UKVisaJobs** for sponsorship-specific UK searches.
- Use **Manual Import** when you already have a specific posting and need direct import.
@ -29,5 +31,6 @@ Many runs combine sources: broad discovery first, then manual import for high-pr
- [Gradcracker](/docs/next/extractors/gradcracker)
- [JobSpy](/docs/next/extractors/jobspy)
- [Adzuna](/docs/next/extractors/adzuna)
- [UKVisaJobs](/docs/next/extractors/ukvisajobs)
- [Manual Import](/docs/next/extractors/manual)

View File

@ -47,6 +47,7 @@ If values are edited manually, the UI shows **Custom**.
- Country selection affects which sources are available.
- UK-only sources are disabled for non-UK countries.
- Adzuna is available only for its supported countries and when App ID/App Key are configured in Settings.
- Glassdoor can be enabled only when:
- selected country supports Glassdoor
- a **Glassdoor city** is set in Advanced settings
@ -98,6 +99,11 @@ For accepted input formats, inference behavior, and limits, see [Manual Import E
- Verify selected country supports Glassdoor.
- Set a Glassdoor city in Advanced settings.
### Adzuna is not selectable
- Set `Adzuna App ID` and `Adzuna App Key` in **Settings > Environment & Accounts**.
- Verify the selected country is one of Adzuna's supported markets.
### Run takes longer than expected
- Reduce term count.

View File

@ -86,6 +86,7 @@ Settings gives you runtime overrides for the key parts of discovery, scoring, ta
- Configure service accounts:
- RxResume email/password
- UKVisaJobs email/password
- Adzuna app ID/app key
- Optional basic authentication for write operations
### Backup

View File

@ -41,6 +41,7 @@ const sidebars: SidebarsConfig = {
"extractors/overview",
"extractors/gradcracker",
"extractors/jobspy",
"extractors/adzuna",
"extractors/manual",
"extractors/ukvisajobs",
],

View File

@ -0,0 +1,15 @@
# Adzuna Extractor
Minimal extractor that pulls jobs from Adzuna's search API and writes a dataset
for orchestrator ingestion.
## Environment
- `ADZUNA_APP_ID` (required)
- `ADZUNA_APP_KEY` (required)
- `ADZUNA_COUNTRY` (default: `gb`)
- `ADZUNA_SEARCH_TERMS` (JSON array or `|` / comma / newline-delimited)
- `ADZUNA_MAX_JOBS_PER_TERM` (default: `50`)
- `ADZUNA_RESULTS_PER_PAGE` (default: `50`, max `50`)
- `ADZUNA_OUTPUT_JSON` (default: `storage/datasets/default/jobs.json`)
- `JOBOPS_EMIT_PROGRESS=1` to emit `JOBOPS_PROGRESS` events

View File

@ -0,0 +1,20 @@
{
"name": "adzuna-extractor",
"version": "0.0.1",
"type": "module",
"description": "Adzuna extractor - fetches jobs from Adzuna search API",
"main": "src/main.ts",
"dependencies": {
"job-ops-shared": "^1.0.0",
"tsx": "^4.4.0"
},
"devDependencies": {
"@types/node": "^24.0.0",
"typescript": "~5.9.0"
},
"scripts": {
"start": "tsx src/main.ts",
"start:dev": "tsx src/main.ts",
"check:types": "tsc --noEmit"
}
}

View File

@ -0,0 +1,245 @@
import { mkdir, writeFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import {
toNumberOrNull,
toStringOrNull,
} from "job-ops-shared/utils/type-conversion";
const API_BASE = "https://api.adzuna.com/v1/api";
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
type AdzunaCompany = { display_name?: unknown };
type AdzunaLocation = { display_name?: unknown };
type AdzunaJob = {
id?: unknown;
title?: unknown;
description?: unknown;
created?: unknown;
redirect_url?: unknown;
company?: AdzunaCompany;
location?: AdzunaLocation;
salary_min?: unknown;
salary_max?: unknown;
contract_time?: unknown;
contract_type?: unknown;
};
type ExtractedJob = {
source: "adzuna";
sourceJobId?: string;
title: string;
employer: string;
jobUrl: string;
applicationLink: string;
location?: string;
salary?: string;
datePosted?: string;
jobDescription?: string;
jobType?: string;
};
function parsePositiveInt(input: string | undefined, fallback: number): number {
const parsed = input ? Number.parseInt(input, 10) : Number.NaN;
if (!Number.isFinite(parsed) || parsed < 1) return fallback;
return parsed;
}
function parseSearchTerms(raw: string | undefined): string[] {
if (!raw || raw.trim().length === 0) return ["web developer"];
const trimmed = raw.trim();
if (trimmed.startsWith("[")) {
try {
const parsed = JSON.parse(trimmed) as unknown;
if (Array.isArray(parsed)) {
const terms = parsed
.map((value) => toStringOrNull(value))
.filter((value): value is string => value !== null);
if (terms.length > 0) return terms;
}
} catch {
// Fall through to delimiter parsing.
}
}
const delimiter = trimmed.includes("|")
? "|"
: trimmed.includes("\n")
? "\n"
: ",";
const terms = trimmed
.split(delimiter)
.map((value) => value.trim())
.filter(Boolean);
return terms.length > 0 ? terms : ["web developer"];
}
function requireEnv(name: string): string {
const value = process.env[name]?.trim();
if (!value) throw new Error(`Missing required environment variable: ${name}`);
return value;
}
function emitProgress(payload: Record<string, unknown>): void {
if (process.env.JOBOPS_EMIT_PROGRESS !== "1") return;
console.log(`${JOBOPS_PROGRESS_PREFIX}${JSON.stringify(payload)}`);
}
function formatSalary(job: AdzunaJob): string | null {
const min = toNumberOrNull(job.salary_min);
const max = toNumberOrNull(job.salary_max);
if (min === null && max === null) return null;
if (min !== null && max !== null) {
return `${Math.round(min)}-${Math.round(max)}`;
}
if (min !== null) return `${Math.round(min)}+`;
if (max !== null) return `${Math.round(max)}`;
return null;
}
function mapJob(raw: AdzunaJob): ExtractedJob | null {
const id = toStringOrNull(raw.id);
const title = toStringOrNull(raw.title) ?? "Unknown Title";
const employer =
toStringOrNull(raw.company?.display_name) ?? "Unknown Employer";
const jobUrl = toStringOrNull(raw.redirect_url);
if (!jobUrl) return null;
const contractType = toStringOrNull(raw.contract_type);
const contractTime = toStringOrNull(raw.contract_time);
const jobType = [contractType, contractTime].filter(Boolean).join(" / ");
return {
source: "adzuna",
sourceJobId: id ?? undefined,
title,
employer,
jobUrl,
applicationLink: jobUrl,
location: toStringOrNull(raw.location?.display_name) ?? undefined,
salary: formatSalary(raw) ?? undefined,
datePosted: toStringOrNull(raw.created) ?? undefined,
jobDescription: toStringOrNull(raw.description) ?? undefined,
jobType: jobType || undefined,
};
}
async function fetchJobsPage(args: {
country: string;
page: number;
appId: string;
appKey: string;
what: string;
resultsPerPage: number;
}): Promise<AdzunaJob[]> {
const url = new URL(`${API_BASE}/jobs/${args.country}/search/${args.page}`);
url.searchParams.set("app_id", args.appId);
url.searchParams.set("app_key", args.appKey);
if (args.what) {
url.searchParams.set("what", args.what);
}
url.searchParams.set("results_per_page", String(args.resultsPerPage));
const response = await fetch(url.toString(), {
headers: { Accept: "application/json" },
});
if (!response.ok) {
throw new Error(`Adzuna request failed with status ${response.status}`);
}
const body = (await response.json()) as { results?: unknown };
if (!Array.isArray(body.results)) return [];
return body.results as AdzunaJob[];
}
async function run(): Promise<void> {
const appId = requireEnv("ADZUNA_APP_ID");
const appKey = requireEnv("ADZUNA_APP_KEY");
const country = (process.env.ADZUNA_COUNTRY || "gb").trim().toLowerCase();
const maxJobsPerTerm = parsePositiveInt(
process.env.ADZUNA_MAX_JOBS_PER_TERM,
50,
);
const configuredResultsPerPage = parsePositiveInt(
process.env.ADZUNA_RESULTS_PER_PAGE,
50,
);
const resultsPerPage = Math.min(50, configuredResultsPerPage);
const searchTerms = parseSearchTerms(process.env.ADZUNA_SEARCH_TERMS);
const outputJson =
process.env.ADZUNA_OUTPUT_JSON ||
join(process.cwd(), "storage/datasets/default/jobs.json");
const jobs: ExtractedJob[] = [];
for (let i = 0; i < searchTerms.length; i += 1) {
const searchTerm = searchTerms[i];
const termIndex = i + 1;
emitProgress({
event: "term_start",
termIndex,
termTotal: searchTerms.length,
searchTerm,
});
let page = 1;
let termCount = 0;
while (termCount < maxJobsPerTerm) {
const remaining = maxJobsPerTerm - termCount;
const take = Math.min(resultsPerPage, remaining);
const pageResults = await fetchJobsPage({
country,
page,
appId,
appKey,
what: searchTerm,
resultsPerPage: take,
});
let mappedOnPage = 0;
for (const raw of pageResults) {
if (termCount >= maxJobsPerTerm) break;
const mapped = mapJob(raw);
if (!mapped) continue;
jobs.push(mapped);
termCount += 1;
mappedOnPage += 1;
}
emitProgress({
event: "page_fetched",
termIndex,
termTotal: searchTerms.length,
searchTerm,
pageNo: page,
resultsOnPage: mappedOnPage,
totalCollected: termCount,
});
if (pageResults.length < take) break;
page += 1;
if (page > 100) break;
}
emitProgress({
event: "term_complete",
termIndex,
termTotal: searchTerms.length,
searchTerm,
jobsFoundTerm: termCount,
});
}
await mkdir(dirname(outputJson), { recursive: true });
await writeFile(outputJson, `${JSON.stringify(jobs, null, 2)}\n`, "utf-8");
console.log(`Adzuna extractor wrote ${jobs.length} jobs`);
}
run().catch((error: unknown) => {
const message = error instanceof Error ? error.message : "Unknown error";
console.error(`Adzuna extractor failed: ${message}`);
process.exitCode = 1;
});

View File

@ -0,0 +1,13 @@
{
"compilerOptions": {
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ES2022",
"outDir": "dist",
"strict": true,
"noUnusedLocals": false,
"lib": ["ES2022", "DOM"],
"types": ["node"]
},
"include": ["./src/**/*"]
}

View File

@ -24,7 +24,7 @@ interface PipelineProgress {
| "failed";
message: string;
detail?: string;
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | null;
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna" | null;
crawlingSourcesCompleted: number;
crawlingSourcesTotal: number;
crawlingTermsProcessed: number;
@ -84,6 +84,7 @@ const sourceLabel: Record<
gradcracker: "Gradcracker",
jobspy: "JobSpy",
ukvisajobs: "UKVisaJobs",
adzuna: "Adzuna",
};
const clamp = (value: number, min: number, max: number) =>

View File

@ -695,6 +695,7 @@ describe("OrchestratorPage", () => {
jobspyResultsWanted: 150,
gradcrackerMaxJobsPerTerm: 150,
ukvisajobsMaxJobs: 150,
adzunaMaxJobsPerTerm: 150,
jobspyCountryIndeed: "united kingdom",
jobspyLocation: "United Kingdom",
});

View File

@ -294,6 +294,7 @@ export const OrchestratorPage: React.FC = () => {
jobspyResultsWanted: limits.jobspyResultsWanted,
gradcrackerMaxJobsPerTerm: limits.gradcrackerMaxJobsPerTerm,
ukvisajobsMaxJobs: limits.ukvisajobsMaxJobs,
adzunaMaxJobsPerTerm: limits.adzunaMaxJobsPerTerm,
jobspyCountryIndeed: values.country,
jobspyLocation,
});

View File

@ -57,6 +57,8 @@ const DEFAULT_FORM_VALUES: UpdateSettingsInput = {
basicAuthPassword: "",
ukvisajobsEmail: "",
ukvisajobsPassword: "",
adzunaAppId: "",
adzunaAppKey: "",
webhookSecret: "",
enableBasicAuth: false,
backupEnabled: null,
@ -96,6 +98,9 @@ const NULL_SETTINGS_PAYLOAD: UpdateSettingsInput = {
basicAuthPassword: null,
ukvisajobsEmail: null,
ukvisajobsPassword: null,
adzunaAppId: null,
adzunaAppKey: null,
adzunaMaxJobsPerTerm: null,
webhookSecret: null,
enableBasicAuth: undefined,
backupEnabled: null,
@ -129,6 +134,8 @@ const mapSettingsToForm = (data: AppSettings): UpdateSettingsInput => ({
basicAuthPassword: "",
ukvisajobsEmail: data.ukvisajobsEmail ?? "",
ukvisajobsPassword: "",
adzunaAppId: data.adzunaAppId ?? "",
adzunaAppKey: "",
webhookSecret: "",
enableBasicAuth: data.basicAuthActive,
backupEnabled: data.overrideBackupEnabled,
@ -235,11 +242,13 @@ const getDerivedSettings = (settings: AppSettings | null) => {
readable: {
rxresumeEmail: settings?.rxresumeEmail ?? "",
ukvisajobsEmail: settings?.ukvisajobsEmail ?? "",
adzunaAppId: settings?.adzunaAppId ?? "",
basicAuthUser: settings?.basicAuthUser ?? "",
},
private: {
rxresumePasswordHint: settings?.rxresumePasswordHint ?? null,
ukvisajobsPasswordHint: settings?.ukvisajobsPasswordHint ?? null,
adzunaAppKeyHint: settings?.adzunaAppKeyHint ?? null,
basicAuthPasswordHint: settings?.basicAuthPasswordHint ?? null,
webhookSecretHint: settings?.webhookSecretHint ?? null,
},
@ -527,6 +536,10 @@ export const SettingsPage: React.FC = () => {
envPayload.ukvisajobsEmail = normalizeString(data.ukvisajobsEmail);
}
if (dirtyFields.adzunaAppId || dirtyFields.adzunaAppKey) {
envPayload.adzunaAppId = normalizeString(data.adzunaAppId);
}
if (data.enableBasicAuth === false) {
envPayload.basicAuthUser = null;
envPayload.basicAuthPassword = null;
@ -568,6 +581,11 @@ export const SettingsPage: React.FC = () => {
if (value !== undefined) envPayload.ukvisajobsPassword = value;
}
if (dirtyFields.adzunaAppKey) {
const value = normalizePrivateInput(data.adzunaAppKey);
if (value !== undefined) envPayload.adzunaAppKey = value;
}
if (dirtyFields.webhookSecret) {
const value = normalizePrivateInput(data.webhookSecret);
if (value !== undefined) envPayload.webhookSecret = value;

View File

@ -153,6 +153,7 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
const rememberedRunBudget =
settings?.jobspyResultsWanted ??
settings?.adzunaMaxJobsPerTerm ??
settings?.gradcrackerMaxJobsPerTerm ??
settings?.ukvisajobsMaxJobs ??
DEFAULT_VALUES.runBudget;
@ -533,7 +534,9 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
? countryAllowed
? GLASSDOOR_LOCATION_REASON
: GLASSDOOR_COUNTRY_REASON
: `${sourceLabel[source]} is available only when country is United Kingdom.`;
: source === "gradcracker" || source === "ukvisajobs"
? `${sourceLabel[source]} is available only when country is United Kingdom.`
: `${sourceLabel[source]} is not available for the selected country.`;
const button = (
<Button

View File

@ -76,4 +76,20 @@ describe("automatic-run utilities", () => {
"api",
]);
});
it("includes adzuna in estimate caps", () => {
const estimate = calculateAutomaticEstimate({
values: {
topN: 10,
minSuitabilityScore: 50,
searchTerms: ["backend", "platform"],
runBudget: 120,
country: "united kingdom",
},
sources: ["adzuna"],
});
expect(estimate.discovered.cap).toBeGreaterThan(0);
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
});
});

View File

@ -61,6 +61,7 @@ export interface ExtractorLimits {
jobspyResultsWanted: number;
gradcrackerMaxJobsPerTerm: number;
ukvisajobsMaxJobs: number;
adzunaMaxJobsPerTerm: number;
}
export function deriveExtractorLimits(args: {
@ -75,19 +76,22 @@ export function deriveExtractorLimits(args: {
const includesGlassdoor = args.sources.includes("glassdoor");
const includesGradcracker = args.sources.includes("gradcracker");
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
const includesAdzuna = args.sources.includes("adzuna");
const weightedContributors =
(includesIndeed ? termCount : 0) +
(includesLinkedIn ? termCount : 0) +
(includesGlassdoor ? termCount : 0) +
(includesGradcracker ? termCount : 0) +
(includesUkVisaJobs ? 1 : 0);
(includesUkVisaJobs ? 1 : 0) +
(includesAdzuna ? termCount : 0);
if (weightedContributors <= 0) {
return {
jobspyResultsWanted: budget,
gradcrackerMaxJobsPerTerm: budget,
ukvisajobsMaxJobs: budget,
adzunaMaxJobsPerTerm: budget,
};
}
@ -98,6 +102,7 @@ export function deriveExtractorLimits(args: {
jobspyResultsWanted: perUnit,
gradcrackerMaxJobsPerTerm: perUnit,
ukvisajobsMaxJobs: Math.min(budget, perUnit + remainder),
adzunaMaxJobsPerTerm: perUnit,
};
}
@ -137,6 +142,7 @@ export function calculateAutomaticEstimate(args: {
const hasIndeed = sources.includes("indeed");
const hasLinkedIn = sources.includes("linkedin");
const hasGlassdoor = sources.includes("glassdoor");
const hasAdzuna = sources.includes("adzuna");
const limits = deriveExtractorLimits({
budget: values.runBudget,
searchTerms: values.searchTerms,
@ -151,8 +157,9 @@ export function calculateAutomaticEstimate(args: {
? limits.gradcrackerMaxJobsPerTerm * termCount
: 0;
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
const adzunaCap = hasAdzuna ? limits.adzunaMaxJobsPerTerm * termCount : 0;
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap;
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap;
const discoveredMin = Math.round(discoveredCap * 0.35);
const discoveredMax = Math.round(discoveredCap * 0.75);
const processedMin = Math.min(values.topN, discoveredMin);

View File

@ -13,6 +13,7 @@ export const orderedSources: JobSource[] = [
"indeed",
"linkedin",
"glassdoor",
"adzuna",
"ukvisajobs",
];
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];

View File

@ -0,0 +1,19 @@
import { createAppSettings } from "@shared/testing/factories.js";
import { describe, expect, it } from "vitest";
import { getEnabledSources } from "./utils";
describe("orchestrator utils", () => {
it("enables adzuna only when both app id and key are configured", () => {
const withCreds = createAppSettings({
adzunaAppId: "app-id",
adzunaAppKeyHint: "key-",
});
const withoutKey = createAppSettings({
adzunaAppId: "app-id",
adzunaAppKeyHint: null,
});
expect(getEnabledSources(withCreds)).toContain("adzuna");
expect(getEnabledSources(withoutKey)).not.toContain("adzuna");
});
});

View File

@ -171,6 +171,9 @@ export const getEnabledSources = (
const hasUkVisaJobsAuth = Boolean(
settings.ukvisajobsEmail?.trim() && settings.ukvisajobsPasswordHint,
);
const hasAdzunaAuth = Boolean(
settings.adzunaAppId?.trim() && settings.adzunaAppKeyHint,
);
for (const source of orderedSources) {
if (source === "gradcracker") {
@ -181,6 +184,10 @@ export const getEnabledSources = (
if (hasUkVisaJobsAuth) enabled.push(source);
continue;
}
if (source === "adzuna") {
if (hasAdzunaAuth) enabled.push(source);
continue;
}
if (
source === "indeed" ||
source === "linkedin" ||

View File

@ -12,6 +12,8 @@ const EnvironmentSettingsHarness = () => {
basicAuthUser: "admin",
rxresumePassword: "",
ukvisajobsPassword: "",
adzunaAppId: "adzuna-id",
adzunaAppKey: "",
basicAuthPassword: "",
webhookSecret: "",
enableBasicAuth: true,
@ -26,11 +28,13 @@ const EnvironmentSettingsHarness = () => {
readable: {
rxresumeEmail: "resume@example.com",
ukvisajobsEmail: "visa@example.com",
adzunaAppId: "adzuna-id",
basicAuthUser: "admin",
},
private: {
rxresumePasswordHint: null,
ukvisajobsPasswordHint: "pass",
adzunaAppKeyHint: "adzu",
basicAuthPasswordHint: "abcd",
webhookSecretHint: "sec-",
},
@ -50,8 +54,10 @@ describe("EnvironmentSettingsSection", () => {
expect(screen.getByDisplayValue("resume@example.com")).toBeInTheDocument();
expect(screen.getByDisplayValue("visa@example.com")).toBeInTheDocument();
expect(screen.getByDisplayValue("adzuna-id")).toBeInTheDocument();
expect(screen.getByText(/pass\*{8}/)).toBeInTheDocument();
expect(screen.getByText(/adzu\*{8}/)).toBeInTheDocument();
expect(screen.getByText(/abcd\*{8}/)).toBeInTheDocument();
expect(screen.getByText("Not set")).toBeInTheDocument();

View File

@ -91,6 +91,28 @@ export const EnvironmentSettingsSection: React.FC<
/>
</div>
</div>
<div className="space-y-4">
<div className="text-sm font-semibold">Adzuna</div>
<div className="grid gap-4 md:grid-cols-2">
<SettingsInput
label="App ID"
inputProps={register("adzunaAppId")}
placeholder="your-app-id"
disabled={isLoading || isSaving}
error={errors.adzunaAppId?.message as string | undefined}
/>
<SettingsInput
label="App Key"
inputProps={register("adzunaAppKey")}
type="password"
placeholder="Enter new app key"
disabled={isLoading || isSaving}
error={errors.adzunaAppKey?.message as string | undefined}
current={formatSecretHint(privateValues.adzunaAppKeyHint)}
/>
</div>
</div>
</div>
<Separator />

View File

@ -25,11 +25,13 @@ export type EnvSettingsValues = {
readable: {
rxresumeEmail: string;
ukvisajobsEmail: string;
adzunaAppId: string;
basicAuthUser: string;
};
private: {
rxresumePasswordHint: string | null;
ukvisajobsPasswordHint: string | null;
adzunaAppKeyHint: string | null;
basicAuthPasswordHint: string | null;
webhookSecretHint: string | null;
};

View File

@ -143,5 +143,6 @@ export const sourceLabel: Record<Job["source"], string> = {
linkedin: "LinkedIn",
glassdoor: "Glassdoor",
ukvisajobs: "UK Visa Jobs",
adzuna: "Adzuna",
manual: "Manual",
};

View File

@ -55,6 +55,17 @@ describe.sequential("Pipeline API routes", () => {
expect(runPipeline).toHaveBeenNthCalledWith(2, {
sources: ["glassdoor"],
});
const adzunaRunRes = await fetch(`${baseUrl}/api/pipeline/run`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ sources: ["adzuna"] }),
});
const adzunaRunBody = await adzunaRunRes.json();
expect(adzunaRunBody.ok).toBe(true);
expect(runPipeline).toHaveBeenNthCalledWith(3, {
sources: ["adzuna"],
});
});
it("returns conflict when cancelling with no active pipeline", async () => {

View File

@ -99,7 +99,14 @@ const runPipelineSchema = z.object({
minSuitabilityScore: z.number().min(0).max(100).optional(),
sources: z
.array(
z.enum(["gradcracker", "indeed", "linkedin", "glassdoor", "ukvisajobs"]),
z.enum([
"gradcracker",
"indeed",
"linkedin",
"glassdoor",
"ukvisajobs",
"adzuna",
]),
)
.min(1)
.optional(),

View File

@ -252,6 +252,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
glassdoor: "https://www.glassdoor.com",
gradcracker: "https://www.gradcracker.com",
ukvisajobs: "https://www.ukvisajobs.com",
adzuna: "https://www.adzuna.com",
manual: "https://example.com",
};

View File

@ -39,6 +39,7 @@ export const jobs = sqliteTable("jobs", {
"linkedin",
"glassdoor",
"ukvisajobs",
"adzuna",
"manual",
],
})

View File

@ -14,7 +14,7 @@ export type PipelineStep =
| "cancelled"
| "failed";
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs";
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna";
export interface PipelineProgress {
step: PipelineStep;

View File

@ -19,6 +19,10 @@ vi.mock("../../services/crawler", () => ({
runCrawler: vi.fn(),
}));
vi.mock("../../services/adzuna", () => ({
runAdzuna: vi.fn(),
}));
vi.mock("../../services/ukvisajobs", () => ({
runUkVisaJobs: vi.fn(),
}));
@ -157,6 +161,63 @@ describe("discoverJobsStep", () => {
).rejects.toThrow("All sources failed: ukvisajobs: boom");
});
it("runs adzuna when selected and country is compatible", async () => {
const settingsRepo = await import("../../repositories/settings");
const adzuna = await import("../../services/adzuna");
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
searchTerms: JSON.stringify(["engineer"]),
jobspyCountryIndeed: "united states",
} as any);
vi.mocked(adzuna.runAdzuna).mockResolvedValue({
success: true,
jobs: [
{
source: "adzuna",
sourceJobId: "adzu-1",
title: "Engineer",
employer: "ACME",
jobUrl: "https://example.com/job",
applicationLink: "https://example.com/job",
},
],
} as any);
const result = await discoverJobsStep({
mergedConfig: {
...config,
sources: ["adzuna"],
},
});
expect(result.discoveredJobs).toHaveLength(1);
expect(vi.mocked(adzuna.runAdzuna)).toHaveBeenCalledWith(
expect.objectContaining({ country: "us" }),
);
});
it("skips adzuna for unsupported countries", async () => {
const settingsRepo = await import("../../repositories/settings");
const adzuna = await import("../../services/adzuna");
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
searchTerms: JSON.stringify(["engineer"]),
jobspyCountryIndeed: "japan",
} as any);
await expect(
discoverJobsStep({
mergedConfig: {
...config,
sources: ["adzuna"],
},
}),
).rejects.toThrow("No compatible sources for selected country: Japan");
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
});
it("maps Gradcracker progress callback into live crawling counters", async () => {
const settingsRepo = await import("../../repositories/settings");
const crawler = await import("../../services/crawler");
@ -340,6 +401,7 @@ describe("discoverJobsStep", () => {
it("does not throw when no sources are requested", async () => {
const settingsRepo = await import("../../repositories/settings");
const adzuna = await import("../../services/adzuna");
const jobSpy = await import("../../services/jobspy");
const crawler = await import("../../services/crawler");
const ukVisa = await import("../../services/ukvisajobs");
@ -359,6 +421,7 @@ describe("discoverJobsStep", () => {
expect(result.discoveredJobs).toEqual([]);
expect(result.sourceErrors).toEqual([]);
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
});

View File

@ -1,12 +1,14 @@
import { logger } from "@infra/logger";
import {
formatCountryLabel,
getAdzunaCountryCode,
isSourceAllowedForCountry,
normalizeCountryKey,
} from "@shared/location-support.js";
import type { CreateJobInput, PipelineConfig } from "@shared/types";
import * as jobsRepo from "../../repositories/jobs";
import * as settingsRepo from "../../repositories/settings";
import { runAdzuna } from "../../services/adzuna";
import { runCrawler } from "../../services/crawler";
import { runJobSpy } from "../../services/jobspy";
import { runUkVisaJobs } from "../../services/ukvisajobs";
@ -72,11 +74,13 @@ export async function discoverJobsStep(args: {
);
const shouldRunJobSpy = jobSpySites.length > 0;
const shouldRunAdzuna = compatibleSources.includes("adzuna");
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
const totalSources =
Number(shouldRunJobSpy) +
Number(shouldRunAdzuna) +
Number(shouldRunGradcracker) +
Number(shouldRunUkVisaJobs);
let completedSources = 0;
@ -149,6 +153,89 @@ export async function discoverJobsStep(args: {
return { discoveredJobs, sourceErrors };
}
if (shouldRunAdzuna) {
progressHelpers.startSource("adzuna", completedSources, totalSources, {
termsTotal: searchTerms.length,
detail: "Adzuna: fetching jobs...",
});
const adzunaCountryCode = getAdzunaCountryCode(selectedCountry);
if (!adzunaCountryCode) {
sourceErrors.push(
`adzuna: unsupported country ${formatCountryLabel(selectedCountry)}`,
);
markSourceComplete();
} else {
const adzunaMaxJobsPerTerm = settings.adzunaMaxJobsPerTerm
? parseInt(settings.adzunaMaxJobsPerTerm, 10)
: 50;
const adzunaResult = await runAdzuna({
country: adzunaCountryCode,
searchTerms,
maxJobsPerTerm: adzunaMaxJobsPerTerm,
onProgress: (event) => {
if (event.type === "term_start") {
progressHelpers.crawlingUpdate({
source: "adzuna",
termsProcessed: Math.max(event.termIndex - 1, 0),
termsTotal: event.termTotal,
phase: "list",
currentUrl: event.searchTerm,
});
updateProgress({
step: "crawling",
detail: `Adzuna: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
});
return;
}
if (event.type === "page_fetched") {
progressHelpers.crawlingUpdate({
source: "adzuna",
termsProcessed: Math.max(event.termIndex - 1, 0),
termsTotal: event.termTotal,
listPagesProcessed: event.pageNo,
jobPagesEnqueued: event.totalCollected,
jobPagesProcessed: event.totalCollected,
phase: "list",
currentUrl: `page ${event.pageNo}`,
});
updateProgress({
step: "crawling",
detail: `Adzuna: term ${event.termIndex}/${event.termTotal}, page ${event.pageNo} (${event.totalCollected} collected)`,
});
return;
}
progressHelpers.crawlingUpdate({
source: "adzuna",
termsProcessed: event.termIndex,
termsTotal: event.termTotal,
phase: "list",
currentUrl: event.searchTerm,
});
updateProgress({
step: "crawling",
detail: `Adzuna: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
});
},
});
if (!adzunaResult.success) {
sourceErrors.push(`adzuna: ${adzunaResult.error ?? "unknown error"}`);
} else {
discoveredJobs.push(...adzunaResult.jobs);
}
markSourceComplete();
}
}
if (args.shouldCancel?.()) {
return { discoveredJobs, sourceErrors };
}
if (shouldRunGradcracker) {
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
detail: "Gradcracker: scraping...",

View File

@ -20,6 +20,7 @@ export type SettingKey =
| "resumeProjects"
| "rxresumeBaseResumeId"
| "ukvisajobsMaxJobs"
| "adzunaMaxJobsPerTerm"
| "gradcrackerMaxJobsPerTerm"
| "searchTerms"
| "jobspyLocation"
@ -36,6 +37,8 @@ export type SettingKey =
| "basicAuthPassword"
| "ukvisajobsEmail"
| "ukvisajobsPassword"
| "adzunaAppId"
| "adzunaAppKey"
| "webhookSecret"
| "backupEnabled"
| "backupHour"

View File

@ -0,0 +1,254 @@
import { spawn, spawnSync } from "node:child_process";
import { readFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { dirname, join } from "node:path";
import { createInterface } from "node:readline";
import { fileURLToPath } from "node:url";
import { logger } from "@infra/logger";
import type { CreateJobInput } from "@shared/types";
import { toNumberOrNull, toStringOrNull } from "@shared/utils/type-conversion";
const __dirname = dirname(fileURLToPath(import.meta.url));
const ADZUNA_DIR = join(__dirname, "../../../../extractors/adzuna");
const DATASET_PATH = join(ADZUNA_DIR, "storage/datasets/default/jobs.json");
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
const require = createRequire(import.meta.url);
const TSX_CLI_PATH = resolveTsxCliPath();
type AdzunaRawJob = Record<string, unknown>;
export type AdzunaProgressEvent =
| {
type: "term_start";
termIndex: number;
termTotal: number;
searchTerm: string;
}
| {
type: "page_fetched";
termIndex: number;
termTotal: number;
searchTerm: string;
pageNo: number;
resultsOnPage: number;
totalCollected: number;
}
| {
type: "term_complete";
termIndex: number;
termTotal: number;
searchTerm: string;
jobsFoundTerm: number;
};
export interface RunAdzunaOptions {
searchTerms?: string[];
country?: string;
maxJobsPerTerm?: number;
onProgress?: (event: AdzunaProgressEvent) => void;
}
export interface AdzunaResult {
success: boolean;
jobs: CreateJobInput[];
error?: string;
}
function resolveTsxCliPath(): string | null {
try {
return require.resolve("tsx/dist/cli.mjs");
} catch {
return null;
}
}
function canRunNpmCommand(): boolean {
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
return !result.error && result.status === 0;
}
function parseAdzunaProgressLine(line: string): AdzunaProgressEvent | null {
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(raw) as Record<string, unknown>;
} catch {
return null;
}
const event = toStringOrNull(parsed.event);
const termIndex = toNumberOrNull(parsed.termIndex);
const termTotal = toNumberOrNull(parsed.termTotal);
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
if (!event || termIndex === null || termTotal === null) return null;
if (event === "term_start") {
return { type: "term_start", termIndex, termTotal, searchTerm };
}
if (event === "page_fetched") {
const pageNo = toNumberOrNull(parsed.pageNo);
if (pageNo === null) return null;
return {
type: "page_fetched",
termIndex,
termTotal,
searchTerm,
pageNo,
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
};
}
if (event === "term_complete") {
return {
type: "term_complete",
termIndex,
termTotal,
searchTerm,
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
};
}
return null;
}
function mapAdzunaRow(row: AdzunaRawJob): CreateJobInput | null {
const jobUrl = toStringOrNull(row.jobUrl);
if (!jobUrl) return null;
return {
source: "adzuna",
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
title: toStringOrNull(row.title) ?? "Unknown Title",
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
jobUrl,
applicationLink:
toStringOrNull(row.applicationLink) ??
toStringOrNull(row.jobUrl) ??
undefined,
location: toStringOrNull(row.location) ?? undefined,
salary: toStringOrNull(row.salary) ?? undefined,
datePosted: toStringOrNull(row.datePosted) ?? undefined,
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
jobType: toStringOrNull(row.jobType) ?? undefined,
};
}
async function readDataset(): Promise<CreateJobInput[]> {
const content = await readFile(DATASET_PATH, "utf-8");
const parsed = JSON.parse(content) as unknown;
if (!Array.isArray(parsed)) return [];
const jobs: CreateJobInput[] = [];
const seen = new Set<string>();
for (const value of parsed) {
if (!value || typeof value !== "object") continue;
const mapped = mapAdzunaRow(value as AdzunaRawJob);
if (!mapped) continue;
const key = mapped.sourceJobId || mapped.jobUrl;
if (seen.has(key)) continue;
seen.add(key);
jobs.push(mapped);
}
return jobs;
}
export async function runAdzuna(
options: RunAdzunaOptions = {},
): Promise<AdzunaResult> {
const appId = process.env.ADZUNA_APP_ID?.trim();
const appKey = process.env.ADZUNA_APP_KEY?.trim();
if (!appId || !appKey) {
return {
success: false,
jobs: [],
error: "Missing Adzuna credentials (ADZUNA_APP_ID / ADZUNA_APP_KEY)",
};
}
const country = (options.country || "gb").trim().toLowerCase();
const maxJobsPerTerm = options.maxJobsPerTerm ?? 50;
const searchTerms =
options.searchTerms && options.searchTerms.length > 0
? options.searchTerms
: ["web developer"];
const useNpmCommand = canRunNpmCommand();
if (!useNpmCommand && !TSX_CLI_PATH) {
return {
success: false,
jobs: [],
error: "Unable to execute Adzuna extractor (npm/tsx unavailable)",
};
}
try {
await new Promise<void>((resolve, reject) => {
const extractorEnv = {
...process.env,
JOBOPS_EMIT_PROGRESS: "1",
ADZUNA_APP_ID: appId,
ADZUNA_APP_KEY: appKey,
ADZUNA_COUNTRY: country,
ADZUNA_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
ADZUNA_SEARCH_TERMS: JSON.stringify(searchTerms),
ADZUNA_OUTPUT_JSON: DATASET_PATH,
};
const child = useNpmCommand
? spawn("npm", ["run", "start"], {
cwd: ADZUNA_DIR,
stdio: ["ignore", "pipe", "pipe"],
env: extractorEnv,
})
: (() => {
const tsxCliPath = TSX_CLI_PATH;
if (!tsxCliPath) {
throw new Error(
"Unable to execute Adzuna extractor (npm/tsx unavailable)",
);
}
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
cwd: ADZUNA_DIR,
stdio: ["ignore", "pipe", "pipe"],
env: extractorEnv,
});
})();
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
const progressEvent = parseAdzunaProgressLine(line);
if (progressEvent) {
options.onProgress?.(progressEvent);
return;
}
stream.write(`${line}\n`);
};
const stdoutRl = child.stdout
? createInterface({ input: child.stdout })
: null;
const stderrRl = child.stderr
? createInterface({ input: child.stderr })
: null;
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
child.on("close", (code) => {
stdoutRl?.close();
stderrRl?.close();
if (code === 0) resolve();
else reject(new Error(`Adzuna extractor exited with code ${code}`));
});
child.on("error", reject);
});
const jobs = await readDataset();
return { success: true, jobs };
} catch (error) {
const message = error instanceof Error ? error.message : "Unknown error";
logger.warn("Adzuna extractor run failed", { error: message });
return { success: false, jobs: [], error: message };
}
}

View File

@ -8,6 +8,7 @@ const readableStringConfig: { settingKey: SettingKey; envKey: string }[] = [
{ settingKey: "llmBaseUrl", envKey: "LLM_BASE_URL" },
{ settingKey: "rxresumeEmail", envKey: "RXRESUME_EMAIL" },
{ settingKey: "ukvisajobsEmail", envKey: "UKVISAJOBS_EMAIL" },
{ settingKey: "adzunaAppId", envKey: "ADZUNA_APP_ID" },
{ settingKey: "basicAuthUser", envKey: "BASIC_AUTH_USER" },
];
@ -37,6 +38,11 @@ const privateStringConfig: {
envKey: "UKVISAJOBS_PASSWORD",
hintKey: "ukvisajobsPasswordHint",
},
{
settingKey: "adzunaAppKey",
envKey: "ADZUNA_APP_KEY",
hintKey: "adzunaAppKeyHint",
},
{
settingKey: "basicAuthPassword",
envKey: "BASIC_AUTH_PASSWORD",

View File

@ -25,6 +25,20 @@ describe("settings-conversion", () => {
expect(resolved.defaultValue).toBe(50);
});
it("round-trips adzuna numeric settings", () => {
process.env.ADZUNA_MAX_JOBS_PER_TERM = "";
const serialized = serializeSettingValue("adzunaMaxJobsPerTerm", 75);
expect(serialized).toBe("75");
const resolved = resolveSettingValue(
"adzunaMaxJobsPerTerm",
serialized ?? undefined,
);
expect(resolved.overrideValue).toBe(75);
expect(resolved.value).toBe(75);
expect(resolved.defaultValue).toBe(50);
});
it("round-trips boolean bit settings", () => {
expect(serializeSettingValue("showSponsorInfo", true)).toBe("1");
expect(serializeSettingValue("showSponsorInfo", false)).toBe("0");

View File

@ -7,6 +7,7 @@ type SettingMetadata<T, Input = T | null | undefined> = {
type SettingsConversionValueMap = {
ukvisajobsMaxJobs: number;
adzunaMaxJobsPerTerm: number;
gradcrackerMaxJobsPerTerm: number;
searchTerms: string[];
jobspyLocation: string;
@ -100,6 +101,13 @@ export const settingsConversionMetadata: SettingsConversionMetadata = {
serialize: serializeNullableNumber,
resolve: resolveWithNullishFallback,
},
adzunaMaxJobsPerTerm: {
defaultValue: () =>
parseInt(process.env.ADZUNA_MAX_JOBS_PER_TERM || "50", 10),
parseOverride: parseIntOrNull,
serialize: serializeNullableNumber,
resolve: resolveWithNullishFallback,
},
gradcrackerMaxJobsPerTerm: {
defaultValue: () => 50,
parseOverride: parseIntOrNull,

View File

@ -35,23 +35,37 @@ describe("applySettingsUpdates", () => {
const plan = await applySettingsUpdates({
model: "gpt-4o-mini",
ukvisajobsMaxJobs: 42,
adzunaMaxJobsPerTerm: 25,
searchTerms: ["backend", "platform"],
llmProvider: "openai",
adzunaAppId: "app-id",
adzunaAppKey: "app-key",
});
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(4);
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(7);
expect(vi.mocked(settingsRepo.setSetting).mock.calls).toEqual(
expect.arrayContaining([
["model", "gpt-4o-mini"],
["ukvisajobsMaxJobs", "42"],
["adzunaMaxJobsPerTerm", "25"],
["searchTerms", '["backend","platform"]'],
["llmProvider", "openai"],
["adzunaAppId", "app-id"],
["adzunaAppKey", "app-key"],
]),
);
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
"LLM_PROVIDER",
"openai",
);
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
"ADZUNA_APP_ID",
"app-id",
);
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
"ADZUNA_APP_KEY",
"app-key",
);
expect(plan.shouldRefreshBackupScheduler).toBe(false);
});

View File

@ -164,6 +164,11 @@ export const settingsUpdateRegistry: Partial<{
actions: [metadataPersistAction("ukvisajobsMaxJobs", value)],
}),
),
adzunaMaxJobsPerTerm: singleAction(({ value }) =>
result({
actions: [metadataPersistAction("adzunaMaxJobsPerTerm", value)],
}),
),
gradcrackerMaxJobsPerTerm: singleAction(({ value }) =>
result({
actions: [metadataPersistAction("gradcrackerMaxJobsPerTerm", value)],
@ -278,6 +283,26 @@ export const settingsUpdateRegistry: Partial<{
],
});
}),
adzunaAppId: singleAction(({ value }) => {
const normalized = toNormalizedStringOrNull(value);
return result({
actions: [
persistAction("adzunaAppId", normalized, () => {
applyEnvValue("ADZUNA_APP_ID", normalized);
}),
],
});
}),
adzunaAppKey: singleAction(({ value }) => {
const normalized = toNormalizedStringOrNull(value);
return result({
actions: [
persistAction("adzunaAppKey", normalized, () => {
applyEnvValue("ADZUNA_APP_KEY", normalized);
}),
],
});
}),
webhookSecret: singleAction(({ value }) => {
const normalized = toNormalizedStringOrNull(value);
return result({

View File

@ -96,6 +96,15 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
const overrideUkvisajobsMaxJobs = ukvisajobsMaxJobsSetting.overrideValue;
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting.value;
const adzunaMaxJobsPerTermSetting = resolveSettingValue(
"adzunaMaxJobsPerTerm",
overrides.adzunaMaxJobsPerTerm,
);
const defaultAdzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.defaultValue;
const overrideAdzunaMaxJobsPerTerm =
adzunaMaxJobsPerTermSetting.overrideValue;
const adzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.value;
const gradcrackerMaxJobsPerTermSetting = resolveSettingValue(
"gradcrackerMaxJobsPerTerm",
overrides.gradcrackerMaxJobsPerTerm,
@ -260,6 +269,9 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
ukvisajobsMaxJobs,
defaultUkvisajobsMaxJobs,
overrideUkvisajobsMaxJobs,
adzunaMaxJobsPerTerm,
defaultAdzunaMaxJobsPerTerm,
overrideAdzunaMaxJobsPerTerm,
gradcrackerMaxJobsPerTerm,
defaultGradcrackerMaxJobsPerTerm,
overrideGradcrackerMaxJobsPerTerm,

26
package-lock.json generated
View File

@ -80,6 +80,28 @@
"node": ">=14.17"
}
},
"extractors/adzuna": {
"name": "adzuna-extractor",
"version": "0.0.1",
"dependencies": {
"job-ops-shared": "^1.0.0",
"tsx": "^4.4.0"
},
"devDependencies": {
"@types/node": "^24.0.0",
"typescript": "~5.9.0"
}
},
"extractors/adzuna/node_modules/@types/node": {
"version": "24.10.13",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz",
"integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"extractors/gradcracker": {
"name": "gradcracker-extractor",
"version": "0.0.1",
@ -8586,6 +8608,10 @@
"node": ">=12.0"
}
},
"node_modules/adzuna-extractor": {
"resolved": "extractors/adzuna",
"link": true
},
"node_modules/agent-base": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",

View File

@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import {
formatCountryLabel,
getAdzunaCountryCode,
getCompatibleSourcesForCountry,
isGlassdoorCountry,
isSourceAllowedForCountry,
@ -52,15 +53,24 @@ describe("location-support", () => {
expect(isSourceAllowedForCountry("linkedin", "worldwide")).toBe(true);
expect(isSourceAllowedForCountry("glassdoor", "united states")).toBe(true);
expect(isSourceAllowedForCountry("glassdoor", "japan")).toBe(false);
expect(isSourceAllowedForCountry("adzuna", "united states")).toBe(true);
expect(isSourceAllowedForCountry("adzuna", "japan")).toBe(false);
});
it("filters incompatible sources while preserving compatible order", () => {
expect(
getCompatibleSourcesForCountry(
["gradcracker", "indeed", "glassdoor", "ukvisajobs", "linkedin"],
[
"gradcracker",
"indeed",
"glassdoor",
"ukvisajobs",
"adzuna",
"linkedin",
],
"united states",
),
).toEqual(["indeed", "glassdoor", "linkedin"]);
).toEqual(["indeed", "glassdoor", "adzuna", "linkedin"]);
});
it("supports glassdoor only in explicitly supported countries", () => {
@ -70,4 +80,10 @@ describe("location-support", () => {
expect(isGlassdoorCountry("japan")).toBe(false);
expect(isGlassdoorCountry("worldwide")).toBe(false);
});
it("maps adzuna country keys to adzuna api country codes", () => {
expect(getAdzunaCountryCode("united states")).toBe("us");
expect(getAdzunaCountryCode("UK")).toBe("gb");
expect(getAdzunaCountryCode("japan")).toBeNull();
});
});

View File

@ -124,6 +124,27 @@ const GLASSDOOR_SUPPORTED_COUNTRIES = new Set(
"vietnam",
].map((country) => normalizeCountryKey(country)),
);
const ADZUNA_COUNTRY_CODE_BY_KEY: Record<string, string> = {
"united kingdom": "gb",
"united states": "us",
austria: "at",
australia: "au",
belgium: "be",
brazil: "br",
canada: "ca",
switzerland: "ch",
germany: "de",
spain: "es",
france: "fr",
india: "in",
italy: "it",
mexico: "mx",
netherlands: "nl",
"new zealand": "nz",
poland: "pl",
singapore: "sg",
"south africa": "za",
};
export function normalizeCountryKey(value: string | null | undefined): string {
const normalized = value?.trim().toLowerCase() ?? "";
@ -155,12 +176,19 @@ export function isGlassdoorCountry(
return GLASSDOOR_SUPPORTED_COUNTRIES.has(normalizeCountryKey(country));
}
export function getAdzunaCountryCode(
country: string | null | undefined,
): string | null {
return ADZUNA_COUNTRY_CODE_BY_KEY[normalizeCountryKey(country)] ?? null;
}
export function isSourceAllowedForCountry(
source: JobSource,
country: string | null | undefined,
): boolean {
if (UK_ONLY_SOURCES.has(source)) return isUkCountry(country);
if (source === "glassdoor") return isGlassdoorCountry(country);
if (source === "adzuna") return getAdzunaCountryCode(country) !== null;
return true;
}

View File

@ -32,6 +32,13 @@ export const updateSettingsSchema = z
resumeProjects: resumeProjectsSchema.nullable().optional(),
rxresumeBaseResumeId: z.string().trim().max(200).nullable().optional(),
ukvisajobsMaxJobs: z.number().int().min(1).max(1000).nullable().optional(),
adzunaMaxJobsPerTerm: z
.number()
.int()
.min(1)
.max(1000)
.nullable()
.optional(),
gradcrackerMaxJobsPerTerm: z
.number()
.int()
@ -64,6 +71,8 @@ export const updateSettingsSchema = z
basicAuthPassword: z.string().trim().max(2000).nullable().optional(),
ukvisajobsEmail: z.string().trim().max(200).nullable().optional(),
ukvisajobsPassword: z.string().trim().max(2000).nullable().optional(),
adzunaAppId: z.string().trim().max(200).nullable().optional(),
adzunaAppKey: z.string().trim().max(2000).nullable().optional(),
webhookSecret: z.string().trim().max(2000).nullable().optional(),
enableBasicAuth: z.boolean().optional(),
backupEnabled: z.boolean().nullable().optional(),

View File

@ -161,6 +161,9 @@ export const createAppSettings = (
ukvisajobsMaxJobs: 50,
defaultUkvisajobsMaxJobs: 50,
overrideUkvisajobsMaxJobs: null,
adzunaMaxJobsPerTerm: 50,
defaultAdzunaMaxJobsPerTerm: 50,
overrideAdzunaMaxJobsPerTerm: null,
gradcrackerMaxJobsPerTerm: 50,
defaultGradcrackerMaxJobsPerTerm: 50,
overrideGradcrackerMaxJobsPerTerm: null,
@ -198,6 +201,8 @@ export const createAppSettings = (
basicAuthPasswordHint: null,
ukvisajobsEmail: null,
ukvisajobsPasswordHint: null,
adzunaAppId: null,
adzunaAppKeyHint: null,
webhookSecretHint: null,
basicAuthActive: false,
backupEnabled: false,

View File

@ -125,6 +125,7 @@ export type JobSource =
| "linkedin"
| "glassdoor"
| "ukvisajobs"
| "adzuna"
| "manual";
export interface Job {
@ -895,6 +896,9 @@ export interface AppSettings {
ukvisajobsMaxJobs: number;
defaultUkvisajobsMaxJobs: number;
overrideUkvisajobsMaxJobs: number | null;
adzunaMaxJobsPerTerm: number;
defaultAdzunaMaxJobsPerTerm: number;
overrideAdzunaMaxJobsPerTerm: number | null;
gradcrackerMaxJobsPerTerm: number;
defaultGradcrackerMaxJobsPerTerm: number;
overrideGradcrackerMaxJobsPerTerm: number | null;
@ -932,6 +936,8 @@ export interface AppSettings {
basicAuthPasswordHint: string | null;
ukvisajobsEmail: string | null;
ukvisajobsPasswordHint: string | null;
adzunaAppId: string | null;
adzunaAppKeyHint: string | null;
webhookSecretHint: string | null;
basicAuthActive: boolean;
// Backup settings