feat: add Adzuna extractor with orchestrator integration (#177)
* feat(settings): add adzuna source fields and country compatibility * feat(discovery): integrate adzuna extractor into pipeline * feat(client): wire adzuna in source selection and run budgeting * docs(extractors): add adzuna guide and configuration notes * chore(workspaces): register adzuna extractor in lockfile * fix(adzuna): run extractor via npm script instead of npx * fix(adzuna): execute extractor via node+tsx without shell * fix(adzuna): prefer npm run start without shell, fallback to tsx * fix(docker): include adzuna extractor workspace in image * chore(adzuna): reuse shared type-conversion utilities * type-check adzuna * formatting * deeedooop * better instructions
This commit is contained in:
parent
4da264eb48
commit
c5c6675f04
@ -39,6 +39,15 @@ UKVISAJOBS_EMAIL=
|
|||||||
UKVISAJOBS_PASSWORD=
|
UKVISAJOBS_PASSWORD=
|
||||||
UKVISAJOBS_HEADLESS=true
|
UKVISAJOBS_HEADLESS=true
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# Adzuna (multi-country API source) - optional
|
||||||
|
# =============================================================================
|
||||||
|
# App credentials from Adzuna developer account.
|
||||||
|
ADZUNA_APP_ID=
|
||||||
|
ADZUNA_APP_KEY=
|
||||||
|
# Optional default per-term cap (can be overridden by UI run budget logic).
|
||||||
|
# ADZUNA_MAX_JOBS_PER_TERM=50
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# JobSpy - Job search configuration
|
# JobSpy - Job search configuration
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|||||||
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@ -49,7 +49,11 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
project: [orchestrator, gradcracker-extractor, ukvisajobs-extractor]
|
project:
|
||||||
|
- orchestrator
|
||||||
|
- adzuna-extractor
|
||||||
|
- gradcracker-extractor
|
||||||
|
- ukvisajobs-extractor
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Setup Node
|
- name: Setup Node
|
||||||
|
|||||||
@ -35,6 +35,7 @@ COPY package*.json ./
|
|||||||
COPY docs-site/package*.json ./docs-site/
|
COPY docs-site/package*.json ./docs-site/
|
||||||
COPY shared/package*.json ./shared/
|
COPY shared/package*.json ./shared/
|
||||||
COPY orchestrator/package*.json ./orchestrator/
|
COPY orchestrator/package*.json ./orchestrator/
|
||||||
|
COPY extractors/adzuna/package*.json ./extractors/adzuna/
|
||||||
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
||||||
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
||||||
|
|
||||||
@ -52,6 +53,7 @@ WORKDIR /app
|
|||||||
COPY shared ./shared
|
COPY shared ./shared
|
||||||
COPY docs-site ./docs-site
|
COPY docs-site ./docs-site
|
||||||
COPY orchestrator ./orchestrator
|
COPY orchestrator ./orchestrator
|
||||||
|
COPY extractors/adzuna ./extractors/adzuna
|
||||||
COPY extractors/gradcracker ./extractors/gradcracker
|
COPY extractors/gradcracker ./extractors/gradcracker
|
||||||
COPY extractors/jobspy ./extractors/jobspy
|
COPY extractors/jobspy ./extractors/jobspy
|
||||||
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
||||||
@ -97,6 +99,7 @@ COPY package*.json ./
|
|||||||
COPY docs-site/package*.json ./docs-site/
|
COPY docs-site/package*.json ./docs-site/
|
||||||
COPY shared/package*.json ./shared/
|
COPY shared/package*.json ./shared/
|
||||||
COPY orchestrator/package*.json ./orchestrator/
|
COPY orchestrator/package*.json ./orchestrator/
|
||||||
|
COPY extractors/adzuna/package*.json ./extractors/adzuna/
|
||||||
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
||||||
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
||||||
|
|
||||||
@ -110,6 +113,7 @@ COPY --from=builder /app/orchestrator/dist ./orchestrator/dist
|
|||||||
COPY --from=builder /app/docs-site/build ./orchestrator/dist/docs
|
COPY --from=builder /app/docs-site/build ./orchestrator/dist/docs
|
||||||
COPY shared ./shared
|
COPY shared ./shared
|
||||||
COPY orchestrator ./orchestrator
|
COPY orchestrator ./orchestrator
|
||||||
|
COPY extractors/adzuna ./extractors/adzuna
|
||||||
COPY extractors/gradcracker ./extractors/gradcracker
|
COPY extractors/gradcracker ./extractors/gradcracker
|
||||||
COPY extractors/jobspy ./extractors/jobspy
|
COPY extractors/jobspy ./extractors/jobspy
|
||||||
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
||||||
|
|||||||
@ -60,7 +60,7 @@ docker compose up -d
|
|||||||
|
|
||||||
## Why JobOps?
|
## Why JobOps?
|
||||||
|
|
||||||
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor** + specialized boards (Gradcracker, UK Visa Jobs).
|
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor, Adzuna** + specialized boards (Gradcracker, UK Visa Jobs).
|
||||||
* **AI Scoring**: Ranks jobs by fit against *your* profile using your preferred LLM (OpenRouter/OpenAI/Gemini).
|
* **AI Scoring**: Ranks jobs by fit against *your* profile using your preferred LLM (OpenRouter/OpenAI/Gemini).
|
||||||
* **Auto-Tailoring**: Generates custom resumes (PDFs) for every application using RxResume v4.
|
* **Auto-Tailoring**: Generates custom resumes (PDFs) for every application using RxResume v4.
|
||||||
* **Email Tracking**: Connect Gmail to auto-detect interviews, offers, and rejections.
|
* **Email Tracking**: Connect Gmail to auto-detect interviews, offers, and rejections.
|
||||||
@ -81,6 +81,7 @@ docker compose up -d
|
|||||||
| **LinkedIn** | Global / General |
|
| **LinkedIn** | Global / General |
|
||||||
| **Indeed** | Global / General |
|
| **Indeed** | Global / General |
|
||||||
| **Glassdoor** | Global / General |
|
| **Glassdoor** | Global / General |
|
||||||
|
| **Adzuna** | Multi-country API source |
|
||||||
| **Gradcracker** | STEM / Grads (UK) |
|
| **Gradcracker** | STEM / Grads (UK) |
|
||||||
| **UK Visa Jobs** | Sponsorship (UK) |
|
| **UK Visa Jobs** | Sponsorship (UK) |
|
||||||
|
|
||||||
|
|||||||
58
docs-site/docs/extractors/adzuna.md
Normal file
58
docs-site/docs/extractors/adzuna.md
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
---
|
||||||
|
id: adzuna
|
||||||
|
title: Adzuna Extractor
|
||||||
|
description: API-based Adzuna extraction with orchestrator ingestion and progress updates.
|
||||||
|
sidebar_position: 6
|
||||||
|
---
|
||||||
|
|
||||||
|
## What it is
|
||||||
|
|
||||||
|
Adzuna is an API-backed extractor implemented in two lean pieces:
|
||||||
|
|
||||||
|
1. `extractors/adzuna/src/main.ts` fetches paginated Adzuna search results and writes `jobs.json`.
|
||||||
|
2. `orchestrator/src/server/services/adzuna.ts` runs the extractor, parses progress lines, and maps rows into `CreateJobInput`.
|
||||||
|
|
||||||
|
It de-duplicates in the existing repository path using `sourceJobId` fallback to `jobUrl`.
|
||||||
|
|
||||||
|
## Why it exists
|
||||||
|
|
||||||
|
Adzuna provides stable API discovery for countries that are not covered by UK-only sources. It adds a lower-maintenance source without introducing new API routes or UI sections.
|
||||||
|
|
||||||
|
## How to use it
|
||||||
|
|
||||||
|
1. Create an Adzuna developer account.
|
||||||
|
2. Open [Adzuna Access Details](https://developer.adzuna.com/admin/access_details).
|
||||||
|
3. Copy your **App ID** and **App Key**.
|
||||||
|
4. In Job Ops, open **Settings** and paste them into `Adzuna App ID` and `Adzuna App Key` under **Environment & Accounts**.
|
||||||
|
5. In **Pipeline Run** (Automatic tab), select a compatible country and enable **Adzuna** in Sources.
|
||||||
|
6. Start the run; Adzuna progress appears in the existing crawl progress stream.
|
||||||
|
|
||||||
|
Default controls:
|
||||||
|
|
||||||
|
- `ADZUNA_APP_ID`
|
||||||
|
- `ADZUNA_APP_KEY`
|
||||||
|
- `ADZUNA_MAX_JOBS_PER_TERM` (default `50`)
|
||||||
|
|
||||||
|
Supported countries in this integration:
|
||||||
|
|
||||||
|
- United Kingdom, United States, Austria, Australia, Belgium, Brazil, Canada, Switzerland, Germany, Spain, France, India, Italy, Mexico, Netherlands, New Zealand, Poland, Singapore, South Africa.
|
||||||
|
|
||||||
|
## Common problems
|
||||||
|
|
||||||
|
### Adzuna is disabled in source selection
|
||||||
|
|
||||||
|
- `Adzuna App ID` and `Adzuna App Key` are missing from Settings (or env).
|
||||||
|
|
||||||
|
### Adzuna is skipped for my selected country
|
||||||
|
|
||||||
|
- The selected country is not in the supported list above.
|
||||||
|
|
||||||
|
### Adzuna fails with authorization errors
|
||||||
|
|
||||||
|
- Verify `ADZUNA_APP_ID` and `ADZUNA_APP_KEY` are valid and active in your Adzuna account.
|
||||||
|
|
||||||
|
## Related pages
|
||||||
|
|
||||||
|
- [Extractors Overview](/docs/next/extractors/overview)
|
||||||
|
- [Pipeline Run](/docs/next/features/pipeline-run)
|
||||||
|
- [Settings](/docs/next/features/settings)
|
||||||
@ -13,12 +13,14 @@ This page helps you choose the right extractor for your run, understand key cons
|
|||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| [Gradcracker](/docs/next/extractors/gradcracker) | UK graduate roles from Gradcracker | Crawling stability depends on page structure and anti-bot behavior; tuned for low concurrency | `GRADCRACKER_SEARCH_TERMS`, `GRADCRACKER_MAX_JOBS_PER_TERM`, `JOBOPS_SKIP_APPLY_FOR_EXISTING` | Scrapes listing metadata, then detail pages and apply URL resolution |
|
| [Gradcracker](/docs/next/extractors/gradcracker) | UK graduate roles from Gradcracker | Crawling stability depends on page structure and anti-bot behavior; tuned for low concurrency | `GRADCRACKER_SEARCH_TERMS`, `GRADCRACKER_MAX_JOBS_PER_TERM`, `JOBOPS_SKIP_APPLY_FOR_EXISTING` | Scrapes listing metadata, then detail pages and apply URL resolution |
|
||||||
| [JobSpy](/docs/next/extractors/jobspy) | Multi-source discovery (Indeed, LinkedIn, Glassdoor) | Requires Python wrapper execution per term; source availability and quality vary by site/location | `JOBSPY_SITES`, `JOBSPY_SEARCH_TERMS`, `JOBSPY_RESULTS_WANTED`, `JOBSPY_HOURS_OLD`, `JOBSPY_LINKEDIN_FETCH_DESCRIPTION` | Produces JSON per term, then orchestrator normalizes and de-duplicates by `jobUrl` |
|
| [JobSpy](/docs/next/extractors/jobspy) | Multi-source discovery (Indeed, LinkedIn, Glassdoor) | Requires Python wrapper execution per term; source availability and quality vary by site/location | `JOBSPY_SITES`, `JOBSPY_SEARCH_TERMS`, `JOBSPY_RESULTS_WANTED`, `JOBSPY_HOURS_OLD`, `JOBSPY_LINKEDIN_FETCH_DESCRIPTION` | Produces JSON per term, then orchestrator normalizes and de-duplicates by `jobUrl` |
|
||||||
|
| [Adzuna](/docs/next/extractors/adzuna) | API-based multi-country discovery with low scraping overhead | Requires valid App ID/App Key; country must be in Adzuna-supported list | `ADZUNA_APP_ID`, `ADZUNA_APP_KEY`, `ADZUNA_MAX_JOBS_PER_TERM` | API pagination to dataset output; orchestrator maps progress and de-duplicates by `sourceJobId`/`jobUrl` |
|
||||||
| [UKVisaJobs](/docs/next/extractors/ukvisajobs) | UK visa sponsorship-focused roles | Requires authenticated session and periodic token/cookie refresh | `UKVISAJOBS_EMAIL`, `UKVISAJOBS_PASSWORD`, `UKVISAJOBS_MAX_JOBS`, `UKVISAJOBS_SEARCH_KEYWORD` | API pagination + dataset output; orchestrator de-dupes and may fetch missing descriptions |
|
| [UKVisaJobs](/docs/next/extractors/ukvisajobs) | UK visa sponsorship-focused roles | Requires authenticated session and periodic token/cookie refresh | `UKVISAJOBS_EMAIL`, `UKVISAJOBS_PASSWORD`, `UKVISAJOBS_MAX_JOBS`, `UKVISAJOBS_SEARCH_KEYWORD` | API pagination + dataset output; orchestrator de-dupes and may fetch missing descriptions |
|
||||||
| [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review |
|
| [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review |
|
||||||
|
|
||||||
## Which extractor should I use?
|
## Which extractor should I use?
|
||||||
|
|
||||||
- Use **JobSpy** for broad first-pass sourcing across common boards.
|
- Use **JobSpy** for broad first-pass sourcing across common boards.
|
||||||
|
- Use **Adzuna** when you want API-first discovery in supported non-UK markets.
|
||||||
- Use **Gradcracker** when targeting graduate pipelines in the UK.
|
- Use **Gradcracker** when targeting graduate pipelines in the UK.
|
||||||
- Use **UKVisaJobs** for sponsorship-specific UK searches.
|
- Use **UKVisaJobs** for sponsorship-specific UK searches.
|
||||||
- Use **Manual Import** when you already have a specific posting and need direct import.
|
- Use **Manual Import** when you already have a specific posting and need direct import.
|
||||||
@ -29,5 +31,6 @@ Many runs combine sources: broad discovery first, then manual import for high-pr
|
|||||||
|
|
||||||
- [Gradcracker](/docs/next/extractors/gradcracker)
|
- [Gradcracker](/docs/next/extractors/gradcracker)
|
||||||
- [JobSpy](/docs/next/extractors/jobspy)
|
- [JobSpy](/docs/next/extractors/jobspy)
|
||||||
|
- [Adzuna](/docs/next/extractors/adzuna)
|
||||||
- [UKVisaJobs](/docs/next/extractors/ukvisajobs)
|
- [UKVisaJobs](/docs/next/extractors/ukvisajobs)
|
||||||
- [Manual Import](/docs/next/extractors/manual)
|
- [Manual Import](/docs/next/extractors/manual)
|
||||||
|
|||||||
@ -47,6 +47,7 @@ If values are edited manually, the UI shows **Custom**.
|
|||||||
|
|
||||||
- Country selection affects which sources are available.
|
- Country selection affects which sources are available.
|
||||||
- UK-only sources are disabled for non-UK countries.
|
- UK-only sources are disabled for non-UK countries.
|
||||||
|
- Adzuna is available only for its supported countries and when App ID/App Key are configured in Settings.
|
||||||
- Glassdoor can be enabled only when:
|
- Glassdoor can be enabled only when:
|
||||||
- selected country supports Glassdoor
|
- selected country supports Glassdoor
|
||||||
- a **Glassdoor city** is set in Advanced settings
|
- a **Glassdoor city** is set in Advanced settings
|
||||||
@ -98,6 +99,11 @@ For accepted input formats, inference behavior, and limits, see [Manual Import E
|
|||||||
- Verify selected country supports Glassdoor.
|
- Verify selected country supports Glassdoor.
|
||||||
- Set a Glassdoor city in Advanced settings.
|
- Set a Glassdoor city in Advanced settings.
|
||||||
|
|
||||||
|
### Adzuna is not selectable
|
||||||
|
|
||||||
|
- Set `Adzuna App ID` and `Adzuna App Key` in **Settings > Environment & Accounts**.
|
||||||
|
- Verify the selected country is one of Adzuna's supported markets.
|
||||||
|
|
||||||
### Run takes longer than expected
|
### Run takes longer than expected
|
||||||
|
|
||||||
- Reduce term count.
|
- Reduce term count.
|
||||||
|
|||||||
@ -86,6 +86,7 @@ Settings gives you runtime overrides for the key parts of discovery, scoring, ta
|
|||||||
- Configure service accounts:
|
- Configure service accounts:
|
||||||
- RxResume email/password
|
- RxResume email/password
|
||||||
- UKVisaJobs email/password
|
- UKVisaJobs email/password
|
||||||
|
- Adzuna app ID/app key
|
||||||
- Optional basic authentication for write operations
|
- Optional basic authentication for write operations
|
||||||
|
|
||||||
### Backup
|
### Backup
|
||||||
|
|||||||
@ -41,6 +41,7 @@ const sidebars: SidebarsConfig = {
|
|||||||
"extractors/overview",
|
"extractors/overview",
|
||||||
"extractors/gradcracker",
|
"extractors/gradcracker",
|
||||||
"extractors/jobspy",
|
"extractors/jobspy",
|
||||||
|
"extractors/adzuna",
|
||||||
"extractors/manual",
|
"extractors/manual",
|
||||||
"extractors/ukvisajobs",
|
"extractors/ukvisajobs",
|
||||||
],
|
],
|
||||||
|
|||||||
15
extractors/adzuna/README.md
Normal file
15
extractors/adzuna/README.md
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
# Adzuna Extractor
|
||||||
|
|
||||||
|
Minimal extractor that pulls jobs from Adzuna's search API and writes a dataset
|
||||||
|
for orchestrator ingestion.
|
||||||
|
|
||||||
|
## Environment
|
||||||
|
|
||||||
|
- `ADZUNA_APP_ID` (required)
|
||||||
|
- `ADZUNA_APP_KEY` (required)
|
||||||
|
- `ADZUNA_COUNTRY` (default: `gb`)
|
||||||
|
- `ADZUNA_SEARCH_TERMS` (JSON array or `|` / comma / newline-delimited)
|
||||||
|
- `ADZUNA_MAX_JOBS_PER_TERM` (default: `50`)
|
||||||
|
- `ADZUNA_RESULTS_PER_PAGE` (default: `50`, max `50`)
|
||||||
|
- `ADZUNA_OUTPUT_JSON` (default: `storage/datasets/default/jobs.json`)
|
||||||
|
- `JOBOPS_EMIT_PROGRESS=1` to emit `JOBOPS_PROGRESS` events
|
||||||
20
extractors/adzuna/package.json
Normal file
20
extractors/adzuna/package.json
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"name": "adzuna-extractor",
|
||||||
|
"version": "0.0.1",
|
||||||
|
"type": "module",
|
||||||
|
"description": "Adzuna extractor - fetches jobs from Adzuna search API",
|
||||||
|
"main": "src/main.ts",
|
||||||
|
"dependencies": {
|
||||||
|
"job-ops-shared": "^1.0.0",
|
||||||
|
"tsx": "^4.4.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^24.0.0",
|
||||||
|
"typescript": "~5.9.0"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"start": "tsx src/main.ts",
|
||||||
|
"start:dev": "tsx src/main.ts",
|
||||||
|
"check:types": "tsc --noEmit"
|
||||||
|
}
|
||||||
|
}
|
||||||
245
extractors/adzuna/src/main.ts
Normal file
245
extractors/adzuna/src/main.ts
Normal file
@ -0,0 +1,245 @@
|
|||||||
|
import { mkdir, writeFile } from "node:fs/promises";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import {
|
||||||
|
toNumberOrNull,
|
||||||
|
toStringOrNull,
|
||||||
|
} from "job-ops-shared/utils/type-conversion";
|
||||||
|
|
||||||
|
const API_BASE = "https://api.adzuna.com/v1/api";
|
||||||
|
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||||
|
|
||||||
|
type AdzunaCompany = { display_name?: unknown };
|
||||||
|
type AdzunaLocation = { display_name?: unknown };
|
||||||
|
type AdzunaJob = {
|
||||||
|
id?: unknown;
|
||||||
|
title?: unknown;
|
||||||
|
description?: unknown;
|
||||||
|
created?: unknown;
|
||||||
|
redirect_url?: unknown;
|
||||||
|
company?: AdzunaCompany;
|
||||||
|
location?: AdzunaLocation;
|
||||||
|
salary_min?: unknown;
|
||||||
|
salary_max?: unknown;
|
||||||
|
contract_time?: unknown;
|
||||||
|
contract_type?: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ExtractedJob = {
|
||||||
|
source: "adzuna";
|
||||||
|
sourceJobId?: string;
|
||||||
|
title: string;
|
||||||
|
employer: string;
|
||||||
|
jobUrl: string;
|
||||||
|
applicationLink: string;
|
||||||
|
location?: string;
|
||||||
|
salary?: string;
|
||||||
|
datePosted?: string;
|
||||||
|
jobDescription?: string;
|
||||||
|
jobType?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function parsePositiveInt(input: string | undefined, fallback: number): number {
|
||||||
|
const parsed = input ? Number.parseInt(input, 10) : Number.NaN;
|
||||||
|
if (!Number.isFinite(parsed) || parsed < 1) return fallback;
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSearchTerms(raw: string | undefined): string[] {
|
||||||
|
if (!raw || raw.trim().length === 0) return ["web developer"];
|
||||||
|
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (trimmed.startsWith("[")) {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(trimmed) as unknown;
|
||||||
|
if (Array.isArray(parsed)) {
|
||||||
|
const terms = parsed
|
||||||
|
.map((value) => toStringOrNull(value))
|
||||||
|
.filter((value): value is string => value !== null);
|
||||||
|
if (terms.length > 0) return terms;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Fall through to delimiter parsing.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const delimiter = trimmed.includes("|")
|
||||||
|
? "|"
|
||||||
|
: trimmed.includes("\n")
|
||||||
|
? "\n"
|
||||||
|
: ",";
|
||||||
|
const terms = trimmed
|
||||||
|
.split(delimiter)
|
||||||
|
.map((value) => value.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
return terms.length > 0 ? terms : ["web developer"];
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireEnv(name: string): string {
|
||||||
|
const value = process.env[name]?.trim();
|
||||||
|
if (!value) throw new Error(`Missing required environment variable: ${name}`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function emitProgress(payload: Record<string, unknown>): void {
|
||||||
|
if (process.env.JOBOPS_EMIT_PROGRESS !== "1") return;
|
||||||
|
console.log(`${JOBOPS_PROGRESS_PREFIX}${JSON.stringify(payload)}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatSalary(job: AdzunaJob): string | null {
|
||||||
|
const min = toNumberOrNull(job.salary_min);
|
||||||
|
const max = toNumberOrNull(job.salary_max);
|
||||||
|
|
||||||
|
if (min === null && max === null) return null;
|
||||||
|
|
||||||
|
if (min !== null && max !== null) {
|
||||||
|
return `${Math.round(min)}-${Math.round(max)}`;
|
||||||
|
}
|
||||||
|
if (min !== null) return `${Math.round(min)}+`;
|
||||||
|
if (max !== null) return `${Math.round(max)}`;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapJob(raw: AdzunaJob): ExtractedJob | null {
|
||||||
|
const id = toStringOrNull(raw.id);
|
||||||
|
const title = toStringOrNull(raw.title) ?? "Unknown Title";
|
||||||
|
const employer =
|
||||||
|
toStringOrNull(raw.company?.display_name) ?? "Unknown Employer";
|
||||||
|
const jobUrl = toStringOrNull(raw.redirect_url);
|
||||||
|
if (!jobUrl) return null;
|
||||||
|
|
||||||
|
const contractType = toStringOrNull(raw.contract_type);
|
||||||
|
const contractTime = toStringOrNull(raw.contract_time);
|
||||||
|
const jobType = [contractType, contractTime].filter(Boolean).join(" / ");
|
||||||
|
|
||||||
|
return {
|
||||||
|
source: "adzuna",
|
||||||
|
sourceJobId: id ?? undefined,
|
||||||
|
title,
|
||||||
|
employer,
|
||||||
|
jobUrl,
|
||||||
|
applicationLink: jobUrl,
|
||||||
|
location: toStringOrNull(raw.location?.display_name) ?? undefined,
|
||||||
|
salary: formatSalary(raw) ?? undefined,
|
||||||
|
datePosted: toStringOrNull(raw.created) ?? undefined,
|
||||||
|
jobDescription: toStringOrNull(raw.description) ?? undefined,
|
||||||
|
jobType: jobType || undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchJobsPage(args: {
|
||||||
|
country: string;
|
||||||
|
page: number;
|
||||||
|
appId: string;
|
||||||
|
appKey: string;
|
||||||
|
what: string;
|
||||||
|
resultsPerPage: number;
|
||||||
|
}): Promise<AdzunaJob[]> {
|
||||||
|
const url = new URL(`${API_BASE}/jobs/${args.country}/search/${args.page}`);
|
||||||
|
url.searchParams.set("app_id", args.appId);
|
||||||
|
url.searchParams.set("app_key", args.appKey);
|
||||||
|
if (args.what) {
|
||||||
|
url.searchParams.set("what", args.what);
|
||||||
|
}
|
||||||
|
url.searchParams.set("results_per_page", String(args.resultsPerPage));
|
||||||
|
|
||||||
|
const response = await fetch(url.toString(), {
|
||||||
|
headers: { Accept: "application/json" },
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`Adzuna request failed with status ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = (await response.json()) as { results?: unknown };
|
||||||
|
if (!Array.isArray(body.results)) return [];
|
||||||
|
return body.results as AdzunaJob[];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function run(): Promise<void> {
|
||||||
|
const appId = requireEnv("ADZUNA_APP_ID");
|
||||||
|
const appKey = requireEnv("ADZUNA_APP_KEY");
|
||||||
|
const country = (process.env.ADZUNA_COUNTRY || "gb").trim().toLowerCase();
|
||||||
|
const maxJobsPerTerm = parsePositiveInt(
|
||||||
|
process.env.ADZUNA_MAX_JOBS_PER_TERM,
|
||||||
|
50,
|
||||||
|
);
|
||||||
|
const configuredResultsPerPage = parsePositiveInt(
|
||||||
|
process.env.ADZUNA_RESULTS_PER_PAGE,
|
||||||
|
50,
|
||||||
|
);
|
||||||
|
const resultsPerPage = Math.min(50, configuredResultsPerPage);
|
||||||
|
const searchTerms = parseSearchTerms(process.env.ADZUNA_SEARCH_TERMS);
|
||||||
|
const outputJson =
|
||||||
|
process.env.ADZUNA_OUTPUT_JSON ||
|
||||||
|
join(process.cwd(), "storage/datasets/default/jobs.json");
|
||||||
|
|
||||||
|
const jobs: ExtractedJob[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < searchTerms.length; i += 1) {
|
||||||
|
const searchTerm = searchTerms[i];
|
||||||
|
const termIndex = i + 1;
|
||||||
|
|
||||||
|
emitProgress({
|
||||||
|
event: "term_start",
|
||||||
|
termIndex,
|
||||||
|
termTotal: searchTerms.length,
|
||||||
|
searchTerm,
|
||||||
|
});
|
||||||
|
|
||||||
|
let page = 1;
|
||||||
|
let termCount = 0;
|
||||||
|
while (termCount < maxJobsPerTerm) {
|
||||||
|
const remaining = maxJobsPerTerm - termCount;
|
||||||
|
const take = Math.min(resultsPerPage, remaining);
|
||||||
|
const pageResults = await fetchJobsPage({
|
||||||
|
country,
|
||||||
|
page,
|
||||||
|
appId,
|
||||||
|
appKey,
|
||||||
|
what: searchTerm,
|
||||||
|
resultsPerPage: take,
|
||||||
|
});
|
||||||
|
|
||||||
|
let mappedOnPage = 0;
|
||||||
|
for (const raw of pageResults) {
|
||||||
|
if (termCount >= maxJobsPerTerm) break;
|
||||||
|
const mapped = mapJob(raw);
|
||||||
|
if (!mapped) continue;
|
||||||
|
jobs.push(mapped);
|
||||||
|
termCount += 1;
|
||||||
|
mappedOnPage += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
emitProgress({
|
||||||
|
event: "page_fetched",
|
||||||
|
termIndex,
|
||||||
|
termTotal: searchTerms.length,
|
||||||
|
searchTerm,
|
||||||
|
pageNo: page,
|
||||||
|
resultsOnPage: mappedOnPage,
|
||||||
|
totalCollected: termCount,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (pageResults.length < take) break;
|
||||||
|
page += 1;
|
||||||
|
if (page > 100) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
emitProgress({
|
||||||
|
event: "term_complete",
|
||||||
|
termIndex,
|
||||||
|
termTotal: searchTerms.length,
|
||||||
|
searchTerm,
|
||||||
|
jobsFoundTerm: termCount,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await mkdir(dirname(outputJson), { recursive: true });
|
||||||
|
await writeFile(outputJson, `${JSON.stringify(jobs, null, 2)}\n`, "utf-8");
|
||||||
|
console.log(`Adzuna extractor wrote ${jobs.length} jobs`);
|
||||||
|
}
|
||||||
|
|
||||||
|
run().catch((error: unknown) => {
|
||||||
|
const message = error instanceof Error ? error.message : "Unknown error";
|
||||||
|
console.error(`Adzuna extractor failed: ${message}`);
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
||||||
13
extractors/adzuna/tsconfig.json
Normal file
13
extractors/adzuna/tsconfig.json
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"module": "NodeNext",
|
||||||
|
"moduleResolution": "NodeNext",
|
||||||
|
"target": "ES2022",
|
||||||
|
"outDir": "dist",
|
||||||
|
"strict": true,
|
||||||
|
"noUnusedLocals": false,
|
||||||
|
"lib": ["ES2022", "DOM"],
|
||||||
|
"types": ["node"]
|
||||||
|
},
|
||||||
|
"include": ["./src/**/*"]
|
||||||
|
}
|
||||||
@ -24,7 +24,7 @@ interface PipelineProgress {
|
|||||||
| "failed";
|
| "failed";
|
||||||
message: string;
|
message: string;
|
||||||
detail?: string;
|
detail?: string;
|
||||||
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | null;
|
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna" | null;
|
||||||
crawlingSourcesCompleted: number;
|
crawlingSourcesCompleted: number;
|
||||||
crawlingSourcesTotal: number;
|
crawlingSourcesTotal: number;
|
||||||
crawlingTermsProcessed: number;
|
crawlingTermsProcessed: number;
|
||||||
@ -84,6 +84,7 @@ const sourceLabel: Record<
|
|||||||
gradcracker: "Gradcracker",
|
gradcracker: "Gradcracker",
|
||||||
jobspy: "JobSpy",
|
jobspy: "JobSpy",
|
||||||
ukvisajobs: "UKVisaJobs",
|
ukvisajobs: "UKVisaJobs",
|
||||||
|
adzuna: "Adzuna",
|
||||||
};
|
};
|
||||||
|
|
||||||
const clamp = (value: number, min: number, max: number) =>
|
const clamp = (value: number, min: number, max: number) =>
|
||||||
|
|||||||
@ -695,6 +695,7 @@ describe("OrchestratorPage", () => {
|
|||||||
jobspyResultsWanted: 150,
|
jobspyResultsWanted: 150,
|
||||||
gradcrackerMaxJobsPerTerm: 150,
|
gradcrackerMaxJobsPerTerm: 150,
|
||||||
ukvisajobsMaxJobs: 150,
|
ukvisajobsMaxJobs: 150,
|
||||||
|
adzunaMaxJobsPerTerm: 150,
|
||||||
jobspyCountryIndeed: "united kingdom",
|
jobspyCountryIndeed: "united kingdom",
|
||||||
jobspyLocation: "United Kingdom",
|
jobspyLocation: "United Kingdom",
|
||||||
});
|
});
|
||||||
|
|||||||
@ -294,6 +294,7 @@ export const OrchestratorPage: React.FC = () => {
|
|||||||
jobspyResultsWanted: limits.jobspyResultsWanted,
|
jobspyResultsWanted: limits.jobspyResultsWanted,
|
||||||
gradcrackerMaxJobsPerTerm: limits.gradcrackerMaxJobsPerTerm,
|
gradcrackerMaxJobsPerTerm: limits.gradcrackerMaxJobsPerTerm,
|
||||||
ukvisajobsMaxJobs: limits.ukvisajobsMaxJobs,
|
ukvisajobsMaxJobs: limits.ukvisajobsMaxJobs,
|
||||||
|
adzunaMaxJobsPerTerm: limits.adzunaMaxJobsPerTerm,
|
||||||
jobspyCountryIndeed: values.country,
|
jobspyCountryIndeed: values.country,
|
||||||
jobspyLocation,
|
jobspyLocation,
|
||||||
});
|
});
|
||||||
|
|||||||
@ -57,6 +57,8 @@ const DEFAULT_FORM_VALUES: UpdateSettingsInput = {
|
|||||||
basicAuthPassword: "",
|
basicAuthPassword: "",
|
||||||
ukvisajobsEmail: "",
|
ukvisajobsEmail: "",
|
||||||
ukvisajobsPassword: "",
|
ukvisajobsPassword: "",
|
||||||
|
adzunaAppId: "",
|
||||||
|
adzunaAppKey: "",
|
||||||
webhookSecret: "",
|
webhookSecret: "",
|
||||||
enableBasicAuth: false,
|
enableBasicAuth: false,
|
||||||
backupEnabled: null,
|
backupEnabled: null,
|
||||||
@ -96,6 +98,9 @@ const NULL_SETTINGS_PAYLOAD: UpdateSettingsInput = {
|
|||||||
basicAuthPassword: null,
|
basicAuthPassword: null,
|
||||||
ukvisajobsEmail: null,
|
ukvisajobsEmail: null,
|
||||||
ukvisajobsPassword: null,
|
ukvisajobsPassword: null,
|
||||||
|
adzunaAppId: null,
|
||||||
|
adzunaAppKey: null,
|
||||||
|
adzunaMaxJobsPerTerm: null,
|
||||||
webhookSecret: null,
|
webhookSecret: null,
|
||||||
enableBasicAuth: undefined,
|
enableBasicAuth: undefined,
|
||||||
backupEnabled: null,
|
backupEnabled: null,
|
||||||
@ -129,6 +134,8 @@ const mapSettingsToForm = (data: AppSettings): UpdateSettingsInput => ({
|
|||||||
basicAuthPassword: "",
|
basicAuthPassword: "",
|
||||||
ukvisajobsEmail: data.ukvisajobsEmail ?? "",
|
ukvisajobsEmail: data.ukvisajobsEmail ?? "",
|
||||||
ukvisajobsPassword: "",
|
ukvisajobsPassword: "",
|
||||||
|
adzunaAppId: data.adzunaAppId ?? "",
|
||||||
|
adzunaAppKey: "",
|
||||||
webhookSecret: "",
|
webhookSecret: "",
|
||||||
enableBasicAuth: data.basicAuthActive,
|
enableBasicAuth: data.basicAuthActive,
|
||||||
backupEnabled: data.overrideBackupEnabled,
|
backupEnabled: data.overrideBackupEnabled,
|
||||||
@ -235,11 +242,13 @@ const getDerivedSettings = (settings: AppSettings | null) => {
|
|||||||
readable: {
|
readable: {
|
||||||
rxresumeEmail: settings?.rxresumeEmail ?? "",
|
rxresumeEmail: settings?.rxresumeEmail ?? "",
|
||||||
ukvisajobsEmail: settings?.ukvisajobsEmail ?? "",
|
ukvisajobsEmail: settings?.ukvisajobsEmail ?? "",
|
||||||
|
adzunaAppId: settings?.adzunaAppId ?? "",
|
||||||
basicAuthUser: settings?.basicAuthUser ?? "",
|
basicAuthUser: settings?.basicAuthUser ?? "",
|
||||||
},
|
},
|
||||||
private: {
|
private: {
|
||||||
rxresumePasswordHint: settings?.rxresumePasswordHint ?? null,
|
rxresumePasswordHint: settings?.rxresumePasswordHint ?? null,
|
||||||
ukvisajobsPasswordHint: settings?.ukvisajobsPasswordHint ?? null,
|
ukvisajobsPasswordHint: settings?.ukvisajobsPasswordHint ?? null,
|
||||||
|
adzunaAppKeyHint: settings?.adzunaAppKeyHint ?? null,
|
||||||
basicAuthPasswordHint: settings?.basicAuthPasswordHint ?? null,
|
basicAuthPasswordHint: settings?.basicAuthPasswordHint ?? null,
|
||||||
webhookSecretHint: settings?.webhookSecretHint ?? null,
|
webhookSecretHint: settings?.webhookSecretHint ?? null,
|
||||||
},
|
},
|
||||||
@ -527,6 +536,10 @@ export const SettingsPage: React.FC = () => {
|
|||||||
envPayload.ukvisajobsEmail = normalizeString(data.ukvisajobsEmail);
|
envPayload.ukvisajobsEmail = normalizeString(data.ukvisajobsEmail);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (dirtyFields.adzunaAppId || dirtyFields.adzunaAppKey) {
|
||||||
|
envPayload.adzunaAppId = normalizeString(data.adzunaAppId);
|
||||||
|
}
|
||||||
|
|
||||||
if (data.enableBasicAuth === false) {
|
if (data.enableBasicAuth === false) {
|
||||||
envPayload.basicAuthUser = null;
|
envPayload.basicAuthUser = null;
|
||||||
envPayload.basicAuthPassword = null;
|
envPayload.basicAuthPassword = null;
|
||||||
@ -568,6 +581,11 @@ export const SettingsPage: React.FC = () => {
|
|||||||
if (value !== undefined) envPayload.ukvisajobsPassword = value;
|
if (value !== undefined) envPayload.ukvisajobsPassword = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (dirtyFields.adzunaAppKey) {
|
||||||
|
const value = normalizePrivateInput(data.adzunaAppKey);
|
||||||
|
if (value !== undefined) envPayload.adzunaAppKey = value;
|
||||||
|
}
|
||||||
|
|
||||||
if (dirtyFields.webhookSecret) {
|
if (dirtyFields.webhookSecret) {
|
||||||
const value = normalizePrivateInput(data.webhookSecret);
|
const value = normalizePrivateInput(data.webhookSecret);
|
||||||
if (value !== undefined) envPayload.webhookSecret = value;
|
if (value !== undefined) envPayload.webhookSecret = value;
|
||||||
|
|||||||
@ -153,6 +153,7 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
|
|||||||
|
|
||||||
const rememberedRunBudget =
|
const rememberedRunBudget =
|
||||||
settings?.jobspyResultsWanted ??
|
settings?.jobspyResultsWanted ??
|
||||||
|
settings?.adzunaMaxJobsPerTerm ??
|
||||||
settings?.gradcrackerMaxJobsPerTerm ??
|
settings?.gradcrackerMaxJobsPerTerm ??
|
||||||
settings?.ukvisajobsMaxJobs ??
|
settings?.ukvisajobsMaxJobs ??
|
||||||
DEFAULT_VALUES.runBudget;
|
DEFAULT_VALUES.runBudget;
|
||||||
@ -533,7 +534,9 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
|
|||||||
? countryAllowed
|
? countryAllowed
|
||||||
? GLASSDOOR_LOCATION_REASON
|
? GLASSDOOR_LOCATION_REASON
|
||||||
: GLASSDOOR_COUNTRY_REASON
|
: GLASSDOOR_COUNTRY_REASON
|
||||||
: `${sourceLabel[source]} is available only when country is United Kingdom.`;
|
: source === "gradcracker" || source === "ukvisajobs"
|
||||||
|
? `${sourceLabel[source]} is available only when country is United Kingdom.`
|
||||||
|
: `${sourceLabel[source]} is not available for the selected country.`;
|
||||||
|
|
||||||
const button = (
|
const button = (
|
||||||
<Button
|
<Button
|
||||||
|
|||||||
@ -76,4 +76,20 @@ describe("automatic-run utilities", () => {
|
|||||||
"api",
|
"api",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("includes adzuna in estimate caps", () => {
|
||||||
|
const estimate = calculateAutomaticEstimate({
|
||||||
|
values: {
|
||||||
|
topN: 10,
|
||||||
|
minSuitabilityScore: 50,
|
||||||
|
searchTerms: ["backend", "platform"],
|
||||||
|
runBudget: 120,
|
||||||
|
country: "united kingdom",
|
||||||
|
},
|
||||||
|
sources: ["adzuna"],
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(estimate.discovered.cap).toBeGreaterThan(0);
|
||||||
|
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -61,6 +61,7 @@ export interface ExtractorLimits {
|
|||||||
jobspyResultsWanted: number;
|
jobspyResultsWanted: number;
|
||||||
gradcrackerMaxJobsPerTerm: number;
|
gradcrackerMaxJobsPerTerm: number;
|
||||||
ukvisajobsMaxJobs: number;
|
ukvisajobsMaxJobs: number;
|
||||||
|
adzunaMaxJobsPerTerm: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function deriveExtractorLimits(args: {
|
export function deriveExtractorLimits(args: {
|
||||||
@ -75,19 +76,22 @@ export function deriveExtractorLimits(args: {
|
|||||||
const includesGlassdoor = args.sources.includes("glassdoor");
|
const includesGlassdoor = args.sources.includes("glassdoor");
|
||||||
const includesGradcracker = args.sources.includes("gradcracker");
|
const includesGradcracker = args.sources.includes("gradcracker");
|
||||||
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
|
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
|
||||||
|
const includesAdzuna = args.sources.includes("adzuna");
|
||||||
|
|
||||||
const weightedContributors =
|
const weightedContributors =
|
||||||
(includesIndeed ? termCount : 0) +
|
(includesIndeed ? termCount : 0) +
|
||||||
(includesLinkedIn ? termCount : 0) +
|
(includesLinkedIn ? termCount : 0) +
|
||||||
(includesGlassdoor ? termCount : 0) +
|
(includesGlassdoor ? termCount : 0) +
|
||||||
(includesGradcracker ? termCount : 0) +
|
(includesGradcracker ? termCount : 0) +
|
||||||
(includesUkVisaJobs ? 1 : 0);
|
(includesUkVisaJobs ? 1 : 0) +
|
||||||
|
(includesAdzuna ? termCount : 0);
|
||||||
|
|
||||||
if (weightedContributors <= 0) {
|
if (weightedContributors <= 0) {
|
||||||
return {
|
return {
|
||||||
jobspyResultsWanted: budget,
|
jobspyResultsWanted: budget,
|
||||||
gradcrackerMaxJobsPerTerm: budget,
|
gradcrackerMaxJobsPerTerm: budget,
|
||||||
ukvisajobsMaxJobs: budget,
|
ukvisajobsMaxJobs: budget,
|
||||||
|
adzunaMaxJobsPerTerm: budget,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -98,6 +102,7 @@ export function deriveExtractorLimits(args: {
|
|||||||
jobspyResultsWanted: perUnit,
|
jobspyResultsWanted: perUnit,
|
||||||
gradcrackerMaxJobsPerTerm: perUnit,
|
gradcrackerMaxJobsPerTerm: perUnit,
|
||||||
ukvisajobsMaxJobs: Math.min(budget, perUnit + remainder),
|
ukvisajobsMaxJobs: Math.min(budget, perUnit + remainder),
|
||||||
|
adzunaMaxJobsPerTerm: perUnit,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -137,6 +142,7 @@ export function calculateAutomaticEstimate(args: {
|
|||||||
const hasIndeed = sources.includes("indeed");
|
const hasIndeed = sources.includes("indeed");
|
||||||
const hasLinkedIn = sources.includes("linkedin");
|
const hasLinkedIn = sources.includes("linkedin");
|
||||||
const hasGlassdoor = sources.includes("glassdoor");
|
const hasGlassdoor = sources.includes("glassdoor");
|
||||||
|
const hasAdzuna = sources.includes("adzuna");
|
||||||
const limits = deriveExtractorLimits({
|
const limits = deriveExtractorLimits({
|
||||||
budget: values.runBudget,
|
budget: values.runBudget,
|
||||||
searchTerms: values.searchTerms,
|
searchTerms: values.searchTerms,
|
||||||
@ -151,8 +157,9 @@ export function calculateAutomaticEstimate(args: {
|
|||||||
? limits.gradcrackerMaxJobsPerTerm * termCount
|
? limits.gradcrackerMaxJobsPerTerm * termCount
|
||||||
: 0;
|
: 0;
|
||||||
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
|
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
|
||||||
|
const adzunaCap = hasAdzuna ? limits.adzunaMaxJobsPerTerm * termCount : 0;
|
||||||
|
|
||||||
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap;
|
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap;
|
||||||
const discoveredMin = Math.round(discoveredCap * 0.35);
|
const discoveredMin = Math.round(discoveredCap * 0.35);
|
||||||
const discoveredMax = Math.round(discoveredCap * 0.75);
|
const discoveredMax = Math.round(discoveredCap * 0.75);
|
||||||
const processedMin = Math.min(values.topN, discoveredMin);
|
const processedMin = Math.min(values.topN, discoveredMin);
|
||||||
|
|||||||
@ -13,6 +13,7 @@ export const orderedSources: JobSource[] = [
|
|||||||
"indeed",
|
"indeed",
|
||||||
"linkedin",
|
"linkedin",
|
||||||
"glassdoor",
|
"glassdoor",
|
||||||
|
"adzuna",
|
||||||
"ukvisajobs",
|
"ukvisajobs",
|
||||||
];
|
];
|
||||||
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];
|
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];
|
||||||
|
|||||||
19
orchestrator/src/client/pages/orchestrator/utils.test.ts
Normal file
19
orchestrator/src/client/pages/orchestrator/utils.test.ts
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
import { createAppSettings } from "@shared/testing/factories.js";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { getEnabledSources } from "./utils";
|
||||||
|
|
||||||
|
describe("orchestrator utils", () => {
|
||||||
|
it("enables adzuna only when both app id and key are configured", () => {
|
||||||
|
const withCreds = createAppSettings({
|
||||||
|
adzunaAppId: "app-id",
|
||||||
|
adzunaAppKeyHint: "key-",
|
||||||
|
});
|
||||||
|
const withoutKey = createAppSettings({
|
||||||
|
adzunaAppId: "app-id",
|
||||||
|
adzunaAppKeyHint: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(getEnabledSources(withCreds)).toContain("adzuna");
|
||||||
|
expect(getEnabledSources(withoutKey)).not.toContain("adzuna");
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -171,6 +171,9 @@ export const getEnabledSources = (
|
|||||||
const hasUkVisaJobsAuth = Boolean(
|
const hasUkVisaJobsAuth = Boolean(
|
||||||
settings.ukvisajobsEmail?.trim() && settings.ukvisajobsPasswordHint,
|
settings.ukvisajobsEmail?.trim() && settings.ukvisajobsPasswordHint,
|
||||||
);
|
);
|
||||||
|
const hasAdzunaAuth = Boolean(
|
||||||
|
settings.adzunaAppId?.trim() && settings.adzunaAppKeyHint,
|
||||||
|
);
|
||||||
|
|
||||||
for (const source of orderedSources) {
|
for (const source of orderedSources) {
|
||||||
if (source === "gradcracker") {
|
if (source === "gradcracker") {
|
||||||
@ -181,6 +184,10 @@ export const getEnabledSources = (
|
|||||||
if (hasUkVisaJobsAuth) enabled.push(source);
|
if (hasUkVisaJobsAuth) enabled.push(source);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (source === "adzuna") {
|
||||||
|
if (hasAdzunaAuth) enabled.push(source);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
source === "indeed" ||
|
source === "indeed" ||
|
||||||
source === "linkedin" ||
|
source === "linkedin" ||
|
||||||
|
|||||||
@ -12,6 +12,8 @@ const EnvironmentSettingsHarness = () => {
|
|||||||
basicAuthUser: "admin",
|
basicAuthUser: "admin",
|
||||||
rxresumePassword: "",
|
rxresumePassword: "",
|
||||||
ukvisajobsPassword: "",
|
ukvisajobsPassword: "",
|
||||||
|
adzunaAppId: "adzuna-id",
|
||||||
|
adzunaAppKey: "",
|
||||||
basicAuthPassword: "",
|
basicAuthPassword: "",
|
||||||
webhookSecret: "",
|
webhookSecret: "",
|
||||||
enableBasicAuth: true,
|
enableBasicAuth: true,
|
||||||
@ -26,11 +28,13 @@ const EnvironmentSettingsHarness = () => {
|
|||||||
readable: {
|
readable: {
|
||||||
rxresumeEmail: "resume@example.com",
|
rxresumeEmail: "resume@example.com",
|
||||||
ukvisajobsEmail: "visa@example.com",
|
ukvisajobsEmail: "visa@example.com",
|
||||||
|
adzunaAppId: "adzuna-id",
|
||||||
basicAuthUser: "admin",
|
basicAuthUser: "admin",
|
||||||
},
|
},
|
||||||
private: {
|
private: {
|
||||||
rxresumePasswordHint: null,
|
rxresumePasswordHint: null,
|
||||||
ukvisajobsPasswordHint: "pass",
|
ukvisajobsPasswordHint: "pass",
|
||||||
|
adzunaAppKeyHint: "adzu",
|
||||||
basicAuthPasswordHint: "abcd",
|
basicAuthPasswordHint: "abcd",
|
||||||
webhookSecretHint: "sec-",
|
webhookSecretHint: "sec-",
|
||||||
},
|
},
|
||||||
@ -50,8 +54,10 @@ describe("EnvironmentSettingsSection", () => {
|
|||||||
|
|
||||||
expect(screen.getByDisplayValue("resume@example.com")).toBeInTheDocument();
|
expect(screen.getByDisplayValue("resume@example.com")).toBeInTheDocument();
|
||||||
expect(screen.getByDisplayValue("visa@example.com")).toBeInTheDocument();
|
expect(screen.getByDisplayValue("visa@example.com")).toBeInTheDocument();
|
||||||
|
expect(screen.getByDisplayValue("adzuna-id")).toBeInTheDocument();
|
||||||
|
|
||||||
expect(screen.getByText(/pass\*{8}/)).toBeInTheDocument();
|
expect(screen.getByText(/pass\*{8}/)).toBeInTheDocument();
|
||||||
|
expect(screen.getByText(/adzu\*{8}/)).toBeInTheDocument();
|
||||||
expect(screen.getByText(/abcd\*{8}/)).toBeInTheDocument();
|
expect(screen.getByText(/abcd\*{8}/)).toBeInTheDocument();
|
||||||
expect(screen.getByText("Not set")).toBeInTheDocument();
|
expect(screen.getByText("Not set")).toBeInTheDocument();
|
||||||
|
|
||||||
|
|||||||
@ -91,6 +91,28 @@ export const EnvironmentSettingsSection: React.FC<
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div className="space-y-4">
|
||||||
|
<div className="text-sm font-semibold">Adzuna</div>
|
||||||
|
<div className="grid gap-4 md:grid-cols-2">
|
||||||
|
<SettingsInput
|
||||||
|
label="App ID"
|
||||||
|
inputProps={register("adzunaAppId")}
|
||||||
|
placeholder="your-app-id"
|
||||||
|
disabled={isLoading || isSaving}
|
||||||
|
error={errors.adzunaAppId?.message as string | undefined}
|
||||||
|
/>
|
||||||
|
<SettingsInput
|
||||||
|
label="App Key"
|
||||||
|
inputProps={register("adzunaAppKey")}
|
||||||
|
type="password"
|
||||||
|
placeholder="Enter new app key"
|
||||||
|
disabled={isLoading || isSaving}
|
||||||
|
error={errors.adzunaAppKey?.message as string | undefined}
|
||||||
|
current={formatSecretHint(privateValues.adzunaAppKeyHint)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<Separator />
|
<Separator />
|
||||||
|
|||||||
@ -25,11 +25,13 @@ export type EnvSettingsValues = {
|
|||||||
readable: {
|
readable: {
|
||||||
rxresumeEmail: string;
|
rxresumeEmail: string;
|
||||||
ukvisajobsEmail: string;
|
ukvisajobsEmail: string;
|
||||||
|
adzunaAppId: string;
|
||||||
basicAuthUser: string;
|
basicAuthUser: string;
|
||||||
};
|
};
|
||||||
private: {
|
private: {
|
||||||
rxresumePasswordHint: string | null;
|
rxresumePasswordHint: string | null;
|
||||||
ukvisajobsPasswordHint: string | null;
|
ukvisajobsPasswordHint: string | null;
|
||||||
|
adzunaAppKeyHint: string | null;
|
||||||
basicAuthPasswordHint: string | null;
|
basicAuthPasswordHint: string | null;
|
||||||
webhookSecretHint: string | null;
|
webhookSecretHint: string | null;
|
||||||
};
|
};
|
||||||
|
|||||||
@ -143,5 +143,6 @@ export const sourceLabel: Record<Job["source"], string> = {
|
|||||||
linkedin: "LinkedIn",
|
linkedin: "LinkedIn",
|
||||||
glassdoor: "Glassdoor",
|
glassdoor: "Glassdoor",
|
||||||
ukvisajobs: "UK Visa Jobs",
|
ukvisajobs: "UK Visa Jobs",
|
||||||
|
adzuna: "Adzuna",
|
||||||
manual: "Manual",
|
manual: "Manual",
|
||||||
};
|
};
|
||||||
|
|||||||
@ -55,6 +55,17 @@ describe.sequential("Pipeline API routes", () => {
|
|||||||
expect(runPipeline).toHaveBeenNthCalledWith(2, {
|
expect(runPipeline).toHaveBeenNthCalledWith(2, {
|
||||||
sources: ["glassdoor"],
|
sources: ["glassdoor"],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const adzunaRunRes = await fetch(`${baseUrl}/api/pipeline/run`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ sources: ["adzuna"] }),
|
||||||
|
});
|
||||||
|
const adzunaRunBody = await adzunaRunRes.json();
|
||||||
|
expect(adzunaRunBody.ok).toBe(true);
|
||||||
|
expect(runPipeline).toHaveBeenNthCalledWith(3, {
|
||||||
|
sources: ["adzuna"],
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns conflict when cancelling with no active pipeline", async () => {
|
it("returns conflict when cancelling with no active pipeline", async () => {
|
||||||
|
|||||||
@ -99,7 +99,14 @@ const runPipelineSchema = z.object({
|
|||||||
minSuitabilityScore: z.number().min(0).max(100).optional(),
|
minSuitabilityScore: z.number().min(0).max(100).optional(),
|
||||||
sources: z
|
sources: z
|
||||||
.array(
|
.array(
|
||||||
z.enum(["gradcracker", "indeed", "linkedin", "glassdoor", "ukvisajobs"]),
|
z.enum([
|
||||||
|
"gradcracker",
|
||||||
|
"indeed",
|
||||||
|
"linkedin",
|
||||||
|
"glassdoor",
|
||||||
|
"ukvisajobs",
|
||||||
|
"adzuna",
|
||||||
|
]),
|
||||||
)
|
)
|
||||||
.min(1)
|
.min(1)
|
||||||
.optional(),
|
.optional(),
|
||||||
|
|||||||
@ -252,6 +252,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
|
|||||||
glassdoor: "https://www.glassdoor.com",
|
glassdoor: "https://www.glassdoor.com",
|
||||||
gradcracker: "https://www.gradcracker.com",
|
gradcracker: "https://www.gradcracker.com",
|
||||||
ukvisajobs: "https://www.ukvisajobs.com",
|
ukvisajobs: "https://www.ukvisajobs.com",
|
||||||
|
adzuna: "https://www.adzuna.com",
|
||||||
manual: "https://example.com",
|
manual: "https://example.com",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -39,6 +39,7 @@ export const jobs = sqliteTable("jobs", {
|
|||||||
"linkedin",
|
"linkedin",
|
||||||
"glassdoor",
|
"glassdoor",
|
||||||
"ukvisajobs",
|
"ukvisajobs",
|
||||||
|
"adzuna",
|
||||||
"manual",
|
"manual",
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|||||||
@ -14,7 +14,7 @@ export type PipelineStep =
|
|||||||
| "cancelled"
|
| "cancelled"
|
||||||
| "failed";
|
| "failed";
|
||||||
|
|
||||||
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs";
|
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna";
|
||||||
|
|
||||||
export interface PipelineProgress {
|
export interface PipelineProgress {
|
||||||
step: PipelineStep;
|
step: PipelineStep;
|
||||||
|
|||||||
@ -19,6 +19,10 @@ vi.mock("../../services/crawler", () => ({
|
|||||||
runCrawler: vi.fn(),
|
runCrawler: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock("../../services/adzuna", () => ({
|
||||||
|
runAdzuna: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
vi.mock("../../services/ukvisajobs", () => ({
|
vi.mock("../../services/ukvisajobs", () => ({
|
||||||
runUkVisaJobs: vi.fn(),
|
runUkVisaJobs: vi.fn(),
|
||||||
}));
|
}));
|
||||||
@ -157,6 +161,63 @@ describe("discoverJobsStep", () => {
|
|||||||
).rejects.toThrow("All sources failed: ukvisajobs: boom");
|
).rejects.toThrow("All sources failed: ukvisajobs: boom");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("runs adzuna when selected and country is compatible", async () => {
|
||||||
|
const settingsRepo = await import("../../repositories/settings");
|
||||||
|
const adzuna = await import("../../services/adzuna");
|
||||||
|
|
||||||
|
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||||
|
searchTerms: JSON.stringify(["engineer"]),
|
||||||
|
jobspyCountryIndeed: "united states",
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
vi.mocked(adzuna.runAdzuna).mockResolvedValue({
|
||||||
|
success: true,
|
||||||
|
jobs: [
|
||||||
|
{
|
||||||
|
source: "adzuna",
|
||||||
|
sourceJobId: "adzu-1",
|
||||||
|
title: "Engineer",
|
||||||
|
employer: "ACME",
|
||||||
|
jobUrl: "https://example.com/job",
|
||||||
|
applicationLink: "https://example.com/job",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
const result = await discoverJobsStep({
|
||||||
|
mergedConfig: {
|
||||||
|
...config,
|
||||||
|
sources: ["adzuna"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.discoveredJobs).toHaveLength(1);
|
||||||
|
expect(vi.mocked(adzuna.runAdzuna)).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({ country: "us" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips adzuna for unsupported countries", async () => {
|
||||||
|
const settingsRepo = await import("../../repositories/settings");
|
||||||
|
const adzuna = await import("../../services/adzuna");
|
||||||
|
|
||||||
|
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||||
|
searchTerms: JSON.stringify(["engineer"]),
|
||||||
|
jobspyCountryIndeed: "japan",
|
||||||
|
} as any);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
discoverJobsStep({
|
||||||
|
mergedConfig: {
|
||||||
|
...config,
|
||||||
|
sources: ["adzuna"],
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
).rejects.toThrow("No compatible sources for selected country: Japan");
|
||||||
|
|
||||||
|
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it("maps Gradcracker progress callback into live crawling counters", async () => {
|
it("maps Gradcracker progress callback into live crawling counters", async () => {
|
||||||
const settingsRepo = await import("../../repositories/settings");
|
const settingsRepo = await import("../../repositories/settings");
|
||||||
const crawler = await import("../../services/crawler");
|
const crawler = await import("../../services/crawler");
|
||||||
@ -340,6 +401,7 @@ describe("discoverJobsStep", () => {
|
|||||||
|
|
||||||
it("does not throw when no sources are requested", async () => {
|
it("does not throw when no sources are requested", async () => {
|
||||||
const settingsRepo = await import("../../repositories/settings");
|
const settingsRepo = await import("../../repositories/settings");
|
||||||
|
const adzuna = await import("../../services/adzuna");
|
||||||
const jobSpy = await import("../../services/jobspy");
|
const jobSpy = await import("../../services/jobspy");
|
||||||
const crawler = await import("../../services/crawler");
|
const crawler = await import("../../services/crawler");
|
||||||
const ukVisa = await import("../../services/ukvisajobs");
|
const ukVisa = await import("../../services/ukvisajobs");
|
||||||
@ -359,6 +421,7 @@ describe("discoverJobsStep", () => {
|
|||||||
expect(result.discoveredJobs).toEqual([]);
|
expect(result.discoveredJobs).toEqual([]);
|
||||||
expect(result.sourceErrors).toEqual([]);
|
expect(result.sourceErrors).toEqual([]);
|
||||||
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
|
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
|
||||||
|
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||||
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
|
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
|
||||||
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
|
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|||||||
@ -1,12 +1,14 @@
|
|||||||
import { logger } from "@infra/logger";
|
import { logger } from "@infra/logger";
|
||||||
import {
|
import {
|
||||||
formatCountryLabel,
|
formatCountryLabel,
|
||||||
|
getAdzunaCountryCode,
|
||||||
isSourceAllowedForCountry,
|
isSourceAllowedForCountry,
|
||||||
normalizeCountryKey,
|
normalizeCountryKey,
|
||||||
} from "@shared/location-support.js";
|
} from "@shared/location-support.js";
|
||||||
import type { CreateJobInput, PipelineConfig } from "@shared/types";
|
import type { CreateJobInput, PipelineConfig } from "@shared/types";
|
||||||
import * as jobsRepo from "../../repositories/jobs";
|
import * as jobsRepo from "../../repositories/jobs";
|
||||||
import * as settingsRepo from "../../repositories/settings";
|
import * as settingsRepo from "../../repositories/settings";
|
||||||
|
import { runAdzuna } from "../../services/adzuna";
|
||||||
import { runCrawler } from "../../services/crawler";
|
import { runCrawler } from "../../services/crawler";
|
||||||
import { runJobSpy } from "../../services/jobspy";
|
import { runJobSpy } from "../../services/jobspy";
|
||||||
import { runUkVisaJobs } from "../../services/ukvisajobs";
|
import { runUkVisaJobs } from "../../services/ukvisajobs";
|
||||||
@ -72,11 +74,13 @@ export async function discoverJobsStep(args: {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const shouldRunJobSpy = jobSpySites.length > 0;
|
const shouldRunJobSpy = jobSpySites.length > 0;
|
||||||
|
const shouldRunAdzuna = compatibleSources.includes("adzuna");
|
||||||
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
|
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
|
||||||
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
|
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
|
||||||
|
|
||||||
const totalSources =
|
const totalSources =
|
||||||
Number(shouldRunJobSpy) +
|
Number(shouldRunJobSpy) +
|
||||||
|
Number(shouldRunAdzuna) +
|
||||||
Number(shouldRunGradcracker) +
|
Number(shouldRunGradcracker) +
|
||||||
Number(shouldRunUkVisaJobs);
|
Number(shouldRunUkVisaJobs);
|
||||||
let completedSources = 0;
|
let completedSources = 0;
|
||||||
@ -149,6 +153,89 @@ export async function discoverJobsStep(args: {
|
|||||||
return { discoveredJobs, sourceErrors };
|
return { discoveredJobs, sourceErrors };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (shouldRunAdzuna) {
|
||||||
|
progressHelpers.startSource("adzuna", completedSources, totalSources, {
|
||||||
|
termsTotal: searchTerms.length,
|
||||||
|
detail: "Adzuna: fetching jobs...",
|
||||||
|
});
|
||||||
|
|
||||||
|
const adzunaCountryCode = getAdzunaCountryCode(selectedCountry);
|
||||||
|
if (!adzunaCountryCode) {
|
||||||
|
sourceErrors.push(
|
||||||
|
`adzuna: unsupported country ${formatCountryLabel(selectedCountry)}`,
|
||||||
|
);
|
||||||
|
markSourceComplete();
|
||||||
|
} else {
|
||||||
|
const adzunaMaxJobsPerTerm = settings.adzunaMaxJobsPerTerm
|
||||||
|
? parseInt(settings.adzunaMaxJobsPerTerm, 10)
|
||||||
|
: 50;
|
||||||
|
|
||||||
|
const adzunaResult = await runAdzuna({
|
||||||
|
country: adzunaCountryCode,
|
||||||
|
searchTerms,
|
||||||
|
maxJobsPerTerm: adzunaMaxJobsPerTerm,
|
||||||
|
onProgress: (event) => {
|
||||||
|
if (event.type === "term_start") {
|
||||||
|
progressHelpers.crawlingUpdate({
|
||||||
|
source: "adzuna",
|
||||||
|
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||||
|
termsTotal: event.termTotal,
|
||||||
|
phase: "list",
|
||||||
|
currentUrl: event.searchTerm,
|
||||||
|
});
|
||||||
|
updateProgress({
|
||||||
|
step: "crawling",
|
||||||
|
detail: `Adzuna: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (event.type === "page_fetched") {
|
||||||
|
progressHelpers.crawlingUpdate({
|
||||||
|
source: "adzuna",
|
||||||
|
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||||
|
termsTotal: event.termTotal,
|
||||||
|
listPagesProcessed: event.pageNo,
|
||||||
|
jobPagesEnqueued: event.totalCollected,
|
||||||
|
jobPagesProcessed: event.totalCollected,
|
||||||
|
phase: "list",
|
||||||
|
currentUrl: `page ${event.pageNo}`,
|
||||||
|
});
|
||||||
|
updateProgress({
|
||||||
|
step: "crawling",
|
||||||
|
detail: `Adzuna: term ${event.termIndex}/${event.termTotal}, page ${event.pageNo} (${event.totalCollected} collected)`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
progressHelpers.crawlingUpdate({
|
||||||
|
source: "adzuna",
|
||||||
|
termsProcessed: event.termIndex,
|
||||||
|
termsTotal: event.termTotal,
|
||||||
|
phase: "list",
|
||||||
|
currentUrl: event.searchTerm,
|
||||||
|
});
|
||||||
|
updateProgress({
|
||||||
|
step: "crawling",
|
||||||
|
detail: `Adzuna: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!adzunaResult.success) {
|
||||||
|
sourceErrors.push(`adzuna: ${adzunaResult.error ?? "unknown error"}`);
|
||||||
|
} else {
|
||||||
|
discoveredJobs.push(...adzunaResult.jobs);
|
||||||
|
}
|
||||||
|
|
||||||
|
markSourceComplete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.shouldCancel?.()) {
|
||||||
|
return { discoveredJobs, sourceErrors };
|
||||||
|
}
|
||||||
|
|
||||||
if (shouldRunGradcracker) {
|
if (shouldRunGradcracker) {
|
||||||
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
|
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
|
||||||
detail: "Gradcracker: scraping...",
|
detail: "Gradcracker: scraping...",
|
||||||
|
|||||||
@ -20,6 +20,7 @@ export type SettingKey =
|
|||||||
| "resumeProjects"
|
| "resumeProjects"
|
||||||
| "rxresumeBaseResumeId"
|
| "rxresumeBaseResumeId"
|
||||||
| "ukvisajobsMaxJobs"
|
| "ukvisajobsMaxJobs"
|
||||||
|
| "adzunaMaxJobsPerTerm"
|
||||||
| "gradcrackerMaxJobsPerTerm"
|
| "gradcrackerMaxJobsPerTerm"
|
||||||
| "searchTerms"
|
| "searchTerms"
|
||||||
| "jobspyLocation"
|
| "jobspyLocation"
|
||||||
@ -36,6 +37,8 @@ export type SettingKey =
|
|||||||
| "basicAuthPassword"
|
| "basicAuthPassword"
|
||||||
| "ukvisajobsEmail"
|
| "ukvisajobsEmail"
|
||||||
| "ukvisajobsPassword"
|
| "ukvisajobsPassword"
|
||||||
|
| "adzunaAppId"
|
||||||
|
| "adzunaAppKey"
|
||||||
| "webhookSecret"
|
| "webhookSecret"
|
||||||
| "backupEnabled"
|
| "backupEnabled"
|
||||||
| "backupHour"
|
| "backupHour"
|
||||||
|
|||||||
254
orchestrator/src/server/services/adzuna.ts
Normal file
254
orchestrator/src/server/services/adzuna.ts
Normal file
@ -0,0 +1,254 @@
|
|||||||
|
import { spawn, spawnSync } from "node:child_process";
|
||||||
|
import { readFile } from "node:fs/promises";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import { createInterface } from "node:readline";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { logger } from "@infra/logger";
|
||||||
|
import type { CreateJobInput } from "@shared/types";
|
||||||
|
import { toNumberOrNull, toStringOrNull } from "@shared/utils/type-conversion";
|
||||||
|
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const ADZUNA_DIR = join(__dirname, "../../../../extractors/adzuna");
|
||||||
|
const DATASET_PATH = join(ADZUNA_DIR, "storage/datasets/default/jobs.json");
|
||||||
|
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const TSX_CLI_PATH = resolveTsxCliPath();
|
||||||
|
|
||||||
|
type AdzunaRawJob = Record<string, unknown>;
|
||||||
|
|
||||||
|
export type AdzunaProgressEvent =
|
||||||
|
| {
|
||||||
|
type: "term_start";
|
||||||
|
termIndex: number;
|
||||||
|
termTotal: number;
|
||||||
|
searchTerm: string;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "page_fetched";
|
||||||
|
termIndex: number;
|
||||||
|
termTotal: number;
|
||||||
|
searchTerm: string;
|
||||||
|
pageNo: number;
|
||||||
|
resultsOnPage: number;
|
||||||
|
totalCollected: number;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: "term_complete";
|
||||||
|
termIndex: number;
|
||||||
|
termTotal: number;
|
||||||
|
searchTerm: string;
|
||||||
|
jobsFoundTerm: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface RunAdzunaOptions {
|
||||||
|
searchTerms?: string[];
|
||||||
|
country?: string;
|
||||||
|
maxJobsPerTerm?: number;
|
||||||
|
onProgress?: (event: AdzunaProgressEvent) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface AdzunaResult {
|
||||||
|
success: boolean;
|
||||||
|
jobs: CreateJobInput[];
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveTsxCliPath(): string | null {
|
||||||
|
try {
|
||||||
|
return require.resolve("tsx/dist/cli.mjs");
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function canRunNpmCommand(): boolean {
|
||||||
|
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
|
||||||
|
return !result.error && result.status === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseAdzunaProgressLine(line: string): AdzunaProgressEvent | null {
|
||||||
|
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||||
|
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||||
|
|
||||||
|
let parsed: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const event = toStringOrNull(parsed.event);
|
||||||
|
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||||
|
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||||
|
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||||
|
if (!event || termIndex === null || termTotal === null) return null;
|
||||||
|
|
||||||
|
if (event === "term_start") {
|
||||||
|
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (event === "page_fetched") {
|
||||||
|
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||||
|
if (pageNo === null) return null;
|
||||||
|
return {
|
||||||
|
type: "page_fetched",
|
||||||
|
termIndex,
|
||||||
|
termTotal,
|
||||||
|
searchTerm,
|
||||||
|
pageNo,
|
||||||
|
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
|
||||||
|
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (event === "term_complete") {
|
||||||
|
return {
|
||||||
|
type: "term_complete",
|
||||||
|
termIndex,
|
||||||
|
termTotal,
|
||||||
|
searchTerm,
|
||||||
|
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapAdzunaRow(row: AdzunaRawJob): CreateJobInput | null {
|
||||||
|
const jobUrl = toStringOrNull(row.jobUrl);
|
||||||
|
if (!jobUrl) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
source: "adzuna",
|
||||||
|
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
|
||||||
|
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||||
|
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
|
||||||
|
jobUrl,
|
||||||
|
applicationLink:
|
||||||
|
toStringOrNull(row.applicationLink) ??
|
||||||
|
toStringOrNull(row.jobUrl) ??
|
||||||
|
undefined,
|
||||||
|
location: toStringOrNull(row.location) ?? undefined,
|
||||||
|
salary: toStringOrNull(row.salary) ?? undefined,
|
||||||
|
datePosted: toStringOrNull(row.datePosted) ?? undefined,
|
||||||
|
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
|
||||||
|
jobType: toStringOrNull(row.jobType) ?? undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readDataset(): Promise<CreateJobInput[]> {
|
||||||
|
const content = await readFile(DATASET_PATH, "utf-8");
|
||||||
|
const parsed = JSON.parse(content) as unknown;
|
||||||
|
if (!Array.isArray(parsed)) return [];
|
||||||
|
|
||||||
|
const jobs: CreateJobInput[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const value of parsed) {
|
||||||
|
if (!value || typeof value !== "object") continue;
|
||||||
|
const mapped = mapAdzunaRow(value as AdzunaRawJob);
|
||||||
|
if (!mapped) continue;
|
||||||
|
const key = mapped.sourceJobId || mapped.jobUrl;
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
jobs.push(mapped);
|
||||||
|
}
|
||||||
|
return jobs;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runAdzuna(
|
||||||
|
options: RunAdzunaOptions = {},
|
||||||
|
): Promise<AdzunaResult> {
|
||||||
|
const appId = process.env.ADZUNA_APP_ID?.trim();
|
||||||
|
const appKey = process.env.ADZUNA_APP_KEY?.trim();
|
||||||
|
if (!appId || !appKey) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
jobs: [],
|
||||||
|
error: "Missing Adzuna credentials (ADZUNA_APP_ID / ADZUNA_APP_KEY)",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const country = (options.country || "gb").trim().toLowerCase();
|
||||||
|
const maxJobsPerTerm = options.maxJobsPerTerm ?? 50;
|
||||||
|
const searchTerms =
|
||||||
|
options.searchTerms && options.searchTerms.length > 0
|
||||||
|
? options.searchTerms
|
||||||
|
: ["web developer"];
|
||||||
|
const useNpmCommand = canRunNpmCommand();
|
||||||
|
if (!useNpmCommand && !TSX_CLI_PATH) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
jobs: [],
|
||||||
|
error: "Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const extractorEnv = {
|
||||||
|
...process.env,
|
||||||
|
JOBOPS_EMIT_PROGRESS: "1",
|
||||||
|
ADZUNA_APP_ID: appId,
|
||||||
|
ADZUNA_APP_KEY: appKey,
|
||||||
|
ADZUNA_COUNTRY: country,
|
||||||
|
ADZUNA_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
|
||||||
|
ADZUNA_SEARCH_TERMS: JSON.stringify(searchTerms),
|
||||||
|
ADZUNA_OUTPUT_JSON: DATASET_PATH,
|
||||||
|
};
|
||||||
|
const child = useNpmCommand
|
||||||
|
? spawn("npm", ["run", "start"], {
|
||||||
|
cwd: ADZUNA_DIR,
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
env: extractorEnv,
|
||||||
|
})
|
||||||
|
: (() => {
|
||||||
|
const tsxCliPath = TSX_CLI_PATH;
|
||||||
|
if (!tsxCliPath) {
|
||||||
|
throw new Error(
|
||||||
|
"Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
|
||||||
|
cwd: ADZUNA_DIR,
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
env: extractorEnv,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||||
|
const progressEvent = parseAdzunaProgressLine(line);
|
||||||
|
if (progressEvent) {
|
||||||
|
options.onProgress?.(progressEvent);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
stream.write(`${line}\n`);
|
||||||
|
};
|
||||||
|
|
||||||
|
const stdoutRl = child.stdout
|
||||||
|
? createInterface({ input: child.stdout })
|
||||||
|
: null;
|
||||||
|
const stderrRl = child.stderr
|
||||||
|
? createInterface({ input: child.stderr })
|
||||||
|
: null;
|
||||||
|
|
||||||
|
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||||
|
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||||
|
|
||||||
|
child.on("close", (code) => {
|
||||||
|
stdoutRl?.close();
|
||||||
|
stderrRl?.close();
|
||||||
|
if (code === 0) resolve();
|
||||||
|
else reject(new Error(`Adzuna extractor exited with code ${code}`));
|
||||||
|
});
|
||||||
|
child.on("error", reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const jobs = await readDataset();
|
||||||
|
return { success: true, jobs };
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : "Unknown error";
|
||||||
|
logger.warn("Adzuna extractor run failed", { error: message });
|
||||||
|
return { success: false, jobs: [], error: message };
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -8,6 +8,7 @@ const readableStringConfig: { settingKey: SettingKey; envKey: string }[] = [
|
|||||||
{ settingKey: "llmBaseUrl", envKey: "LLM_BASE_URL" },
|
{ settingKey: "llmBaseUrl", envKey: "LLM_BASE_URL" },
|
||||||
{ settingKey: "rxresumeEmail", envKey: "RXRESUME_EMAIL" },
|
{ settingKey: "rxresumeEmail", envKey: "RXRESUME_EMAIL" },
|
||||||
{ settingKey: "ukvisajobsEmail", envKey: "UKVISAJOBS_EMAIL" },
|
{ settingKey: "ukvisajobsEmail", envKey: "UKVISAJOBS_EMAIL" },
|
||||||
|
{ settingKey: "adzunaAppId", envKey: "ADZUNA_APP_ID" },
|
||||||
{ settingKey: "basicAuthUser", envKey: "BASIC_AUTH_USER" },
|
{ settingKey: "basicAuthUser", envKey: "BASIC_AUTH_USER" },
|
||||||
];
|
];
|
||||||
|
|
||||||
@ -37,6 +38,11 @@ const privateStringConfig: {
|
|||||||
envKey: "UKVISAJOBS_PASSWORD",
|
envKey: "UKVISAJOBS_PASSWORD",
|
||||||
hintKey: "ukvisajobsPasswordHint",
|
hintKey: "ukvisajobsPasswordHint",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
settingKey: "adzunaAppKey",
|
||||||
|
envKey: "ADZUNA_APP_KEY",
|
||||||
|
hintKey: "adzunaAppKeyHint",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
settingKey: "basicAuthPassword",
|
settingKey: "basicAuthPassword",
|
||||||
envKey: "BASIC_AUTH_PASSWORD",
|
envKey: "BASIC_AUTH_PASSWORD",
|
||||||
|
|||||||
@ -25,6 +25,20 @@ describe("settings-conversion", () => {
|
|||||||
expect(resolved.defaultValue).toBe(50);
|
expect(resolved.defaultValue).toBe(50);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("round-trips adzuna numeric settings", () => {
|
||||||
|
process.env.ADZUNA_MAX_JOBS_PER_TERM = "";
|
||||||
|
const serialized = serializeSettingValue("adzunaMaxJobsPerTerm", 75);
|
||||||
|
expect(serialized).toBe("75");
|
||||||
|
|
||||||
|
const resolved = resolveSettingValue(
|
||||||
|
"adzunaMaxJobsPerTerm",
|
||||||
|
serialized ?? undefined,
|
||||||
|
);
|
||||||
|
expect(resolved.overrideValue).toBe(75);
|
||||||
|
expect(resolved.value).toBe(75);
|
||||||
|
expect(resolved.defaultValue).toBe(50);
|
||||||
|
});
|
||||||
|
|
||||||
it("round-trips boolean bit settings", () => {
|
it("round-trips boolean bit settings", () => {
|
||||||
expect(serializeSettingValue("showSponsorInfo", true)).toBe("1");
|
expect(serializeSettingValue("showSponsorInfo", true)).toBe("1");
|
||||||
expect(serializeSettingValue("showSponsorInfo", false)).toBe("0");
|
expect(serializeSettingValue("showSponsorInfo", false)).toBe("0");
|
||||||
|
|||||||
@ -7,6 +7,7 @@ type SettingMetadata<T, Input = T | null | undefined> = {
|
|||||||
|
|
||||||
type SettingsConversionValueMap = {
|
type SettingsConversionValueMap = {
|
||||||
ukvisajobsMaxJobs: number;
|
ukvisajobsMaxJobs: number;
|
||||||
|
adzunaMaxJobsPerTerm: number;
|
||||||
gradcrackerMaxJobsPerTerm: number;
|
gradcrackerMaxJobsPerTerm: number;
|
||||||
searchTerms: string[];
|
searchTerms: string[];
|
||||||
jobspyLocation: string;
|
jobspyLocation: string;
|
||||||
@ -100,6 +101,13 @@ export const settingsConversionMetadata: SettingsConversionMetadata = {
|
|||||||
serialize: serializeNullableNumber,
|
serialize: serializeNullableNumber,
|
||||||
resolve: resolveWithNullishFallback,
|
resolve: resolveWithNullishFallback,
|
||||||
},
|
},
|
||||||
|
adzunaMaxJobsPerTerm: {
|
||||||
|
defaultValue: () =>
|
||||||
|
parseInt(process.env.ADZUNA_MAX_JOBS_PER_TERM || "50", 10),
|
||||||
|
parseOverride: parseIntOrNull,
|
||||||
|
serialize: serializeNullableNumber,
|
||||||
|
resolve: resolveWithNullishFallback,
|
||||||
|
},
|
||||||
gradcrackerMaxJobsPerTerm: {
|
gradcrackerMaxJobsPerTerm: {
|
||||||
defaultValue: () => 50,
|
defaultValue: () => 50,
|
||||||
parseOverride: parseIntOrNull,
|
parseOverride: parseIntOrNull,
|
||||||
|
|||||||
@ -35,23 +35,37 @@ describe("applySettingsUpdates", () => {
|
|||||||
const plan = await applySettingsUpdates({
|
const plan = await applySettingsUpdates({
|
||||||
model: "gpt-4o-mini",
|
model: "gpt-4o-mini",
|
||||||
ukvisajobsMaxJobs: 42,
|
ukvisajobsMaxJobs: 42,
|
||||||
|
adzunaMaxJobsPerTerm: 25,
|
||||||
searchTerms: ["backend", "platform"],
|
searchTerms: ["backend", "platform"],
|
||||||
llmProvider: "openai",
|
llmProvider: "openai",
|
||||||
|
adzunaAppId: "app-id",
|
||||||
|
adzunaAppKey: "app-key",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(4);
|
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(7);
|
||||||
expect(vi.mocked(settingsRepo.setSetting).mock.calls).toEqual(
|
expect(vi.mocked(settingsRepo.setSetting).mock.calls).toEqual(
|
||||||
expect.arrayContaining([
|
expect.arrayContaining([
|
||||||
["model", "gpt-4o-mini"],
|
["model", "gpt-4o-mini"],
|
||||||
["ukvisajobsMaxJobs", "42"],
|
["ukvisajobsMaxJobs", "42"],
|
||||||
|
["adzunaMaxJobsPerTerm", "25"],
|
||||||
["searchTerms", '["backend","platform"]'],
|
["searchTerms", '["backend","platform"]'],
|
||||||
["llmProvider", "openai"],
|
["llmProvider", "openai"],
|
||||||
|
["adzunaAppId", "app-id"],
|
||||||
|
["adzunaAppKey", "app-key"],
|
||||||
]),
|
]),
|
||||||
);
|
);
|
||||||
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||||
"LLM_PROVIDER",
|
"LLM_PROVIDER",
|
||||||
"openai",
|
"openai",
|
||||||
);
|
);
|
||||||
|
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||||
|
"ADZUNA_APP_ID",
|
||||||
|
"app-id",
|
||||||
|
);
|
||||||
|
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||||
|
"ADZUNA_APP_KEY",
|
||||||
|
"app-key",
|
||||||
|
);
|
||||||
expect(plan.shouldRefreshBackupScheduler).toBe(false);
|
expect(plan.shouldRefreshBackupScheduler).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@ -164,6 +164,11 @@ export const settingsUpdateRegistry: Partial<{
|
|||||||
actions: [metadataPersistAction("ukvisajobsMaxJobs", value)],
|
actions: [metadataPersistAction("ukvisajobsMaxJobs", value)],
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
|
adzunaMaxJobsPerTerm: singleAction(({ value }) =>
|
||||||
|
result({
|
||||||
|
actions: [metadataPersistAction("adzunaMaxJobsPerTerm", value)],
|
||||||
|
}),
|
||||||
|
),
|
||||||
gradcrackerMaxJobsPerTerm: singleAction(({ value }) =>
|
gradcrackerMaxJobsPerTerm: singleAction(({ value }) =>
|
||||||
result({
|
result({
|
||||||
actions: [metadataPersistAction("gradcrackerMaxJobsPerTerm", value)],
|
actions: [metadataPersistAction("gradcrackerMaxJobsPerTerm", value)],
|
||||||
@ -278,6 +283,26 @@ export const settingsUpdateRegistry: Partial<{
|
|||||||
],
|
],
|
||||||
});
|
});
|
||||||
}),
|
}),
|
||||||
|
adzunaAppId: singleAction(({ value }) => {
|
||||||
|
const normalized = toNormalizedStringOrNull(value);
|
||||||
|
return result({
|
||||||
|
actions: [
|
||||||
|
persistAction("adzunaAppId", normalized, () => {
|
||||||
|
applyEnvValue("ADZUNA_APP_ID", normalized);
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
adzunaAppKey: singleAction(({ value }) => {
|
||||||
|
const normalized = toNormalizedStringOrNull(value);
|
||||||
|
return result({
|
||||||
|
actions: [
|
||||||
|
persistAction("adzunaAppKey", normalized, () => {
|
||||||
|
applyEnvValue("ADZUNA_APP_KEY", normalized);
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}),
|
||||||
webhookSecret: singleAction(({ value }) => {
|
webhookSecret: singleAction(({ value }) => {
|
||||||
const normalized = toNormalizedStringOrNull(value);
|
const normalized = toNormalizedStringOrNull(value);
|
||||||
return result({
|
return result({
|
||||||
|
|||||||
@ -96,6 +96,15 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
|
|||||||
const overrideUkvisajobsMaxJobs = ukvisajobsMaxJobsSetting.overrideValue;
|
const overrideUkvisajobsMaxJobs = ukvisajobsMaxJobsSetting.overrideValue;
|
||||||
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting.value;
|
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting.value;
|
||||||
|
|
||||||
|
const adzunaMaxJobsPerTermSetting = resolveSettingValue(
|
||||||
|
"adzunaMaxJobsPerTerm",
|
||||||
|
overrides.adzunaMaxJobsPerTerm,
|
||||||
|
);
|
||||||
|
const defaultAdzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.defaultValue;
|
||||||
|
const overrideAdzunaMaxJobsPerTerm =
|
||||||
|
adzunaMaxJobsPerTermSetting.overrideValue;
|
||||||
|
const adzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.value;
|
||||||
|
|
||||||
const gradcrackerMaxJobsPerTermSetting = resolveSettingValue(
|
const gradcrackerMaxJobsPerTermSetting = resolveSettingValue(
|
||||||
"gradcrackerMaxJobsPerTerm",
|
"gradcrackerMaxJobsPerTerm",
|
||||||
overrides.gradcrackerMaxJobsPerTerm,
|
overrides.gradcrackerMaxJobsPerTerm,
|
||||||
@ -260,6 +269,9 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
|
|||||||
ukvisajobsMaxJobs,
|
ukvisajobsMaxJobs,
|
||||||
defaultUkvisajobsMaxJobs,
|
defaultUkvisajobsMaxJobs,
|
||||||
overrideUkvisajobsMaxJobs,
|
overrideUkvisajobsMaxJobs,
|
||||||
|
adzunaMaxJobsPerTerm,
|
||||||
|
defaultAdzunaMaxJobsPerTerm,
|
||||||
|
overrideAdzunaMaxJobsPerTerm,
|
||||||
gradcrackerMaxJobsPerTerm,
|
gradcrackerMaxJobsPerTerm,
|
||||||
defaultGradcrackerMaxJobsPerTerm,
|
defaultGradcrackerMaxJobsPerTerm,
|
||||||
overrideGradcrackerMaxJobsPerTerm,
|
overrideGradcrackerMaxJobsPerTerm,
|
||||||
|
|||||||
26
package-lock.json
generated
26
package-lock.json
generated
@ -80,6 +80,28 @@
|
|||||||
"node": ">=14.17"
|
"node": ">=14.17"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"extractors/adzuna": {
|
||||||
|
"name": "adzuna-extractor",
|
||||||
|
"version": "0.0.1",
|
||||||
|
"dependencies": {
|
||||||
|
"job-ops-shared": "^1.0.0",
|
||||||
|
"tsx": "^4.4.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^24.0.0",
|
||||||
|
"typescript": "~5.9.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"extractors/adzuna/node_modules/@types/node": {
|
||||||
|
"version": "24.10.13",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz",
|
||||||
|
"integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==",
|
||||||
|
"dev": true,
|
||||||
|
"license": "MIT",
|
||||||
|
"dependencies": {
|
||||||
|
"undici-types": "~7.16.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"extractors/gradcracker": {
|
"extractors/gradcracker": {
|
||||||
"name": "gradcracker-extractor",
|
"name": "gradcracker-extractor",
|
||||||
"version": "0.0.1",
|
"version": "0.0.1",
|
||||||
@ -8586,6 +8608,10 @@
|
|||||||
"node": ">=12.0"
|
"node": ">=12.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/adzuna-extractor": {
|
||||||
|
"resolved": "extractors/adzuna",
|
||||||
|
"link": true
|
||||||
|
},
|
||||||
"node_modules/agent-base": {
|
"node_modules/agent-base": {
|
||||||
"version": "7.1.4",
|
"version": "7.1.4",
|
||||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import {
|
import {
|
||||||
formatCountryLabel,
|
formatCountryLabel,
|
||||||
|
getAdzunaCountryCode,
|
||||||
getCompatibleSourcesForCountry,
|
getCompatibleSourcesForCountry,
|
||||||
isGlassdoorCountry,
|
isGlassdoorCountry,
|
||||||
isSourceAllowedForCountry,
|
isSourceAllowedForCountry,
|
||||||
@ -52,15 +53,24 @@ describe("location-support", () => {
|
|||||||
expect(isSourceAllowedForCountry("linkedin", "worldwide")).toBe(true);
|
expect(isSourceAllowedForCountry("linkedin", "worldwide")).toBe(true);
|
||||||
expect(isSourceAllowedForCountry("glassdoor", "united states")).toBe(true);
|
expect(isSourceAllowedForCountry("glassdoor", "united states")).toBe(true);
|
||||||
expect(isSourceAllowedForCountry("glassdoor", "japan")).toBe(false);
|
expect(isSourceAllowedForCountry("glassdoor", "japan")).toBe(false);
|
||||||
|
expect(isSourceAllowedForCountry("adzuna", "united states")).toBe(true);
|
||||||
|
expect(isSourceAllowedForCountry("adzuna", "japan")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("filters incompatible sources while preserving compatible order", () => {
|
it("filters incompatible sources while preserving compatible order", () => {
|
||||||
expect(
|
expect(
|
||||||
getCompatibleSourcesForCountry(
|
getCompatibleSourcesForCountry(
|
||||||
["gradcracker", "indeed", "glassdoor", "ukvisajobs", "linkedin"],
|
[
|
||||||
|
"gradcracker",
|
||||||
|
"indeed",
|
||||||
|
"glassdoor",
|
||||||
|
"ukvisajobs",
|
||||||
|
"adzuna",
|
||||||
|
"linkedin",
|
||||||
|
],
|
||||||
"united states",
|
"united states",
|
||||||
),
|
),
|
||||||
).toEqual(["indeed", "glassdoor", "linkedin"]);
|
).toEqual(["indeed", "glassdoor", "adzuna", "linkedin"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("supports glassdoor only in explicitly supported countries", () => {
|
it("supports glassdoor only in explicitly supported countries", () => {
|
||||||
@ -70,4 +80,10 @@ describe("location-support", () => {
|
|||||||
expect(isGlassdoorCountry("japan")).toBe(false);
|
expect(isGlassdoorCountry("japan")).toBe(false);
|
||||||
expect(isGlassdoorCountry("worldwide")).toBe(false);
|
expect(isGlassdoorCountry("worldwide")).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("maps adzuna country keys to adzuna api country codes", () => {
|
||||||
|
expect(getAdzunaCountryCode("united states")).toBe("us");
|
||||||
|
expect(getAdzunaCountryCode("UK")).toBe("gb");
|
||||||
|
expect(getAdzunaCountryCode("japan")).toBeNull();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -124,6 +124,27 @@ const GLASSDOOR_SUPPORTED_COUNTRIES = new Set(
|
|||||||
"vietnam",
|
"vietnam",
|
||||||
].map((country) => normalizeCountryKey(country)),
|
].map((country) => normalizeCountryKey(country)),
|
||||||
);
|
);
|
||||||
|
const ADZUNA_COUNTRY_CODE_BY_KEY: Record<string, string> = {
|
||||||
|
"united kingdom": "gb",
|
||||||
|
"united states": "us",
|
||||||
|
austria: "at",
|
||||||
|
australia: "au",
|
||||||
|
belgium: "be",
|
||||||
|
brazil: "br",
|
||||||
|
canada: "ca",
|
||||||
|
switzerland: "ch",
|
||||||
|
germany: "de",
|
||||||
|
spain: "es",
|
||||||
|
france: "fr",
|
||||||
|
india: "in",
|
||||||
|
italy: "it",
|
||||||
|
mexico: "mx",
|
||||||
|
netherlands: "nl",
|
||||||
|
"new zealand": "nz",
|
||||||
|
poland: "pl",
|
||||||
|
singapore: "sg",
|
||||||
|
"south africa": "za",
|
||||||
|
};
|
||||||
|
|
||||||
export function normalizeCountryKey(value: string | null | undefined): string {
|
export function normalizeCountryKey(value: string | null | undefined): string {
|
||||||
const normalized = value?.trim().toLowerCase() ?? "";
|
const normalized = value?.trim().toLowerCase() ?? "";
|
||||||
@ -155,12 +176,19 @@ export function isGlassdoorCountry(
|
|||||||
return GLASSDOOR_SUPPORTED_COUNTRIES.has(normalizeCountryKey(country));
|
return GLASSDOOR_SUPPORTED_COUNTRIES.has(normalizeCountryKey(country));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getAdzunaCountryCode(
|
||||||
|
country: string | null | undefined,
|
||||||
|
): string | null {
|
||||||
|
return ADZUNA_COUNTRY_CODE_BY_KEY[normalizeCountryKey(country)] ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
export function isSourceAllowedForCountry(
|
export function isSourceAllowedForCountry(
|
||||||
source: JobSource,
|
source: JobSource,
|
||||||
country: string | null | undefined,
|
country: string | null | undefined,
|
||||||
): boolean {
|
): boolean {
|
||||||
if (UK_ONLY_SOURCES.has(source)) return isUkCountry(country);
|
if (UK_ONLY_SOURCES.has(source)) return isUkCountry(country);
|
||||||
if (source === "glassdoor") return isGlassdoorCountry(country);
|
if (source === "glassdoor") return isGlassdoorCountry(country);
|
||||||
|
if (source === "adzuna") return getAdzunaCountryCode(country) !== null;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -32,6 +32,13 @@ export const updateSettingsSchema = z
|
|||||||
resumeProjects: resumeProjectsSchema.nullable().optional(),
|
resumeProjects: resumeProjectsSchema.nullable().optional(),
|
||||||
rxresumeBaseResumeId: z.string().trim().max(200).nullable().optional(),
|
rxresumeBaseResumeId: z.string().trim().max(200).nullable().optional(),
|
||||||
ukvisajobsMaxJobs: z.number().int().min(1).max(1000).nullable().optional(),
|
ukvisajobsMaxJobs: z.number().int().min(1).max(1000).nullable().optional(),
|
||||||
|
adzunaMaxJobsPerTerm: z
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.min(1)
|
||||||
|
.max(1000)
|
||||||
|
.nullable()
|
||||||
|
.optional(),
|
||||||
gradcrackerMaxJobsPerTerm: z
|
gradcrackerMaxJobsPerTerm: z
|
||||||
.number()
|
.number()
|
||||||
.int()
|
.int()
|
||||||
@ -64,6 +71,8 @@ export const updateSettingsSchema = z
|
|||||||
basicAuthPassword: z.string().trim().max(2000).nullable().optional(),
|
basicAuthPassword: z.string().trim().max(2000).nullable().optional(),
|
||||||
ukvisajobsEmail: z.string().trim().max(200).nullable().optional(),
|
ukvisajobsEmail: z.string().trim().max(200).nullable().optional(),
|
||||||
ukvisajobsPassword: z.string().trim().max(2000).nullable().optional(),
|
ukvisajobsPassword: z.string().trim().max(2000).nullable().optional(),
|
||||||
|
adzunaAppId: z.string().trim().max(200).nullable().optional(),
|
||||||
|
adzunaAppKey: z.string().trim().max(2000).nullable().optional(),
|
||||||
webhookSecret: z.string().trim().max(2000).nullable().optional(),
|
webhookSecret: z.string().trim().max(2000).nullable().optional(),
|
||||||
enableBasicAuth: z.boolean().optional(),
|
enableBasicAuth: z.boolean().optional(),
|
||||||
backupEnabled: z.boolean().nullable().optional(),
|
backupEnabled: z.boolean().nullable().optional(),
|
||||||
|
|||||||
@ -161,6 +161,9 @@ export const createAppSettings = (
|
|||||||
ukvisajobsMaxJobs: 50,
|
ukvisajobsMaxJobs: 50,
|
||||||
defaultUkvisajobsMaxJobs: 50,
|
defaultUkvisajobsMaxJobs: 50,
|
||||||
overrideUkvisajobsMaxJobs: null,
|
overrideUkvisajobsMaxJobs: null,
|
||||||
|
adzunaMaxJobsPerTerm: 50,
|
||||||
|
defaultAdzunaMaxJobsPerTerm: 50,
|
||||||
|
overrideAdzunaMaxJobsPerTerm: null,
|
||||||
gradcrackerMaxJobsPerTerm: 50,
|
gradcrackerMaxJobsPerTerm: 50,
|
||||||
defaultGradcrackerMaxJobsPerTerm: 50,
|
defaultGradcrackerMaxJobsPerTerm: 50,
|
||||||
overrideGradcrackerMaxJobsPerTerm: null,
|
overrideGradcrackerMaxJobsPerTerm: null,
|
||||||
@ -198,6 +201,8 @@ export const createAppSettings = (
|
|||||||
basicAuthPasswordHint: null,
|
basicAuthPasswordHint: null,
|
||||||
ukvisajobsEmail: null,
|
ukvisajobsEmail: null,
|
||||||
ukvisajobsPasswordHint: null,
|
ukvisajobsPasswordHint: null,
|
||||||
|
adzunaAppId: null,
|
||||||
|
adzunaAppKeyHint: null,
|
||||||
webhookSecretHint: null,
|
webhookSecretHint: null,
|
||||||
basicAuthActive: false,
|
basicAuthActive: false,
|
||||||
backupEnabled: false,
|
backupEnabled: false,
|
||||||
|
|||||||
@ -125,6 +125,7 @@ export type JobSource =
|
|||||||
| "linkedin"
|
| "linkedin"
|
||||||
| "glassdoor"
|
| "glassdoor"
|
||||||
| "ukvisajobs"
|
| "ukvisajobs"
|
||||||
|
| "adzuna"
|
||||||
| "manual";
|
| "manual";
|
||||||
|
|
||||||
export interface Job {
|
export interface Job {
|
||||||
@ -895,6 +896,9 @@ export interface AppSettings {
|
|||||||
ukvisajobsMaxJobs: number;
|
ukvisajobsMaxJobs: number;
|
||||||
defaultUkvisajobsMaxJobs: number;
|
defaultUkvisajobsMaxJobs: number;
|
||||||
overrideUkvisajobsMaxJobs: number | null;
|
overrideUkvisajobsMaxJobs: number | null;
|
||||||
|
adzunaMaxJobsPerTerm: number;
|
||||||
|
defaultAdzunaMaxJobsPerTerm: number;
|
||||||
|
overrideAdzunaMaxJobsPerTerm: number | null;
|
||||||
gradcrackerMaxJobsPerTerm: number;
|
gradcrackerMaxJobsPerTerm: number;
|
||||||
defaultGradcrackerMaxJobsPerTerm: number;
|
defaultGradcrackerMaxJobsPerTerm: number;
|
||||||
overrideGradcrackerMaxJobsPerTerm: number | null;
|
overrideGradcrackerMaxJobsPerTerm: number | null;
|
||||||
@ -932,6 +936,8 @@ export interface AppSettings {
|
|||||||
basicAuthPasswordHint: string | null;
|
basicAuthPasswordHint: string | null;
|
||||||
ukvisajobsEmail: string | null;
|
ukvisajobsEmail: string | null;
|
||||||
ukvisajobsPasswordHint: string | null;
|
ukvisajobsPasswordHint: string | null;
|
||||||
|
adzunaAppId: string | null;
|
||||||
|
adzunaAppKeyHint: string | null;
|
||||||
webhookSecretHint: string | null;
|
webhookSecretHint: string | null;
|
||||||
basicAuthActive: boolean;
|
basicAuthActive: boolean;
|
||||||
// Backup settings
|
// Backup settings
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user