feat: add Adzuna extractor with orchestrator integration (#177)
* feat(settings): add adzuna source fields and country compatibility * feat(discovery): integrate adzuna extractor into pipeline * feat(client): wire adzuna in source selection and run budgeting * docs(extractors): add adzuna guide and configuration notes * chore(workspaces): register adzuna extractor in lockfile * fix(adzuna): run extractor via npm script instead of npx * fix(adzuna): execute extractor via node+tsx without shell * fix(adzuna): prefer npm run start without shell, fallback to tsx * fix(docker): include adzuna extractor workspace in image * chore(adzuna): reuse shared type-conversion utilities * type-check adzuna * formatting * deeedooop * better instructions
This commit is contained in:
parent
4da264eb48
commit
c5c6675f04
@ -39,6 +39,15 @@ UKVISAJOBS_EMAIL=
|
||||
UKVISAJOBS_PASSWORD=
|
||||
UKVISAJOBS_HEADLESS=true
|
||||
|
||||
# =============================================================================
|
||||
# Adzuna (multi-country API source) - optional
|
||||
# =============================================================================
|
||||
# App credentials from Adzuna developer account.
|
||||
ADZUNA_APP_ID=
|
||||
ADZUNA_APP_KEY=
|
||||
# Optional default per-term cap (can be overridden by UI run budget logic).
|
||||
# ADZUNA_MAX_JOBS_PER_TERM=50
|
||||
|
||||
# =============================================================================
|
||||
# JobSpy - Job search configuration
|
||||
# =============================================================================
|
||||
|
||||
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@ -49,7 +49,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
project: [orchestrator, gradcracker-extractor, ukvisajobs-extractor]
|
||||
project:
|
||||
- orchestrator
|
||||
- adzuna-extractor
|
||||
- gradcracker-extractor
|
||||
- ukvisajobs-extractor
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Setup Node
|
||||
|
||||
@ -35,6 +35,7 @@ COPY package*.json ./
|
||||
COPY docs-site/package*.json ./docs-site/
|
||||
COPY shared/package*.json ./shared/
|
||||
COPY orchestrator/package*.json ./orchestrator/
|
||||
COPY extractors/adzuna/package*.json ./extractors/adzuna/
|
||||
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
||||
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
||||
|
||||
@ -52,6 +53,7 @@ WORKDIR /app
|
||||
COPY shared ./shared
|
||||
COPY docs-site ./docs-site
|
||||
COPY orchestrator ./orchestrator
|
||||
COPY extractors/adzuna ./extractors/adzuna
|
||||
COPY extractors/gradcracker ./extractors/gradcracker
|
||||
COPY extractors/jobspy ./extractors/jobspy
|
||||
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
||||
@ -97,6 +99,7 @@ COPY package*.json ./
|
||||
COPY docs-site/package*.json ./docs-site/
|
||||
COPY shared/package*.json ./shared/
|
||||
COPY orchestrator/package*.json ./orchestrator/
|
||||
COPY extractors/adzuna/package*.json ./extractors/adzuna/
|
||||
COPY extractors/gradcracker/package*.json ./extractors/gradcracker/
|
||||
COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/
|
||||
|
||||
@ -110,6 +113,7 @@ COPY --from=builder /app/orchestrator/dist ./orchestrator/dist
|
||||
COPY --from=builder /app/docs-site/build ./orchestrator/dist/docs
|
||||
COPY shared ./shared
|
||||
COPY orchestrator ./orchestrator
|
||||
COPY extractors/adzuna ./extractors/adzuna
|
||||
COPY extractors/gradcracker ./extractors/gradcracker
|
||||
COPY extractors/jobspy ./extractors/jobspy
|
||||
COPY extractors/ukvisajobs ./extractors/ukvisajobs
|
||||
|
||||
@ -60,7 +60,7 @@ docker compose up -d
|
||||
|
||||
## Why JobOps?
|
||||
|
||||
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor** + specialized boards (Gradcracker, UK Visa Jobs).
|
||||
* **Universal Scraping**: Supports **LinkedIn, Indeed, Glassdoor, Adzuna** + specialized boards (Gradcracker, UK Visa Jobs).
|
||||
* **AI Scoring**: Ranks jobs by fit against *your* profile using your preferred LLM (OpenRouter/OpenAI/Gemini).
|
||||
* **Auto-Tailoring**: Generates custom resumes (PDFs) for every application using RxResume v4.
|
||||
* **Email Tracking**: Connect Gmail to auto-detect interviews, offers, and rejections.
|
||||
@ -81,6 +81,7 @@ docker compose up -d
|
||||
| **LinkedIn** | Global / General |
|
||||
| **Indeed** | Global / General |
|
||||
| **Glassdoor** | Global / General |
|
||||
| **Adzuna** | Multi-country API source |
|
||||
| **Gradcracker** | STEM / Grads (UK) |
|
||||
| **UK Visa Jobs** | Sponsorship (UK) |
|
||||
|
||||
|
||||
58
docs-site/docs/extractors/adzuna.md
Normal file
58
docs-site/docs/extractors/adzuna.md
Normal file
@ -0,0 +1,58 @@
|
||||
---
|
||||
id: adzuna
|
||||
title: Adzuna Extractor
|
||||
description: API-based Adzuna extraction with orchestrator ingestion and progress updates.
|
||||
sidebar_position: 6
|
||||
---
|
||||
|
||||
## What it is
|
||||
|
||||
Adzuna is an API-backed extractor implemented in two lean pieces:
|
||||
|
||||
1. `extractors/adzuna/src/main.ts` fetches paginated Adzuna search results and writes `jobs.json`.
|
||||
2. `orchestrator/src/server/services/adzuna.ts` runs the extractor, parses progress lines, and maps rows into `CreateJobInput`.
|
||||
|
||||
It de-duplicates in the existing repository path using `sourceJobId` fallback to `jobUrl`.
|
||||
|
||||
## Why it exists
|
||||
|
||||
Adzuna provides stable API discovery for countries that are not covered by UK-only sources. It adds a lower-maintenance source without introducing new API routes or UI sections.
|
||||
|
||||
## How to use it
|
||||
|
||||
1. Create an Adzuna developer account.
|
||||
2. Open [Adzuna Access Details](https://developer.adzuna.com/admin/access_details).
|
||||
3. Copy your **App ID** and **App Key**.
|
||||
4. In Job Ops, open **Settings** and paste them into `Adzuna App ID` and `Adzuna App Key` under **Environment & Accounts**.
|
||||
5. In **Pipeline Run** (Automatic tab), select a compatible country and enable **Adzuna** in Sources.
|
||||
6. Start the run; Adzuna progress appears in the existing crawl progress stream.
|
||||
|
||||
Default controls:
|
||||
|
||||
- `ADZUNA_APP_ID`
|
||||
- `ADZUNA_APP_KEY`
|
||||
- `ADZUNA_MAX_JOBS_PER_TERM` (default `50`)
|
||||
|
||||
Supported countries in this integration:
|
||||
|
||||
- United Kingdom, United States, Austria, Australia, Belgium, Brazil, Canada, Switzerland, Germany, Spain, France, India, Italy, Mexico, Netherlands, New Zealand, Poland, Singapore, South Africa.
|
||||
|
||||
## Common problems
|
||||
|
||||
### Adzuna is disabled in source selection
|
||||
|
||||
- `Adzuna App ID` and `Adzuna App Key` are missing from Settings (or env).
|
||||
|
||||
### Adzuna is skipped for my selected country
|
||||
|
||||
- The selected country is not in the supported list above.
|
||||
|
||||
### Adzuna fails with authorization errors
|
||||
|
||||
- Verify `ADZUNA_APP_ID` and `ADZUNA_APP_KEY` are valid and active in your Adzuna account.
|
||||
|
||||
## Related pages
|
||||
|
||||
- [Extractors Overview](/docs/next/extractors/overview)
|
||||
- [Pipeline Run](/docs/next/features/pipeline-run)
|
||||
- [Settings](/docs/next/features/settings)
|
||||
@ -13,12 +13,14 @@ This page helps you choose the right extractor for your run, understand key cons
|
||||
| --- | --- | --- | --- | --- |
|
||||
| [Gradcracker](/docs/next/extractors/gradcracker) | UK graduate roles from Gradcracker | Crawling stability depends on page structure and anti-bot behavior; tuned for low concurrency | `GRADCRACKER_SEARCH_TERMS`, `GRADCRACKER_MAX_JOBS_PER_TERM`, `JOBOPS_SKIP_APPLY_FOR_EXISTING` | Scrapes listing metadata, then detail pages and apply URL resolution |
|
||||
| [JobSpy](/docs/next/extractors/jobspy) | Multi-source discovery (Indeed, LinkedIn, Glassdoor) | Requires Python wrapper execution per term; source availability and quality vary by site/location | `JOBSPY_SITES`, `JOBSPY_SEARCH_TERMS`, `JOBSPY_RESULTS_WANTED`, `JOBSPY_HOURS_OLD`, `JOBSPY_LINKEDIN_FETCH_DESCRIPTION` | Produces JSON per term, then orchestrator normalizes and de-duplicates by `jobUrl` |
|
||||
| [Adzuna](/docs/next/extractors/adzuna) | API-based multi-country discovery with low scraping overhead | Requires valid App ID/App Key; country must be in Adzuna-supported list | `ADZUNA_APP_ID`, `ADZUNA_APP_KEY`, `ADZUNA_MAX_JOBS_PER_TERM` | API pagination to dataset output; orchestrator maps progress and de-duplicates by `sourceJobId`/`jobUrl` |
|
||||
| [UKVisaJobs](/docs/next/extractors/ukvisajobs) | UK visa sponsorship-focused roles | Requires authenticated session and periodic token/cookie refresh | `UKVISAJOBS_EMAIL`, `UKVISAJOBS_PASSWORD`, `UKVISAJOBS_MAX_JOBS`, `UKVISAJOBS_SEARCH_KEYWORD` | API pagination + dataset output; orchestrator de-dupes and may fetch missing descriptions |
|
||||
| [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review |
|
||||
|
||||
## Which extractor should I use?
|
||||
|
||||
- Use **JobSpy** for broad first-pass sourcing across common boards.
|
||||
- Use **Adzuna** when you want API-first discovery in supported non-UK markets.
|
||||
- Use **Gradcracker** when targeting graduate pipelines in the UK.
|
||||
- Use **UKVisaJobs** for sponsorship-specific UK searches.
|
||||
- Use **Manual Import** when you already have a specific posting and need direct import.
|
||||
@ -29,5 +31,6 @@ Many runs combine sources: broad discovery first, then manual import for high-pr
|
||||
|
||||
- [Gradcracker](/docs/next/extractors/gradcracker)
|
||||
- [JobSpy](/docs/next/extractors/jobspy)
|
||||
- [Adzuna](/docs/next/extractors/adzuna)
|
||||
- [UKVisaJobs](/docs/next/extractors/ukvisajobs)
|
||||
- [Manual Import](/docs/next/extractors/manual)
|
||||
|
||||
@ -47,6 +47,7 @@ If values are edited manually, the UI shows **Custom**.
|
||||
|
||||
- Country selection affects which sources are available.
|
||||
- UK-only sources are disabled for non-UK countries.
|
||||
- Adzuna is available only for its supported countries and when App ID/App Key are configured in Settings.
|
||||
- Glassdoor can be enabled only when:
|
||||
- selected country supports Glassdoor
|
||||
- a **Glassdoor city** is set in Advanced settings
|
||||
@ -98,6 +99,11 @@ For accepted input formats, inference behavior, and limits, see [Manual Import E
|
||||
- Verify selected country supports Glassdoor.
|
||||
- Set a Glassdoor city in Advanced settings.
|
||||
|
||||
### Adzuna is not selectable
|
||||
|
||||
- Set `Adzuna App ID` and `Adzuna App Key` in **Settings > Environment & Accounts**.
|
||||
- Verify the selected country is one of Adzuna's supported markets.
|
||||
|
||||
### Run takes longer than expected
|
||||
|
||||
- Reduce term count.
|
||||
|
||||
@ -86,6 +86,7 @@ Settings gives you runtime overrides for the key parts of discovery, scoring, ta
|
||||
- Configure service accounts:
|
||||
- RxResume email/password
|
||||
- UKVisaJobs email/password
|
||||
- Adzuna app ID/app key
|
||||
- Optional basic authentication for write operations
|
||||
|
||||
### Backup
|
||||
|
||||
@ -41,6 +41,7 @@ const sidebars: SidebarsConfig = {
|
||||
"extractors/overview",
|
||||
"extractors/gradcracker",
|
||||
"extractors/jobspy",
|
||||
"extractors/adzuna",
|
||||
"extractors/manual",
|
||||
"extractors/ukvisajobs",
|
||||
],
|
||||
|
||||
15
extractors/adzuna/README.md
Normal file
15
extractors/adzuna/README.md
Normal file
@ -0,0 +1,15 @@
|
||||
# Adzuna Extractor
|
||||
|
||||
Minimal extractor that pulls jobs from Adzuna's search API and writes a dataset
|
||||
for orchestrator ingestion.
|
||||
|
||||
## Environment
|
||||
|
||||
- `ADZUNA_APP_ID` (required)
|
||||
- `ADZUNA_APP_KEY` (required)
|
||||
- `ADZUNA_COUNTRY` (default: `gb`)
|
||||
- `ADZUNA_SEARCH_TERMS` (JSON array or `|` / comma / newline-delimited)
|
||||
- `ADZUNA_MAX_JOBS_PER_TERM` (default: `50`)
|
||||
- `ADZUNA_RESULTS_PER_PAGE` (default: `50`, max `50`)
|
||||
- `ADZUNA_OUTPUT_JSON` (default: `storage/datasets/default/jobs.json`)
|
||||
- `JOBOPS_EMIT_PROGRESS=1` to emit `JOBOPS_PROGRESS` events
|
||||
20
extractors/adzuna/package.json
Normal file
20
extractors/adzuna/package.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "adzuna-extractor",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"description": "Adzuna extractor - fetches jobs from Adzuna search API",
|
||||
"main": "src/main.ts",
|
||||
"dependencies": {
|
||||
"job-ops-shared": "^1.0.0",
|
||||
"tsx": "^4.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
"typescript": "~5.9.0"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "tsx src/main.ts",
|
||||
"start:dev": "tsx src/main.ts",
|
||||
"check:types": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
245
extractors/adzuna/src/main.ts
Normal file
245
extractors/adzuna/src/main.ts
Normal file
@ -0,0 +1,245 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import {
|
||||
toNumberOrNull,
|
||||
toStringOrNull,
|
||||
} from "job-ops-shared/utils/type-conversion";
|
||||
|
||||
const API_BASE = "https://api.adzuna.com/v1/api";
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
|
||||
type AdzunaCompany = { display_name?: unknown };
|
||||
type AdzunaLocation = { display_name?: unknown };
|
||||
type AdzunaJob = {
|
||||
id?: unknown;
|
||||
title?: unknown;
|
||||
description?: unknown;
|
||||
created?: unknown;
|
||||
redirect_url?: unknown;
|
||||
company?: AdzunaCompany;
|
||||
location?: AdzunaLocation;
|
||||
salary_min?: unknown;
|
||||
salary_max?: unknown;
|
||||
contract_time?: unknown;
|
||||
contract_type?: unknown;
|
||||
};
|
||||
|
||||
type ExtractedJob = {
|
||||
source: "adzuna";
|
||||
sourceJobId?: string;
|
||||
title: string;
|
||||
employer: string;
|
||||
jobUrl: string;
|
||||
applicationLink: string;
|
||||
location?: string;
|
||||
salary?: string;
|
||||
datePosted?: string;
|
||||
jobDescription?: string;
|
||||
jobType?: string;
|
||||
};
|
||||
|
||||
function parsePositiveInt(input: string | undefined, fallback: number): number {
|
||||
const parsed = input ? Number.parseInt(input, 10) : Number.NaN;
|
||||
if (!Number.isFinite(parsed) || parsed < 1) return fallback;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseSearchTerms(raw: string | undefined): string[] {
|
||||
if (!raw || raw.trim().length === 0) return ["web developer"];
|
||||
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown;
|
||||
if (Array.isArray(parsed)) {
|
||||
const terms = parsed
|
||||
.map((value) => toStringOrNull(value))
|
||||
.filter((value): value is string => value !== null);
|
||||
if (terms.length > 0) return terms;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to delimiter parsing.
|
||||
}
|
||||
}
|
||||
|
||||
const delimiter = trimmed.includes("|")
|
||||
? "|"
|
||||
: trimmed.includes("\n")
|
||||
? "\n"
|
||||
: ",";
|
||||
const terms = trimmed
|
||||
.split(delimiter)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
return terms.length > 0 ? terms : ["web developer"];
|
||||
}
|
||||
|
||||
function requireEnv(name: string): string {
|
||||
const value = process.env[name]?.trim();
|
||||
if (!value) throw new Error(`Missing required environment variable: ${name}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function emitProgress(payload: Record<string, unknown>): void {
|
||||
if (process.env.JOBOPS_EMIT_PROGRESS !== "1") return;
|
||||
console.log(`${JOBOPS_PROGRESS_PREFIX}${JSON.stringify(payload)}`);
|
||||
}
|
||||
|
||||
function formatSalary(job: AdzunaJob): string | null {
|
||||
const min = toNumberOrNull(job.salary_min);
|
||||
const max = toNumberOrNull(job.salary_max);
|
||||
|
||||
if (min === null && max === null) return null;
|
||||
|
||||
if (min !== null && max !== null) {
|
||||
return `${Math.round(min)}-${Math.round(max)}`;
|
||||
}
|
||||
if (min !== null) return `${Math.round(min)}+`;
|
||||
if (max !== null) return `${Math.round(max)}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapJob(raw: AdzunaJob): ExtractedJob | null {
|
||||
const id = toStringOrNull(raw.id);
|
||||
const title = toStringOrNull(raw.title) ?? "Unknown Title";
|
||||
const employer =
|
||||
toStringOrNull(raw.company?.display_name) ?? "Unknown Employer";
|
||||
const jobUrl = toStringOrNull(raw.redirect_url);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
const contractType = toStringOrNull(raw.contract_type);
|
||||
const contractTime = toStringOrNull(raw.contract_time);
|
||||
const jobType = [contractType, contractTime].filter(Boolean).join(" / ");
|
||||
|
||||
return {
|
||||
source: "adzuna",
|
||||
sourceJobId: id ?? undefined,
|
||||
title,
|
||||
employer,
|
||||
jobUrl,
|
||||
applicationLink: jobUrl,
|
||||
location: toStringOrNull(raw.location?.display_name) ?? undefined,
|
||||
salary: formatSalary(raw) ?? undefined,
|
||||
datePosted: toStringOrNull(raw.created) ?? undefined,
|
||||
jobDescription: toStringOrNull(raw.description) ?? undefined,
|
||||
jobType: jobType || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchJobsPage(args: {
|
||||
country: string;
|
||||
page: number;
|
||||
appId: string;
|
||||
appKey: string;
|
||||
what: string;
|
||||
resultsPerPage: number;
|
||||
}): Promise<AdzunaJob[]> {
|
||||
const url = new URL(`${API_BASE}/jobs/${args.country}/search/${args.page}`);
|
||||
url.searchParams.set("app_id", args.appId);
|
||||
url.searchParams.set("app_key", args.appKey);
|
||||
if (args.what) {
|
||||
url.searchParams.set("what", args.what);
|
||||
}
|
||||
url.searchParams.set("results_per_page", String(args.resultsPerPage));
|
||||
|
||||
const response = await fetch(url.toString(), {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Adzuna request failed with status ${response.status}`);
|
||||
}
|
||||
|
||||
const body = (await response.json()) as { results?: unknown };
|
||||
if (!Array.isArray(body.results)) return [];
|
||||
return body.results as AdzunaJob[];
|
||||
}
|
||||
|
||||
async function run(): Promise<void> {
|
||||
const appId = requireEnv("ADZUNA_APP_ID");
|
||||
const appKey = requireEnv("ADZUNA_APP_KEY");
|
||||
const country = (process.env.ADZUNA_COUNTRY || "gb").trim().toLowerCase();
|
||||
const maxJobsPerTerm = parsePositiveInt(
|
||||
process.env.ADZUNA_MAX_JOBS_PER_TERM,
|
||||
50,
|
||||
);
|
||||
const configuredResultsPerPage = parsePositiveInt(
|
||||
process.env.ADZUNA_RESULTS_PER_PAGE,
|
||||
50,
|
||||
);
|
||||
const resultsPerPage = Math.min(50, configuredResultsPerPage);
|
||||
const searchTerms = parseSearchTerms(process.env.ADZUNA_SEARCH_TERMS);
|
||||
const outputJson =
|
||||
process.env.ADZUNA_OUTPUT_JSON ||
|
||||
join(process.cwd(), "storage/datasets/default/jobs.json");
|
||||
|
||||
const jobs: ExtractedJob[] = [];
|
||||
|
||||
for (let i = 0; i < searchTerms.length; i += 1) {
|
||||
const searchTerm = searchTerms[i];
|
||||
const termIndex = i + 1;
|
||||
|
||||
emitProgress({
|
||||
event: "term_start",
|
||||
termIndex,
|
||||
termTotal: searchTerms.length,
|
||||
searchTerm,
|
||||
});
|
||||
|
||||
let page = 1;
|
||||
let termCount = 0;
|
||||
while (termCount < maxJobsPerTerm) {
|
||||
const remaining = maxJobsPerTerm - termCount;
|
||||
const take = Math.min(resultsPerPage, remaining);
|
||||
const pageResults = await fetchJobsPage({
|
||||
country,
|
||||
page,
|
||||
appId,
|
||||
appKey,
|
||||
what: searchTerm,
|
||||
resultsPerPage: take,
|
||||
});
|
||||
|
||||
let mappedOnPage = 0;
|
||||
for (const raw of pageResults) {
|
||||
if (termCount >= maxJobsPerTerm) break;
|
||||
const mapped = mapJob(raw);
|
||||
if (!mapped) continue;
|
||||
jobs.push(mapped);
|
||||
termCount += 1;
|
||||
mappedOnPage += 1;
|
||||
}
|
||||
|
||||
emitProgress({
|
||||
event: "page_fetched",
|
||||
termIndex,
|
||||
termTotal: searchTerms.length,
|
||||
searchTerm,
|
||||
pageNo: page,
|
||||
resultsOnPage: mappedOnPage,
|
||||
totalCollected: termCount,
|
||||
});
|
||||
|
||||
if (pageResults.length < take) break;
|
||||
page += 1;
|
||||
if (page > 100) break;
|
||||
}
|
||||
|
||||
emitProgress({
|
||||
event: "term_complete",
|
||||
termIndex,
|
||||
termTotal: searchTerms.length,
|
||||
searchTerm,
|
||||
jobsFoundTerm: termCount,
|
||||
});
|
||||
}
|
||||
|
||||
await mkdir(dirname(outputJson), { recursive: true });
|
||||
await writeFile(outputJson, `${JSON.stringify(jobs, null, 2)}\n`, "utf-8");
|
||||
console.log(`Adzuna extractor wrote ${jobs.length} jobs`);
|
||||
}
|
||||
|
||||
run().catch((error: unknown) => {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
console.error(`Adzuna extractor failed: ${message}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
13
extractors/adzuna/tsconfig.json
Normal file
13
extractors/adzuna/tsconfig.json
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"strict": true,
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["ES2022", "DOM"],
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
@ -24,7 +24,7 @@ interface PipelineProgress {
|
||||
| "failed";
|
||||
message: string;
|
||||
detail?: string;
|
||||
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | null;
|
||||
crawlingSource: "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna" | null;
|
||||
crawlingSourcesCompleted: number;
|
||||
crawlingSourcesTotal: number;
|
||||
crawlingTermsProcessed: number;
|
||||
@ -84,6 +84,7 @@ const sourceLabel: Record<
|
||||
gradcracker: "Gradcracker",
|
||||
jobspy: "JobSpy",
|
||||
ukvisajobs: "UKVisaJobs",
|
||||
adzuna: "Adzuna",
|
||||
};
|
||||
|
||||
const clamp = (value: number, min: number, max: number) =>
|
||||
|
||||
@ -695,6 +695,7 @@ describe("OrchestratorPage", () => {
|
||||
jobspyResultsWanted: 150,
|
||||
gradcrackerMaxJobsPerTerm: 150,
|
||||
ukvisajobsMaxJobs: 150,
|
||||
adzunaMaxJobsPerTerm: 150,
|
||||
jobspyCountryIndeed: "united kingdom",
|
||||
jobspyLocation: "United Kingdom",
|
||||
});
|
||||
|
||||
@ -294,6 +294,7 @@ export const OrchestratorPage: React.FC = () => {
|
||||
jobspyResultsWanted: limits.jobspyResultsWanted,
|
||||
gradcrackerMaxJobsPerTerm: limits.gradcrackerMaxJobsPerTerm,
|
||||
ukvisajobsMaxJobs: limits.ukvisajobsMaxJobs,
|
||||
adzunaMaxJobsPerTerm: limits.adzunaMaxJobsPerTerm,
|
||||
jobspyCountryIndeed: values.country,
|
||||
jobspyLocation,
|
||||
});
|
||||
|
||||
@ -57,6 +57,8 @@ const DEFAULT_FORM_VALUES: UpdateSettingsInput = {
|
||||
basicAuthPassword: "",
|
||||
ukvisajobsEmail: "",
|
||||
ukvisajobsPassword: "",
|
||||
adzunaAppId: "",
|
||||
adzunaAppKey: "",
|
||||
webhookSecret: "",
|
||||
enableBasicAuth: false,
|
||||
backupEnabled: null,
|
||||
@ -96,6 +98,9 @@ const NULL_SETTINGS_PAYLOAD: UpdateSettingsInput = {
|
||||
basicAuthPassword: null,
|
||||
ukvisajobsEmail: null,
|
||||
ukvisajobsPassword: null,
|
||||
adzunaAppId: null,
|
||||
adzunaAppKey: null,
|
||||
adzunaMaxJobsPerTerm: null,
|
||||
webhookSecret: null,
|
||||
enableBasicAuth: undefined,
|
||||
backupEnabled: null,
|
||||
@ -129,6 +134,8 @@ const mapSettingsToForm = (data: AppSettings): UpdateSettingsInput => ({
|
||||
basicAuthPassword: "",
|
||||
ukvisajobsEmail: data.ukvisajobsEmail ?? "",
|
||||
ukvisajobsPassword: "",
|
||||
adzunaAppId: data.adzunaAppId ?? "",
|
||||
adzunaAppKey: "",
|
||||
webhookSecret: "",
|
||||
enableBasicAuth: data.basicAuthActive,
|
||||
backupEnabled: data.overrideBackupEnabled,
|
||||
@ -235,11 +242,13 @@ const getDerivedSettings = (settings: AppSettings | null) => {
|
||||
readable: {
|
||||
rxresumeEmail: settings?.rxresumeEmail ?? "",
|
||||
ukvisajobsEmail: settings?.ukvisajobsEmail ?? "",
|
||||
adzunaAppId: settings?.adzunaAppId ?? "",
|
||||
basicAuthUser: settings?.basicAuthUser ?? "",
|
||||
},
|
||||
private: {
|
||||
rxresumePasswordHint: settings?.rxresumePasswordHint ?? null,
|
||||
ukvisajobsPasswordHint: settings?.ukvisajobsPasswordHint ?? null,
|
||||
adzunaAppKeyHint: settings?.adzunaAppKeyHint ?? null,
|
||||
basicAuthPasswordHint: settings?.basicAuthPasswordHint ?? null,
|
||||
webhookSecretHint: settings?.webhookSecretHint ?? null,
|
||||
},
|
||||
@ -527,6 +536,10 @@ export const SettingsPage: React.FC = () => {
|
||||
envPayload.ukvisajobsEmail = normalizeString(data.ukvisajobsEmail);
|
||||
}
|
||||
|
||||
if (dirtyFields.adzunaAppId || dirtyFields.adzunaAppKey) {
|
||||
envPayload.adzunaAppId = normalizeString(data.adzunaAppId);
|
||||
}
|
||||
|
||||
if (data.enableBasicAuth === false) {
|
||||
envPayload.basicAuthUser = null;
|
||||
envPayload.basicAuthPassword = null;
|
||||
@ -568,6 +581,11 @@ export const SettingsPage: React.FC = () => {
|
||||
if (value !== undefined) envPayload.ukvisajobsPassword = value;
|
||||
}
|
||||
|
||||
if (dirtyFields.adzunaAppKey) {
|
||||
const value = normalizePrivateInput(data.adzunaAppKey);
|
||||
if (value !== undefined) envPayload.adzunaAppKey = value;
|
||||
}
|
||||
|
||||
if (dirtyFields.webhookSecret) {
|
||||
const value = normalizePrivateInput(data.webhookSecret);
|
||||
if (value !== undefined) envPayload.webhookSecret = value;
|
||||
|
||||
@ -153,6 +153,7 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
|
||||
|
||||
const rememberedRunBudget =
|
||||
settings?.jobspyResultsWanted ??
|
||||
settings?.adzunaMaxJobsPerTerm ??
|
||||
settings?.gradcrackerMaxJobsPerTerm ??
|
||||
settings?.ukvisajobsMaxJobs ??
|
||||
DEFAULT_VALUES.runBudget;
|
||||
@ -533,7 +534,9 @@ export const AutomaticRunTab: React.FC<AutomaticRunTabProps> = ({
|
||||
? countryAllowed
|
||||
? GLASSDOOR_LOCATION_REASON
|
||||
: GLASSDOOR_COUNTRY_REASON
|
||||
: `${sourceLabel[source]} is available only when country is United Kingdom.`;
|
||||
: source === "gradcracker" || source === "ukvisajobs"
|
||||
? `${sourceLabel[source]} is available only when country is United Kingdom.`
|
||||
: `${sourceLabel[source]} is not available for the selected country.`;
|
||||
|
||||
const button = (
|
||||
<Button
|
||||
|
||||
@ -76,4 +76,20 @@ describe("automatic-run utilities", () => {
|
||||
"api",
|
||||
]);
|
||||
});
|
||||
|
||||
it("includes adzuna in estimate caps", () => {
|
||||
const estimate = calculateAutomaticEstimate({
|
||||
values: {
|
||||
topN: 10,
|
||||
minSuitabilityScore: 50,
|
||||
searchTerms: ["backend", "platform"],
|
||||
runBudget: 120,
|
||||
country: "united kingdom",
|
||||
},
|
||||
sources: ["adzuna"],
|
||||
});
|
||||
|
||||
expect(estimate.discovered.cap).toBeGreaterThan(0);
|
||||
expect(estimate.discovered.cap).toBeLessThanOrEqual(120);
|
||||
});
|
||||
});
|
||||
|
||||
@ -61,6 +61,7 @@ export interface ExtractorLimits {
|
||||
jobspyResultsWanted: number;
|
||||
gradcrackerMaxJobsPerTerm: number;
|
||||
ukvisajobsMaxJobs: number;
|
||||
adzunaMaxJobsPerTerm: number;
|
||||
}
|
||||
|
||||
export function deriveExtractorLimits(args: {
|
||||
@ -75,19 +76,22 @@ export function deriveExtractorLimits(args: {
|
||||
const includesGlassdoor = args.sources.includes("glassdoor");
|
||||
const includesGradcracker = args.sources.includes("gradcracker");
|
||||
const includesUkVisaJobs = args.sources.includes("ukvisajobs");
|
||||
const includesAdzuna = args.sources.includes("adzuna");
|
||||
|
||||
const weightedContributors =
|
||||
(includesIndeed ? termCount : 0) +
|
||||
(includesLinkedIn ? termCount : 0) +
|
||||
(includesGlassdoor ? termCount : 0) +
|
||||
(includesGradcracker ? termCount : 0) +
|
||||
(includesUkVisaJobs ? 1 : 0);
|
||||
(includesUkVisaJobs ? 1 : 0) +
|
||||
(includesAdzuna ? termCount : 0);
|
||||
|
||||
if (weightedContributors <= 0) {
|
||||
return {
|
||||
jobspyResultsWanted: budget,
|
||||
gradcrackerMaxJobsPerTerm: budget,
|
||||
ukvisajobsMaxJobs: budget,
|
||||
adzunaMaxJobsPerTerm: budget,
|
||||
};
|
||||
}
|
||||
|
||||
@ -98,6 +102,7 @@ export function deriveExtractorLimits(args: {
|
||||
jobspyResultsWanted: perUnit,
|
||||
gradcrackerMaxJobsPerTerm: perUnit,
|
||||
ukvisajobsMaxJobs: Math.min(budget, perUnit + remainder),
|
||||
adzunaMaxJobsPerTerm: perUnit,
|
||||
};
|
||||
}
|
||||
|
||||
@ -137,6 +142,7 @@ export function calculateAutomaticEstimate(args: {
|
||||
const hasIndeed = sources.includes("indeed");
|
||||
const hasLinkedIn = sources.includes("linkedin");
|
||||
const hasGlassdoor = sources.includes("glassdoor");
|
||||
const hasAdzuna = sources.includes("adzuna");
|
||||
const limits = deriveExtractorLimits({
|
||||
budget: values.runBudget,
|
||||
searchTerms: values.searchTerms,
|
||||
@ -151,8 +157,9 @@ export function calculateAutomaticEstimate(args: {
|
||||
? limits.gradcrackerMaxJobsPerTerm * termCount
|
||||
: 0;
|
||||
const ukvisaCap = hasUkVisaJobs ? limits.ukvisajobsMaxJobs : 0;
|
||||
const adzunaCap = hasAdzuna ? limits.adzunaMaxJobsPerTerm * termCount : 0;
|
||||
|
||||
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap;
|
||||
const discoveredCap = jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap;
|
||||
const discoveredMin = Math.round(discoveredCap * 0.35);
|
||||
const discoveredMax = Math.round(discoveredCap * 0.75);
|
||||
const processedMin = Math.min(values.topN, discoveredMin);
|
||||
|
||||
@ -13,6 +13,7 @@ export const orderedSources: JobSource[] = [
|
||||
"indeed",
|
||||
"linkedin",
|
||||
"glassdoor",
|
||||
"adzuna",
|
||||
"ukvisajobs",
|
||||
];
|
||||
export const orderedFilterSources: JobSource[] = [...orderedSources, "manual"];
|
||||
|
||||
19
orchestrator/src/client/pages/orchestrator/utils.test.ts
Normal file
19
orchestrator/src/client/pages/orchestrator/utils.test.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import { createAppSettings } from "@shared/testing/factories.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getEnabledSources } from "./utils";
|
||||
|
||||
describe("orchestrator utils", () => {
|
||||
it("enables adzuna only when both app id and key are configured", () => {
|
||||
const withCreds = createAppSettings({
|
||||
adzunaAppId: "app-id",
|
||||
adzunaAppKeyHint: "key-",
|
||||
});
|
||||
const withoutKey = createAppSettings({
|
||||
adzunaAppId: "app-id",
|
||||
adzunaAppKeyHint: null,
|
||||
});
|
||||
|
||||
expect(getEnabledSources(withCreds)).toContain("adzuna");
|
||||
expect(getEnabledSources(withoutKey)).not.toContain("adzuna");
|
||||
});
|
||||
});
|
||||
@ -171,6 +171,9 @@ export const getEnabledSources = (
|
||||
const hasUkVisaJobsAuth = Boolean(
|
||||
settings.ukvisajobsEmail?.trim() && settings.ukvisajobsPasswordHint,
|
||||
);
|
||||
const hasAdzunaAuth = Boolean(
|
||||
settings.adzunaAppId?.trim() && settings.adzunaAppKeyHint,
|
||||
);
|
||||
|
||||
for (const source of orderedSources) {
|
||||
if (source === "gradcracker") {
|
||||
@ -181,6 +184,10 @@ export const getEnabledSources = (
|
||||
if (hasUkVisaJobsAuth) enabled.push(source);
|
||||
continue;
|
||||
}
|
||||
if (source === "adzuna") {
|
||||
if (hasAdzunaAuth) enabled.push(source);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
source === "indeed" ||
|
||||
source === "linkedin" ||
|
||||
|
||||
@ -12,6 +12,8 @@ const EnvironmentSettingsHarness = () => {
|
||||
basicAuthUser: "admin",
|
||||
rxresumePassword: "",
|
||||
ukvisajobsPassword: "",
|
||||
adzunaAppId: "adzuna-id",
|
||||
adzunaAppKey: "",
|
||||
basicAuthPassword: "",
|
||||
webhookSecret: "",
|
||||
enableBasicAuth: true,
|
||||
@ -26,11 +28,13 @@ const EnvironmentSettingsHarness = () => {
|
||||
readable: {
|
||||
rxresumeEmail: "resume@example.com",
|
||||
ukvisajobsEmail: "visa@example.com",
|
||||
adzunaAppId: "adzuna-id",
|
||||
basicAuthUser: "admin",
|
||||
},
|
||||
private: {
|
||||
rxresumePasswordHint: null,
|
||||
ukvisajobsPasswordHint: "pass",
|
||||
adzunaAppKeyHint: "adzu",
|
||||
basicAuthPasswordHint: "abcd",
|
||||
webhookSecretHint: "sec-",
|
||||
},
|
||||
@ -50,8 +54,10 @@ describe("EnvironmentSettingsSection", () => {
|
||||
|
||||
expect(screen.getByDisplayValue("resume@example.com")).toBeInTheDocument();
|
||||
expect(screen.getByDisplayValue("visa@example.com")).toBeInTheDocument();
|
||||
expect(screen.getByDisplayValue("adzuna-id")).toBeInTheDocument();
|
||||
|
||||
expect(screen.getByText(/pass\*{8}/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/adzu\*{8}/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/abcd\*{8}/)).toBeInTheDocument();
|
||||
expect(screen.getByText("Not set")).toBeInTheDocument();
|
||||
|
||||
|
||||
@ -91,6 +91,28 @@ export const EnvironmentSettingsSection: React.FC<
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-4">
|
||||
<div className="text-sm font-semibold">Adzuna</div>
|
||||
<div className="grid gap-4 md:grid-cols-2">
|
||||
<SettingsInput
|
||||
label="App ID"
|
||||
inputProps={register("adzunaAppId")}
|
||||
placeholder="your-app-id"
|
||||
disabled={isLoading || isSaving}
|
||||
error={errors.adzunaAppId?.message as string | undefined}
|
||||
/>
|
||||
<SettingsInput
|
||||
label="App Key"
|
||||
inputProps={register("adzunaAppKey")}
|
||||
type="password"
|
||||
placeholder="Enter new app key"
|
||||
disabled={isLoading || isSaving}
|
||||
error={errors.adzunaAppKey?.message as string | undefined}
|
||||
current={formatSecretHint(privateValues.adzunaAppKeyHint)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<Separator />
|
||||
|
||||
@ -25,11 +25,13 @@ export type EnvSettingsValues = {
|
||||
readable: {
|
||||
rxresumeEmail: string;
|
||||
ukvisajobsEmail: string;
|
||||
adzunaAppId: string;
|
||||
basicAuthUser: string;
|
||||
};
|
||||
private: {
|
||||
rxresumePasswordHint: string | null;
|
||||
ukvisajobsPasswordHint: string | null;
|
||||
adzunaAppKeyHint: string | null;
|
||||
basicAuthPasswordHint: string | null;
|
||||
webhookSecretHint: string | null;
|
||||
};
|
||||
|
||||
@ -143,5 +143,6 @@ export const sourceLabel: Record<Job["source"], string> = {
|
||||
linkedin: "LinkedIn",
|
||||
glassdoor: "Glassdoor",
|
||||
ukvisajobs: "UK Visa Jobs",
|
||||
adzuna: "Adzuna",
|
||||
manual: "Manual",
|
||||
};
|
||||
|
||||
@ -55,6 +55,17 @@ describe.sequential("Pipeline API routes", () => {
|
||||
expect(runPipeline).toHaveBeenNthCalledWith(2, {
|
||||
sources: ["glassdoor"],
|
||||
});
|
||||
|
||||
const adzunaRunRes = await fetch(`${baseUrl}/api/pipeline/run`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ sources: ["adzuna"] }),
|
||||
});
|
||||
const adzunaRunBody = await adzunaRunRes.json();
|
||||
expect(adzunaRunBody.ok).toBe(true);
|
||||
expect(runPipeline).toHaveBeenNthCalledWith(3, {
|
||||
sources: ["adzuna"],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns conflict when cancelling with no active pipeline", async () => {
|
||||
|
||||
@ -99,7 +99,14 @@ const runPipelineSchema = z.object({
|
||||
minSuitabilityScore: z.number().min(0).max(100).optional(),
|
||||
sources: z
|
||||
.array(
|
||||
z.enum(["gradcracker", "indeed", "linkedin", "glassdoor", "ukvisajobs"]),
|
||||
z.enum([
|
||||
"gradcracker",
|
||||
"indeed",
|
||||
"linkedin",
|
||||
"glassdoor",
|
||||
"ukvisajobs",
|
||||
"adzuna",
|
||||
]),
|
||||
)
|
||||
.min(1)
|
||||
.optional(),
|
||||
|
||||
@ -252,6 +252,7 @@ export const DEMO_SOURCE_BASE_URLS: Record<JobSource, string> = {
|
||||
glassdoor: "https://www.glassdoor.com",
|
||||
gradcracker: "https://www.gradcracker.com",
|
||||
ukvisajobs: "https://www.ukvisajobs.com",
|
||||
adzuna: "https://www.adzuna.com",
|
||||
manual: "https://example.com",
|
||||
};
|
||||
|
||||
|
||||
@ -39,6 +39,7 @@ export const jobs = sqliteTable("jobs", {
|
||||
"linkedin",
|
||||
"glassdoor",
|
||||
"ukvisajobs",
|
||||
"adzuna",
|
||||
"manual",
|
||||
],
|
||||
})
|
||||
|
||||
@ -14,7 +14,7 @@ export type PipelineStep =
|
||||
| "cancelled"
|
||||
| "failed";
|
||||
|
||||
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs";
|
||||
export type CrawlSource = "gradcracker" | "jobspy" | "ukvisajobs" | "adzuna";
|
||||
|
||||
export interface PipelineProgress {
|
||||
step: PipelineStep;
|
||||
|
||||
@ -19,6 +19,10 @@ vi.mock("../../services/crawler", () => ({
|
||||
runCrawler: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../services/adzuna", () => ({
|
||||
runAdzuna: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../services/ukvisajobs", () => ({
|
||||
runUkVisaJobs: vi.fn(),
|
||||
}));
|
||||
@ -157,6 +161,63 @@ describe("discoverJobsStep", () => {
|
||||
).rejects.toThrow("All sources failed: ukvisajobs: boom");
|
||||
});
|
||||
|
||||
it("runs adzuna when selected and country is compatible", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const adzuna = await import("../../services/adzuna");
|
||||
|
||||
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||
searchTerms: JSON.stringify(["engineer"]),
|
||||
jobspyCountryIndeed: "united states",
|
||||
} as any);
|
||||
|
||||
vi.mocked(adzuna.runAdzuna).mockResolvedValue({
|
||||
success: true,
|
||||
jobs: [
|
||||
{
|
||||
source: "adzuna",
|
||||
sourceJobId: "adzu-1",
|
||||
title: "Engineer",
|
||||
employer: "ACME",
|
||||
jobUrl: "https://example.com/job",
|
||||
applicationLink: "https://example.com/job",
|
||||
},
|
||||
],
|
||||
} as any);
|
||||
|
||||
const result = await discoverJobsStep({
|
||||
mergedConfig: {
|
||||
...config,
|
||||
sources: ["adzuna"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.discoveredJobs).toHaveLength(1);
|
||||
expect(vi.mocked(adzuna.runAdzuna)).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ country: "us" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips adzuna for unsupported countries", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const adzuna = await import("../../services/adzuna");
|
||||
|
||||
vi.mocked(settingsRepo.getAllSettings).mockResolvedValue({
|
||||
searchTerms: JSON.stringify(["engineer"]),
|
||||
jobspyCountryIndeed: "japan",
|
||||
} as any);
|
||||
|
||||
await expect(
|
||||
discoverJobsStep({
|
||||
mergedConfig: {
|
||||
...config,
|
||||
sources: ["adzuna"],
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow("No compatible sources for selected country: Japan");
|
||||
|
||||
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("maps Gradcracker progress callback into live crawling counters", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const crawler = await import("../../services/crawler");
|
||||
@ -340,6 +401,7 @@ describe("discoverJobsStep", () => {
|
||||
|
||||
it("does not throw when no sources are requested", async () => {
|
||||
const settingsRepo = await import("../../repositories/settings");
|
||||
const adzuna = await import("../../services/adzuna");
|
||||
const jobSpy = await import("../../services/jobspy");
|
||||
const crawler = await import("../../services/crawler");
|
||||
const ukVisa = await import("../../services/ukvisajobs");
|
||||
@ -359,6 +421,7 @@ describe("discoverJobsStep", () => {
|
||||
expect(result.discoveredJobs).toEqual([]);
|
||||
expect(result.sourceErrors).toEqual([]);
|
||||
expect(vi.mocked(jobSpy.runJobSpy)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(adzuna.runAdzuna)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(crawler.runCrawler)).not.toHaveBeenCalled();
|
||||
expect(vi.mocked(ukVisa.runUkVisaJobs)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import { logger } from "@infra/logger";
|
||||
import {
|
||||
formatCountryLabel,
|
||||
getAdzunaCountryCode,
|
||||
isSourceAllowedForCountry,
|
||||
normalizeCountryKey,
|
||||
} from "@shared/location-support.js";
|
||||
import type { CreateJobInput, PipelineConfig } from "@shared/types";
|
||||
import * as jobsRepo from "../../repositories/jobs";
|
||||
import * as settingsRepo from "../../repositories/settings";
|
||||
import { runAdzuna } from "../../services/adzuna";
|
||||
import { runCrawler } from "../../services/crawler";
|
||||
import { runJobSpy } from "../../services/jobspy";
|
||||
import { runUkVisaJobs } from "../../services/ukvisajobs";
|
||||
@ -72,11 +74,13 @@ export async function discoverJobsStep(args: {
|
||||
);
|
||||
|
||||
const shouldRunJobSpy = jobSpySites.length > 0;
|
||||
const shouldRunAdzuna = compatibleSources.includes("adzuna");
|
||||
const shouldRunGradcracker = compatibleSources.includes("gradcracker");
|
||||
const shouldRunUkVisaJobs = compatibleSources.includes("ukvisajobs");
|
||||
|
||||
const totalSources =
|
||||
Number(shouldRunJobSpy) +
|
||||
Number(shouldRunAdzuna) +
|
||||
Number(shouldRunGradcracker) +
|
||||
Number(shouldRunUkVisaJobs);
|
||||
let completedSources = 0;
|
||||
@ -149,6 +153,89 @@ export async function discoverJobsStep(args: {
|
||||
return { discoveredJobs, sourceErrors };
|
||||
}
|
||||
|
||||
if (shouldRunAdzuna) {
|
||||
progressHelpers.startSource("adzuna", completedSources, totalSources, {
|
||||
termsTotal: searchTerms.length,
|
||||
detail: "Adzuna: fetching jobs...",
|
||||
});
|
||||
|
||||
const adzunaCountryCode = getAdzunaCountryCode(selectedCountry);
|
||||
if (!adzunaCountryCode) {
|
||||
sourceErrors.push(
|
||||
`adzuna: unsupported country ${formatCountryLabel(selectedCountry)}`,
|
||||
);
|
||||
markSourceComplete();
|
||||
} else {
|
||||
const adzunaMaxJobsPerTerm = settings.adzunaMaxJobsPerTerm
|
||||
? parseInt(settings.adzunaMaxJobsPerTerm, 10)
|
||||
: 50;
|
||||
|
||||
const adzunaResult = await runAdzuna({
|
||||
country: adzunaCountryCode,
|
||||
searchTerms,
|
||||
maxJobsPerTerm: adzunaMaxJobsPerTerm,
|
||||
onProgress: (event) => {
|
||||
if (event.type === "term_start") {
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "adzuna",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
phase: "list",
|
||||
currentUrl: event.searchTerm,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Adzuna: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (event.type === "page_fetched") {
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "adzuna",
|
||||
termsProcessed: Math.max(event.termIndex - 1, 0),
|
||||
termsTotal: event.termTotal,
|
||||
listPagesProcessed: event.pageNo,
|
||||
jobPagesEnqueued: event.totalCollected,
|
||||
jobPagesProcessed: event.totalCollected,
|
||||
phase: "list",
|
||||
currentUrl: `page ${event.pageNo}`,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Adzuna: term ${event.termIndex}/${event.termTotal}, page ${event.pageNo} (${event.totalCollected} collected)`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
progressHelpers.crawlingUpdate({
|
||||
source: "adzuna",
|
||||
termsProcessed: event.termIndex,
|
||||
termsTotal: event.termTotal,
|
||||
phase: "list",
|
||||
currentUrl: event.searchTerm,
|
||||
});
|
||||
updateProgress({
|
||||
step: "crawling",
|
||||
detail: `Adzuna: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
if (!adzunaResult.success) {
|
||||
sourceErrors.push(`adzuna: ${adzunaResult.error ?? "unknown error"}`);
|
||||
} else {
|
||||
discoveredJobs.push(...adzunaResult.jobs);
|
||||
}
|
||||
|
||||
markSourceComplete();
|
||||
}
|
||||
}
|
||||
|
||||
if (args.shouldCancel?.()) {
|
||||
return { discoveredJobs, sourceErrors };
|
||||
}
|
||||
|
||||
if (shouldRunGradcracker) {
|
||||
progressHelpers.startSource("gradcracker", completedSources, totalSources, {
|
||||
detail: "Gradcracker: scraping...",
|
||||
|
||||
@ -20,6 +20,7 @@ export type SettingKey =
|
||||
| "resumeProjects"
|
||||
| "rxresumeBaseResumeId"
|
||||
| "ukvisajobsMaxJobs"
|
||||
| "adzunaMaxJobsPerTerm"
|
||||
| "gradcrackerMaxJobsPerTerm"
|
||||
| "searchTerms"
|
||||
| "jobspyLocation"
|
||||
@ -36,6 +37,8 @@ export type SettingKey =
|
||||
| "basicAuthPassword"
|
||||
| "ukvisajobsEmail"
|
||||
| "ukvisajobsPassword"
|
||||
| "adzunaAppId"
|
||||
| "adzunaAppKey"
|
||||
| "webhookSecret"
|
||||
| "backupEnabled"
|
||||
| "backupHour"
|
||||
|
||||
254
orchestrator/src/server/services/adzuna.ts
Normal file
254
orchestrator/src/server/services/adzuna.ts
Normal file
@ -0,0 +1,254 @@
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { logger } from "@infra/logger";
|
||||
import type { CreateJobInput } from "@shared/types";
|
||||
import { toNumberOrNull, toStringOrNull } from "@shared/utils/type-conversion";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ADZUNA_DIR = join(__dirname, "../../../../extractors/adzuna");
|
||||
const DATASET_PATH = join(ADZUNA_DIR, "storage/datasets/default/jobs.json");
|
||||
const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
|
||||
const require = createRequire(import.meta.url);
|
||||
const TSX_CLI_PATH = resolveTsxCliPath();
|
||||
|
||||
type AdzunaRawJob = Record<string, unknown>;
|
||||
|
||||
export type AdzunaProgressEvent =
|
||||
| {
|
||||
type: "term_start";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
}
|
||||
| {
|
||||
type: "page_fetched";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
pageNo: number;
|
||||
resultsOnPage: number;
|
||||
totalCollected: number;
|
||||
}
|
||||
| {
|
||||
type: "term_complete";
|
||||
termIndex: number;
|
||||
termTotal: number;
|
||||
searchTerm: string;
|
||||
jobsFoundTerm: number;
|
||||
};
|
||||
|
||||
export interface RunAdzunaOptions {
|
||||
searchTerms?: string[];
|
||||
country?: string;
|
||||
maxJobsPerTerm?: number;
|
||||
onProgress?: (event: AdzunaProgressEvent) => void;
|
||||
}
|
||||
|
||||
export interface AdzunaResult {
|
||||
success: boolean;
|
||||
jobs: CreateJobInput[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function resolveTsxCliPath(): string | null {
|
||||
try {
|
||||
return require.resolve("tsx/dist/cli.mjs");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function canRunNpmCommand(): boolean {
|
||||
const result = spawnSync("npm", ["--version"], { stdio: "ignore" });
|
||||
return !result.error && result.status === 0;
|
||||
}
|
||||
|
||||
function parseAdzunaProgressLine(line: string): AdzunaProgressEvent | null {
|
||||
if (!line.startsWith(JOBOPS_PROGRESS_PREFIX)) return null;
|
||||
const raw = line.slice(JOBOPS_PROGRESS_PREFIX.length).trim();
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
const event = toStringOrNull(parsed.event);
|
||||
const termIndex = toNumberOrNull(parsed.termIndex);
|
||||
const termTotal = toNumberOrNull(parsed.termTotal);
|
||||
const searchTerm = toStringOrNull(parsed.searchTerm) ?? "";
|
||||
if (!event || termIndex === null || termTotal === null) return null;
|
||||
|
||||
if (event === "term_start") {
|
||||
return { type: "term_start", termIndex, termTotal, searchTerm };
|
||||
}
|
||||
|
||||
if (event === "page_fetched") {
|
||||
const pageNo = toNumberOrNull(parsed.pageNo);
|
||||
if (pageNo === null) return null;
|
||||
return {
|
||||
type: "page_fetched",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
pageNo,
|
||||
resultsOnPage: toNumberOrNull(parsed.resultsOnPage) ?? 0,
|
||||
totalCollected: toNumberOrNull(parsed.totalCollected) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (event === "term_complete") {
|
||||
return {
|
||||
type: "term_complete",
|
||||
termIndex,
|
||||
termTotal,
|
||||
searchTerm,
|
||||
jobsFoundTerm: toNumberOrNull(parsed.jobsFoundTerm) ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function mapAdzunaRow(row: AdzunaRawJob): CreateJobInput | null {
|
||||
const jobUrl = toStringOrNull(row.jobUrl);
|
||||
if (!jobUrl) return null;
|
||||
|
||||
return {
|
||||
source: "adzuna",
|
||||
sourceJobId: toStringOrNull(row.sourceJobId) ?? undefined,
|
||||
title: toStringOrNull(row.title) ?? "Unknown Title",
|
||||
employer: toStringOrNull(row.employer) ?? "Unknown Employer",
|
||||
jobUrl,
|
||||
applicationLink:
|
||||
toStringOrNull(row.applicationLink) ??
|
||||
toStringOrNull(row.jobUrl) ??
|
||||
undefined,
|
||||
location: toStringOrNull(row.location) ?? undefined,
|
||||
salary: toStringOrNull(row.salary) ?? undefined,
|
||||
datePosted: toStringOrNull(row.datePosted) ?? undefined,
|
||||
jobDescription: toStringOrNull(row.jobDescription) ?? undefined,
|
||||
jobType: toStringOrNull(row.jobType) ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function readDataset(): Promise<CreateJobInput[]> {
|
||||
const content = await readFile(DATASET_PATH, "utf-8");
|
||||
const parsed = JSON.parse(content) as unknown;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const jobs: CreateJobInput[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const value of parsed) {
|
||||
if (!value || typeof value !== "object") continue;
|
||||
const mapped = mapAdzunaRow(value as AdzunaRawJob);
|
||||
if (!mapped) continue;
|
||||
const key = mapped.sourceJobId || mapped.jobUrl;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
jobs.push(mapped);
|
||||
}
|
||||
return jobs;
|
||||
}
|
||||
|
||||
export async function runAdzuna(
|
||||
options: RunAdzunaOptions = {},
|
||||
): Promise<AdzunaResult> {
|
||||
const appId = process.env.ADZUNA_APP_ID?.trim();
|
||||
const appKey = process.env.ADZUNA_APP_KEY?.trim();
|
||||
if (!appId || !appKey) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Missing Adzuna credentials (ADZUNA_APP_ID / ADZUNA_APP_KEY)",
|
||||
};
|
||||
}
|
||||
|
||||
const country = (options.country || "gb").trim().toLowerCase();
|
||||
const maxJobsPerTerm = options.maxJobsPerTerm ?? 50;
|
||||
const searchTerms =
|
||||
options.searchTerms && options.searchTerms.length > 0
|
||||
? options.searchTerms
|
||||
: ["web developer"];
|
||||
const useNpmCommand = canRunNpmCommand();
|
||||
if (!useNpmCommand && !TSX_CLI_PATH) {
|
||||
return {
|
||||
success: false,
|
||||
jobs: [],
|
||||
error: "Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const extractorEnv = {
|
||||
...process.env,
|
||||
JOBOPS_EMIT_PROGRESS: "1",
|
||||
ADZUNA_APP_ID: appId,
|
||||
ADZUNA_APP_KEY: appKey,
|
||||
ADZUNA_COUNTRY: country,
|
||||
ADZUNA_MAX_JOBS_PER_TERM: String(maxJobsPerTerm),
|
||||
ADZUNA_SEARCH_TERMS: JSON.stringify(searchTerms),
|
||||
ADZUNA_OUTPUT_JSON: DATASET_PATH,
|
||||
};
|
||||
const child = useNpmCommand
|
||||
? spawn("npm", ["run", "start"], {
|
||||
cwd: ADZUNA_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
})
|
||||
: (() => {
|
||||
const tsxCliPath = TSX_CLI_PATH;
|
||||
if (!tsxCliPath) {
|
||||
throw new Error(
|
||||
"Unable to execute Adzuna extractor (npm/tsx unavailable)",
|
||||
);
|
||||
}
|
||||
return spawn(process.execPath, [tsxCliPath, "src/main.ts"], {
|
||||
cwd: ADZUNA_DIR,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: extractorEnv,
|
||||
});
|
||||
})();
|
||||
|
||||
const handleLine = (line: string, stream: NodeJS.WriteStream) => {
|
||||
const progressEvent = parseAdzunaProgressLine(line);
|
||||
if (progressEvent) {
|
||||
options.onProgress?.(progressEvent);
|
||||
return;
|
||||
}
|
||||
stream.write(`${line}\n`);
|
||||
};
|
||||
|
||||
const stdoutRl = child.stdout
|
||||
? createInterface({ input: child.stdout })
|
||||
: null;
|
||||
const stderrRl = child.stderr
|
||||
? createInterface({ input: child.stderr })
|
||||
: null;
|
||||
|
||||
stdoutRl?.on("line", (line) => handleLine(line, process.stdout));
|
||||
stderrRl?.on("line", (line) => handleLine(line, process.stderr));
|
||||
|
||||
child.on("close", (code) => {
|
||||
stdoutRl?.close();
|
||||
stderrRl?.close();
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`Adzuna extractor exited with code ${code}`));
|
||||
});
|
||||
child.on("error", reject);
|
||||
});
|
||||
|
||||
const jobs = await readDataset();
|
||||
return { success: true, jobs };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Unknown error";
|
||||
logger.warn("Adzuna extractor run failed", { error: message });
|
||||
return { success: false, jobs: [], error: message };
|
||||
}
|
||||
}
|
||||
@ -8,6 +8,7 @@ const readableStringConfig: { settingKey: SettingKey; envKey: string }[] = [
|
||||
{ settingKey: "llmBaseUrl", envKey: "LLM_BASE_URL" },
|
||||
{ settingKey: "rxresumeEmail", envKey: "RXRESUME_EMAIL" },
|
||||
{ settingKey: "ukvisajobsEmail", envKey: "UKVISAJOBS_EMAIL" },
|
||||
{ settingKey: "adzunaAppId", envKey: "ADZUNA_APP_ID" },
|
||||
{ settingKey: "basicAuthUser", envKey: "BASIC_AUTH_USER" },
|
||||
];
|
||||
|
||||
@ -37,6 +38,11 @@ const privateStringConfig: {
|
||||
envKey: "UKVISAJOBS_PASSWORD",
|
||||
hintKey: "ukvisajobsPasswordHint",
|
||||
},
|
||||
{
|
||||
settingKey: "adzunaAppKey",
|
||||
envKey: "ADZUNA_APP_KEY",
|
||||
hintKey: "adzunaAppKeyHint",
|
||||
},
|
||||
{
|
||||
settingKey: "basicAuthPassword",
|
||||
envKey: "BASIC_AUTH_PASSWORD",
|
||||
|
||||
@ -25,6 +25,20 @@ describe("settings-conversion", () => {
|
||||
expect(resolved.defaultValue).toBe(50);
|
||||
});
|
||||
|
||||
it("round-trips adzuna numeric settings", () => {
|
||||
process.env.ADZUNA_MAX_JOBS_PER_TERM = "";
|
||||
const serialized = serializeSettingValue("adzunaMaxJobsPerTerm", 75);
|
||||
expect(serialized).toBe("75");
|
||||
|
||||
const resolved = resolveSettingValue(
|
||||
"adzunaMaxJobsPerTerm",
|
||||
serialized ?? undefined,
|
||||
);
|
||||
expect(resolved.overrideValue).toBe(75);
|
||||
expect(resolved.value).toBe(75);
|
||||
expect(resolved.defaultValue).toBe(50);
|
||||
});
|
||||
|
||||
it("round-trips boolean bit settings", () => {
|
||||
expect(serializeSettingValue("showSponsorInfo", true)).toBe("1");
|
||||
expect(serializeSettingValue("showSponsorInfo", false)).toBe("0");
|
||||
|
||||
@ -7,6 +7,7 @@ type SettingMetadata<T, Input = T | null | undefined> = {
|
||||
|
||||
type SettingsConversionValueMap = {
|
||||
ukvisajobsMaxJobs: number;
|
||||
adzunaMaxJobsPerTerm: number;
|
||||
gradcrackerMaxJobsPerTerm: number;
|
||||
searchTerms: string[];
|
||||
jobspyLocation: string;
|
||||
@ -100,6 +101,13 @@ export const settingsConversionMetadata: SettingsConversionMetadata = {
|
||||
serialize: serializeNullableNumber,
|
||||
resolve: resolveWithNullishFallback,
|
||||
},
|
||||
adzunaMaxJobsPerTerm: {
|
||||
defaultValue: () =>
|
||||
parseInt(process.env.ADZUNA_MAX_JOBS_PER_TERM || "50", 10),
|
||||
parseOverride: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
resolve: resolveWithNullishFallback,
|
||||
},
|
||||
gradcrackerMaxJobsPerTerm: {
|
||||
defaultValue: () => 50,
|
||||
parseOverride: parseIntOrNull,
|
||||
|
||||
@ -35,23 +35,37 @@ describe("applySettingsUpdates", () => {
|
||||
const plan = await applySettingsUpdates({
|
||||
model: "gpt-4o-mini",
|
||||
ukvisajobsMaxJobs: 42,
|
||||
adzunaMaxJobsPerTerm: 25,
|
||||
searchTerms: ["backend", "platform"],
|
||||
llmProvider: "openai",
|
||||
adzunaAppId: "app-id",
|
||||
adzunaAppKey: "app-key",
|
||||
});
|
||||
|
||||
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(4);
|
||||
expect(settingsRepo.setSetting).toHaveBeenCalledTimes(7);
|
||||
expect(vi.mocked(settingsRepo.setSetting).mock.calls).toEqual(
|
||||
expect.arrayContaining([
|
||||
["model", "gpt-4o-mini"],
|
||||
["ukvisajobsMaxJobs", "42"],
|
||||
["adzunaMaxJobsPerTerm", "25"],
|
||||
["searchTerms", '["backend","platform"]'],
|
||||
["llmProvider", "openai"],
|
||||
["adzunaAppId", "app-id"],
|
||||
["adzunaAppKey", "app-key"],
|
||||
]),
|
||||
);
|
||||
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||
"LLM_PROVIDER",
|
||||
"openai",
|
||||
);
|
||||
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||
"ADZUNA_APP_ID",
|
||||
"app-id",
|
||||
);
|
||||
expect(envSettings.applyEnvValue).toHaveBeenCalledWith(
|
||||
"ADZUNA_APP_KEY",
|
||||
"app-key",
|
||||
);
|
||||
expect(plan.shouldRefreshBackupScheduler).toBe(false);
|
||||
});
|
||||
|
||||
|
||||
@ -164,6 +164,11 @@ export const settingsUpdateRegistry: Partial<{
|
||||
actions: [metadataPersistAction("ukvisajobsMaxJobs", value)],
|
||||
}),
|
||||
),
|
||||
adzunaMaxJobsPerTerm: singleAction(({ value }) =>
|
||||
result({
|
||||
actions: [metadataPersistAction("adzunaMaxJobsPerTerm", value)],
|
||||
}),
|
||||
),
|
||||
gradcrackerMaxJobsPerTerm: singleAction(({ value }) =>
|
||||
result({
|
||||
actions: [metadataPersistAction("gradcrackerMaxJobsPerTerm", value)],
|
||||
@ -278,6 +283,26 @@ export const settingsUpdateRegistry: Partial<{
|
||||
],
|
||||
});
|
||||
}),
|
||||
adzunaAppId: singleAction(({ value }) => {
|
||||
const normalized = toNormalizedStringOrNull(value);
|
||||
return result({
|
||||
actions: [
|
||||
persistAction("adzunaAppId", normalized, () => {
|
||||
applyEnvValue("ADZUNA_APP_ID", normalized);
|
||||
}),
|
||||
],
|
||||
});
|
||||
}),
|
||||
adzunaAppKey: singleAction(({ value }) => {
|
||||
const normalized = toNormalizedStringOrNull(value);
|
||||
return result({
|
||||
actions: [
|
||||
persistAction("adzunaAppKey", normalized, () => {
|
||||
applyEnvValue("ADZUNA_APP_KEY", normalized);
|
||||
}),
|
||||
],
|
||||
});
|
||||
}),
|
||||
webhookSecret: singleAction(({ value }) => {
|
||||
const normalized = toNormalizedStringOrNull(value);
|
||||
return result({
|
||||
|
||||
@ -96,6 +96,15 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
|
||||
const overrideUkvisajobsMaxJobs = ukvisajobsMaxJobsSetting.overrideValue;
|
||||
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting.value;
|
||||
|
||||
const adzunaMaxJobsPerTermSetting = resolveSettingValue(
|
||||
"adzunaMaxJobsPerTerm",
|
||||
overrides.adzunaMaxJobsPerTerm,
|
||||
);
|
||||
const defaultAdzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.defaultValue;
|
||||
const overrideAdzunaMaxJobsPerTerm =
|
||||
adzunaMaxJobsPerTermSetting.overrideValue;
|
||||
const adzunaMaxJobsPerTerm = adzunaMaxJobsPerTermSetting.value;
|
||||
|
||||
const gradcrackerMaxJobsPerTermSetting = resolveSettingValue(
|
||||
"gradcrackerMaxJobsPerTerm",
|
||||
overrides.gradcrackerMaxJobsPerTerm,
|
||||
@ -260,6 +269,9 @@ export async function getEffectiveSettings(): Promise<AppSettings> {
|
||||
ukvisajobsMaxJobs,
|
||||
defaultUkvisajobsMaxJobs,
|
||||
overrideUkvisajobsMaxJobs,
|
||||
adzunaMaxJobsPerTerm,
|
||||
defaultAdzunaMaxJobsPerTerm,
|
||||
overrideAdzunaMaxJobsPerTerm,
|
||||
gradcrackerMaxJobsPerTerm,
|
||||
defaultGradcrackerMaxJobsPerTerm,
|
||||
overrideGradcrackerMaxJobsPerTerm,
|
||||
|
||||
26
package-lock.json
generated
26
package-lock.json
generated
@ -80,6 +80,28 @@
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"extractors/adzuna": {
|
||||
"name": "adzuna-extractor",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"job-ops-shared": "^1.0.0",
|
||||
"tsx": "^4.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.0",
|
||||
"typescript": "~5.9.0"
|
||||
}
|
||||
},
|
||||
"extractors/adzuna/node_modules/@types/node": {
|
||||
"version": "24.10.13",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.13.tgz",
|
||||
"integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"extractors/gradcracker": {
|
||||
"name": "gradcracker-extractor",
|
||||
"version": "0.0.1",
|
||||
@ -8586,6 +8608,10 @@
|
||||
"node": ">=12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adzuna-extractor": {
|
||||
"resolved": "extractors/adzuna",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "7.1.4",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz",
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formatCountryLabel,
|
||||
getAdzunaCountryCode,
|
||||
getCompatibleSourcesForCountry,
|
||||
isGlassdoorCountry,
|
||||
isSourceAllowedForCountry,
|
||||
@ -52,15 +53,24 @@ describe("location-support", () => {
|
||||
expect(isSourceAllowedForCountry("linkedin", "worldwide")).toBe(true);
|
||||
expect(isSourceAllowedForCountry("glassdoor", "united states")).toBe(true);
|
||||
expect(isSourceAllowedForCountry("glassdoor", "japan")).toBe(false);
|
||||
expect(isSourceAllowedForCountry("adzuna", "united states")).toBe(true);
|
||||
expect(isSourceAllowedForCountry("adzuna", "japan")).toBe(false);
|
||||
});
|
||||
|
||||
it("filters incompatible sources while preserving compatible order", () => {
|
||||
expect(
|
||||
getCompatibleSourcesForCountry(
|
||||
["gradcracker", "indeed", "glassdoor", "ukvisajobs", "linkedin"],
|
||||
[
|
||||
"gradcracker",
|
||||
"indeed",
|
||||
"glassdoor",
|
||||
"ukvisajobs",
|
||||
"adzuna",
|
||||
"linkedin",
|
||||
],
|
||||
"united states",
|
||||
),
|
||||
).toEqual(["indeed", "glassdoor", "linkedin"]);
|
||||
).toEqual(["indeed", "glassdoor", "adzuna", "linkedin"]);
|
||||
});
|
||||
|
||||
it("supports glassdoor only in explicitly supported countries", () => {
|
||||
@ -70,4 +80,10 @@ describe("location-support", () => {
|
||||
expect(isGlassdoorCountry("japan")).toBe(false);
|
||||
expect(isGlassdoorCountry("worldwide")).toBe(false);
|
||||
});
|
||||
|
||||
it("maps adzuna country keys to adzuna api country codes", () => {
|
||||
expect(getAdzunaCountryCode("united states")).toBe("us");
|
||||
expect(getAdzunaCountryCode("UK")).toBe("gb");
|
||||
expect(getAdzunaCountryCode("japan")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@ -124,6 +124,27 @@ const GLASSDOOR_SUPPORTED_COUNTRIES = new Set(
|
||||
"vietnam",
|
||||
].map((country) => normalizeCountryKey(country)),
|
||||
);
|
||||
const ADZUNA_COUNTRY_CODE_BY_KEY: Record<string, string> = {
|
||||
"united kingdom": "gb",
|
||||
"united states": "us",
|
||||
austria: "at",
|
||||
australia: "au",
|
||||
belgium: "be",
|
||||
brazil: "br",
|
||||
canada: "ca",
|
||||
switzerland: "ch",
|
||||
germany: "de",
|
||||
spain: "es",
|
||||
france: "fr",
|
||||
india: "in",
|
||||
italy: "it",
|
||||
mexico: "mx",
|
||||
netherlands: "nl",
|
||||
"new zealand": "nz",
|
||||
poland: "pl",
|
||||
singapore: "sg",
|
||||
"south africa": "za",
|
||||
};
|
||||
|
||||
export function normalizeCountryKey(value: string | null | undefined): string {
|
||||
const normalized = value?.trim().toLowerCase() ?? "";
|
||||
@ -155,12 +176,19 @@ export function isGlassdoorCountry(
|
||||
return GLASSDOOR_SUPPORTED_COUNTRIES.has(normalizeCountryKey(country));
|
||||
}
|
||||
|
||||
export function getAdzunaCountryCode(
|
||||
country: string | null | undefined,
|
||||
): string | null {
|
||||
return ADZUNA_COUNTRY_CODE_BY_KEY[normalizeCountryKey(country)] ?? null;
|
||||
}
|
||||
|
||||
export function isSourceAllowedForCountry(
|
||||
source: JobSource,
|
||||
country: string | null | undefined,
|
||||
): boolean {
|
||||
if (UK_ONLY_SOURCES.has(source)) return isUkCountry(country);
|
||||
if (source === "glassdoor") return isGlassdoorCountry(country);
|
||||
if (source === "adzuna") return getAdzunaCountryCode(country) !== null;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -32,6 +32,13 @@ export const updateSettingsSchema = z
|
||||
resumeProjects: resumeProjectsSchema.nullable().optional(),
|
||||
rxresumeBaseResumeId: z.string().trim().max(200).nullable().optional(),
|
||||
ukvisajobsMaxJobs: z.number().int().min(1).max(1000).nullable().optional(),
|
||||
adzunaMaxJobsPerTerm: z
|
||||
.number()
|
||||
.int()
|
||||
.min(1)
|
||||
.max(1000)
|
||||
.nullable()
|
||||
.optional(),
|
||||
gradcrackerMaxJobsPerTerm: z
|
||||
.number()
|
||||
.int()
|
||||
@ -64,6 +71,8 @@ export const updateSettingsSchema = z
|
||||
basicAuthPassword: z.string().trim().max(2000).nullable().optional(),
|
||||
ukvisajobsEmail: z.string().trim().max(200).nullable().optional(),
|
||||
ukvisajobsPassword: z.string().trim().max(2000).nullable().optional(),
|
||||
adzunaAppId: z.string().trim().max(200).nullable().optional(),
|
||||
adzunaAppKey: z.string().trim().max(2000).nullable().optional(),
|
||||
webhookSecret: z.string().trim().max(2000).nullable().optional(),
|
||||
enableBasicAuth: z.boolean().optional(),
|
||||
backupEnabled: z.boolean().nullable().optional(),
|
||||
|
||||
@ -161,6 +161,9 @@ export const createAppSettings = (
|
||||
ukvisajobsMaxJobs: 50,
|
||||
defaultUkvisajobsMaxJobs: 50,
|
||||
overrideUkvisajobsMaxJobs: null,
|
||||
adzunaMaxJobsPerTerm: 50,
|
||||
defaultAdzunaMaxJobsPerTerm: 50,
|
||||
overrideAdzunaMaxJobsPerTerm: null,
|
||||
gradcrackerMaxJobsPerTerm: 50,
|
||||
defaultGradcrackerMaxJobsPerTerm: 50,
|
||||
overrideGradcrackerMaxJobsPerTerm: null,
|
||||
@ -198,6 +201,8 @@ export const createAppSettings = (
|
||||
basicAuthPasswordHint: null,
|
||||
ukvisajobsEmail: null,
|
||||
ukvisajobsPasswordHint: null,
|
||||
adzunaAppId: null,
|
||||
adzunaAppKeyHint: null,
|
||||
webhookSecretHint: null,
|
||||
basicAuthActive: false,
|
||||
backupEnabled: false,
|
||||
|
||||
@ -125,6 +125,7 @@ export type JobSource =
|
||||
| "linkedin"
|
||||
| "glassdoor"
|
||||
| "ukvisajobs"
|
||||
| "adzuna"
|
||||
| "manual";
|
||||
|
||||
export interface Job {
|
||||
@ -895,6 +896,9 @@ export interface AppSettings {
|
||||
ukvisajobsMaxJobs: number;
|
||||
defaultUkvisajobsMaxJobs: number;
|
||||
overrideUkvisajobsMaxJobs: number | null;
|
||||
adzunaMaxJobsPerTerm: number;
|
||||
defaultAdzunaMaxJobsPerTerm: number;
|
||||
overrideAdzunaMaxJobsPerTerm: number | null;
|
||||
gradcrackerMaxJobsPerTerm: number;
|
||||
defaultGradcrackerMaxJobsPerTerm: number;
|
||||
overrideGradcrackerMaxJobsPerTerm: number | null;
|
||||
@ -932,6 +936,8 @@ export interface AppSettings {
|
||||
basicAuthPasswordHint: string | null;
|
||||
ukvisajobsEmail: string | null;
|
||||
ukvisajobsPasswordHint: string | null;
|
||||
adzunaAppId: string | null;
|
||||
adzunaAppKeyHint: string | null;
|
||||
webhookSecretHint: string | null;
|
||||
basicAuthActive: boolean;
|
||||
// Backup settings
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user