ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

in extractors

Browse Source
This commit is contained in:
DaKheera47
2026-01-25 13:34:16 +00:00
parent aaab9b5124
commit 4ffaf06b1d
9 changed files with 626 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files
biome.json
+2 -9
View File
@@ -5,10 +5,7 @@
"indentWidth": 2 "indentWidth": 2
}, },
"files": { "files": {
"includes": [ "includes": ["**", "!!**/dist"]
"**",
"!!**/dist"
]
}, },
"css": { "css": {
"parser": { "parser": {
@@ -17,11 +14,7 @@
}, },
"overrides": [ "overrides": [
{ {
"includes": [ "includes": ["**/*.test.ts", "**/*.test.tsx", "**/test-utils.ts"],
"**/*.test.ts",
"**/*.test.tsx",
"**/test-utils.ts"
],
"linter": { "linter": {
"rules": { "rules": {
"suspicious": { "suspicious": {
extractors/gradcracker/src/main.ts
+11 -14
View File
@@ -2,9 +2,8 @@
import { launchOptions } from "camoufox-js"; import { launchOptions } from "camoufox-js";
import { PlaywrightCrawler } from "crawlee"; import { PlaywrightCrawler } from "crawlee";
import { firefox } from "playwright"; import { firefox } from "playwright";
import { router } from "./routes.js";
import { initJobOpsProgress } from "./progress.js"; import { initJobOpsProgress } from "./progress.js";
import { router } from "./routes.js";
// locations // locations
const locations = [ const locations = [
@@ -17,10 +16,7 @@ const locations = [
]; ];
// roles // roles
const defaultRoles = [ const defaultRoles = ["web-development", "software-systems"];
"web-development",
"software-systems",
];
let roles = defaultRoles; let roles = defaultRoles;
const envRolesRaw = process.env.GRADCRACKER_SEARCH_TERMS; const envRolesRaw = process.env.GRADCRACKER_SEARCH_TERMS;
@@ -29,15 +25,16 @@ if (envRolesRaw) {
try { try {
const parsed = JSON.parse(envRolesRaw) as string[]; const parsed = JSON.parse(envRolesRaw) as string[];
if (Array.isArray(parsed) && parsed.length > 0) { if (Array.isArray(parsed) && parsed.length > 0) {
roles = parsed.map(term => roles = parsed.map((term) =>
term.toLowerCase() term
.replace(/[^a-z0-9]+/g, '-') .toLowerCase()
.replace(/^-+|-+$/g, '') .replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, ""),
); );
console.log(`Using configured search terms: ${roles.join(', ')}`); console.log(`Using configured search terms: ${roles.join(", ")}`);
} }
} catch (e) { } catch (e) {
console.warn('Failed to parse GRADCRACKER_SEARCH_TERMS', e); console.warn("Failed to parse GRADCRACKER_SEARCH_TERMS", e);
} }
} }
@@ -46,12 +43,12 @@ const gradcrackerUrls = locations.flatMap((location) => {
return roles.map((role) => { return roles.map((role) => {
return { return {
url: `https://www.gradcracker.com/search/computing-technology/${role}-graduate-jobs-in-${location}?order=dateAdded`, url: `https://www.gradcracker.com/search/computing-technology/${role}-graduate-jobs-in-${location}?order=dateAdded`,
role role,
}; };
}); });
}); });
console.log(`Total gradcracker URLs: ${gradcrackerUrls.length}`) console.log(`Total gradcracker URLs: ${gradcrackerUrls.length}`);
const startUrls = gradcrackerUrls.map(({ url, role }) => ({ const startUrls = gradcrackerUrls.map(({ url, role }) => ({
url, url,
extractors/gradcracker/src/progress.ts
+1 -2
View File
@@ -26,7 +26,7 @@ interface JobOpsCrawlProgressState {
const PROGRESS_PREFIX = "JOBOPS_PROGRESS "; const PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
const isEnabled = () => process.env.JOBOPS_EMIT_PROGRESS === "1"; const isEnabled = () => process.env.JOBOPS_EMIT_PROGRESS === "1";
let state: JobOpsCrawlProgressState = { const state: JobOpsCrawlProgressState = {
listPagesProcessed: 0, listPagesProcessed: 0,
jobCardsFound: 0, jobCardsFound: 0,
jobPagesEnqueued: 0, jobPagesEnqueued: 0,
@@ -80,4 +80,3 @@ export function markJobPageDone(params: { currentUrl: string }): void {
state.currentUrl = params.currentUrl; state.currentUrl = params.currentUrl;
emit(); emit();
} }
extractors/gradcracker/src/routes.ts
+36 -17
View File
@@ -1,5 +1,5 @@
import { createPlaywrightRouter, log } from "crawlee";
import { readFileSync } from "node:fs"; import { readFileSync } from "node:fs";
import { createPlaywrightRouter, log } from "crawlee";
import { markJobPageDone, markListPageDone } from "./progress.js"; import { markJobPageDone, markListPageDone } from "./progress.js";
function normalizeUrl(raw: string | null | undefined): string | null { function normalizeUrl(raw: string | null | undefined): string | null {
@@ -17,8 +17,7 @@ function normalizeUrl(raw: string | null | undefined): string | null {
function getExistingJobUrlSet(): Set<string> { function getExistingJobUrlSet(): Set<string> {
const filePath = process.env.JOBOPS_EXISTING_JOB_URLS_FILE; const filePath = process.env.JOBOPS_EXISTING_JOB_URLS_FILE;
const raw = const raw = filePath
filePath
? (() => { ? (() => {
try { try {
return readFileSync(filePath, "utf-8"); return readFileSync(filePath, "utf-8");
@@ -41,12 +40,16 @@ function getExistingJobUrlSet(): Set<string> {
} }
} }
const SKIP_APPLY_FOR_EXISTING = process.env.JOBOPS_SKIP_APPLY_FOR_EXISTING === "1"; const SKIP_APPLY_FOR_EXISTING =
process.env.JOBOPS_SKIP_APPLY_FOR_EXISTING === "1";
const EXISTING_JOB_URLS = getExistingJobUrlSet(); const EXISTING_JOB_URLS = getExistingJobUrlSet();
// Global counters for max jobs per search term // Global counters for max jobs per search term
const jobCounts = new Map<string, number>(); const jobCounts = new Map<string, number>();
const MAX_JOBS_PER_TERM = parseInt(process.env.GRADCRACKER_MAX_JOBS_PER_TERM || "0", 10); const MAX_JOBS_PER_TERM = parseInt(
process.env.GRADCRACKER_MAX_JOBS_PER_TERM || "0",
10,
);
interface Job { interface Job {
title: string | null; title: string | null;
@@ -72,7 +75,9 @@ router.addHandler(
if (MAX_JOBS_PER_TERM > 0) { if (MAX_JOBS_PER_TERM > 0) {
const currentCount = jobCounts.get(role) || 0; const currentCount = jobCounts.get(role) || 0;
if (currentCount >= MAX_JOBS_PER_TERM) { if (currentCount >= MAX_JOBS_PER_TERM) {
log.info(`Max jobs (${MAX_JOBS_PER_TERM}) already enqueued for role "${role}". Skipping list page.`); log.info(
`Max jobs (${MAX_JOBS_PER_TERM}) already enqueued for role "${role}". Skipping list page.`,
);
markListPageDone({ markListPageDone({
currentUrl: request.url, currentUrl: request.url,
jobCardsFound: 0, jobCardsFound: 0,
@@ -120,7 +125,8 @@ router.addHandler(
let disciplines: string | null = null; let disciplines: string | null = null;
try { try {
const disciplinesEl = article.locator("h3"); const disciplinesEl = article.locator("h3");
disciplines = (await disciplinesEl.textContent({ timeout: 1000 }))?.trim() ?? null; disciplines =
(await disciplinesEl.textContent({ timeout: 1000 }))?.trim() ?? null;
} catch { } catch {
// h3 not found or timed out - that's okay, disciplines is optional // h3 not found or timed out - that's okay, disciplines is optional
} }
@@ -195,7 +201,9 @@ router.addHandler(
if (MAX_JOBS_PER_TERM > 0) { if (MAX_JOBS_PER_TERM > 0) {
const currentCount = jobCounts.get(role) || 0; const currentCount = jobCounts.get(role) || 0;
if (currentCount >= MAX_JOBS_PER_TERM) { if (currentCount >= MAX_JOBS_PER_TERM) {
log.info(`Reached max jobs limit (${MAX_JOBS_PER_TERM}) for role "${role}" while processing list. Stopping.`); log.info(
`Reached max jobs limit (${MAX_JOBS_PER_TERM}) for role "${role}" while processing list. Stopping.`,
);
break; break;
} }
jobCounts.set(role, currentCount + 1); jobCounts.set(role, currentCount + 1);
@@ -205,7 +213,7 @@ router.addHandler(
urls: [jobUrl], urls: [jobUrl],
userData: { userData: {
...jobs[jobs.length - 1], ...jobs[jobs.length - 1],
label: "gradcracker-single-job-page" label: "gradcracker-single-job-page",
}, },
}); });
enqueuedJobs++; enqueuedJobs++;
@@ -216,7 +224,7 @@ router.addHandler(
log.info(`Extracted ${jobs.length} jobs`); log.info(`Extracted ${jobs.length} jobs`);
if (SKIP_APPLY_FOR_EXISTING && skippedKnownJobs > 0) { if (SKIP_APPLY_FOR_EXISTING && skippedKnownJobs > 0) {
log.info( log.info(
`Skipping ${skippedKnownJobs} already-known job pages; enqueued ${enqueuedJobs} new job pages.` `Skipping ${skippedKnownJobs} already-known job pages; enqueued ${enqueuedJobs} new job pages.`,
); );
} }
@@ -226,7 +234,7 @@ router.addHandler(
jobPagesEnqueued: enqueuedJobs, jobPagesEnqueued: enqueuedJobs,
jobPagesSkipped: skippedKnownJobs, jobPagesSkipped: skippedKnownJobs,
}); });
} },
); );
router.addHandler( router.addHandler(
@@ -261,7 +269,9 @@ router.addHandler(
// Prefer page-scoped popup detection. Using the browser context's "page" event // Prefer page-scoped popup detection. Using the browser context's "page" event
// can accidentally capture unrelated pages created by other concurrent requests. // can accidentally capture unrelated pages created by other concurrent requests.
const popupPromise = page.waitForEvent("popup", { timeout: 8000 }).catch(() => null); const popupPromise = page
.waitForEvent("popup", { timeout: 8000 })
.catch(() => null);
const navigationPromise = page const navigationPromise = page
.waitForNavigation({ timeout: 8000, waitUntil: "domcontentloaded" }) .waitForNavigation({ timeout: 8000, waitUntil: "domcontentloaded" })
.catch(() => null); .catch(() => null);
@@ -271,7 +281,12 @@ router.addHandler(
await applyButton.click(); await applyButton.click();
// Wait for URL to stabilize (same URL for 3 consecutive checks) // Wait for URL to stabilize (same URL for 3 consecutive checks)
const waitForUrlStable = async (targetPage: typeof page, maxWaitMs = 10000, checkIntervalMs = 100, requiredStableChecks = 3) => { const waitForUrlStable = async (
targetPage: typeof page,
maxWaitMs = 10000,
checkIntervalMs = 100,
requiredStableChecks = 3,
) => {
let lastUrl = targetPage.url(); let lastUrl = targetPage.url();
let stableCount = 0; let stableCount = 0;
const startTime = Date.now(); const startTime = Date.now();
@@ -298,11 +313,15 @@ router.addHandler(
const targetPage = maybePopup ?? page; const targetPage = maybePopup ?? page;
if (maybePopup) { if (maybePopup) {
await maybePopup.waitForLoadState("domcontentloaded", { timeout: 15000 }).catch(() => null); await maybePopup
.waitForLoadState("domcontentloaded", { timeout: 15000 })
.catch(() => null);
// If the popup initially opens as about:blank, give it a moment to redirect. // If the popup initially opens as about:blank, give it a moment to redirect.
if (maybePopup.url() === "about:blank") { if (maybePopup.url() === "about:blank") {
await maybePopup await maybePopup
.waitForURL((u) => u.toString() !== "about:blank", { timeout: 15000 }) .waitForURL((u) => u.toString() !== "about:blank", {
timeout: 15000,
})
.catch(() => null); .catch(() => null);
} }
} else { } else {
@@ -317,7 +336,7 @@ router.addHandler(
if (applicationLink === originalUrl) { if (applicationLink === originalUrl) {
log.info( log.info(
`Apply click did not change URL (still Gradcracker): ${applicationLink}` `Apply click did not change URL (still Gradcracker): ${applicationLink}`,
); );
} else { } else {
log.info(`Captured application URL: ${applicationLink}`); log.info(`Captured application URL: ${applicationLink}`);
@@ -342,5 +361,5 @@ router.addHandler(
}); });
markJobPageDone({ currentUrl: request.url }); markJobPageDone({ currentUrl: request.url });
} },
); );
extractors/ukvisajobs/src/main.ts
+161 -108
View File
@@ -13,17 +13,18 @@
* UKVISAJOBS_REFRESH_ONLY - Set to "1" to refresh tokens and exit * UKVISAJOBS_REFRESH_ONLY - Set to "1" to refresh tokens and exit
*/ */
import { mkdir, writeFile, readFile } from 'fs/promises'; import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, dirname } from 'path'; import { dirname, join } from "node:path";
import { fileURLToPath } from 'url'; import { fileURLToPath } from "node:url";
import type { Request } from 'playwright'; import type { Request } from "playwright";
const __dirname = dirname(fileURLToPath(import.meta.url)); const __dirname = dirname(fileURLToPath(import.meta.url));
const API_URL = 'https://my.ukvisajobs.com/ukvisa-api/api/fetch-jobs-data'; const API_URL = "https://my.ukvisajobs.com/ukvisa-api/api/fetch-jobs-data";
const SIGNIN_URL = 'https://my.ukvisajobs.com/signin'; const SIGNIN_URL = "https://my.ukvisajobs.com/signin";
const OPEN_JOBS_URL = 'https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&visaAcceptance=false&applicants_outside_uk=false&pageNo=1'; const OPEN_JOBS_URL =
const AUTH_CACHE_PATH = join(__dirname, '../storage/ukvisajobs-auth.json'); "https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&visaAcceptance=false&applicants_outside_uk=false&pageNo=1";
const AUTH_CACHE_PATH = join(__dirname, "../storage/ukvisajobs-auth.json");
const JOBS_PER_PAGE = 15; const JOBS_PER_PAGE = 15;
const DEFAULT_MAX_JOBS = 50; const DEFAULT_MAX_JOBS = 50;
const MAX_ALLOWED_JOBS = 200; const MAX_ALLOWED_JOBS = 200;
@@ -64,7 +65,7 @@ interface UkVisaJobsApiResponse {
} }
interface ExtractedJob { interface ExtractedJob {
source: 'ukvisajobs'; source: "ukvisajobs";
sourceJobId: string; sourceJobId: string;
title: string; title: string;
employer: string; employer: string;
@@ -87,7 +88,7 @@ interface UkVisaJobsAuthSession {
csrfToken: string; csrfToken: string;
ciSession: string; ciSession: string;
fetchedAt: string; fetchedAt: string;
source: 'cache' | 'browser'; source: "cache" | "browser";
} }
class UkVisaJobsAuthError extends Error { class UkVisaJobsAuthError extends Error {
@@ -96,7 +97,7 @@ class UkVisaJobsAuthError extends Error {
constructor(message: string, status: number, responseText: string) { constructor(message: string, status: number, responseText: string) {
super(message); super(message);
this.name = 'UkVisaJobsAuthError'; this.name = "UkVisaJobsAuthError";
this.status = status; this.status = status;
this.responseText = responseText; this.responseText = responseText;
} }
@@ -104,18 +105,19 @@ class UkVisaJobsAuthError extends Error {
function toStringOrNull(value: unknown): string | null { function toStringOrNull(value: unknown): string | null {
if (value === null || value === undefined) return null; if (value === null || value === undefined) return null;
if (typeof value === 'string') { if (typeof value === "string") {
const trimmed = value.trim(); const trimmed = value.trim();
return trimmed.length > 0 ? trimmed : null; return trimmed.length > 0 ? trimmed : null;
} }
if (typeof value === 'number' || typeof value === 'boolean') return String(value); if (typeof value === "number" || typeof value === "boolean")
return String(value);
return null; return null;
} }
function toNumberOrNull(value: unknown): number | null { function toNumberOrNull(value: unknown): number | null {
if (value === null || value === undefined) return null; if (value === null || value === undefined) return null;
if (typeof value === 'number') return Number.isFinite(value) ? value : null; if (typeof value === "number") return Number.isFinite(value) ? value : null;
if (typeof value === 'string') { if (typeof value === "string") {
const trimmed = value.trim(); const trimmed = value.trim();
if (!trimmed) return null; if (!trimmed) return null;
const parsed = Number(trimmed); const parsed = Number(trimmed);
@@ -127,29 +129,30 @@ function toNumberOrNull(value: unknown): number | null {
async function fetchPage( async function fetchPage(
pageNo: number, pageNo: number,
session: UkVisaJobsAuthSession, session: UkVisaJobsAuthSession,
options: { searchKeyword?: string } = {} options: { searchKeyword?: string } = {},
): Promise<UkVisaJobsApiResponse> { ): Promise<UkVisaJobsApiResponse> {
// Use native FormData API (Node.js 18+) // Use native FormData API (Node.js 18+)
const formData = new FormData(); const formData = new FormData();
formData.append('is_global', '0'); formData.append("is_global", "0");
formData.append('sortBy', 'desc'); formData.append("sortBy", "desc");
formData.append('pageNo', String(pageNo)); formData.append("pageNo", String(pageNo));
formData.append('visaAcceptance', 'false'); formData.append("visaAcceptance", "false");
formData.append('applicants_outside_uk', 'false'); formData.append("applicants_outside_uk", "false");
formData.append('searchKeyword', options.searchKeyword || 'null'); formData.append("searchKeyword", options.searchKeyword || "null");
formData.append('token', session.token); formData.append("token", session.token);
const cookies = buildCookieHeader(session); const cookies = buildCookieHeader(session);
const response = await fetch(API_URL, { const response = await fetch(API_URL, {
method: 'POST', method: "POST",
headers: { headers: {
'accept': 'application/json, text/plain, */*', accept: "application/json, text/plain, */*",
'accept-language': 'en-US,en;q=0.9', "accept-language": "en-US,en;q=0.9",
'cookie': cookies, cookie: cookies,
'origin': 'https://my.ukvisajobs.com', origin: "https://my.ukvisajobs.com",
'referer': `https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&pageNo=${pageNo}&visaAcceptance=false&applicants_outside_uk=false`, referer: `https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&pageNo=${pageNo}&visaAcceptance=false&applicants_outside_uk=false`,
'user-agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36', "user-agent":
"Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36",
}, },
body: formData, body: formData,
}); });
@@ -160,10 +163,12 @@ async function fetchPage(
throw new UkVisaJobsAuthError( throw new UkVisaJobsAuthError(
`UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`, `UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`,
response.status, response.status,
text text,
); );
} }
throw new Error(`UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`); throw new Error(
`UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`,
);
} }
return response.json() as Promise<UkVisaJobsApiResponse>; return response.json() as Promise<UkVisaJobsApiResponse>;
@@ -171,11 +176,16 @@ async function fetchPage(
function mapJob(raw: UkVisaJobsApiJob): ExtractedJob { function mapJob(raw: UkVisaJobsApiJob): ExtractedJob {
// Build salary string from min/max // Build salary string from min/max
let salary: string | undefined = undefined; let salary: string | undefined;
const minSalary = toNumberOrNull(raw.min_salary); const minSalary = toNumberOrNull(raw.min_salary);
const maxSalary = toNumberOrNull(raw.max_salary); const maxSalary = toNumberOrNull(raw.max_salary);
if (minSalary !== null && minSalary > 0 && maxSalary !== null && maxSalary > 0) { if (
minSalary !== null &&
minSalary > 0 &&
maxSalary !== null &&
maxSalary > 0
) {
salary = `£${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}`; salary = `£${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}`;
if (raw.salary_interval) { if (raw.salary_interval) {
salary += ` / ${raw.salary_interval}`; salary += ` / ${raw.salary_interval}`;
@@ -189,24 +199,30 @@ function mapJob(raw: UkVisaJobsApiJob): ExtractedJob {
// Build a description from visa sponsorship fields // Build a description from visa sponsorship fields
const visaInfo: string[] = []; const visaInfo: string[] = [];
if (raw.visa_acceptance?.toLowerCase() === 'yes') visaInfo.push('Visa acceptance: Yes'); if (raw.visa_acceptance?.toLowerCase() === "yes")
if (raw.applicants_outside_uk?.toLowerCase() === 'yes') visaInfo.push('Accepts applicants outside UK'); visaInfo.push("Visa acceptance: Yes");
if (raw.likely_to_sponsor?.toLowerCase() === 'yes') visaInfo.push('Likely to sponsor'); if (raw.applicants_outside_uk?.toLowerCase() === "yes")
if (raw.definitely_sponsored?.toLowerCase() === 'yes') visaInfo.push('Definitely sponsored'); visaInfo.push("Accepts applicants outside UK");
if (raw.new_entrant?.toLowerCase() === 'yes') visaInfo.push('New entrant friendly'); if (raw.likely_to_sponsor?.toLowerCase() === "yes")
if (raw.student_graduate?.toLowerCase() === 'yes') visaInfo.push('Student/Graduate friendly'); visaInfo.push("Likely to sponsor");
if (raw.definitely_sponsored?.toLowerCase() === "yes")
visaInfo.push("Definitely sponsored");
if (raw.new_entrant?.toLowerCase() === "yes")
visaInfo.push("New entrant friendly");
if (raw.student_graduate?.toLowerCase() === "yes")
visaInfo.push("Student/Graduate friendly");
const description = raw.description const description = raw.description
? raw.description ? raw.description
: visaInfo.length > 0 : visaInfo.length > 0
? `Visa sponsorship info: ${visaInfo.join(', ')}` ? `Visa sponsorship info: ${visaInfo.join(", ")}`
: undefined; : undefined;
return { return {
source: 'ukvisajobs', source: "ukvisajobs",
sourceJobId: raw.id, sourceJobId: raw.id,
title: raw.title || 'Unknown Title', title: raw.title || "Unknown Title",
employer: raw.company_name || 'Unknown Employer', employer: raw.company_name || "Unknown Employer",
employerUrl: toStringOrNull(raw.company_link) ?? undefined, employerUrl: toStringOrNull(raw.company_link) ?? undefined,
jobUrl: raw.job_link, jobUrl: raw.job_link,
applicationLink: raw.job_link, applicationLink: raw.job_link,
@@ -226,7 +242,7 @@ function buildCookieHeader(session: UkVisaJobsAuthSession): string {
if (session.csrfToken) cookieParts.push(`csrf_token=${session.csrfToken}`); if (session.csrfToken) cookieParts.push(`csrf_token=${session.csrfToken}`);
if (session.ciSession) cookieParts.push(`ci_session=${session.ciSession}`); if (session.ciSession) cookieParts.push(`ci_session=${session.ciSession}`);
if (session.authToken) cookieParts.push(`authToken=${session.authToken}`); if (session.authToken) cookieParts.push(`authToken=${session.authToken}`);
return cookieParts.join('; '); return cookieParts.join("; ");
} }
function getLoginCredentials(): { email: string; password: string } | null { function getLoginCredentials(): { email: string; password: string } | null {
@@ -238,23 +254,25 @@ function getLoginCredentials(): { email: string; password: string } | null {
async function loadCachedAuthSession(): Promise<UkVisaJobsAuthSession | null> { async function loadCachedAuthSession(): Promise<UkVisaJobsAuthSession | null> {
try { try {
const data = await readFile(AUTH_CACHE_PATH, 'utf8'); const data = await readFile(AUTH_CACHE_PATH, "utf8");
const parsed = JSON.parse(data) as UkVisaJobsAuthSession; const parsed = JSON.parse(data) as UkVisaJobsAuthSession;
if (!parsed?.token) return null; if (!parsed?.token) return null;
return { return {
token: parsed.token, token: parsed.token,
authToken: parsed.authToken || parsed.token, authToken: parsed.authToken || parsed.token,
csrfToken: parsed.csrfToken || '', csrfToken: parsed.csrfToken || "",
ciSession: parsed.ciSession || '', ciSession: parsed.ciSession || "",
fetchedAt: parsed.fetchedAt || new Date().toISOString(), fetchedAt: parsed.fetchedAt || new Date().toISOString(),
source: 'cache', source: "cache",
}; };
} catch (error) { } catch (_error) {
return null; return null;
} }
} }
async function saveCachedAuthSession(session: UkVisaJobsAuthSession): Promise<void> { async function saveCachedAuthSession(
session: UkVisaJobsAuthSession,
): Promise<void> {
const payload = { const payload = {
token: session.token, token: session.token,
authToken: session.authToken, authToken: session.authToken,
@@ -273,17 +291,18 @@ function extractMultipartField(body: string, field: string): string | null {
if (index === -1) return null; if (index === -1) return null;
const afterName = body.slice(index + nameToken.length); const afterName = body.slice(index + nameToken.length);
let separatorIndex = afterName.indexOf('\r\n\r\n'); let separatorIndex = afterName.indexOf("\r\n\r\n");
let separatorLength = 4; let separatorLength = 4;
if (separatorIndex === -1) { if (separatorIndex === -1) {
separatorIndex = afterName.indexOf('\n\n'); separatorIndex = afterName.indexOf("\n\n");
separatorLength = 2; separatorLength = 2;
} }
if (separatorIndex === -1) return null; if (separatorIndex === -1) return null;
const valueStart = index + nameToken.length + separatorIndex + separatorLength; const valueStart =
index + nameToken.length + separatorIndex + separatorLength;
const remainder = body.slice(valueStart); const remainder = body.slice(valueStart);
const endIndex = remainder.indexOf('\r\n'); const endIndex = remainder.indexOf("\r\n");
if (endIndex === -1) return remainder.trim(); if (endIndex === -1) return remainder.trim();
return remainder.slice(0, endIndex).trim(); return remainder.slice(0, endIndex).trim();
} }
@@ -291,13 +310,13 @@ function extractMultipartField(body: string, field: string): string | null {
function extractTokenFromRequest(request: Request): string | null { function extractTokenFromRequest(request: Request): string | null {
const postData = request.postData(); const postData = request.postData();
if (!postData) return null; if (!postData) return null;
const multipartToken = extractMultipartField(postData, 'token'); const multipartToken = extractMultipartField(postData, "token");
if (multipartToken) return multipartToken; if (multipartToken) return multipartToken;
try { try {
const params = new URLSearchParams(postData); const params = new URLSearchParams(postData);
const token = params.get('token'); const token = params.get("token");
return token || null; return token || null;
} catch (error) { } catch (_error) {
return null; return null;
} }
} }
@@ -306,60 +325,75 @@ function isAuthErrorResponse(status: number, bodyText: string): boolean {
if (status === 401 || status === 403) return true; if (status === 401 || status === 403) return true;
if (status !== 400) return false; if (status !== 400) return false;
try { try {
const parsed = JSON.parse(bodyText) as { errorType?: string; message?: string }; const parsed = JSON.parse(bodyText) as {
if (parsed?.errorType === 'expired') return true; errorType?: string;
if (parsed?.message && parsed.message.toLowerCase().includes('expired')) return true; message?: string;
} catch (error) { };
if (parsed?.errorType === "expired") return true;
if (parsed?.message?.toLowerCase().includes("expired")) return true;
} catch (_error) {
// ignore JSON parse failures // ignore JSON parse failures
} }
return bodyText.toLowerCase().includes('expired'); return bodyText.toLowerCase().includes("expired");
} }
async function loginWithBrowser(email: string, password: string): Promise<UkVisaJobsAuthSession> { async function loginWithBrowser(
email: string,
password: string,
): Promise<UkVisaJobsAuthSession> {
const [{ launchOptions }, { firefox }] = await Promise.all([ const [{ launchOptions }, { firefox }] = await Promise.all([
import('camoufox-js'), import("camoufox-js"),
import('playwright'), import("playwright"),
]); ]);
const headless = process.env.UKVISAJOBS_HEADLESS !== 'false'; const headless = process.env.UKVISAJOBS_HEADLESS !== "false";
const browser = await firefox.launch(await launchOptions({ const browser = await firefox.launch(
await launchOptions({
headless, headless,
humanize: true, humanize: true,
geoip: true, geoip: true,
})); }),
);
const context = await browser.newContext(); const context = await browser.newContext();
const page = await context.newPage(); const page = await context.newPage();
try { try {
await page.goto(SIGNIN_URL, { waitUntil: 'domcontentloaded' }); await page.goto(SIGNIN_URL, { waitUntil: "domcontentloaded" });
await page.waitForSelector('#email', { timeout: 15000 }); await page.waitForSelector("#email", { timeout: 15000 });
await page.fill('#email', email); await page.fill("#email", email);
await page.fill('#password', password); await page.fill("#password", password);
await page.keyboard.press('Enter'); await page.keyboard.press("Enter");
await page.waitForTimeout(7000); await page.waitForTimeout(7000);
const requestPromise = page.waitForRequest( const requestPromise = page.waitForRequest(
(request) => request.url().includes('/ukvisa-api/api/fetch-jobs-data') && request.method() === 'POST', (request) =>
{ timeout: 30000 } request.url().includes("/ukvisa-api/api/fetch-jobs-data") &&
request.method() === "POST",
{ timeout: 30000 },
); );
await page.goto(OPEN_JOBS_URL, { waitUntil: 'networkidle' }); await page.goto(OPEN_JOBS_URL, { waitUntil: "networkidle" });
await page.waitForTimeout(5000); await page.waitForTimeout(5000);
let fetchRequest: Request | null = null; let fetchRequest: Request | null = null;
try { try {
fetchRequest = await requestPromise; fetchRequest = await requestPromise;
} catch (error) { } catch (_error) {
fetchRequest = null; fetchRequest = null;
} }
const cookies = await context.cookies('https://my.ukvisajobs.com'); const cookies = await context.cookies("https://my.ukvisajobs.com");
const csrfToken = cookies.find((cookie) => cookie.name === 'csrf_token')?.value || ''; const csrfToken =
const ciSession = cookies.find((cookie) => cookie.name === 'ci_session')?.value || ''; cookies.find((cookie) => cookie.name === "csrf_token")?.value || "";
const authToken = cookies.find((cookie) => cookie.name === 'authToken')?.value || ''; const ciSession =
const token = fetchRequest ? extractTokenFromRequest(fetchRequest) : authToken; cookies.find((cookie) => cookie.name === "ci_session")?.value || "";
const authToken =
cookies.find((cookie) => cookie.name === "authToken")?.value || "";
const token = fetchRequest
? extractTokenFromRequest(fetchRequest)
: authToken;
if (!token) { if (!token) {
throw new Error('Failed to locate auth token from browser session.'); throw new Error("Failed to locate auth token from browser session.");
} }
return { return {
@@ -368,7 +402,7 @@ async function loginWithBrowser(email: string, password: string): Promise<UkVisa
csrfToken, csrfToken,
ciSession, ciSession,
fetchedAt: new Date().toISOString(), fetchedAt: new Date().toISOString(),
source: 'browser', source: "browser",
}; };
} finally { } finally {
await browser.close(); await browser.close();
@@ -376,38 +410,48 @@ async function loginWithBrowser(email: string, password: string): Promise<UkVisa
} }
async function main(): Promise<void> { async function main(): Promise<void> {
console.log('🇬🇧 UK Visa Jobs Extractor starting...'); console.log("🇬🇧 UK Visa Jobs Extractor starting...");
const credentials = getLoginCredentials(); const credentials = getLoginCredentials();
const searchKeyword = process.env.UKVISAJOBS_SEARCH_KEYWORD || undefined; const searchKeyword = process.env.UKVISAJOBS_SEARCH_KEYWORD || undefined;
const refreshOnly = process.env.UKVISAJOBS_REFRESH_ONLY === '1'; const refreshOnly = process.env.UKVISAJOBS_REFRESH_ONLY === "1";
let authSession = await loadCachedAuthSession(); let authSession = await loadCachedAuthSession();
if (refreshOnly) { if (refreshOnly) {
if (!credentials) { if (!credentials) {
console.error('ERROR: UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD must be set'); console.error(
"ERROR: UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD must be set",
);
process.exit(1); process.exit(1);
} }
console.log(' Refresh-only mode: logging in to refresh tokens...'); console.log(" Refresh-only mode: logging in to refresh tokens...");
authSession = await loginWithBrowser(credentials.email, credentials.password); authSession = await loginWithBrowser(
credentials.email,
credentials.password,
);
await saveCachedAuthSession(authSession); await saveCachedAuthSession(authSession);
console.log(' Auth session refreshed.'); console.log(" Auth session refreshed.");
return; return;
} }
if (!authSession) { if (!authSession) {
if (!credentials) { if (!credentials) {
console.error('ERROR: UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD must be set'); console.error(
"ERROR: UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD must be set",
);
process.exit(1); process.exit(1);
} }
console.log(' No cached session found. Logging in to refresh tokens...'); console.log(" No cached session found. Logging in to refresh tokens...");
authSession = await loginWithBrowser(credentials.email, credentials.password); authSession = await loginWithBrowser(
credentials.email,
credentials.password,
);
await saveCachedAuthSession(authSession); await saveCachedAuthSession(authSession);
} }
const cookies = buildCookieHeader(authSession); const cookies = buildCookieHeader(authSession);
console.log(` Auth source: ${authSession.source}`); console.log(` Auth source: ${authSession.source}`);
console.log(` Cookies configured: ${cookies ? 'Yes' : 'No'}`); console.log(` Cookies configured: ${cookies ? "Yes" : "No"}`);
console.log(` Token length: ${authSession.token.length}`); console.log(` Token length: ${authSession.token.length}`);
// Get max jobs from environment // Get max jobs from environment
@@ -435,20 +479,30 @@ async function main(): Promise<void> {
} catch (error) { } catch (error) {
if (!credentials) { if (!credentials) {
if (error instanceof UkVisaJobsAuthError) { if (error instanceof UkVisaJobsAuthError) {
throw new Error('UKVisaJobs auth expired. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.'); throw new Error(
"UKVisaJobs auth expired. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.",
);
} }
throw error; throw error;
} }
const reason = error instanceof UkVisaJobsAuthError ? 'Auth expired.' : 'Fetch failed.'; const reason =
error instanceof UkVisaJobsAuthError
? "Auth expired."
: "Fetch failed.";
console.log(` ${reason} Refreshing tokens and retrying...`); console.log(` ${reason} Refreshing tokens and retrying...`);
authSession = await loginWithBrowser(credentials.email, credentials.password); authSession = await loginWithBrowser(
credentials.email,
credentials.password,
);
await saveCachedAuthSession(authSession); await saveCachedAuthSession(authSession);
response = await fetchPage(pageNo, authSession, { searchKeyword }); response = await fetchPage(pageNo, authSession, { searchKeyword });
} }
if (response.status !== 1) { if (response.status !== 1) {
console.warn(` ⚠️ API returned status ${response.status} on page ${pageNo}`); console.warn(
` ⚠️ API returned status ${response.status} on page ${pageNo}`,
);
break; break;
} }
@@ -487,33 +541,32 @@ async function main(): Promise<void> {
console.log(`✅ Scraped ${allJobs.length} jobs`); console.log(`✅ Scraped ${allJobs.length} jobs`);
// Write output to storage directory (similar to Crawlee dataset structure) // Write output to storage directory (similar to Crawlee dataset structure)
const storageDir = join(__dirname, '../storage/datasets/default'); const storageDir = join(__dirname, "../storage/datasets/default");
await mkdir(storageDir, { recursive: true }); await mkdir(storageDir, { recursive: true });
// Write each job as a separate JSON file (Crawlee dataset format) // Write each job as a separate JSON file (Crawlee dataset format)
for (let i = 0; i < allJobs.length; i++) { for (let i = 0; i < allJobs.length; i++) {
const filename = join(storageDir, `${String(i + 1).padStart(6, '0')}.json`); const filename = join(
storageDir,
`${String(i + 1).padStart(6, "0")}.json`,
);
await writeFile(filename, JSON.stringify(allJobs[i], null, 2)); await writeFile(filename, JSON.stringify(allJobs[i], null, 2));
} }
// Also write a combined output file for easier consumption // Also write a combined output file for easier consumption
const outputFile = join(storageDir, 'jobs.json'); const outputFile = join(storageDir, "jobs.json");
await writeFile(outputFile, JSON.stringify(allJobs, null, 2)); await writeFile(outputFile, JSON.stringify(allJobs, null, 2));
console.log(` Output written to: ${storageDir}`); console.log(` Output written to: ${storageDir}`);
console.log(` Jobs file: ${outputFile}`); console.log(` Jobs file: ${outputFile}`);
} catch (error) { } catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error'; const message = error instanceof Error ? error.message : "Unknown error";
console.error(`❌ Error: ${message}`); console.error(`❌ Error: ${message}`);
process.exit(1); process.exit(1);
} }
} }
main().catch((error) => { main().catch((error) => {
console.error('Fatal error:', error); console.error("Fatal error:", error);
process.exit(1); process.exit(1);
}); });