diff --git a/orchestrator/src/client/App.tsx b/orchestrator/src/client/App.tsx index 8fb2815..94dab4c 100644 --- a/orchestrator/src/client/App.tsx +++ b/orchestrator/src/client/App.tsx @@ -8,14 +8,16 @@ import { Route, Routes } from "react-router-dom"; import { Toaster } from "@/components/ui/sonner"; import { OrchestratorPage } from "./pages/OrchestratorPage"; import { SettingsPage } from "./pages/SettingsPage"; +import { UkVisaJobsPage } from "./pages/UkVisaJobsPage"; export const App: React.FC = () => ( <> } /> } /> + } /> -); \ No newline at end of file +); diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index 344cdbd..2bf3373 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -11,6 +11,9 @@ import type { PipelineRun, AppSettings, ResumeProjectsSettings, + UkVisaJobsSearchResponse, + UkVisaJobsImportResponse, + CreateJobInput, } from '../../shared/types'; const API_BASE = '/api'; @@ -108,6 +111,26 @@ export async function runPipeline(config?: { }); } +// UK Visa Jobs API +export async function searchUkVisaJobs(input: { + searchTerm?: string; + page?: number; +}): Promise { + return fetchApi('/ukvisajobs/search', { + method: 'POST', + body: JSON.stringify(input), + }); +} + +export async function importUkVisaJobs(input: { + jobs: CreateJobInput[]; +}): Promise { + return fetchApi('/ukvisajobs/import', { + method: 'POST', + body: JSON.stringify(input), + }); +} + // Settings & Profile API export async function getSettings(): Promise { return fetchApi('/settings'); diff --git a/orchestrator/src/client/pages/OrchestratorPage.tsx b/orchestrator/src/client/pages/OrchestratorPage.tsx index b01f742..f55e51b 100644 --- a/orchestrator/src/client/pages/OrchestratorPage.tsx +++ b/orchestrator/src/client/pages/OrchestratorPage.tsx @@ -639,6 +639,11 @@ export const OrchestratorPage: React.FC = () => { + + + + + + +
+
+
+
+ + setSearchTermInput(event.target.value)} + placeholder="e.g. data analyst" + className="h-10" + /> +

+ Single keyword or phrase. Leave blank to fetch the newest jobs. +

+
+ +
+ +
+
+ + {errorMessage && ( +
+ {errorMessage} +
+ )} + + + +
+
+ Last run: {lastRunAt ? formatDateTime(lastRunAt) : "No searches yet"} +
+
+ + {totalJobs} total + + + {results.length} on page + + + Page {page} of {totalPages} + + {lastSearchTerm && Term: {lastSearchTerm}} +
+
+
+ +
+
+ {results.length === 0 ? ( +
+
No results yet
+

+ Run a search to fetch fresh UK Visa Jobs listings. +

+
+ ) : ( + <> +
+
+ { + if (checked === true) { + setSelectedJobIds(new Set(results.map((job) => jobKey(job)))); + } else { + setSelectedJobIds(new Set()); + } + }} + aria-label="Select all jobs on this page" + /> + Select page + + {selectedCount} selected +
+ +
+
+ {results.map((job) => { + const key = jobKey(job); + const isSelected = key === selectedJobId; + const isChecked = selectedJobIds.has(key); + const description = job.jobDescription ? clampText(stripHtml(job.jobDescription)) : "No description."; + + return ( +
setSelectedJobId(key)} + onKeyDown={(event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + setSelectedJobId(key); + } + }} + className={cn( + "flex w-full items-start gap-4 px-4 py-3 text-left transition-colors", + isSelected ? "bg-muted/40" : "hover:bg-muted/30", + )} + aria-pressed={isSelected} + > +
event.stopPropagation()} + onKeyDown={(event) => event.stopPropagation()} + role="presentation" + > + { + setSelectedJobIds((current) => { + const next = new Set(current); + if (checked) { + next.add(key); + } else { + next.delete(key); + } + return next; + }); + }} + aria-label={`Select ${job.title}`} + /> +
+ + + +
+
+
{job.title}
+
{job.employer}
+
+
{description}
+
+ {job.location && ( + + + {job.location} + + )} + {job.salary && ( + + + {job.salary} + + )} + {job.deadline && ( + + + {formatDate(job.deadline)} + + )} +
+
+ {job.jobType && ( + + {job.jobType} + + )} + {job.jobLevel && ( + + {job.jobLevel} + + )} +
+
+
+ ); + })} +
+
+ + Showing {summaryCounts.startIndex}-{summaryCounts.endIndex} of {totalJobs} + +
+ + + Page {page} of {totalPages} + + +
+
+ + )} +
+ +
+ {!selectedJob ? ( +
+
Select a job
+

Pick a job from the list to inspect details.

+
+ ) : ( +
+
+
+
{selectedJob.title}
+
{selectedJob.employer}
+
+ + UK Visa Jobs + +
+ +
+ {selectedJob.location && ( + + + {selectedJob.location} + + )} + {selectedDeadline && ( + + + {selectedDeadline} + + )} + {selectedPosted && ( + + + Posted {selectedPosted} + + )} + {selectedJob.salary && ( + + + {selectedJob.salary} + + )} + {selectedJob.degreeRequired && ( + + + {selectedJob.degreeRequired} + + )} +
+ +
+
+
Job type
+
{selectedJob.jobType || "Not set"}
+
+
+
Job level
+
{selectedJob.jobLevel || "Not set"}
+
+
+
Location
+
{selectedJob.location || "Not set"}
+
+
+
Deadline
+
{selectedDeadline || "Not set"}
+
+
+ + + + + +
+
+ Description +
+
+ {selectedDescription} +
+
+
+ )} +
+
+
+ + ); +}; diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts index df9f496..7797415 100644 --- a/orchestrator/src/server/api/routes.ts +++ b/orchestrator/src/server/api/routes.ts @@ -9,6 +9,7 @@ import * as pipelineRepo from '../repositories/pipeline.js'; import * as settingsRepo from '../repositories/settings.js'; import { runPipeline, processJob, summarizeJob, generateFinalPdf, getPipelineStatus, subscribeToProgress, getProgress } from '../pipeline/index.js'; import { createNotionEntry } from '../services/notion.js'; +import { fetchUkVisaJobsPage } from '../services/ukvisajobs.js'; import { clearDatabase } from '../db/clear.js'; import { extractProjectsFromProfile, @@ -16,9 +17,10 @@ import { normalizeResumeProjectsSettings, resolveResumeProjectsSettings, } from '../services/resumeProjects.js'; -import type { Job, JobStatus, ApiResponse, JobsListResponse, PipelineStatusResponse } from '../../shared/types.js'; +import type { Job, JobStatus, ApiResponse, JobsListResponse, PipelineStatusResponse, UkVisaJobsSearchResponse, UkVisaJobsImportResponse } from '../../shared/types.js'; export const apiRouter = Router(); +let isUkVisaJobsSearchRunning = false; async function notifyJobCompleteWebhook(job: Job) { const overrideWebhookUrl = await settingsRepo.getSetting('jobCompleteWebhookUrl') @@ -642,6 +644,129 @@ apiRouter.post('/pipeline/run', async (req: Request, res: Response) => { } }); +// ============================================================================ +// UK Visa Jobs API +// ============================================================================ + +const ukVisaJobsSearchSchema = z.object({ + query: z.string().trim().min(1).max(200).optional(), + searchTerm: z.string().trim().min(1).max(200).optional(), + searchTerms: z.array(z.string().trim().min(1).max(200)).max(20).optional(), + page: z.number().int().min(1).optional(), +}); + +/** + * POST /api/ukvisajobs/search - Run a UKVisaJobs search without importing into the DB + */ +apiRouter.post('/ukvisajobs/search', async (req: Request, res: Response) => { + let lockAcquired = false; + + try { + const input = ukVisaJobsSearchSchema.parse(req.body ?? {}); + + if (isUkVisaJobsSearchRunning) { + return res.status(409).json({ success: false, error: 'UK Visa Jobs search is already running' }); + } + + const { isRunning } = getPipelineStatus(); + if (isRunning) { + return res.status(409).json({ success: false, error: 'Pipeline is running. Stop it before running UK Visa Jobs search.' }); + } + + isUkVisaJobsSearchRunning = true; + lockAcquired = true; + + const rawTerms = input.searchTerms ?? []; + if (rawTerms.length > 1) { + return res.status(400).json({ success: false, error: 'Pagination supports a single search term.' }); + } + + const searchTerm = input.searchTerm ?? input.query ?? rawTerms[0]; + const page = input.page ?? 1; + + const result = await fetchUkVisaJobsPage({ + searchKeyword: searchTerm, + page, + }); + + const totalPages = Math.max(1, Math.ceil(result.totalJobs / result.pageSize)); + + const response: ApiResponse = { + success: true, + data: { + jobs: result.jobs, + totalJobs: result.totalJobs, + page: result.page, + pageSize: result.pageSize, + totalPages, + }, + }; + + res.json(response); + } catch (error) { + if (error instanceof z.ZodError) { + return res.status(400).json({ success: false, error: error.message }); + } + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, error: message }); + } finally { + if (lockAcquired) { + isUkVisaJobsSearchRunning = false; + } + } +}); + +const ukVisaJobsImportSchema = z.object({ + jobs: z.array(z.object({ + title: z.string().trim().min(1).max(500), + employer: z.string().trim().min(1).max(500), + jobUrl: z.string().trim().min(1).max(2000), + sourceJobId: z.string().trim().min(1).max(200).optional(), + employerUrl: z.string().trim().min(1).max(2000).optional(), + applicationLink: z.string().trim().min(1).max(2000).optional(), + location: z.string().trim().max(200).optional(), + deadline: z.string().trim().max(100).optional(), + salary: z.string().trim().max(200).optional(), + jobDescription: z.string().trim().max(20000).optional(), + datePosted: z.string().trim().max(100).optional(), + degreeRequired: z.string().trim().max(200).optional(), + jobType: z.string().trim().max(200).optional(), + jobLevel: z.string().trim().max(200).optional(), + })).min(1).max(200), +}); + +/** + * POST /api/ukvisajobs/import - Import selected UKVisaJobs results into the DB + */ +apiRouter.post('/ukvisajobs/import', async (req: Request, res: Response) => { + try { + const input = ukVisaJobsImportSchema.parse(req.body ?? {}); + + const jobs = input.jobs.map((job) => ({ + ...job, + source: 'ukvisajobs' as const, + })); + + const result = await jobsRepo.bulkCreateJobs(jobs); + + const response: ApiResponse = { + success: true, + data: { + created: result.created, + skipped: result.skipped, + }, + }; + + res.json(response); + } catch (error) { + if (error instanceof z.ZodError) { + return res.status(400).json({ success: false, error: error.message }); + } + const message = error instanceof Error ? error.message : 'Unknown error'; + res.status(500).json({ success: false, error: message }); + } +}); + // ============================================================================ // Webhook for n8n // ============================================================================ diff --git a/orchestrator/src/server/services/ukvisajobs.ts b/orchestrator/src/server/services/ukvisajobs.ts index 1e727c5..c1497c8 100644 --- a/orchestrator/src/server/services/ukvisajobs.ts +++ b/orchestrator/src/server/services/ukvisajobs.ts @@ -14,6 +14,9 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const UKVISAJOBS_DIR = join(__dirname, '../../../../extractors/ukvisajobs'); const STORAGE_DIR = join(UKVISAJOBS_DIR, 'storage/datasets/default'); const AUTH_CACHE_PATH = join(UKVISAJOBS_DIR, 'storage/ukvisajobs-auth.json'); +const UKVISAJOBS_API_URL = 'https://my.ukvisajobs.com/ukvisa-api/api/fetch-jobs-data'; +const UKVISAJOBS_PAGE_SIZE = 15; +let isUkVisaJobsRunning = false; interface UkVisaJobsAuthSession { token?: string; @@ -37,6 +40,117 @@ export interface UkVisaJobsResult { error?: string; } +function toStringOrNull(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (typeof value === 'string') { + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; + } + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return null; +} + +function toNumberOrNull(value: unknown): number | null { + if (value === null || value === undefined) return null; + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed) return null; + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} + +function buildCookieHeader(session: UkVisaJobsAuthSession): string { + const cookieParts: string[] = []; + if (session.csrfToken) cookieParts.push(`csrf_token=${session.csrfToken}`); + if (session.ciSession) cookieParts.push(`ci_session=${session.ciSession}`); + const token = session.authToken || session.token; + if (token) cookieParts.push(`authToken=${token}`); + return cookieParts.join('; '); +} + +function buildVisaInfoDescription(raw: UkVisaJobsApiJob): string | undefined { + const visaInfo: string[] = []; + if (raw.visa_acceptance?.toLowerCase() === 'yes') visaInfo.push('Visa acceptance: Yes'); + if (raw.applicants_outside_uk?.toLowerCase() === 'yes') visaInfo.push('Accepts applicants outside UK'); + if (raw.likely_to_sponsor?.toLowerCase() === 'yes') visaInfo.push('Likely to sponsor'); + if (raw.definitely_sponsored?.toLowerCase() === 'yes') visaInfo.push('Definitely sponsored'); + if (raw.new_entrant?.toLowerCase() === 'yes') visaInfo.push('New entrant friendly'); + if (raw.student_graduate?.toLowerCase() === 'yes') visaInfo.push('Student/Graduate friendly'); + if (visaInfo.length === 0) return undefined; + return `Visa sponsorship info: ${visaInfo.join(', ')}`; +} + +function formatSalary(raw: UkVisaJobsApiJob): string | undefined { + const minSalary = toNumberOrNull(raw.min_salary); + const maxSalary = toNumberOrNull(raw.max_salary); + const interval = toStringOrNull(raw.salary_interval); + + if (minSalary && maxSalary && maxSalary > 0) { + return `GBP ${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}${interval ? ` / ${interval}` : ''}`; + } + if (maxSalary && maxSalary > 0) { + return `GBP ${maxSalary.toLocaleString()}${interval ? ` / ${interval}` : ''}`; + } + return undefined; +} + +function mapApiJob(raw: UkVisaJobsApiJob): CreateJobInput { + const description = toStringOrNull(raw.description) ?? buildVisaInfoDescription(raw); + return { + source: 'ukvisajobs', + sourceJobId: toStringOrNull(raw.id) ?? undefined, + title: toStringOrNull(raw.title) ?? 'Unknown Title', + employer: toStringOrNull(raw.company_name) ?? 'Unknown Employer', + employerUrl: toStringOrNull(raw.company_link) ?? undefined, + jobUrl: toStringOrNull(raw.job_link) ?? '', + applicationLink: toStringOrNull(raw.job_link) ?? undefined, + location: toStringOrNull(raw.city) ?? undefined, + deadline: toStringOrNull(raw.job_expire) ?? undefined, + salary: formatSalary(raw), + jobDescription: description ?? undefined, + datePosted: toStringOrNull(raw.created_date) ?? undefined, + degreeRequired: toStringOrNull(raw.degree_requirement) ?? undefined, + jobType: toStringOrNull(raw.job_type) ?? undefined, + jobLevel: toStringOrNull(raw.job_level) ?? undefined, + }; +} + +interface UkVisaJobsApiJob { + id: string; + title: string; + company_name: string; + company_link?: string; + job_link: string; + city?: string; + created_date?: string; + job_expire?: string; + description?: string; + min_salary?: string; + max_salary?: string; + salary_interval?: string; + salary_method?: string; + degree_requirement?: string; + job_type?: string; + job_level?: string; + job_industry?: string; + visa_acceptance?: string; + applicants_outside_uk?: string; + likely_to_sponsor?: string; + definitely_sponsored?: string; + new_entrant?: string; + student_graduate?: string; +} + +interface UkVisaJobsApiResponse { + status: number; + totalJobs: number; + query?: string; + jobs: UkVisaJobsApiJob[]; +} + /** * Basic HTML to text conversion to extract job description. */ @@ -134,7 +248,85 @@ async function clearStorageDataset(): Promise { } } +export async function fetchUkVisaJobsPage(options: { searchKeyword?: string; page?: number } = {}): Promise<{ + jobs: CreateJobInput[]; + totalJobs: number; + page: number; + pageSize: number; +}> { + const page = options.page && options.page > 0 ? options.page : 1; + const authSession = await loadCachedAuthSession(); + const token = authSession?.token || authSession?.authToken; + + if (!token) { + throw new Error('UK Visa Jobs auth session missing. Run the extractor to refresh tokens.'); + } + + const formData = new FormData(); + formData.append('is_global', '0'); + formData.append('sortBy', 'desc'); + formData.append('pageNo', String(page)); + formData.append('visaAcceptance', 'false'); + formData.append('applicants_outside_uk', 'false'); + formData.append('searchKeyword', options.searchKeyword ? options.searchKeyword : 'null'); + formData.append('token', token); + + const cookies = buildCookieHeader({ + token: authSession?.token, + authToken: authSession?.authToken, + csrfToken: authSession?.csrfToken, + ciSession: authSession?.ciSession, + }); + + const response = await fetch(UKVISAJOBS_API_URL, { + method: 'POST', + headers: { + 'accept': 'application/json, text/plain, */*', + 'cookie': cookies, + 'origin': 'https://my.ukvisajobs.com', + 'referer': `https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&pageNo=${page}&visaAcceptance=false&applicants_outside_uk=false`, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + body: formData, + }); + + const text = await response.text(); + if (!response.ok) { + throw new Error(`UK Visa Jobs API returned ${response.status}: ${text}`); + } + + let data: UkVisaJobsApiResponse; + try { + data = JSON.parse(text) as UkVisaJobsApiResponse; + } catch (error) { + throw new Error('UK Visa Jobs API returned an invalid response.'); + } + + if (data.status !== 1) { + throw new Error(`UK Visa Jobs API returned status ${data.status}`); + } + + const jobs = (data.jobs || []) + .map(mapApiJob) + .filter((job) => Boolean(job.jobUrl)); + + const totalJobs = Number.isFinite(data.totalJobs) ? data.totalJobs : jobs.length; + + return { + jobs, + totalJobs, + page, + pageSize: UKVISAJOBS_PAGE_SIZE, + }; +} + export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise { + if (isUkVisaJobsRunning) { + return { success: false, jobs: [], error: 'UK Visa Jobs extractor is already running' }; + } + + isUkVisaJobsRunning = true; + try { console.log('🇬🇧 Running UK Visa Jobs extractor...'); // Determine terms to run @@ -226,6 +418,9 @@ export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise console.log(`✅ UK Visa Jobs: imported total ${allJobs.length} unique jobs`); return { success: true, jobs: allJobs }; + } finally { + isUkVisaJobsRunning = false; + } } /** diff --git a/orchestrator/src/shared/types.ts b/orchestrator/src/shared/types.ts index 7815db1..964f5ab 100644 --- a/orchestrator/src/shared/types.ts +++ b/orchestrator/src/shared/types.ts @@ -170,6 +170,19 @@ export interface JobsListResponse { byStatus: Record; } +export interface UkVisaJobsSearchResponse { + jobs: CreateJobInput[]; + totalJobs: number; + page: number; + pageSize: number; + totalPages: number; +} + +export interface UkVisaJobsImportResponse { + created: number; + skipped: number; +} + export interface PipelineStatusResponse { isRunning: boolean; lastRun: PipelineRun | null;