From 3d7a014891297149ffcb00555d8e72b1c50cf34c Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 9 Feb 2026 18:39:33 +0000 Subject: [PATCH] Remove /ukvisajobs page and related API surface (#115) * Initial plan * Remove ukvisajobs page and API Co-authored-by: DaKheera47 <53654735+DaKheera47@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: DaKheera47 <53654735+DaKheera47@users.noreply.github.com> --- orchestrator/src/client/App.tsx | 2 - orchestrator/src/client/api/client.ts | 29 - .../src/client/components/navigation.ts | 9 +- .../src/client/pages/UkVisaJobsPage.tsx | 705 ------------------ orchestrator/src/server/api/routes.ts | 2 - .../src/server/api/routes/test-utils.ts | 4 - .../src/server/api/routes/ukvisajobs.test.ts | 94 --- .../src/server/api/routes/ukvisajobs.ts | 151 ---- .../src/server/services/ukvisajobs.ts | 272 ------- shared/src/types.ts | 13 - 10 files changed, 1 insertion(+), 1280 deletions(-) delete mode 100644 orchestrator/src/client/pages/UkVisaJobsPage.tsx delete mode 100644 orchestrator/src/server/api/routes/ukvisajobs.test.ts delete mode 100644 orchestrator/src/server/api/routes/ukvisajobs.ts diff --git a/orchestrator/src/client/App.tsx b/orchestrator/src/client/App.tsx index 7d75f33..4541a38 100644 --- a/orchestrator/src/client/App.tsx +++ b/orchestrator/src/client/App.tsx @@ -13,7 +13,6 @@ import { HomePage } from "./pages/HomePage"; import { JobPage } from "./pages/JobPage"; import { OrchestratorPage } from "./pages/OrchestratorPage"; import { SettingsPage } from "./pages/SettingsPage"; -import { UkVisaJobsPage } from "./pages/UkVisaJobsPage"; import { VisaSponsorsPage } from "./pages/VisaSponsorsPage"; export const App: React.FC = () => { @@ -54,7 +53,6 @@ export const App: React.FC = () => { } /> } /> } /> - } /> } /> } /> } /> diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index df4d748..71b9702 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -10,7 +10,6 @@ import type { BackupInfo, BulkJobActionRequest, BulkJobActionResponse, - CreateJobInput, DemoInfoResponse, Job, JobOutcome, @@ -27,8 +26,6 @@ import type { StageEvent, StageEventMetadata, StageTransitionTarget, - UkVisaJobsImportResponse, - UkVisaJobsSearchResponse, ValidationResult, VisaSponsor, VisaSponsorSearchResponse, @@ -337,32 +334,6 @@ export async function getDemoInfo(): Promise { return fetchApi("/demo/info"); } -// UK Visa Jobs API -export async function searchUkVisaJobs(input: { - searchTerm?: string; - page?: number; -}): Promise { - if (input.searchTerm?.trim()) { - trackEvent("ukvisajobs_search", { - searchTerm: input.searchTerm.trim(), - page: input.page ?? 1, - }); - } - return fetchApi("/ukvisajobs/search", { - method: "POST", - body: JSON.stringify(input), - }); -} - -export async function importUkVisaJobs(input: { - jobs: CreateJobInput[]; -}): Promise { - return fetchApi("/ukvisajobs/import", { - method: "POST", - body: JSON.stringify(input), - }); -} - // Manual Job Import API export async function fetchJobFromUrl(input: { url: string; diff --git a/orchestrator/src/client/components/navigation.ts b/orchestrator/src/client/components/navigation.ts index 8c1ecfa..0f3f2cb 100644 --- a/orchestrator/src/client/components/navigation.ts +++ b/orchestrator/src/client/components/navigation.ts @@ -1,10 +1,4 @@ -import { - Briefcase, - Home, - LayoutDashboard, - Settings, - Shield, -} from "lucide-react"; +import { Home, LayoutDashboard, Settings, Shield } from "lucide-react"; export type NavLink = { to: string; @@ -22,7 +16,6 @@ export const NAV_LINKS: NavLink[] = [ activePaths: ["/ready", "/discovered", "/applied", "/all"], }, { to: "/visa-sponsors", label: "Visa Sponsors", icon: Shield }, - { to: "/ukvisajobs", label: "UK Visa Jobs", icon: Briefcase }, { to: "/settings", label: "Settings", icon: Settings }, ]; diff --git a/orchestrator/src/client/pages/UkVisaJobsPage.tsx b/orchestrator/src/client/pages/UkVisaJobsPage.tsx deleted file mode 100644 index 790277d..0000000 --- a/orchestrator/src/client/pages/UkVisaJobsPage.tsx +++ /dev/null @@ -1,705 +0,0 @@ -/** - * UK Visa Jobs search page. - */ - -import { isNavActive, NAV_LINKS } from "@client/components/navigation"; -import type { CreateJobInput } from "@shared/types.js"; -import { - Briefcase, - Calendar, - ChevronLeft, - ChevronRight, - Clock, - Database, - DollarSign, - ExternalLink, - GraduationCap, - Loader2, - MapPin, - Menu, - Search, -} from "lucide-react"; -import type React from "react"; -import { useEffect, useMemo, useState } from "react"; -import { useLocation, useNavigate } from "react-router-dom"; -import { toast } from "sonner"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { Checkbox } from "@/components/ui/checkbox"; -import { Drawer, DrawerClose, DrawerContent } from "@/components/ui/drawer"; -import { Input } from "@/components/ui/input"; -import { Separator } from "@/components/ui/separator"; -import { - Sheet, - SheetContent, - SheetHeader, - SheetTitle, - SheetTrigger, -} from "@/components/ui/sheet"; -import { cn, formatDate, formatDateTime, stripHtml } from "@/lib/utils"; -import * as api from "../api"; - -const clampText = (value: string, max = 160) => - value.length > max ? `${value.slice(0, max).trim()}...` : value; - -const jobKey = (job: CreateJobInput) => job.sourceJobId || job.jobUrl; - -export const UkVisaJobsPage: React.FC = () => { - const location = useLocation(); - const navigate = useNavigate(); - const [navOpen, setNavOpen] = useState(false); - const [searchTermInput, setSearchTermInput] = useState(""); - const [activeSearchTerm, setActiveSearchTerm] = useState(null); - const [results, setResults] = useState([]); - const [selectedJobId, setSelectedJobId] = useState(null); - const [selectedJobIds, setSelectedJobIds] = useState>(new Set()); - const [isSearching, setIsSearching] = useState(false); - const [isImporting, setIsImporting] = useState(false); - const [lastRunAt, setLastRunAt] = useState(null); - const [lastSearchTerm, setLastSearchTerm] = useState(null); - const [page, setPage] = useState(1); - const [pageSize, setPageSize] = useState(15); - const [totalJobs, setTotalJobs] = useState(0); - const [totalPages, setTotalPages] = useState(1); - const [errorMessage, setErrorMessage] = useState(null); - const [isDetailDrawerOpen, setIsDetailDrawerOpen] = useState(false); - const [isDesktop, setIsDesktop] = useState(() => - typeof window !== "undefined" - ? window.matchMedia("(min-width: 1024px)").matches - : false, - ); - useEffect(() => { - if (results.length === 0) { - setSelectedJobId(null); - return; - } - const firstKey = jobKey(results[0]); - if ( - !selectedJobId || - !results.some((job) => jobKey(job) === selectedJobId) - ) { - setSelectedJobId(firstKey); - } - }, [results, selectedJobId]); - - useEffect(() => { - if (!selectedJobId) { - setIsDetailDrawerOpen(false); - } - }, [selectedJobId]); - - useEffect(() => { - if (typeof window === "undefined") return; - const media = window.matchMedia("(min-width: 1024px)"); - const handleChange = () => setIsDesktop(media.matches); - handleChange(); - if (media.addEventListener) { - media.addEventListener("change", handleChange); - return () => media.removeEventListener("change", handleChange); - } - media.addListener(handleChange); - return () => media.removeListener(handleChange); - }, []); - - useEffect(() => { - if (isDesktop && isDetailDrawerOpen) { - setIsDetailDrawerOpen(false); - } - }, [isDesktop, isDetailDrawerOpen]); - - useEffect(() => { - setSelectedJobIds(new Set()); - }, []); - - const selectedJob = useMemo( - () => - selectedJobId - ? (results.find((job) => jobKey(job) === selectedJobId) ?? null) - : null, - [results, selectedJobId], - ); - - const summaryCounts = useMemo(() => { - const startIndex = totalJobs === 0 ? 0 : (page - 1) * pageSize + 1; - const endIndex = totalJobs === 0 ? 0 : Math.min(page * pageSize, totalJobs); - return { - startIndex, - endIndex, - }; - }, [page, pageSize, totalJobs]); - - const runSearch = async ({ - term, - pageNumber, - }: { - term: string | null; - pageNumber: number; - }) => { - try { - setIsSearching(true); - setErrorMessage(null); - const response = await api.searchUkVisaJobs({ - searchTerm: term ?? undefined, - page: pageNumber, - }); - setResults(response.jobs); - setPage(response.page); - setPageSize(response.pageSize); - setTotalJobs(response.totalJobs); - setTotalPages(response.totalPages); - setLastRunAt(new Date().toISOString()); - if (response.jobs.length === 0) { - toast.message("No UK Visa Jobs found for this search."); - } - } catch (error) { - const message = - error instanceof Error ? error.message : "UK Visa Jobs search failed"; - setErrorMessage(message); - toast.error(message); - } finally { - setIsSearching(false); - } - }; - - const handleSearch = async (event: React.FormEvent) => { - event.preventDefault(); - setErrorMessage(null); - - const terms = searchTermInput - .split(/[\n,]+/) - .map((term) => term.trim()) - .filter(Boolean); - const term = terms[0] || null; - - if (terms.length > 1) { - toast.message("Using the first term for pagination."); - } - - setActiveSearchTerm(term); - setLastSearchTerm(term); - setPage(1); - - await runSearch({ term, pageNumber: 1 }); - }; - - const handlePageChange = (nextPage: number) => { - if (isSearching) return; - if (nextPage < 1 || nextPage > totalPages) return; - setPage(nextPage); - void runSearch({ term: activeSearchTerm, pageNumber: nextPage }); - }; - - const handleImportSelected = async () => { - const selectedJobs = results.filter((job) => - selectedJobIds.has(jobKey(job)), - ); - if (selectedJobs.length === 0) return; - - try { - setIsImporting(true); - const response = await api.importUkVisaJobs({ jobs: selectedJobs }); - toast.success(`Imported ${response.created} jobs`, { - description: response.skipped - ? `${response.skipped} skipped (duplicates)` - : undefined, - }); - setSelectedJobIds(new Set()); - } catch (error) { - const message = - error instanceof Error ? error.message : "Failed to import jobs"; - toast.error(message); - } finally { - setIsImporting(false); - } - }; - - const selectedDescription = useMemo(() => { - if (!selectedJob?.jobDescription) return "No description available."; - const cleaned = stripHtml(selectedJob.jobDescription); - return cleaned || "No description available."; - }, [selectedJob]); - - const selectedJobLink = selectedJob - ? selectedJob.applicationLink || selectedJob.jobUrl - : "#"; - const selectedDeadline = selectedJob - ? formatDate(selectedJob.deadline) - : null; - const selectedPosted = selectedJob - ? formatDate(selectedJob.datePosted) - : null; - const selectedCount = selectedJobIds.size; - const allSelected = - results.length > 0 && - results.every((job) => selectedJobIds.has(jobKey(job))); - const selectAllState = allSelected - ? true - : selectedCount > 0 - ? "indeterminate" - : false; - const canGoPrev = page > 1; - const canGoNext = page < totalPages; - - const handleSelectJob = (jobId: string) => { - setSelectedJobId(jobId); - if (!isDesktop) { - setIsDetailDrawerOpen(true); - } - }; - - const detailPanelContent = !selectedJob ? ( -
-
Select a job
-

- Pick a job from the list to inspect details. -

-
- ) : ( -
-
-
-
- {selectedJob.title} -
-
- {selectedJob.employer} -
-
- - UK Visa Jobs - -
- -
- {selectedJob.location && ( - - - {selectedJob.location} - - )} - {selectedDeadline && ( - - - {selectedDeadline} - - )} - {selectedPosted && ( - - - Posted {selectedPosted} - - )} - {selectedJob.salary && ( - - - {selectedJob.salary} - - )} - {selectedJob.degreeRequired && ( - - - {selectedJob.degreeRequired} - - )} -
- -
-
-
Job type
-
{selectedJob.jobType || "Not set"}
-
-
-
Job level
-
{selectedJob.jobLevel || "Not set"}
-
-
-
Location
-
{selectedJob.location || "Not set"}
-
-
-
Deadline
-
{selectedDeadline || "Not set"}
-
-
- - - - - -
-
- Description -
-
- {selectedDescription} -
-
-
- ); - - return ( - <> -
-
-
- - - - - - - JobOps - - - - - -
- -
-
-
- UK Visa Jobs -
-
- Live search console -
-
-
-
-
- -
-
-
-
- - setSearchTermInput(event.target.value)} - placeholder="e.g. data analyst" - className="h-10" - /> -

- Note: Search is limited to job titles only due to API - constraints. -

-
- -
- -
-
- - {errorMessage && ( -
- {errorMessage} -
- )} - - - -
-
- Last run:{" "} - {lastRunAt ? formatDateTime(lastRunAt) : "No searches yet"} -
-
- - {totalJobs} total - - - {results.length} on page - - - Page {page} of {totalPages} - - {lastSearchTerm && ( - Term: {lastSearchTerm} - )} -
-
-
- -
-
- {isSearching && results.length > 0 && ( -
- - Fetching UK Visa Jobs... -
- )} - {results.length === 0 ? ( -
- {isSearching ? ( - <> - -
Searching...
-

- Fetching fresh UK Visa Jobs listings. -

- - ) : ( - <> -
- No results yet -
-

- Run a search to fetch fresh UK Visa Jobs listings. -

- - )} -
- ) : ( - <> -
-
- { - if (checked === true) { - setSelectedJobIds( - new Set(results.map((job) => jobKey(job))), - ); - } else { - setSelectedJobIds(new Set()); - } - }} - aria-label="Select all jobs on this page" - /> - Select page - - {selectedCount} selected -
- -
-
- {results.map((job) => { - const key = jobKey(job); - const isSelected = key === selectedJobId; - const isChecked = selectedJobIds.has(key); - const description = job.jobDescription - ? clampText(stripHtml(job.jobDescription)) - : "No description."; - - return ( -
-
- { - setSelectedJobIds((current) => { - const next = new Set(current); - if (checked) { - next.add(key); - } else { - next.delete(key); - } - return next; - }); - }} - aria-label={`Select ${job.title}`} - /> -
- -
- ); - })} -
-
- - Showing {summaryCounts.startIndex}-{summaryCounts.endIndex}{" "} - of {totalJobs} - -
- - - Page {page} of {totalPages} - - -
-
- - )} -
- -
- {detailPanelContent} -
-
-
- - - -
-
- Job details -
- - - -
-
- {detailPanelContent} -
- - - - ); -}; diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts index ee864f0..c0d7a7a 100644 --- a/orchestrator/src/server/api/routes.ts +++ b/orchestrator/src/server/api/routes.ts @@ -12,7 +12,6 @@ import { onboardingRouter } from "./routes/onboarding"; import { pipelineRouter } from "./routes/pipeline"; import { profileRouter } from "./routes/profile"; import { settingsRouter } from "./routes/settings"; -import { ukVisaJobsRouter } from "./routes/ukvisajobs"; import { visaSponsorsRouter } from "./routes/visa-sponsors"; import { webhookRouter } from "./routes/webhook"; @@ -23,7 +22,6 @@ apiRouter.use("/demo", demoRouter); apiRouter.use("/settings", settingsRouter); apiRouter.use("/pipeline", pipelineRouter); apiRouter.use("/manual-jobs", manualJobsRouter); -apiRouter.use("/ukvisajobs", ukVisaJobsRouter); apiRouter.use("/webhook", webhookRouter); apiRouter.use("/profile", profileRouter); apiRouter.use("/database", databaseRouter); diff --git a/orchestrator/src/server/api/routes/test-utils.ts b/orchestrator/src/server/api/routes/test-utils.ts index d86c183..9901767 100644 --- a/orchestrator/src/server/api/routes/test-utils.ts +++ b/orchestrator/src/server/api/routes/test-utils.ts @@ -67,10 +67,6 @@ vi.mock("../../services/profile", () => ({ getProfile: vi.fn().mockResolvedValue({}), })); -vi.mock("../../services/ukvisajobs", () => ({ - fetchUkVisaJobsPage: vi.fn(), -})); - vi.mock("../../services/visa-sponsors/index", () => ({ getStatus: vi.fn(), searchSponsors: vi.fn(), diff --git a/orchestrator/src/server/api/routes/ukvisajobs.test.ts b/orchestrator/src/server/api/routes/ukvisajobs.test.ts deleted file mode 100644 index b0f3a8e..0000000 --- a/orchestrator/src/server/api/routes/ukvisajobs.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { Server } from "node:http"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { startServer, stopServer } from "./test-utils"; - -describe.sequential("UK Visa Jobs API routes", () => { - let server: Server; - let baseUrl: string; - let closeDb: () => void; - let tempDir: string; - - beforeEach(async () => { - ({ server, baseUrl, closeDb, tempDir } = await startServer()); - }); - - afterEach(async () => { - await stopServer({ server, closeDb, tempDir }); - }); - - it("enforces pagination rules for search", async () => { - const badRes = await fetch(`${baseUrl}/api/ukvisajobs/search`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ searchTerms: ["one", "two"] }), - }); - expect(badRes.status).toBe(400); - }); - - it("searches UK Visa Jobs with valid payloads", async () => { - const { fetchUkVisaJobsPage } = await import("../../services/ukvisajobs"); - vi.mocked(fetchUkVisaJobsPage).mockResolvedValue({ - jobs: [ - { - source: "ukvisajobs", - title: "Engineer", - employer: "Acme", - jobUrl: "https://example.com/visa/1", - }, - ], - totalJobs: 3, - page: 1, - pageSize: 2, - }); - - const res = await fetch(`${baseUrl}/api/ukvisajobs/search`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ query: "engineer" }), - }); - const body = await res.json(); - expect(body.ok).toBe(true); - expect(body.data.totalPages).toBe(2); - expect(fetchUkVisaJobsPage).toHaveBeenCalledWith({ - searchKeyword: "engineer", - page: 1, - }); - }); - - it("blocks search when pipeline is running", async () => { - const { getPipelineStatus } = await import("../../pipeline/index"); - vi.mocked(getPipelineStatus).mockReturnValue({ isRunning: true }); - - const res = await fetch(`${baseUrl}/api/ukvisajobs/search`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ query: "engineer" }), - }); - expect(res.status).toBe(409); - }); - - it("imports UK Visa Jobs and reports created vs skipped", async () => { - const res = await fetch(`${baseUrl}/api/ukvisajobs/import`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - jobs: [ - { - title: "Engineer", - employer: "Acme", - jobUrl: "https://example.com/visa/2", - }, - { - title: "Engineer Duplicate", - employer: "Acme", - jobUrl: "https://example.com/visa/2", - }, - ], - }), - }); - const body = await res.json(); - expect(body.ok).toBe(true); - expect(body.data.created).toBe(1); - expect(body.data.skipped).toBe(1); - }); -}); diff --git a/orchestrator/src/server/api/routes/ukvisajobs.ts b/orchestrator/src/server/api/routes/ukvisajobs.ts deleted file mode 100644 index c781122..0000000 --- a/orchestrator/src/server/api/routes/ukvisajobs.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { - ApiResponse, - UkVisaJobsImportResponse, - UkVisaJobsSearchResponse, -} from "@shared/types"; -import { type Request, type Response, Router } from "express"; -import { z } from "zod"; - -import { getPipelineStatus } from "../../pipeline/index"; -import * as jobsRepo from "../../repositories/jobs"; -import { fetchUkVisaJobsPage } from "../../services/ukvisajobs"; - -export const ukVisaJobsRouter = Router(); -let isUkVisaJobsSearchRunning = false; - -const ukVisaJobsSearchSchema = z.object({ - query: z.string().trim().min(1).max(200).optional(), - searchTerm: z.string().trim().min(1).max(200).optional(), - searchTerms: z.array(z.string().trim().min(1).max(200)).max(20).optional(), - page: z.number().int().min(1).optional(), -}); - -/** - * POST /api/ukvisajobs/search - Run a UKVisaJobs search without importing into the DB - */ -ukVisaJobsRouter.post("/search", async (req: Request, res: Response) => { - let lockAcquired = false; - - try { - const input = ukVisaJobsSearchSchema.parse(req.body ?? {}); - - if (isUkVisaJobsSearchRunning) { - return res.status(409).json({ - success: false, - error: "UK Visa Jobs search is already running", - }); - } - - const { isRunning } = getPipelineStatus(); - if (isRunning) { - return res.status(409).json({ - success: false, - error: - "Pipeline is running. Stop it before running UK Visa Jobs search.", - }); - } - - isUkVisaJobsSearchRunning = true; - lockAcquired = true; - - const rawTerms = input.searchTerms ?? []; - if (rawTerms.length > 1) { - return res.status(400).json({ - success: false, - error: "Pagination supports a single search term.", - }); - } - - const searchTerm = input.searchTerm ?? input.query ?? rawTerms[0]; - const page = input.page ?? 1; - - const result = await fetchUkVisaJobsPage({ - searchKeyword: searchTerm, - page, - }); - - const totalPages = Math.max( - 1, - Math.ceil(result.totalJobs / result.pageSize), - ); - - const response: ApiResponse = { - ok: true, - data: { - jobs: result.jobs, - totalJobs: result.totalJobs, - page: result.page, - pageSize: result.pageSize, - totalPages, - }, - }; - - res.json(response); - } catch (error) { - if (error instanceof z.ZodError) { - return res.status(400).json({ success: false, error: error.message }); - } - const message = error instanceof Error ? error.message : "Unknown error"; - res.status(500).json({ success: false, error: message }); - } finally { - if (lockAcquired) { - isUkVisaJobsSearchRunning = false; - } - } -}); - -const ukVisaJobsImportSchema = z.object({ - jobs: z - .array( - z.object({ - title: z.string().trim().min(1).max(500), - employer: z.string().trim().min(1).max(500), - jobUrl: z.string().trim().min(1).max(2000), - sourceJobId: z.string().trim().min(1).max(200).optional(), - employerUrl: z.string().trim().min(1).max(2000).optional(), - applicationLink: z.string().trim().min(1).max(2000).optional(), - location: z.string().trim().max(200).optional(), - deadline: z.string().trim().max(100).optional(), - salary: z.string().trim().max(200).optional(), - jobDescription: z.string().trim().max(20000).optional(), - datePosted: z.string().trim().max(100).optional(), - degreeRequired: z.string().trim().max(200).optional(), - jobType: z.string().trim().max(200).optional(), - jobLevel: z.string().trim().max(200).optional(), - }), - ) - .min(1) - .max(200), -}); - -/** - * POST /api/ukvisajobs/import - Import selected UKVisaJobs results into the DB - */ -ukVisaJobsRouter.post("/import", async (req: Request, res: Response) => { - try { - const input = ukVisaJobsImportSchema.parse(req.body ?? {}); - - const jobs = input.jobs.map((job) => ({ - ...job, - source: "ukvisajobs" as const, - })); - - const result = await jobsRepo.bulkCreateJobs(jobs); - - const response: ApiResponse = { - ok: true, - data: { - created: result.created, - skipped: result.skipped, - }, - }; - - res.json(response); - } catch (error) { - if (error instanceof z.ZodError) { - return res.status(400).json({ success: false, error: error.message }); - } - const message = error instanceof Error ? error.message : "Unknown error"; - res.status(500).json({ success: false, error: message }); - } -}); diff --git a/orchestrator/src/server/services/ukvisajobs.ts b/orchestrator/src/server/services/ukvisajobs.ts index eaf2f08..652d299 100644 --- a/orchestrator/src/server/services/ukvisajobs.ts +++ b/orchestrator/src/server/services/ukvisajobs.ts @@ -16,9 +16,6 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const UKVISAJOBS_DIR = join(__dirname, "../../../../extractors/ukvisajobs"); const STORAGE_DIR = join(UKVISAJOBS_DIR, "storage/datasets/default"); const AUTH_CACHE_PATH = join(UKVISAJOBS_DIR, "storage/ukvisajobs-auth.json"); -const UKVISAJOBS_API_URL = - "https://my.ukvisajobs.com/ukvisa-api/api/fetch-jobs-data"; -const UKVISAJOBS_PAGE_SIZE = 15; const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS "; let isUkVisaJobsRunning = false; @@ -173,102 +170,6 @@ export function parseUkVisaJobsProgressLine( return null; } -function buildCookieHeader(session: UkVisaJobsAuthSession): string { - const cookieParts: string[] = []; - if (session.csrfToken) cookieParts.push(`csrf_token=${session.csrfToken}`); - if (session.ciSession) cookieParts.push(`ci_session=${session.ciSession}`); - const token = session.authToken || session.token; - if (token) cookieParts.push(`authToken=${token}`); - return cookieParts.join("; "); -} - -function buildVisaInfoDescription(raw: UkVisaJobsApiJob): string | undefined { - const visaInfo: string[] = []; - if (raw.visa_acceptance?.toLowerCase() === "yes") - visaInfo.push("Visa acceptance: Yes"); - if (raw.applicants_outside_uk?.toLowerCase() === "yes") - visaInfo.push("Accepts applicants outside UK"); - if (raw.likely_to_sponsor?.toLowerCase() === "yes") - visaInfo.push("Likely to sponsor"); - if (raw.definitely_sponsored?.toLowerCase() === "yes") - visaInfo.push("Definitely sponsored"); - if (raw.new_entrant?.toLowerCase() === "yes") - visaInfo.push("New entrant friendly"); - if (raw.student_graduate?.toLowerCase() === "yes") - visaInfo.push("Student/Graduate friendly"); - if (visaInfo.length === 0) return undefined; - return `Visa sponsorship info: ${visaInfo.join(", ")}`; -} - -function formatSalary(raw: UkVisaJobsApiJob): string | undefined { - const minSalary = toNumberOrNull(raw.min_salary); - const maxSalary = toNumberOrNull(raw.max_salary); - const interval = toStringOrNull(raw.salary_interval); - - if (minSalary && maxSalary && maxSalary > 0) { - return `GBP ${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}${interval ? ` / ${interval}` : ""}`; - } - if (maxSalary && maxSalary > 0) { - return `GBP ${maxSalary.toLocaleString()}${interval ? ` / ${interval}` : ""}`; - } - return undefined; -} - -function mapApiJob(raw: UkVisaJobsApiJob): CreateJobInput { - const description = - toStringOrNull(raw.description) ?? buildVisaInfoDescription(raw); - return { - source: "ukvisajobs", - sourceJobId: toStringOrNull(raw.id) ?? undefined, - title: toStringOrNull(raw.title) ?? "Unknown Title", - employer: toStringOrNull(raw.company_name) ?? "Unknown Employer", - employerUrl: toStringOrNull(raw.company_link) ?? undefined, - jobUrl: toStringOrNull(raw.job_link) ?? "", - applicationLink: toStringOrNull(raw.job_link) ?? undefined, - location: toStringOrNull(raw.city) ?? undefined, - deadline: toStringOrNull(raw.job_expire) ?? undefined, - salary: formatSalary(raw), - jobDescription: description ?? undefined, - datePosted: toStringOrNull(raw.created_date) ?? undefined, - degreeRequired: toStringOrNull(raw.degree_requirement) ?? undefined, - jobType: toStringOrNull(raw.job_type) ?? undefined, - jobLevel: toStringOrNull(raw.job_level) ?? undefined, - }; -} - -interface UkVisaJobsApiJob { - id: string; - title: string; - company_name: string; - company_link?: string; - job_link: string; - city?: string; - created_date?: string; - job_expire?: string; - description?: string; - min_salary?: string; - max_salary?: string; - salary_interval?: string; - salary_method?: string; - degree_requirement?: string; - job_type?: string; - job_level?: string; - job_industry?: string; - visa_acceptance?: string; - applicants_outside_uk?: string; - likely_to_sponsor?: string; - definitely_sponsored?: string; - new_entrant?: string; - student_graduate?: string; -} - -interface UkVisaJobsApiResponse { - status: number; - totalJobs: number; - query?: string; - jobs: UkVisaJobsApiJob[]; -} - /** * Basic HTML to text conversion to extract job description. */ @@ -361,79 +262,6 @@ async function loadCachedAuthSession(): Promise { } } -function getAuthToken(session: UkVisaJobsAuthSession | null): string | null { - if (!session) return null; - return session.authToken || session.token || null; -} - -function hasAuthToken( - session: UkVisaJobsAuthSession | null, -): session is UkVisaJobsAuthSession { - return Boolean(session && (session.authToken || session.token)); -} - -function isAuthErrorResponse(status: number, bodyText: string): boolean { - if (status === 401 || status === 403) return true; - if (status !== 400) return false; - try { - const parsed = JSON.parse(bodyText) as { - errorType?: string; - message?: string; - }; - if (parsed?.errorType === "expired") return true; - if (parsed?.message?.toLowerCase().includes("expired")) return true; - } catch { - // Ignore parse errors - } - return bodyText.toLowerCase().includes("expired"); -} - -async function refreshUkVisaJobsAuthSession(): Promise { - const email = process.env.UKVISAJOBS_EMAIL; - const password = process.env.UKVISAJOBS_PASSWORD; - if (!email || !password) { - throw new Error( - "UK Visa Jobs auth expired. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.", - ); - } - - await new Promise((resolve, reject) => { - const child = spawn("npx", ["tsx", "src/main.ts"], { - cwd: UKVISAJOBS_DIR, - stdio: "inherit", - env: { - ...process.env, - UKVISAJOBS_REFRESH_ONLY: "1", - }, - }); - - child.on("close", (code) => { - if (code === 0) resolve(); - else - reject(new Error(`UK Visa Jobs auth refresh exited with code ${code}`)); - }); - child.on("error", reject); - }); -} - -async function loadAuthSessionOrRefresh(): Promise { - let authSession = await loadCachedAuthSession(); - if (hasAuthToken(authSession)) { - return authSession; - } - - await refreshUkVisaJobsAuthSession(); - - authSession = await loadCachedAuthSession(); - if (!hasAuthToken(authSession)) { - throw new Error( - "UK Visa Jobs auth session missing. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.", - ); - } - - return authSession; -} - /** * Clear previous extraction results. */ @@ -445,106 +273,6 @@ async function clearStorageDataset(): Promise { } } -export async function fetchUkVisaJobsPage( - options: { searchKeyword?: string; page?: number } = {}, -): Promise<{ - jobs: CreateJobInput[]; - totalJobs: number; - page: number; - pageSize: number; -}> { - const page = options.page && options.page > 0 ? options.page : 1; - let authSession = await loadAuthSessionOrRefresh(); - - const fetchWithSession = async (session: UkVisaJobsAuthSession) => { - const token = getAuthToken(session); - if (!token) { - throw new Error( - "UK Visa Jobs auth session missing. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.", - ); - } - - const formData = new FormData(); - formData.append("is_global", "0"); - formData.append("sortBy", "desc"); - formData.append("pageNo", String(page)); - formData.append("visaAcceptance", "false"); - formData.append("applicants_outside_uk", "false"); - formData.append( - "searchKeyword", - options.searchKeyword ? options.searchKeyword : "null", - ); - formData.append("token", token); - - const cookies = buildCookieHeader({ - token: session?.token, - authToken: session?.authToken, - csrfToken: session?.csrfToken, - ciSession: session?.ciSession, - }); - - const response = await fetch(UKVISAJOBS_API_URL, { - method: "POST", - headers: { - accept: "application/json, text/plain, */*", - cookie: cookies, - origin: "https://my.ukvisajobs.com", - referer: `https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&pageNo=${page}&visaAcceptance=false&applicants_outside_uk=false`, - "user-agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - }, - body: formData, - }); - - const text = await response.text(); - return { response, text }; - }; - - let { response, text } = await fetchWithSession(authSession); - - if (!response.ok && isAuthErrorResponse(response.status, text)) { - await refreshUkVisaJobsAuthSession(); - const refreshedSession = await loadCachedAuthSession(); - if (!hasAuthToken(refreshedSession)) { - throw new Error( - "UK Visa Jobs auth session missing. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.", - ); - } - authSession = refreshedSession; - ({ response, text } = await fetchWithSession(authSession)); - } - - if (!response.ok) { - throw new Error(`UK Visa Jobs API returned ${response.status}: ${text}`); - } - - let data: UkVisaJobsApiResponse; - try { - data = JSON.parse(text) as UkVisaJobsApiResponse; - } catch (_error) { - throw new Error("UK Visa Jobs API returned an invalid response."); - } - - if (data.status !== 1) { - throw new Error(`UK Visa Jobs API returned status ${data.status}`); - } - - const jobs = (data.jobs || []) - .map(mapApiJob) - .filter((job) => Boolean(job.jobUrl)); - - const totalJobs = Number.isFinite(data.totalJobs) - ? data.totalJobs - : jobs.length; - - return { - jobs, - totalJobs, - page, - pageSize: UKVISAJOBS_PAGE_SIZE, - }; -} - export async function runUkVisaJobs( options: RunUkVisaJobsOptions = {}, ): Promise { diff --git a/shared/src/types.ts b/shared/src/types.ts index f52767b..762bcad 100644 --- a/shared/src/types.ts +++ b/shared/src/types.ts @@ -369,19 +369,6 @@ export interface BulkJobActionResponse { results: BulkJobActionResult[]; } -export interface UkVisaJobsSearchResponse { - jobs: CreateJobInput[]; - totalJobs: number; - page: number; - pageSize: number; - totalPages: number; -} - -export interface UkVisaJobsImportResponse { - created: number; - skipped: number; -} - // Visa Sponsors types export interface VisaSponsor { organisationName: string;