From bd7baafbecc66b8fa54fbc44786944194c9758c3 Mon Sep 17 00:00:00 2001 From: DaKheera47 Date: Fri, 26 Dec 2025 20:47:28 +0000 Subject: [PATCH] passing max ukvisajobs --- .env.example | 1 - docker-compose.yml | 1 - extractors/ukvisajobs/src/main.ts | 2 +- orchestrator/src/client/api/client.ts | 17 +++--- .../src/client/pages/SettingsPage.tsx | 60 ++++++++++++++++++- orchestrator/src/server/api/routes.ts | 22 +++++++ .../src/server/pipeline/orchestrator.ts | 6 +- .../src/server/repositories/settings.ts | 1 + orchestrator/src/shared/types.ts | 3 + 9 files changed, 100 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index 3a3d8d1..5b7fbf6 100644 --- a/.env.example +++ b/.env.example @@ -47,4 +47,3 @@ UKVISAJOBS_TOKEN= UKVISAJOBS_AUTH_TOKEN= UKVISAJOBS_CSRF_TOKEN= UKVISAJOBS_CI_SESSION= -UKVISAJOBS_MAX_JOBS=50 diff --git a/docker-compose.yml b/docker-compose.yml index 01f7350..8e771a0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,6 @@ services: - UKVISAJOBS_AUTH_TOKEN=${UKVISAJOBS_AUTH_TOKEN:-} - UKVISAJOBS_CSRF_TOKEN=${UKVISAJOBS_CSRF_TOKEN:-} - UKVISAJOBS_CI_SESSION=${UKVISAJOBS_CI_SESSION:-} - - UKVISAJOBS_MAX_JOBS=${UKVISAJOBS_MAX_JOBS:-50} - UKVISAJOBS_SEARCH_KEYWORD=${UKVISAJOBS_SEARCH_KEYWORD:-} # Python path (uses system python in container) diff --git a/extractors/ukvisajobs/src/main.ts b/extractors/ukvisajobs/src/main.ts index 4e3dabe..471cd37 100644 --- a/extractors/ukvisajobs/src/main.ts +++ b/extractors/ukvisajobs/src/main.ts @@ -9,7 +9,7 @@ * UKVISAJOBS_AUTH_TOKEN - Auth cookie token (defaults to UKVISAJOBS_TOKEN) * UKVISAJOBS_CSRF_TOKEN - CSRF token cookie * UKVISAJOBS_CI_SESSION - CI session cookie - * UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200) + * UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200) - Set via UI Settings * UKVISAJOBS_SEARCH_KEYWORD - Optional search filter */ diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index 09dea72..43d624e 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -2,10 +2,10 @@ * API client for the orchestrator backend. */ -import type { - Job, - ApiResponse, - JobsListResponse, +import type { + Job, + ApiResponse, + JobsListResponse, PipelineStatusResponse, JobSource, PipelineRun, @@ -26,13 +26,13 @@ async function fetchApi( ...options?.headers, }, }); - + const data: ApiResponse = await response.json(); - + if (!data.success) { throw new Error(data.error || 'API request failed'); } - + return data.data as T; } @@ -47,7 +47,7 @@ export async function getJob(id: string): Promise { } export async function updateJob( - id: string, + id: string, update: Partial ): Promise { return fetchApi(`/jobs/${id}`, { @@ -105,6 +105,7 @@ export async function updateSettings(update: { pipelineWebhookUrl?: string | null jobCompleteWebhookUrl?: string | null resumeProjects?: ResumeProjectsSettings | null + ukvisajobsMaxJobs?: number | null }): Promise { return fetchApi('/settings', { method: 'PATCH', diff --git a/orchestrator/src/client/pages/SettingsPage.tsx b/orchestrator/src/client/pages/SettingsPage.tsx index 41bc64e..f90bb2a 100644 --- a/orchestrator/src/client/pages/SettingsPage.tsx +++ b/orchestrator/src/client/pages/SettingsPage.tsx @@ -42,6 +42,7 @@ export const SettingsPage: React.FC = () => { const [pipelineWebhookUrlDraft, setPipelineWebhookUrlDraft] = useState("") const [jobCompleteWebhookUrlDraft, setJobCompleteWebhookUrlDraft] = useState("") const [resumeProjectsDraft, setResumeProjectsDraft] = useState(null) + const [ukvisajobsMaxJobsDraft, setUkvisajobsMaxJobsDraft] = useState(null) const [isSaving, setIsSaving] = useState(false) const [isLoading, setIsLoading] = useState(true) @@ -57,6 +58,7 @@ export const SettingsPage: React.FC = () => { setPipelineWebhookUrlDraft(data.overridePipelineWebhookUrl ?? "") setJobCompleteWebhookUrlDraft(data.overrideJobCompleteWebhookUrl ?? "") setResumeProjectsDraft(data.resumeProjects) + setUkvisajobsMaxJobsDraft(data.overrideUkvisajobsMaxJobs) }) .catch((error) => { const message = error instanceof Error ? error.message : "Failed to load settings" @@ -81,6 +83,9 @@ export const SettingsPage: React.FC = () => { const effectiveJobCompleteWebhookUrl = settings?.jobCompleteWebhookUrl ?? "" const defaultJobCompleteWebhookUrl = settings?.defaultJobCompleteWebhookUrl ?? "" const overrideJobCompleteWebhookUrl = settings?.overrideJobCompleteWebhookUrl + const effectiveUkvisajobsMaxJobs = settings?.ukvisajobsMaxJobs ?? 50 + const defaultUkvisajobsMaxJobs = settings?.defaultUkvisajobsMaxJobs ?? 50 + const overrideUkvisajobsMaxJobs = settings?.overrideUkvisajobsMaxJobs const profileProjects = settings?.profileProjects ?? [] const maxProjectsTotal = profileProjects.length const lockedCount = resumeProjectsDraft?.lockedProjectIds.length ?? 0 @@ -93,11 +98,13 @@ export const SettingsPage: React.FC = () => { const currentWebhook = (overridePipelineWebhookUrl ?? "").trim() const nextJobCompleteWebhook = jobCompleteWebhookUrlDraft.trim() const currentJobCompleteWebhook = (overrideJobCompleteWebhookUrl ?? "").trim() + const ukvisajobsChanged = ukvisajobsMaxJobsDraft !== (overrideUkvisajobsMaxJobs ?? null) return ( next !== current || nextWebhook !== currentWebhook || nextJobCompleteWebhook !== currentJobCompleteWebhook || - !resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects) + !resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects) || + ukvisajobsChanged ) }, [ settings, @@ -108,6 +115,8 @@ export const SettingsPage: React.FC = () => { overridePipelineWebhookUrl, overrideJobCompleteWebhookUrl, resumeProjectsDraft, + ukvisajobsMaxJobsDraft, + overrideUkvisajobsMaxJobs, ]) const handleSave = async () => { @@ -120,17 +129,20 @@ export const SettingsPage: React.FC = () => { const resumeProjectsOverride = resumeProjectsEqual(resumeProjectsDraft, settings.defaultResumeProjects) ? null : resumeProjectsDraft + const ukvisajobsMaxJobsOverride = ukvisajobsMaxJobsDraft === defaultUkvisajobsMaxJobs ? null : ukvisajobsMaxJobsDraft const updated = await api.updateSettings({ model: trimmed.length > 0 ? trimmed : null, pipelineWebhookUrl: webhookTrimmed.length > 0 ? webhookTrimmed : null, jobCompleteWebhookUrl: jobCompleteTrimmed.length > 0 ? jobCompleteTrimmed : null, resumeProjects: resumeProjectsOverride, + ukvisajobsMaxJobs: ukvisajobsMaxJobsOverride, }) setSettings(updated) setModelDraft(updated.overrideModel ?? "") setPipelineWebhookUrlDraft(updated.overridePipelineWebhookUrl ?? "") setJobCompleteWebhookUrlDraft(updated.overrideJobCompleteWebhookUrl ?? "") setResumeProjectsDraft(updated.resumeProjects) + setUkvisajobsMaxJobsDraft(updated.overrideUkvisajobsMaxJobs) toast.success("Settings saved") } catch (error) { const message = error instanceof Error ? error.message : "Failed to save settings" @@ -148,12 +160,14 @@ export const SettingsPage: React.FC = () => { pipelineWebhookUrl: null, jobCompleteWebhookUrl: null, resumeProjects: null, + ukvisajobsMaxJobs: null, }) setSettings(updated) setModelDraft("") setPipelineWebhookUrlDraft("") setJobCompleteWebhookUrlDraft("") setResumeProjectsDraft(updated.resumeProjects) + setUkvisajobsMaxJobsDraft(null) toast.success("Reset to default") } catch (error) { const message = error instanceof Error ? error.message : "Failed to reset settings" @@ -272,6 +286,50 @@ export const SettingsPage: React.FC = () => { + + + UKVisaJobs Extractor + + + +
+
Max jobs to fetch
+ { + const value = parseInt(event.target.value, 10) + if (Number.isNaN(value)) { + setUkvisajobsMaxJobsDraft(null) + } else { + setUkvisajobsMaxJobsDraft(Math.min(200, Math.max(1, value))) + } + }} + disabled={isLoading || isSaving} + /> +
+ Maximum number of jobs to fetch from UKVisaJobs per pipeline run. Range: 1-200. +
+
+ + + +
+
+
Effective
+
{effectiveUkvisajobsMaxJobs}
+
+
+
Default
+
{defaultUkvisajobsMaxJobs}
+
+
+ + + Resume Projects diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts index 2b0a475..1bda882 100644 --- a/orchestrator/src/server/api/routes.ts +++ b/orchestrator/src/server/api/routes.ts @@ -238,6 +238,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => { const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects'); const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw }); + const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs'); + const defaultUkvisajobsMaxJobs = 50; + const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null; + const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs; + res.json({ success: true, data: { @@ -251,6 +256,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => { defaultJobCompleteWebhookUrl, overrideJobCompleteWebhookUrl, ...resumeProjectsData, + ukvisajobsMaxJobs, + defaultUkvisajobsMaxJobs, + overrideUkvisajobsMaxJobs, }, }); } catch (error) { @@ -268,6 +276,7 @@ const updateSettingsSchema = z.object({ lockedProjectIds: z.array(z.string().trim().min(1)).max(200), aiSelectableProjectIds: z.array(z.string().trim().min(1)).max(200), }).nullable().optional(), + ukvisajobsMaxJobs: z.number().int().min(1).max(200).nullable().optional(), }); /** @@ -306,6 +315,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { } } + if ('ukvisajobsMaxJobs' in input) { + const ukvisajobsMaxJobs = input.ukvisajobsMaxJobs ?? null; + await settingsRepo.setSetting('ukvisajobsMaxJobs', ukvisajobsMaxJobs !== null ? String(ukvisajobsMaxJobs) : null); + } + const overrideModel = await settingsRepo.getSetting('model'); const defaultModel = process.env.MODEL || 'openai/gpt-4o-mini'; const model = overrideModel || defaultModel; @@ -323,6 +337,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects'); const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw }); + const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs'); + const defaultUkvisajobsMaxJobs = 50; + const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null; + const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs; + res.json({ success: true, data: { @@ -336,6 +355,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { defaultJobCompleteWebhookUrl, overrideJobCompleteWebhookUrl, ...resumeProjectsData, + ukvisajobsMaxJobs, + defaultUkvisajobsMaxJobs, + overrideUkvisajobsMaxJobs, }, }); } catch (error) { diff --git a/orchestrator/src/server/pipeline/orchestrator.ts b/orchestrator/src/server/pipeline/orchestrator.ts index 100588a..fb2348e 100644 --- a/orchestrator/src/server/pipeline/orchestrator.ts +++ b/orchestrator/src/server/pipeline/orchestrator.ts @@ -162,7 +162,11 @@ export async function runPipeline(config: Partial = {}): Promise detail: 'UKVisaJobs: scraping visa-sponsoring jobs...', }); - const ukVisaResult = await runUkVisaJobs({ maxJobs: 50 }); + // Read max jobs setting from database (default to 50 if not set) + const ukvisajobsMaxJobsSetting = await settingsRepo.getSetting('ukvisajobsMaxJobs'); + const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting ? parseInt(ukvisajobsMaxJobsSetting, 10) : 50; + + const ukVisaResult = await runUkVisaJobs({ maxJobs: ukvisajobsMaxJobs }); if (!ukVisaResult.success) { sourceErrors.push(`ukvisajobs: ${ukVisaResult.error ?? 'unknown error'}`); } else { diff --git a/orchestrator/src/server/repositories/settings.ts b/orchestrator/src/server/repositories/settings.ts index 15c9d1c..57d8be6 100644 --- a/orchestrator/src/server/repositories/settings.ts +++ b/orchestrator/src/server/repositories/settings.ts @@ -11,6 +11,7 @@ export type SettingKey = 'model' | 'pipelineWebhookUrl' | 'jobCompleteWebhookUrl' | 'resumeProjects' + | 'ukvisajobsMaxJobs' export async function getSetting(key: SettingKey): Promise { const [row] = await db.select().from(settings).where(eq(settings.key, key)) diff --git a/orchestrator/src/shared/types.ts b/orchestrator/src/shared/types.ts index a7ad297..4abc143 100644 --- a/orchestrator/src/shared/types.ts +++ b/orchestrator/src/shared/types.ts @@ -201,4 +201,7 @@ export interface AppSettings { resumeProjects: ResumeProjectsSettings; defaultResumeProjects: ResumeProjectsSettings; overrideResumeProjects: ResumeProjectsSettings | null; + ukvisajobsMaxJobs: number; + defaultUkvisajobsMaxJobs: number; + overrideUkvisajobsMaxJobs: number | null; }