passing max ukvisajobs

.env.example

@@ -47,4 +47,3 @@ UKVISAJOBS_TOKEN=
 UKVISAJOBS_AUTH_TOKEN=
 UKVISAJOBS_CSRF_TOKEN=
 UKVISAJOBS_CI_SESSION=
-UKVISAJOBS_MAX_JOBS=50

docker-compose.yml

@@ -55,7 +55,6 @@ services:
       - UKVISAJOBS_AUTH_TOKEN=${UKVISAJOBS_AUTH_TOKEN:-}
       - UKVISAJOBS_CSRF_TOKEN=${UKVISAJOBS_CSRF_TOKEN:-}
       - UKVISAJOBS_CI_SESSION=${UKVISAJOBS_CI_SESSION:-}
-      - UKVISAJOBS_MAX_JOBS=${UKVISAJOBS_MAX_JOBS:-50}
       - UKVISAJOBS_SEARCH_KEYWORD=${UKVISAJOBS_SEARCH_KEYWORD:-}
 
       # Python path (uses system python in container)

extractors/ukvisajobs/src/main.ts

@@ -9,7 +9,7 @@
  *   UKVISAJOBS_AUTH_TOKEN - Auth cookie token (defaults to UKVISAJOBS_TOKEN)
  *   UKVISAJOBS_CSRF_TOKEN - CSRF token cookie
  *   UKVISAJOBS_CI_SESSION - CI session cookie
- *   UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200)
+ *   UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200) - Set via UI Settings
  *   UKVISAJOBS_SEARCH_KEYWORD - Optional search filter
  */
 

orchestrator/src/client/api/client.ts

@@ -2,10 +2,10 @@
  * API client for the orchestrator backend.
  */
 
-import type { 
+import type {
-  Job, 
+  Job,
-  ApiResponse, 
+  ApiResponse,
-  JobsListResponse, 
+  JobsListResponse,
   PipelineStatusResponse,
   JobSource,
   PipelineRun,
@@ -26,13 +26,13 @@ async function fetchApi<T>(
       ...options?.headers,
     },
   });
-  
+
   const data: ApiResponse<T> = await response.json();
-  
+
   if (!data.success) {
     throw new Error(data.error || 'API request failed');
   }
-  
+
   return data.data as T;
 }
 
@@ -47,7 +47,7 @@ export async function getJob(id: string): Promise<Job> {
 }
 
 export async function updateJob(
-  id: string, 
+  id: string,
   update: Partial<Job>
 ): Promise<Job> {
   return fetchApi<Job>(`/jobs/${id}`, {
@@ -105,6 +105,7 @@ export async function updateSettings(update: {
   pipelineWebhookUrl?: string | null
   jobCompleteWebhookUrl?: string | null
   resumeProjects?: ResumeProjectsSettings | null
+  ukvisajobsMaxJobs?: number | null
 }): Promise<AppSettings> {
   return fetchApi<AppSettings>('/settings', {
     method: 'PATCH',

orchestrator/src/client/pages/SettingsPage.tsx

@@ -42,6 +42,7 @@ export const SettingsPage: React.FC = () => {
   const [pipelineWebhookUrlDraft, setPipelineWebhookUrlDraft] = useState("")
   const [jobCompleteWebhookUrlDraft, setJobCompleteWebhookUrlDraft] = useState("")
   const [resumeProjectsDraft, setResumeProjectsDraft] = useState<ResumeProjectsSettings | null>(null)
+  const [ukvisajobsMaxJobsDraft, setUkvisajobsMaxJobsDraft] = useState<number | null>(null)
   const [isSaving, setIsSaving] = useState(false)
   const [isLoading, setIsLoading] = useState(true)
 
@@ -57,6 +58,7 @@ export const SettingsPage: React.FC = () => {
         setPipelineWebhookUrlDraft(data.overridePipelineWebhookUrl ?? "")
         setJobCompleteWebhookUrlDraft(data.overrideJobCompleteWebhookUrl ?? "")
         setResumeProjectsDraft(data.resumeProjects)
+        setUkvisajobsMaxJobsDraft(data.overrideUkvisajobsMaxJobs)
       })
       .catch((error) => {
         const message = error instanceof Error ? error.message : "Failed to load settings"
@@ -81,6 +83,9 @@ export const SettingsPage: React.FC = () => {
   const effectiveJobCompleteWebhookUrl = settings?.jobCompleteWebhookUrl ?? ""
   const defaultJobCompleteWebhookUrl = settings?.defaultJobCompleteWebhookUrl ?? ""
   const overrideJobCompleteWebhookUrl = settings?.overrideJobCompleteWebhookUrl
+  const effectiveUkvisajobsMaxJobs = settings?.ukvisajobsMaxJobs ?? 50
+  const defaultUkvisajobsMaxJobs = settings?.defaultUkvisajobsMaxJobs ?? 50
+  const overrideUkvisajobsMaxJobs = settings?.overrideUkvisajobsMaxJobs
   const profileProjects = settings?.profileProjects ?? []
   const maxProjectsTotal = profileProjects.length
   const lockedCount = resumeProjectsDraft?.lockedProjectIds.length ?? 0
@@ -93,11 +98,13 @@ export const SettingsPage: React.FC = () => {
     const currentWebhook = (overridePipelineWebhookUrl ?? "").trim()
     const nextJobCompleteWebhook = jobCompleteWebhookUrlDraft.trim()
     const currentJobCompleteWebhook = (overrideJobCompleteWebhookUrl ?? "").trim()
+    const ukvisajobsChanged = ukvisajobsMaxJobsDraft !== (overrideUkvisajobsMaxJobs ?? null)
     return (
       next !== current ||
       nextWebhook !== currentWebhook ||
       nextJobCompleteWebhook !== currentJobCompleteWebhook ||
-      !resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects)
+      !resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects) ||
+      ukvisajobsChanged
     )
   }, [
     settings,
@@ -108,6 +115,8 @@ export const SettingsPage: React.FC = () => {
     overridePipelineWebhookUrl,
     overrideJobCompleteWebhookUrl,
     resumeProjectsDraft,
+    ukvisajobsMaxJobsDraft,
+    overrideUkvisajobsMaxJobs,
   ])
 
   const handleSave = async () => {
@@ -120,17 +129,20 @@ export const SettingsPage: React.FC = () => {
       const resumeProjectsOverride = resumeProjectsEqual(resumeProjectsDraft, settings.defaultResumeProjects)
         ? null
         : resumeProjectsDraft
+      const ukvisajobsMaxJobsOverride = ukvisajobsMaxJobsDraft === defaultUkvisajobsMaxJobs ? null : ukvisajobsMaxJobsDraft
       const updated = await api.updateSettings({
         model: trimmed.length > 0 ? trimmed : null,
         pipelineWebhookUrl: webhookTrimmed.length > 0 ? webhookTrimmed : null,
         jobCompleteWebhookUrl: jobCompleteTrimmed.length > 0 ? jobCompleteTrimmed : null,
         resumeProjects: resumeProjectsOverride,
+        ukvisajobsMaxJobs: ukvisajobsMaxJobsOverride,
       })
       setSettings(updated)
       setModelDraft(updated.overrideModel ?? "")
       setPipelineWebhookUrlDraft(updated.overridePipelineWebhookUrl ?? "")
       setJobCompleteWebhookUrlDraft(updated.overrideJobCompleteWebhookUrl ?? "")
       setResumeProjectsDraft(updated.resumeProjects)
+      setUkvisajobsMaxJobsDraft(updated.overrideUkvisajobsMaxJobs)
       toast.success("Settings saved")
     } catch (error) {
       const message = error instanceof Error ? error.message : "Failed to save settings"
@@ -148,12 +160,14 @@ export const SettingsPage: React.FC = () => {
         pipelineWebhookUrl: null,
         jobCompleteWebhookUrl: null,
         resumeProjects: null,
+        ukvisajobsMaxJobs: null,
       })
       setSettings(updated)
       setModelDraft("")
       setPipelineWebhookUrlDraft("")
       setJobCompleteWebhookUrlDraft("")
       setResumeProjectsDraft(updated.resumeProjects)
+      setUkvisajobsMaxJobsDraft(null)
       toast.success("Reset to default")
     } catch (error) {
       const message = error instanceof Error ? error.message : "Failed to reset settings"
@@ -272,6 +286,50 @@ export const SettingsPage: React.FC = () => {
         </CardContent>
       </Card>
 
+      <Card>
+        <CardHeader>
+          <CardTitle className="text-base">UKVisaJobs Extractor</CardTitle>
+        </CardHeader>
+
+        <CardContent className="space-y-4">
+          <div className="space-y-2">
+            <div className="text-sm font-medium">Max jobs to fetch</div>
+            <Input
+              type="number"
+              inputMode="numeric"
+              min={1}
+              max={200}
+              value={ukvisajobsMaxJobsDraft ?? defaultUkvisajobsMaxJobs}
+              onChange={(event) => {
+                const value = parseInt(event.target.value, 10)
+                if (Number.isNaN(value)) {
+                  setUkvisajobsMaxJobsDraft(null)
+                } else {
+                  setUkvisajobsMaxJobsDraft(Math.min(200, Math.max(1, value)))
+                }
+              }}
+              disabled={isLoading || isSaving}
+            />
+            <div className="text-xs text-muted-foreground">
+              Maximum number of jobs to fetch from UKVisaJobs per pipeline run. Range: 1-200.
+            </div>
+          </div>
+
+          <Separator />
+
+          <div className="grid gap-2 text-sm sm:grid-cols-2">
+            <div>
+              <div className="text-xs text-muted-foreground">Effective</div>
+              <div className="break-words font-mono text-xs">{effectiveUkvisajobsMaxJobs}</div>
+            </div>
+            <div>
+              <div className="text-xs text-muted-foreground">Default</div>
+              <div className="break-words font-mono text-xs">{defaultUkvisajobsMaxJobs}</div>
+            </div>
+          </div>
+        </CardContent>
+      </Card>
+
       <Card>
         <CardHeader>
           <CardTitle className="text-base">Resume Projects</CardTitle>

orchestrator/src/server/api/routes.ts

@@ -238,6 +238,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
     const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects');
     const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw });
 
+    const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs');
+    const defaultUkvisajobsMaxJobs = 50;
+    const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null;
+    const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs;
+
     res.json({
       success: true,
       data: {
@@ -251,6 +256,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
         defaultJobCompleteWebhookUrl,
         overrideJobCompleteWebhookUrl,
         ...resumeProjectsData,
+        ukvisajobsMaxJobs,
+        defaultUkvisajobsMaxJobs,
+        overrideUkvisajobsMaxJobs,
       },
     });
   } catch (error) {
@@ -268,6 +276,7 @@ const updateSettingsSchema = z.object({
     lockedProjectIds: z.array(z.string().trim().min(1)).max(200),
     aiSelectableProjectIds: z.array(z.string().trim().min(1)).max(200),
   }).nullable().optional(),
+  ukvisajobsMaxJobs: z.number().int().min(1).max(200).nullable().optional(),
 });
 
 /**
@@ -306,6 +315,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
       }
     }
 
+    if ('ukvisajobsMaxJobs' in input) {
+      const ukvisajobsMaxJobs = input.ukvisajobsMaxJobs ?? null;
+      await settingsRepo.setSetting('ukvisajobsMaxJobs', ukvisajobsMaxJobs !== null ? String(ukvisajobsMaxJobs) : null);
+    }
+
     const overrideModel = await settingsRepo.getSetting('model');
     const defaultModel = process.env.MODEL || 'openai/gpt-4o-mini';
     const model = overrideModel || defaultModel;
@@ -323,6 +337,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
     const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects');
     const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw });
 
+    const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs');
+    const defaultUkvisajobsMaxJobs = 50;
+    const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null;
+    const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs;
+
     res.json({
       success: true,
       data: {
@@ -336,6 +355,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
         defaultJobCompleteWebhookUrl,
         overrideJobCompleteWebhookUrl,
         ...resumeProjectsData,
+        ukvisajobsMaxJobs,
+        defaultUkvisajobsMaxJobs,
+        overrideUkvisajobsMaxJobs,
       },
     });
   } catch (error) {

orchestrator/src/server/pipeline/orchestrator.ts

@@ -162,7 +162,11 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
         detail: 'UKVisaJobs: scraping visa-sponsoring jobs...',
       });
 
-      const ukVisaResult = await runUkVisaJobs({ maxJobs: 50 });
+      // Read max jobs setting from database (default to 50 if not set)
+      const ukvisajobsMaxJobsSetting = await settingsRepo.getSetting('ukvisajobsMaxJobs');
+      const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting ? parseInt(ukvisajobsMaxJobsSetting, 10) : 50;
+
+      const ukVisaResult = await runUkVisaJobs({ maxJobs: ukvisajobsMaxJobs });
       if (!ukVisaResult.success) {
         sourceErrors.push(`ukvisajobs: ${ukVisaResult.error ?? 'unknown error'}`);
       } else {

orchestrator/src/server/repositories/settings.ts

@@ -11,6 +11,7 @@ export type SettingKey = 'model'
   | 'pipelineWebhookUrl'
   | 'jobCompleteWebhookUrl'
   | 'resumeProjects'
+  | 'ukvisajobsMaxJobs'
 
 export async function getSetting(key: SettingKey): Promise<string | null> {
   const [row] = await db.select().from(settings).where(eq(settings.key, key))

orchestrator/src/shared/types.ts

@@ -201,4 +201,7 @@ export interface AppSettings {
   resumeProjects: ResumeProjectsSettings;
   defaultResumeProjects: ResumeProjectsSettings;
   overrideResumeProjects: ResumeProjectsSettings | null;
+  ukvisajobsMaxJobs: number;
+  defaultUkvisajobsMaxJobs: number;
+  overrideUkvisajobsMaxJobs: number | null;
 }