ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

passing max ukvisajobs

Browse Source
This commit is contained in:
DaKheera47 2025-12-26 20:47:28 +00:00
parent 878a2f0f54
commit bd7baafbec
9 changed files with 100 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.example
View File

@ -47,4 +47,3 @@ UKVISAJOBS_TOKEN=
UKVISAJOBS_AUTH_TOKEN=
UKVISAJOBS_CSRF_TOKEN=
UKVISAJOBS_CI_SESSION=
UKVISAJOBS_MAX_JOBS=50

1
docker-compose.yml
View File

@ -55,7 +55,6 @@ services:
- UKVISAJOBS_AUTH_TOKEN=${UKVISAJOBS_AUTH_TOKEN:-}
- UKVISAJOBS_CSRF_TOKEN=${UKVISAJOBS_CSRF_TOKEN:-}
- UKVISAJOBS_CI_SESSION=${UKVISAJOBS_CI_SESSION:-}
- UKVISAJOBS_MAX_JOBS=${UKVISAJOBS_MAX_JOBS:-50}
- UKVISAJOBS_SEARCH_KEYWORD=${UKVISAJOBS_SEARCH_KEYWORD:-}
# Python path (uses system python in container)

2
extractors/ukvisajobs/src/main.ts
View File

@ -9,7 +9,7 @@
* UKVISAJOBS_AUTH_TOKEN - Auth cookie token (defaults to UKVISAJOBS_TOKEN)
* UKVISAJOBS_CSRF_TOKEN - CSRF token cookie
* UKVISAJOBS_CI_SESSION - CI session cookie
* UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200)
* UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200) - Set via UI Settings
* UKVISAJOBS_SEARCH_KEYWORD - Optional search filter
*/

17
orchestrator/src/client/api/client.ts
View File

@ -2,10 +2,10 @@
* API client for the orchestrator backend.
*/
import type {
Job,
ApiResponse,
JobsListResponse,
import type {
Job,
ApiResponse,
JobsListResponse,
PipelineStatusResponse,
JobSource,
PipelineRun,
@ -26,13 +26,13 @@ async function fetchApi<T>(
...options?.headers,
},
});
const data: ApiResponse<T> = await response.json();
if (!data.success) {
throw new Error(data.error || 'API request failed');
}
return data.data as T;
}
@ -47,7 +47,7 @@ export async function getJob(id: string): Promise<Job> {
}
export async function updateJob(
id: string,
id: string,
update: Partial<Job>
): Promise<Job> {
return fetchApi<Job>(`/jobs/${id}`, {
@ -105,6 +105,7 @@ export async function updateSettings(update: {
pipelineWebhookUrl?: string | null
jobCompleteWebhookUrl?: string | null
resumeProjects?: ResumeProjectsSettings | null
ukvisajobsMaxJobs?: number | null
}): Promise<AppSettings> {
return fetchApi<AppSettings>('/settings', {
method: 'PATCH',

60
orchestrator/src/client/pages/SettingsPage.tsx
View File

@ -42,6 +42,7 @@ export const SettingsPage: React.FC = () => {
const [pipelineWebhookUrlDraft, setPipelineWebhookUrlDraft] = useState("")
const [jobCompleteWebhookUrlDraft, setJobCompleteWebhookUrlDraft] = useState("")
const [resumeProjectsDraft, setResumeProjectsDraft] = useState<ResumeProjectsSettings | null>(null)
const [ukvisajobsMaxJobsDraft, setUkvisajobsMaxJobsDraft] = useState<number | null>(null)
const [isSaving, setIsSaving] = useState(false)
const [isLoading, setIsLoading] = useState(true)
@ -57,6 +58,7 @@ export const SettingsPage: React.FC = () => {
setPipelineWebhookUrlDraft(data.overridePipelineWebhookUrl ?? "")
setJobCompleteWebhookUrlDraft(data.overrideJobCompleteWebhookUrl ?? "")
setResumeProjectsDraft(data.resumeProjects)
setUkvisajobsMaxJobsDraft(data.overrideUkvisajobsMaxJobs)
})
.catch((error) => {
const message = error instanceof Error ? error.message : "Failed to load settings"
@ -81,6 +83,9 @@ export const SettingsPage: React.FC = () => {
const effectiveJobCompleteWebhookUrl = settings?.jobCompleteWebhookUrl ?? ""
const defaultJobCompleteWebhookUrl = settings?.defaultJobCompleteWebhookUrl ?? ""
const overrideJobCompleteWebhookUrl = settings?.overrideJobCompleteWebhookUrl
const effectiveUkvisajobsMaxJobs = settings?.ukvisajobsMaxJobs ?? 50
const defaultUkvisajobsMaxJobs = settings?.defaultUkvisajobsMaxJobs ?? 50
const overrideUkvisajobsMaxJobs = settings?.overrideUkvisajobsMaxJobs
const profileProjects = settings?.profileProjects ?? []
const maxProjectsTotal = profileProjects.length
const lockedCount = resumeProjectsDraft?.lockedProjectIds.length ?? 0
@ -93,11 +98,13 @@ export const SettingsPage: React.FC = () => {
const currentWebhook = (overridePipelineWebhookUrl ?? "").trim()
const nextJobCompleteWebhook = jobCompleteWebhookUrlDraft.trim()
const currentJobCompleteWebhook = (overrideJobCompleteWebhookUrl ?? "").trim()
const ukvisajobsChanged = ukvisajobsMaxJobsDraft !== (overrideUkvisajobsMaxJobs ?? null)
return (
next !== current ||
nextWebhook !== currentWebhook ||
nextJobCompleteWebhook !== currentJobCompleteWebhook ||
!resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects)
!resumeProjectsEqual(resumeProjectsDraft, settings.resumeProjects) ||
ukvisajobsChanged
)
}, [
settings,
@ -108,6 +115,8 @@ export const SettingsPage: React.FC = () => {
overridePipelineWebhookUrl,
overrideJobCompleteWebhookUrl,
resumeProjectsDraft,
ukvisajobsMaxJobsDraft,
overrideUkvisajobsMaxJobs,
])
const handleSave = async () => {
@ -120,17 +129,20 @@ export const SettingsPage: React.FC = () => {
const resumeProjectsOverride = resumeProjectsEqual(resumeProjectsDraft, settings.defaultResumeProjects)
? null
: resumeProjectsDraft
const ukvisajobsMaxJobsOverride = ukvisajobsMaxJobsDraft === defaultUkvisajobsMaxJobs ? null : ukvisajobsMaxJobsDraft
const updated = await api.updateSettings({
model: trimmed.length > 0 ? trimmed : null,
pipelineWebhookUrl: webhookTrimmed.length > 0 ? webhookTrimmed : null,
jobCompleteWebhookUrl: jobCompleteTrimmed.length > 0 ? jobCompleteTrimmed : null,
resumeProjects: resumeProjectsOverride,
ukvisajobsMaxJobs: ukvisajobsMaxJobsOverride,
})
setSettings(updated)
setModelDraft(updated.overrideModel ?? "")
setPipelineWebhookUrlDraft(updated.overridePipelineWebhookUrl ?? "")
setJobCompleteWebhookUrlDraft(updated.overrideJobCompleteWebhookUrl ?? "")
setResumeProjectsDraft(updated.resumeProjects)
setUkvisajobsMaxJobsDraft(updated.overrideUkvisajobsMaxJobs)
toast.success("Settings saved")
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to save settings"
@ -148,12 +160,14 @@ export const SettingsPage: React.FC = () => {
pipelineWebhookUrl: null,
jobCompleteWebhookUrl: null,
resumeProjects: null,
ukvisajobsMaxJobs: null,
})
setSettings(updated)
setModelDraft("")
setPipelineWebhookUrlDraft("")
setJobCompleteWebhookUrlDraft("")
setResumeProjectsDraft(updated.resumeProjects)
setUkvisajobsMaxJobsDraft(null)
toast.success("Reset to default")
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to reset settings"
@ -272,6 +286,50 @@ export const SettingsPage: React.FC = () => {
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle className="text-base">UKVisaJobs Extractor</CardTitle>
</CardHeader>
<CardContent className="space-y-4">
<div className="space-y-2">
<div className="text-sm font-medium">Max jobs to fetch</div>
<Input
type="number"
inputMode="numeric"
min={1}
max={200}
value={ukvisajobsMaxJobsDraft ?? defaultUkvisajobsMaxJobs}
onChange={(event) => {
const value = parseInt(event.target.value, 10)
if (Number.isNaN(value)) {
setUkvisajobsMaxJobsDraft(null)
} else {
setUkvisajobsMaxJobsDraft(Math.min(200, Math.max(1, value)))
}
}}
disabled={isLoading || isSaving}
/>
<div className="text-xs text-muted-foreground">
Maximum number of jobs to fetch from UKVisaJobs per pipeline run. Range: 1-200.
</div>
</div>
<Separator />
<div className="grid gap-2 text-sm sm:grid-cols-2">
<div>
<div className="text-xs text-muted-foreground">Effective</div>
<div className="break-words font-mono text-xs">{effectiveUkvisajobsMaxJobs}</div>
</div>
<div>
<div className="text-xs text-muted-foreground">Default</div>
<div className="break-words font-mono text-xs">{defaultUkvisajobsMaxJobs}</div>
</div>
</div>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle className="text-base">Resume Projects</CardTitle>

22
orchestrator/src/server/api/routes.ts
View File

@ -238,6 +238,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects');
const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw });
const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs');
const defaultUkvisajobsMaxJobs = 50;
const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null;
const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs;
res.json({
success: true,
data: {
@ -251,6 +256,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
defaultJobCompleteWebhookUrl,
overrideJobCompleteWebhookUrl,
...resumeProjectsData,
ukvisajobsMaxJobs,
defaultUkvisajobsMaxJobs,
overrideUkvisajobsMaxJobs,
},
});
} catch (error) {
@ -268,6 +276,7 @@ const updateSettingsSchema = z.object({
lockedProjectIds: z.array(z.string().trim().min(1)).max(200),
aiSelectableProjectIds: z.array(z.string().trim().min(1)).max(200),
}).nullable().optional(),
ukvisajobsMaxJobs: z.number().int().min(1).max(200).nullable().optional(),
});
/**
@ -306,6 +315,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
}
}
if ('ukvisajobsMaxJobs' in input) {
const ukvisajobsMaxJobs = input.ukvisajobsMaxJobs ?? null;
await settingsRepo.setSetting('ukvisajobsMaxJobs', ukvisajobsMaxJobs !== null ? String(ukvisajobsMaxJobs) : null);
}
const overrideModel = await settingsRepo.getSetting('model');
const defaultModel = process.env.MODEL || 'openai/gpt-4o-mini';
const model = overrideModel || defaultModel;
@ -323,6 +337,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
const overrideResumeProjectsRaw = await settingsRepo.getSetting('resumeProjects');
const resumeProjectsData = resolveResumeProjectsSettings({ catalog, overrideRaw: overrideResumeProjectsRaw });
const overrideUkvisajobsMaxJobsRaw = await settingsRepo.getSetting('ukvisajobsMaxJobs');
const defaultUkvisajobsMaxJobs = 50;
const overrideUkvisajobsMaxJobs = overrideUkvisajobsMaxJobsRaw ? parseInt(overrideUkvisajobsMaxJobsRaw, 10) : null;
const ukvisajobsMaxJobs = overrideUkvisajobsMaxJobs ?? defaultUkvisajobsMaxJobs;
res.json({
success: true,
data: {
@ -336,6 +355,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
defaultJobCompleteWebhookUrl,
overrideJobCompleteWebhookUrl,
...resumeProjectsData,
ukvisajobsMaxJobs,
defaultUkvisajobsMaxJobs,
overrideUkvisajobsMaxJobs,
},
});
} catch (error) {

6
orchestrator/src/server/pipeline/orchestrator.ts
View File

@ -162,7 +162,11 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
detail: 'UKVisaJobs: scraping visa-sponsoring jobs...',
});
const ukVisaResult = await runUkVisaJobs({ maxJobs: 50 });
// Read max jobs setting from database (default to 50 if not set)
const ukvisajobsMaxJobsSetting = await settingsRepo.getSetting('ukvisajobsMaxJobs');
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting ? parseInt(ukvisajobsMaxJobsSetting, 10) : 50;
const ukVisaResult = await runUkVisaJobs({ maxJobs: ukvisajobsMaxJobs });
if (!ukVisaResult.success) {
sourceErrors.push(`ukvisajobs: ${ukVisaResult.error ?? 'unknown error'}`);
} else {

1
orchestrator/src/server/repositories/settings.ts
View File

@ -11,6 +11,7 @@ export type SettingKey = 'model'
| 'pipelineWebhookUrl'
| 'jobCompleteWebhookUrl'
| 'resumeProjects'
| 'ukvisajobsMaxJobs'
export async function getSetting(key: SettingKey): Promise<string | null> {
const [row] = await db.select().from(settings).where(eq(settings.key, key))

3
orchestrator/src/shared/types.ts
View File

@ -201,4 +201,7 @@ export interface AppSettings {
resumeProjects: ResumeProjectsSettings;
defaultResumeProjects: ResumeProjectsSettings;
overrideResumeProjects: ResumeProjectsSettings | null;
ukvisajobsMaxJobs: number;
defaultUkvisajobsMaxJobs: number;
overrideUkvisajobsMaxJobs: number | null;
}