diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts index 572e7ae..b6318bf 100644 --- a/orchestrator/src/client/api/client.ts +++ b/orchestrator/src/client/api/client.ts @@ -155,6 +155,7 @@ export async function updateSettings(update: { jobspyResultsWanted?: number | null jobspyHoursOld?: number | null jobspyCountryIndeed?: string | null + jobspySites?: string[] | null jobspyLinkedinFetchDescription?: boolean | null }): Promise { return fetchApi('/settings', { diff --git a/orchestrator/src/client/pages/SettingsPage.tsx b/orchestrator/src/client/pages/SettingsPage.tsx index 1e759f6..33de3bc 100644 --- a/orchestrator/src/client/pages/SettingsPage.tsx +++ b/orchestrator/src/client/pages/SettingsPage.tsx @@ -69,6 +69,7 @@ export const SettingsPage: React.FC = () => { const [jobspyResultsWantedDraft, setJobspyResultsWantedDraft] = useState(null) const [jobspyHoursOldDraft, setJobspyHoursOldDraft] = useState(null) const [jobspyCountryIndeedDraft, setJobspyCountryIndeedDraft] = useState(null) + const [jobspySitesDraft, setJobspySitesDraft] = useState(null) const [jobspyLinkedinFetchDescriptionDraft, setJobspyLinkedinFetchDescriptionDraft] = useState(null) const [isSaving, setIsSaving] = useState(false) const [isLoading, setIsLoading] = useState(true) @@ -94,6 +95,7 @@ export const SettingsPage: React.FC = () => { setJobspyResultsWantedDraft(data.overrideJobspyResultsWanted) setJobspyHoursOldDraft(data.overrideJobspyHoursOld) setJobspyCountryIndeedDraft(data.overrideJobspyCountryIndeed) + setJobspySitesDraft(data.overrideJobspySites) setJobspyLinkedinFetchDescriptionDraft(data.overrideJobspyLinkedinFetchDescription) }) .catch((error) => { @@ -143,6 +145,9 @@ export const SettingsPage: React.FC = () => { const effectiveJobspyCountryIndeed = settings?.jobspyCountryIndeed ?? "" const defaultJobspyCountryIndeed = settings?.defaultJobspyCountryIndeed ?? "" const overrideJobspyCountryIndeed = settings?.overrideJobspyCountryIndeed + const effectiveJobspySites = settings?.jobspySites ?? ["indeed", "linkedin"] + const defaultJobspySites = settings?.defaultJobspySites ?? ["indeed", "linkedin"] + const overrideJobspySites = settings?.overrideJobspySites const effectiveJobspyLinkedinFetchDescription = settings?.jobspyLinkedinFetchDescription ?? true const defaultJobspyLinkedinFetchDescription = settings?.defaultJobspyLinkedinFetchDescription ?? true const overrideJobspyLinkedinFetchDescription = settings?.overrideJobspyLinkedinFetchDescription @@ -180,6 +185,7 @@ export const SettingsPage: React.FC = () => { jobspyResultsWantedDraft !== (overrideJobspyResultsWanted ?? null) || jobspyHoursOldDraft !== (overrideJobspyHoursOld ?? null) || jobspyCountryIndeedDraft !== (overrideJobspyCountryIndeed ?? null) || + JSON.stringify((jobspySitesDraft ?? []).slice().sort()) !== JSON.stringify((overrideJobspySites ?? []).slice().sort()) || jobspyLinkedinFetchDescriptionDraft !== (overrideJobspyLinkedinFetchDescription ?? null) ) }, [ @@ -205,11 +211,13 @@ export const SettingsPage: React.FC = () => { jobspyResultsWantedDraft, jobspyHoursOldDraft, jobspyCountryIndeedDraft, + jobspySitesDraft, jobspyLinkedinFetchDescriptionDraft, overrideJobspyLocation, overrideJobspyResultsWanted, overrideJobspyHoursOld, overrideJobspyCountryIndeed, + overrideJobspySites, overrideJobspyLinkedinFetchDescription, ]) @@ -232,6 +240,7 @@ export const SettingsPage: React.FC = () => { const jobspyResultsWantedOverride = jobspyResultsWantedDraft === defaultJobspyResultsWanted ? null : jobspyResultsWantedDraft const jobspyHoursOldOverride = jobspyHoursOldDraft === defaultJobspyHoursOld ? null : jobspyHoursOldDraft const jobspyCountryIndeedOverride = jobspyCountryIndeedDraft === defaultJobspyCountryIndeed ? null : jobspyCountryIndeedDraft + const jobspySitesOverride = arraysEqual((jobspySitesDraft ?? []).slice().sort(), (defaultJobspySites ?? []).slice().sort()) ? null : jobspySitesDraft const jobspyLinkedinFetchDescriptionOverride = jobspyLinkedinFetchDescriptionDraft === defaultJobspyLinkedinFetchDescription ? null : jobspyLinkedinFetchDescriptionDraft const updated = await api.updateSettings({ model: trimmed.length > 0 ? trimmed : null, @@ -247,6 +256,7 @@ export const SettingsPage: React.FC = () => { jobspyResultsWanted: jobspyResultsWantedOverride, jobspyHoursOld: jobspyHoursOldOverride, jobspyCountryIndeed: jobspyCountryIndeedOverride, + jobspySites: jobspySitesOverride, jobspyLinkedinFetchDescription: jobspyLinkedinFetchDescriptionOverride, }) setSettings(updated) @@ -263,6 +273,7 @@ export const SettingsPage: React.FC = () => { setJobspyResultsWantedDraft(updated.overrideJobspyResultsWanted) setJobspyHoursOldDraft(updated.overrideJobspyHoursOld) setJobspyCountryIndeedDraft(updated.overrideJobspyCountryIndeed) + setJobspySitesDraft(updated.overrideJobspySites) setJobspyLinkedinFetchDescriptionDraft(updated.overrideJobspyLinkedinFetchDescription) toast.success("Settings saved") } catch (error) { @@ -314,6 +325,7 @@ export const SettingsPage: React.FC = () => { jobspyResultsWanted: null, jobspyHoursOld: null, jobspyCountryIndeed: null, + jobspySites: null, jobspyLinkedinFetchDescription: null, }) setSettings(updated) @@ -330,6 +342,7 @@ export const SettingsPage: React.FC = () => { setJobspyResultsWantedDraft(null) setJobspyHoursOldDraft(null) setJobspyCountryIndeedDraft(null) + setJobspySitesDraft(null) setJobspyLinkedinFetchDescriptionDraft(null) toast.success("Reset to default") } catch (error) { @@ -598,6 +611,55 @@ export const SettingsPage: React.FC = () => {
+
+
Scraped Sites
+
+
+ { + const current = jobspySitesDraft ?? defaultJobspySites + let next = [...current] + if (checked) { + if (!next.includes('indeed')) next.push('indeed') + } else { + next = next.filter(s => s !== 'indeed') + } + setJobspySitesDraft(next) + }} + disabled={isLoading || isSaving} + /> + +
+
+ { + const current = jobspySitesDraft ?? defaultJobspySites + let next = [...current] + if (checked) { + if (!next.includes('linkedin')) next.push('linkedin') + } else { + next = next.filter(s => s !== 'linkedin') + } + setJobspySitesDraft(next) + }} + disabled={isLoading || isSaving} + /> + +
+
+
+ Select which sites JobSpy should scrape. +
+
+ Effective: {(effectiveJobspySites || []).join(', ') || "None"} + Default: {(defaultJobspySites || []).join(', ')} +
+
+
Location
diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts index 6a61782..3b0a397 100644 --- a/orchestrator/src/server/api/routes.ts +++ b/orchestrator/src/server/api/routes.ts @@ -304,7 +304,7 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => { const overrideSearchTerms = overrideSearchTermsRaw ? JSON.parse(overrideSearchTermsRaw) as string[] : null; const searchTerms = overrideSearchTerms ?? defaultSearchTerms; - // JobSpy settings + // JobSpy settings (GET) const overrideJobspyLocation = await settingsRepo.getSetting('jobspyLocation'); const defaultJobspyLocation = process.env.JOBSPY_LOCATION || 'UK'; const jobspyLocation = overrideJobspyLocation || defaultJobspyLocation; @@ -323,6 +323,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => { const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK'; const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed; + const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites'); + const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean); + const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null; + const jobspySites = overrideJobspySites ?? defaultJobspySites; + const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription'); const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1'; const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw @@ -367,6 +372,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => { jobspyCountryIndeed, defaultJobspyCountryIndeed, overrideJobspyCountryIndeed, + jobspySites, + defaultJobspySites, + overrideJobspySites, jobspyLinkedinFetchDescription, defaultJobspyLinkedinFetchDescription, overrideJobspyLinkedinFetchDescription, @@ -396,6 +404,7 @@ const updateSettingsSchema = z.object({ jobspyResultsWanted: z.number().int().min(1).max(500).nullable().optional(), jobspyHoursOld: z.number().int().min(1).max(168).nullable().optional(), jobspyCountryIndeed: z.string().trim().min(1).max(100).nullable().optional(), + jobspySites: z.array(z.string().trim().min(1).max(50)).max(10).nullable().optional(), jobspyLinkedinFetchDescription: z.boolean().nullable().optional(), }); @@ -475,6 +484,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { await settingsRepo.setSetting('jobspyCountryIndeed', value); } + if ('jobspySites' in input) { + const value = input.jobspySites ?? null; + await settingsRepo.setSetting('jobspySites', value !== null ? JSON.stringify(value) : null); + } + if ('jobspyLinkedinFetchDescription' in input) { const value = input.jobspyLinkedinFetchDescription ?? null; await settingsRepo.setSetting('jobspyLinkedinFetchDescription', value !== null ? (value ? '1' : '0') : null); @@ -537,6 +551,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK'; const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed; + const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites'); + const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean); + const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null; + const jobspySites = overrideJobspySites ?? defaultJobspySites; + const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription'); const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1'; const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw @@ -581,6 +600,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { jobspyCountryIndeed, defaultJobspyCountryIndeed, overrideJobspyCountryIndeed, + jobspySites, + defaultJobspySites, + overrideJobspySites, jobspyLinkedinFetchDescription, defaultJobspyLinkedinFetchDescription, overrideJobspyLinkedinFetchDescription, @@ -588,6 +610,8 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => { }); } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; + // PATCH usually returns 500 for unknown, but let's stick to what was there (400?) + // Wait, the file said 400? Let's verify line 608. res.status(400).json({ success: false, error: message }); } }); diff --git a/orchestrator/src/server/pipeline/orchestrator.ts b/orchestrator/src/server/pipeline/orchestrator.ts index 1dd0ed0..aaf6b38 100644 --- a/orchestrator/src/server/pipeline/orchestrator.ts +++ b/orchestrator/src/server/pipeline/orchestrator.ts @@ -127,10 +127,23 @@ export async function runPipeline(config: Partial = {}): Promise } // Run JobSpy (Indeed/LinkedIn) if selected - const jobSpySites = mergedConfig.sources.filter( + let jobSpySites = mergedConfig.sources.filter( (s): s is 'indeed' | 'linkedin' => s === 'indeed' || s === 'linkedin' ); + // Apply setting override for JobSpy sites + const jobspySitesSettingRaw = await settingsRepo.getSetting('jobspySites'); + if (jobspySitesSettingRaw) { + try { + const allowed = JSON.parse(jobspySitesSettingRaw); + if (Array.isArray(allowed)) { + jobSpySites = jobSpySites.filter((s) => allowed.includes(s)); + } + } catch { + // ignore JSON parse error + } + } + if (jobSpySites.length > 0) { updateProgress({ step: 'crawling', diff --git a/orchestrator/src/server/repositories/settings.ts b/orchestrator/src/server/repositories/settings.ts index c22c228..4bede73 100644 --- a/orchestrator/src/server/repositories/settings.ts +++ b/orchestrator/src/server/repositories/settings.ts @@ -20,6 +20,7 @@ export type SettingKey = 'model' | 'jobspyResultsWanted' | 'jobspyHoursOld' | 'jobspyCountryIndeed' + | 'jobspySites' | 'jobspyLinkedinFetchDescription' export async function getSetting(key: SettingKey): Promise { diff --git a/orchestrator/src/shared/types.ts b/orchestrator/src/shared/types.ts index 4e8d5f7..697233f 100644 --- a/orchestrator/src/shared/types.ts +++ b/orchestrator/src/shared/types.ts @@ -247,6 +247,9 @@ export interface AppSettings { jobspyCountryIndeed: string; defaultJobspyCountryIndeed: string; overrideJobspyCountryIndeed: string | null; + jobspySites: string[]; + defaultJobspySites: string[]; + overrideJobspySites: string[] | null; jobspyLinkedinFetchDescription: boolean; defaultJobspyLinkedinFetchDescription: boolean; overrideJobspyLinkedinFetchDescription: boolean | null;