more jobspy settings

orchestrator/src/client/api/client.ts

@@ -155,6 +155,7 @@ export async function updateSettings(update: {
   jobspyResultsWanted?: number | null
   jobspyHoursOld?: number | null
   jobspyCountryIndeed?: string | null
+  jobspySites?: string[] | null
   jobspyLinkedinFetchDescription?: boolean | null
 }): Promise<AppSettings> {
   return fetchApi<AppSettings>('/settings', {

orchestrator/src/client/pages/SettingsPage.tsx

@@ -69,6 +69,7 @@ export const SettingsPage: React.FC = () => {
   const [jobspyResultsWantedDraft, setJobspyResultsWantedDraft] = useState<number | null>(null)
   const [jobspyHoursOldDraft, setJobspyHoursOldDraft] = useState<number | null>(null)
   const [jobspyCountryIndeedDraft, setJobspyCountryIndeedDraft] = useState<string | null>(null)
+  const [jobspySitesDraft, setJobspySitesDraft] = useState<string[] | null>(null)
   const [jobspyLinkedinFetchDescriptionDraft, setJobspyLinkedinFetchDescriptionDraft] = useState<boolean | null>(null)
   const [isSaving, setIsSaving] = useState(false)
   const [isLoading, setIsLoading] = useState(true)
@@ -94,6 +95,7 @@ export const SettingsPage: React.FC = () => {
         setJobspyResultsWantedDraft(data.overrideJobspyResultsWanted)
         setJobspyHoursOldDraft(data.overrideJobspyHoursOld)
         setJobspyCountryIndeedDraft(data.overrideJobspyCountryIndeed)
+        setJobspySitesDraft(data.overrideJobspySites)
         setJobspyLinkedinFetchDescriptionDraft(data.overrideJobspyLinkedinFetchDescription)
       })
       .catch((error) => {
@@ -143,6 +145,9 @@ export const SettingsPage: React.FC = () => {
   const effectiveJobspyCountryIndeed = settings?.jobspyCountryIndeed ?? ""
   const defaultJobspyCountryIndeed = settings?.defaultJobspyCountryIndeed ?? ""
   const overrideJobspyCountryIndeed = settings?.overrideJobspyCountryIndeed
+  const effectiveJobspySites = settings?.jobspySites ?? ["indeed", "linkedin"]
+  const defaultJobspySites = settings?.defaultJobspySites ?? ["indeed", "linkedin"]
+  const overrideJobspySites = settings?.overrideJobspySites
   const effectiveJobspyLinkedinFetchDescription = settings?.jobspyLinkedinFetchDescription ?? true
   const defaultJobspyLinkedinFetchDescription = settings?.defaultJobspyLinkedinFetchDescription ?? true
   const overrideJobspyLinkedinFetchDescription = settings?.overrideJobspyLinkedinFetchDescription
@@ -180,6 +185,7 @@ export const SettingsPage: React.FC = () => {
       jobspyResultsWantedDraft !== (overrideJobspyResultsWanted ?? null) ||
       jobspyHoursOldDraft !== (overrideJobspyHoursOld ?? null) ||
       jobspyCountryIndeedDraft !== (overrideJobspyCountryIndeed ?? null) ||
+      JSON.stringify((jobspySitesDraft ?? []).slice().sort()) !== JSON.stringify((overrideJobspySites ?? []).slice().sort()) ||
       jobspyLinkedinFetchDescriptionDraft !== (overrideJobspyLinkedinFetchDescription ?? null)
     )
   }, [
@@ -205,11 +211,13 @@ export const SettingsPage: React.FC = () => {
     jobspyResultsWantedDraft,
     jobspyHoursOldDraft,
     jobspyCountryIndeedDraft,
+    jobspySitesDraft,
     jobspyLinkedinFetchDescriptionDraft,
     overrideJobspyLocation,
     overrideJobspyResultsWanted,
     overrideJobspyHoursOld,
     overrideJobspyCountryIndeed,
+    overrideJobspySites,
     overrideJobspyLinkedinFetchDescription,
   ])
 
@@ -232,6 +240,7 @@ export const SettingsPage: React.FC = () => {
       const jobspyResultsWantedOverride = jobspyResultsWantedDraft === defaultJobspyResultsWanted ? null : jobspyResultsWantedDraft
       const jobspyHoursOldOverride = jobspyHoursOldDraft === defaultJobspyHoursOld ? null : jobspyHoursOldDraft
       const jobspyCountryIndeedOverride = jobspyCountryIndeedDraft === defaultJobspyCountryIndeed ? null : jobspyCountryIndeedDraft
+      const jobspySitesOverride = arraysEqual((jobspySitesDraft ?? []).slice().sort(), (defaultJobspySites ?? []).slice().sort()) ? null : jobspySitesDraft
       const jobspyLinkedinFetchDescriptionOverride = jobspyLinkedinFetchDescriptionDraft === defaultJobspyLinkedinFetchDescription ? null : jobspyLinkedinFetchDescriptionDraft
       const updated = await api.updateSettings({
         model: trimmed.length > 0 ? trimmed : null,
@@ -247,6 +256,7 @@ export const SettingsPage: React.FC = () => {
         jobspyResultsWanted: jobspyResultsWantedOverride,
         jobspyHoursOld: jobspyHoursOldOverride,
         jobspyCountryIndeed: jobspyCountryIndeedOverride,
+        jobspySites: jobspySitesOverride,
         jobspyLinkedinFetchDescription: jobspyLinkedinFetchDescriptionOverride,
       })
       setSettings(updated)
@@ -263,6 +273,7 @@ export const SettingsPage: React.FC = () => {
       setJobspyResultsWantedDraft(updated.overrideJobspyResultsWanted)
       setJobspyHoursOldDraft(updated.overrideJobspyHoursOld)
       setJobspyCountryIndeedDraft(updated.overrideJobspyCountryIndeed)
+      setJobspySitesDraft(updated.overrideJobspySites)
       setJobspyLinkedinFetchDescriptionDraft(updated.overrideJobspyLinkedinFetchDescription)
       toast.success("Settings saved")
     } catch (error) {
@@ -314,6 +325,7 @@ export const SettingsPage: React.FC = () => {
         jobspyResultsWanted: null,
         jobspyHoursOld: null,
         jobspyCountryIndeed: null,
+        jobspySites: null,
         jobspyLinkedinFetchDescription: null,
       })
       setSettings(updated)
@@ -330,6 +342,7 @@ export const SettingsPage: React.FC = () => {
       setJobspyResultsWantedDraft(null)
       setJobspyHoursOldDraft(null)
       setJobspyCountryIndeedDraft(null)
+      setJobspySitesDraft(null)
       setJobspyLinkedinFetchDescriptionDraft(null)
       toast.success("Reset to default")
     } catch (error) {
@@ -598,6 +611,55 @@ export const SettingsPage: React.FC = () => {
           </AccordionTrigger>
           <AccordionContent className="pb-4">
             <div className="space-y-6">
+              <div className="space-y-3">
+                  <div className="text-sm font-medium">Scraped Sites</div>
+                  <div className="flex gap-6">
+                    <div className="flex items-center space-x-2">
+                        <Checkbox 
+                            id="site-indeed" 
+                            checked={jobspySitesDraft?.includes('indeed') ?? defaultJobspySites.includes('indeed')}
+                            onCheckedChange={(checked) => {
+                                const current = jobspySitesDraft ?? defaultJobspySites
+                                let next = [...current]
+                                if (checked) {
+                                    if (!next.includes('indeed')) next.push('indeed')
+                                } else {
+                                    next = next.filter(s => s !== 'indeed')
+                                }
+                                setJobspySitesDraft(next)
+                            }}
+                            disabled={isLoading || isSaving}
+                        />
+                        <label htmlFor="site-indeed" className="text-sm leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70">Indeed</label>
+                    </div>
+                     <div className="flex items-center space-x-2">
+                        <Checkbox 
+                            id="site-linkedin" 
+                            checked={jobspySitesDraft?.includes('linkedin') ?? defaultJobspySites.includes('linkedin')}
+                            onCheckedChange={(checked) => {
+                                const current = jobspySitesDraft ?? defaultJobspySites
+                                let next = [...current]
+                                if (checked) {
+                                    if (!next.includes('linkedin')) next.push('linkedin')
+                                } else {
+                                    next = next.filter(s => s !== 'linkedin')
+                                }
+                                setJobspySitesDraft(next)
+                            }}
+                            disabled={isLoading || isSaving}
+                        />
+                        <label htmlFor="site-linkedin" className="text-sm leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70">LinkedIn</label>
+                    </div>
+                  </div>
+                   <div className="text-xs text-muted-foreground">
+                    Select which sites JobSpy should scrape.
+                  </div>
+                   <div className="flex gap-2 text-xs text-muted-foreground">
+                    <span>Effective: {(effectiveJobspySites || []).join(', ') || "None"}</span>
+                    <span>Default: {(defaultJobspySites || []).join(', ')}</span>
+                  </div>
+              </div>
+
               <div className="grid gap-6 md:grid-cols-2">
                 <div className="space-y-2">
                   <div className="text-sm font-medium">Location</div>

orchestrator/src/server/api/routes.ts

@@ -304,7 +304,7 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
     const overrideSearchTerms = overrideSearchTermsRaw ? JSON.parse(overrideSearchTermsRaw) as string[] : null;
     const searchTerms = overrideSearchTerms ?? defaultSearchTerms;
 
-    // JobSpy settings
+    // JobSpy settings (GET)
     const overrideJobspyLocation = await settingsRepo.getSetting('jobspyLocation');
     const defaultJobspyLocation = process.env.JOBSPY_LOCATION || 'UK';
     const jobspyLocation = overrideJobspyLocation || defaultJobspyLocation;
@@ -323,6 +323,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
     const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK';
     const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed;
 
+    const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites');
+    const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean);
+    const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null;
+    const jobspySites = overrideJobspySites ?? defaultJobspySites;
+
     const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription');
     const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1';
     const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw
@@ -367,6 +372,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
         jobspyCountryIndeed,
         defaultJobspyCountryIndeed,
         overrideJobspyCountryIndeed,
+        jobspySites,
+        defaultJobspySites,
+        overrideJobspySites,
         jobspyLinkedinFetchDescription,
         defaultJobspyLinkedinFetchDescription,
         overrideJobspyLinkedinFetchDescription,
@@ -396,6 +404,7 @@ const updateSettingsSchema = z.object({
   jobspyResultsWanted: z.number().int().min(1).max(500).nullable().optional(),
   jobspyHoursOld: z.number().int().min(1).max(168).nullable().optional(),
   jobspyCountryIndeed: z.string().trim().min(1).max(100).nullable().optional(),
+  jobspySites: z.array(z.string().trim().min(1).max(50)).max(10).nullable().optional(),
   jobspyLinkedinFetchDescription: z.boolean().nullable().optional(),
 });
 
@@ -475,6 +484,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
       await settingsRepo.setSetting('jobspyCountryIndeed', value);
     }
 
+    if ('jobspySites' in input) {
+      const value = input.jobspySites ?? null;
+      await settingsRepo.setSetting('jobspySites', value !== null ? JSON.stringify(value) : null);
+    }
+
     if ('jobspyLinkedinFetchDescription' in input) {
       const value = input.jobspyLinkedinFetchDescription ?? null;
       await settingsRepo.setSetting('jobspyLinkedinFetchDescription', value !== null ? (value ? '1' : '0') : null);
@@ -537,6 +551,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
     const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK';
     const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed;
 
+    const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites');
+    const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean);
+    const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null;
+    const jobspySites = overrideJobspySites ?? defaultJobspySites;
+
     const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription');
     const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1';
     const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw
@@ -581,6 +600,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
         jobspyCountryIndeed,
         defaultJobspyCountryIndeed,
         overrideJobspyCountryIndeed,
+        jobspySites,
+        defaultJobspySites,
+        overrideJobspySites,
         jobspyLinkedinFetchDescription,
         defaultJobspyLinkedinFetchDescription,
         overrideJobspyLinkedinFetchDescription,
@@ -588,6 +610,8 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
     });
   } catch (error) {
     const message = error instanceof Error ? error.message : 'Unknown error';
+    // PATCH usually returns 500 for unknown, but let's stick to what was there (400?)
+    // Wait, the file said 400? Let's verify line 608.
     res.status(400).json({ success: false, error: message });
   }
 });

orchestrator/src/server/pipeline/orchestrator.ts

@@ -127,10 +127,23 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
     }
 
     // Run JobSpy (Indeed/LinkedIn) if selected
-    const jobSpySites = mergedConfig.sources.filter(
+    let jobSpySites = mergedConfig.sources.filter(
       (s): s is 'indeed' | 'linkedin' => s === 'indeed' || s === 'linkedin'
     );
 
+    // Apply setting override for JobSpy sites
+    const jobspySitesSettingRaw = await settingsRepo.getSetting('jobspySites');
+    if (jobspySitesSettingRaw) {
+      try {
+        const allowed = JSON.parse(jobspySitesSettingRaw);
+        if (Array.isArray(allowed)) {
+          jobSpySites = jobSpySites.filter((s) => allowed.includes(s));
+        }
+      } catch {
+        // ignore JSON parse error
+      }
+    }
+
     if (jobSpySites.length > 0) {
       updateProgress({
         step: 'crawling',

orchestrator/src/server/repositories/settings.ts

@@ -20,6 +20,7 @@ export type SettingKey = 'model'
   | 'jobspyResultsWanted'
   | 'jobspyHoursOld'
   | 'jobspyCountryIndeed'
+  | 'jobspySites'
   | 'jobspyLinkedinFetchDescription'
 
 export async function getSetting(key: SettingKey): Promise<string | null> {

orchestrator/src/shared/types.ts

@@ -247,6 +247,9 @@ export interface AppSettings {
   jobspyCountryIndeed: string;
   defaultJobspyCountryIndeed: string;
   overrideJobspyCountryIndeed: string | null;
+  jobspySites: string[];
+  defaultJobspySites: string[];
+  overrideJobspySites: string[] | null;
   jobspyLinkedinFetchDescription: boolean;
   defaultJobspyLinkedinFetchDescription: boolean;
   overrideJobspyLinkedinFetchDescription: boolean | null;