ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more jobspy settings

Browse Source
This commit is contained in:
DaKheera47 2026-01-11 18:15:17 +00:00
parent ed23722ffa
commit e54b5f2178
6 changed files with 106 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
orchestrator/src/client/api/client.ts
View File

@ -155,6 +155,7 @@ export async function updateSettings(update: {
jobspyResultsWanted?: number | null
jobspyHoursOld?: number | null
jobspyCountryIndeed?: string | null
jobspySites?: string[] | null
jobspyLinkedinFetchDescription?: boolean | null
}): Promise<AppSettings> {
return fetchApi<AppSettings>('/settings', {

62
orchestrator/src/client/pages/SettingsPage.tsx
View File

@ -69,6 +69,7 @@ export const SettingsPage: React.FC = () => {
const [jobspyResultsWantedDraft, setJobspyResultsWantedDraft] = useState<number | null>(null)
const [jobspyHoursOldDraft, setJobspyHoursOldDraft] = useState<number | null>(null)
const [jobspyCountryIndeedDraft, setJobspyCountryIndeedDraft] = useState<string | null>(null)
const [jobspySitesDraft, setJobspySitesDraft] = useState<string[] | null>(null)
const [jobspyLinkedinFetchDescriptionDraft, setJobspyLinkedinFetchDescriptionDraft] = useState<boolean | null>(null)
const [isSaving, setIsSaving] = useState(false)
const [isLoading, setIsLoading] = useState(true)
@ -94,6 +95,7 @@ export const SettingsPage: React.FC = () => {
setJobspyResultsWantedDraft(data.overrideJobspyResultsWanted)
setJobspyHoursOldDraft(data.overrideJobspyHoursOld)
setJobspyCountryIndeedDraft(data.overrideJobspyCountryIndeed)
setJobspySitesDraft(data.overrideJobspySites)
setJobspyLinkedinFetchDescriptionDraft(data.overrideJobspyLinkedinFetchDescription)
})
.catch((error) => {
@ -143,6 +145,9 @@ export const SettingsPage: React.FC = () => {
const effectiveJobspyCountryIndeed = settings?.jobspyCountryIndeed ?? ""
const defaultJobspyCountryIndeed = settings?.defaultJobspyCountryIndeed ?? ""
const overrideJobspyCountryIndeed = settings?.overrideJobspyCountryIndeed
const effectiveJobspySites = settings?.jobspySites ?? ["indeed", "linkedin"]
const defaultJobspySites = settings?.defaultJobspySites ?? ["indeed", "linkedin"]
const overrideJobspySites = settings?.overrideJobspySites
const effectiveJobspyLinkedinFetchDescription = settings?.jobspyLinkedinFetchDescription ?? true
const defaultJobspyLinkedinFetchDescription = settings?.defaultJobspyLinkedinFetchDescription ?? true
const overrideJobspyLinkedinFetchDescription = settings?.overrideJobspyLinkedinFetchDescription
@ -180,6 +185,7 @@ export const SettingsPage: React.FC = () => {
jobspyResultsWantedDraft !== (overrideJobspyResultsWanted ?? null) ||
jobspyHoursOldDraft !== (overrideJobspyHoursOld ?? null) ||
jobspyCountryIndeedDraft !== (overrideJobspyCountryIndeed ?? null) ||
JSON.stringify((jobspySitesDraft ?? []).slice().sort()) !== JSON.stringify((overrideJobspySites ?? []).slice().sort()) ||
jobspyLinkedinFetchDescriptionDraft !== (overrideJobspyLinkedinFetchDescription ?? null)
)
}, [
@ -205,11 +211,13 @@ export const SettingsPage: React.FC = () => {
jobspyResultsWantedDraft,
jobspyHoursOldDraft,
jobspyCountryIndeedDraft,
jobspySitesDraft,
jobspyLinkedinFetchDescriptionDraft,
overrideJobspyLocation,
overrideJobspyResultsWanted,
overrideJobspyHoursOld,
overrideJobspyCountryIndeed,
overrideJobspySites,
overrideJobspyLinkedinFetchDescription,
])
@ -232,6 +240,7 @@ export const SettingsPage: React.FC = () => {
const jobspyResultsWantedOverride = jobspyResultsWantedDraft === defaultJobspyResultsWanted ? null : jobspyResultsWantedDraft
const jobspyHoursOldOverride = jobspyHoursOldDraft === defaultJobspyHoursOld ? null : jobspyHoursOldDraft
const jobspyCountryIndeedOverride = jobspyCountryIndeedDraft === defaultJobspyCountryIndeed ? null : jobspyCountryIndeedDraft
const jobspySitesOverride = arraysEqual((jobspySitesDraft ?? []).slice().sort(), (defaultJobspySites ?? []).slice().sort()) ? null : jobspySitesDraft
const jobspyLinkedinFetchDescriptionOverride = jobspyLinkedinFetchDescriptionDraft === defaultJobspyLinkedinFetchDescription ? null : jobspyLinkedinFetchDescriptionDraft
const updated = await api.updateSettings({
model: trimmed.length > 0 ? trimmed : null,
@ -247,6 +256,7 @@ export const SettingsPage: React.FC = () => {
jobspyResultsWanted: jobspyResultsWantedOverride,
jobspyHoursOld: jobspyHoursOldOverride,
jobspyCountryIndeed: jobspyCountryIndeedOverride,
jobspySites: jobspySitesOverride,
jobspyLinkedinFetchDescription: jobspyLinkedinFetchDescriptionOverride,
})
setSettings(updated)
@ -263,6 +273,7 @@ export const SettingsPage: React.FC = () => {
setJobspyResultsWantedDraft(updated.overrideJobspyResultsWanted)
setJobspyHoursOldDraft(updated.overrideJobspyHoursOld)
setJobspyCountryIndeedDraft(updated.overrideJobspyCountryIndeed)
setJobspySitesDraft(updated.overrideJobspySites)
setJobspyLinkedinFetchDescriptionDraft(updated.overrideJobspyLinkedinFetchDescription)
toast.success("Settings saved")
} catch (error) {
@ -314,6 +325,7 @@ export const SettingsPage: React.FC = () => {
jobspyResultsWanted: null,
jobspyHoursOld: null,
jobspyCountryIndeed: null,
jobspySites: null,
jobspyLinkedinFetchDescription: null,
})
setSettings(updated)
@ -330,6 +342,7 @@ export const SettingsPage: React.FC = () => {
setJobspyResultsWantedDraft(null)
setJobspyHoursOldDraft(null)
setJobspyCountryIndeedDraft(null)
setJobspySitesDraft(null)
setJobspyLinkedinFetchDescriptionDraft(null)
toast.success("Reset to default")
} catch (error) {
@ -598,6 +611,55 @@ export const SettingsPage: React.FC = () => {
</AccordionTrigger>
<AccordionContent className="pb-4">
<div className="space-y-6">
<div className="space-y-3">
<div className="text-sm font-medium">Scraped Sites</div>
<div className="flex gap-6">
<div className="flex items-center space-x-2">
<Checkbox
id="site-indeed"
checked={jobspySitesDraft?.includes('indeed') ?? defaultJobspySites.includes('indeed')}
onCheckedChange={(checked) => {
const current = jobspySitesDraft ?? defaultJobspySites
let next = [...current]
if (checked) {
if (!next.includes('indeed')) next.push('indeed')
} else {
next = next.filter(s => s !== 'indeed')
}
setJobspySitesDraft(next)
}}
disabled={isLoading || isSaving}
/>
<label htmlFor="site-indeed" className="text-sm leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70">Indeed</label>
</div>
<div className="flex items-center space-x-2">
<Checkbox
id="site-linkedin"
checked={jobspySitesDraft?.includes('linkedin') ?? defaultJobspySites.includes('linkedin')}
onCheckedChange={(checked) => {
const current = jobspySitesDraft ?? defaultJobspySites
let next = [...current]
if (checked) {
if (!next.includes('linkedin')) next.push('linkedin')
} else {
next = next.filter(s => s !== 'linkedin')
}
setJobspySitesDraft(next)
}}
disabled={isLoading || isSaving}
/>
<label htmlFor="site-linkedin" className="text-sm leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70">LinkedIn</label>
</div>
</div>
<div className="text-xs text-muted-foreground">
Select which sites JobSpy should scrape.
</div>
<div className="flex gap-2 text-xs text-muted-foreground">
<span>Effective: {(effectiveJobspySites || []).join(', ') || "None"}</span>
<span>Default: {(defaultJobspySites || []).join(', ')}</span>
</div>
</div>
<div className="grid gap-6 md:grid-cols-2">
<div className="space-y-2">
<div className="text-sm font-medium">Location</div>

26
orchestrator/src/server/api/routes.ts
View File

@ -304,7 +304,7 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
const overrideSearchTerms = overrideSearchTermsRaw ? JSON.parse(overrideSearchTermsRaw) as string[] : null;
const searchTerms = overrideSearchTerms ?? defaultSearchTerms;
// JobSpy settings
// JobSpy settings (GET)
const overrideJobspyLocation = await settingsRepo.getSetting('jobspyLocation');
const defaultJobspyLocation = process.env.JOBSPY_LOCATION || 'UK';
const jobspyLocation = overrideJobspyLocation || defaultJobspyLocation;
@ -323,6 +323,11 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK';
const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed;
const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites');
const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean);
const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null;
const jobspySites = overrideJobspySites ?? defaultJobspySites;
const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription');
const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1';
const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw
@ -367,6 +372,9 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
jobspyCountryIndeed,
defaultJobspyCountryIndeed,
overrideJobspyCountryIndeed,
jobspySites,
defaultJobspySites,
overrideJobspySites,
jobspyLinkedinFetchDescription,
defaultJobspyLinkedinFetchDescription,
overrideJobspyLinkedinFetchDescription,
@ -396,6 +404,7 @@ const updateSettingsSchema = z.object({
jobspyResultsWanted: z.number().int().min(1).max(500).nullable().optional(),
jobspyHoursOld: z.number().int().min(1).max(168).nullable().optional(),
jobspyCountryIndeed: z.string().trim().min(1).max(100).nullable().optional(),
jobspySites: z.array(z.string().trim().min(1).max(50)).max(10).nullable().optional(),
jobspyLinkedinFetchDescription: z.boolean().nullable().optional(),
});
@ -475,6 +484,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
await settingsRepo.setSetting('jobspyCountryIndeed', value);
}
if ('jobspySites' in input) {
const value = input.jobspySites ?? null;
await settingsRepo.setSetting('jobspySites', value !== null ? JSON.stringify(value) : null);
}
if ('jobspyLinkedinFetchDescription' in input) {
const value = input.jobspyLinkedinFetchDescription ?? null;
await settingsRepo.setSetting('jobspyLinkedinFetchDescription', value !== null ? (value ? '1' : '0') : null);
@ -537,6 +551,11 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
const defaultJobspyCountryIndeed = process.env.JOBSPY_COUNTRY_INDEED || 'UK';
const jobspyCountryIndeed = overrideJobspyCountryIndeed || defaultJobspyCountryIndeed;
const overrideJobspySitesRaw = await settingsRepo.getSetting('jobspySites');
const defaultJobspySites = (process.env.JOBSPY_SITES || 'indeed,linkedin').split(',').map(s => s.trim()).filter(Boolean);
const overrideJobspySites = overrideJobspySitesRaw ? JSON.parse(overrideJobspySitesRaw) as string[] : null;
const jobspySites = overrideJobspySites ?? defaultJobspySites;
const overrideJobspyLinkedinFetchDescriptionRaw = await settingsRepo.getSetting('jobspyLinkedinFetchDescription');
const defaultJobspyLinkedinFetchDescription = (process.env.JOBSPY_LINKEDIN_FETCH_DESCRIPTION || '1') === '1';
const overrideJobspyLinkedinFetchDescription = overrideJobspyLinkedinFetchDescriptionRaw
@ -581,6 +600,9 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
jobspyCountryIndeed,
defaultJobspyCountryIndeed,
overrideJobspyCountryIndeed,
jobspySites,
defaultJobspySites,
overrideJobspySites,
jobspyLinkedinFetchDescription,
defaultJobspyLinkedinFetchDescription,
overrideJobspyLinkedinFetchDescription,
@ -588,6 +610,8 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
// PATCH usually returns 500 for unknown, but let's stick to what was there (400?)
// Wait, the file said 400? Let's verify line 608.
res.status(400).json({ success: false, error: message });
}
});

15
orchestrator/src/server/pipeline/orchestrator.ts
View File

@ -127,10 +127,23 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
}
// Run JobSpy (Indeed/LinkedIn) if selected
const jobSpySites = mergedConfig.sources.filter(
let jobSpySites = mergedConfig.sources.filter(
(s): s is 'indeed' | 'linkedin' => s === 'indeed' || s === 'linkedin'
);
// Apply setting override for JobSpy sites
const jobspySitesSettingRaw = await settingsRepo.getSetting('jobspySites');
if (jobspySitesSettingRaw) {
try {
const allowed = JSON.parse(jobspySitesSettingRaw);
if (Array.isArray(allowed)) {
jobSpySites = jobSpySites.filter((s) => allowed.includes(s));
}
} catch {
// ignore JSON parse error
}
}
if (jobSpySites.length > 0) {
updateProgress({
step: 'crawling',

1
orchestrator/src/server/repositories/settings.ts
View File

@ -20,6 +20,7 @@ export type SettingKey = 'model'
| 'jobspyResultsWanted'
| 'jobspyHoursOld'
| 'jobspyCountryIndeed'
| 'jobspySites'
| 'jobspyLinkedinFetchDescription'
export async function getSetting(key: SettingKey): Promise<string | null> {

3
orchestrator/src/shared/types.ts
View File

@ -247,6 +247,9 @@ export interface AppSettings {
jobspyCountryIndeed: string;
defaultJobspyCountryIndeed: string;
overrideJobspyCountryIndeed: string | null;
jobspySites: string[];
defaultJobspySites: string[];
overrideJobspySites: string[] | null;
jobspyLinkedinFetchDescription: boolean;
defaultJobspyLinkedinFetchDescription: boolean;
overrideJobspyLinkedinFetchDescription: boolean | null;