ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parallelize

Browse Source
This commit is contained in:
DaKheera47 2026-01-22 16:25:05 +00:00
parent 0844071b7a
commit 71649c5a13
10 changed files with 98 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
orchestrator/src/server/api/routes/settings.ts
View File

@ -33,150 +33,158 @@ settingsRouter.get('/', async (_req: Request, res: Response) => {
settingsRouter.patch('/', async (req: Request, res: Response) => {
try {
const input = updateSettingsSchema.parse(req.body);
const promises: Promise<void>[] = [];
if ('model' in input) {
const model = input.model ?? null;
await settingsRepo.setSetting('model', model);
promises.push(settingsRepo.setSetting('model', input.model ?? null));
}
if ('modelScorer' in input) {
await settingsRepo.setSetting('modelScorer', input.modelScorer ?? null);
promises.push(settingsRepo.setSetting('modelScorer', input.modelScorer ?? null));
}
if ('modelTailoring' in input) {
await settingsRepo.setSetting('modelTailoring', input.modelTailoring ?? null);
promises.push(settingsRepo.setSetting('modelTailoring', input.modelTailoring ?? null));
}
if ('modelProjectSelection' in input) {
await settingsRepo.setSetting('modelProjectSelection', input.modelProjectSelection ?? null);
promises.push(settingsRepo.setSetting('modelProjectSelection', input.modelProjectSelection ?? null));
}
if ('pipelineWebhookUrl' in input) {
const pipelineWebhookUrl = input.pipelineWebhookUrl ?? null;
await settingsRepo.setSetting('pipelineWebhookUrl', pipelineWebhookUrl);
promises.push(settingsRepo.setSetting('pipelineWebhookUrl', input.pipelineWebhookUrl ?? null));
}
if ('jobCompleteWebhookUrl' in input) {
const webhookUrl = input.jobCompleteWebhookUrl ?? null;
await settingsRepo.setSetting('jobCompleteWebhookUrl', webhookUrl);
promises.push(settingsRepo.setSetting('jobCompleteWebhookUrl', input.jobCompleteWebhookUrl ?? null));
}
if ('resumeProjects' in input) {
const resumeProjects = input.resumeProjects ?? null;
if (resumeProjects === null) {
await settingsRepo.setSetting('resumeProjects', null);
promises.push(settingsRepo.setSetting('resumeProjects', null));
} else {
const rawProfile = await getProfile();
promises.push((async () => {
const rawProfile = await getProfile();
if (rawProfile === null || typeof rawProfile !== 'object' || Array.isArray(rawProfile)) {
throw new Error('Invalid resume profile format: expected a non-null object');
}
if (rawProfile === null || typeof rawProfile !== 'object' || Array.isArray(rawProfile)) {
throw new Error('Invalid resume profile format: expected a non-null object');
}
const profile = rawProfile as Record<string, unknown>;
const { catalog } = extractProjectsFromProfile(profile);
const allowed = new Set(catalog.map((p) => p.id));
const normalized = normalizeResumeProjectsSettings(resumeProjects, allowed);
await settingsRepo.setSetting('resumeProjects', JSON.stringify(normalized));
const profile = rawProfile as Record<string, unknown>;
const { catalog } = extractProjectsFromProfile(profile);
const allowed = new Set(catalog.map((p) => p.id));
const normalized = normalizeResumeProjectsSettings(resumeProjects, allowed);
await settingsRepo.setSetting('resumeProjects', JSON.stringify(normalized));
})());
}
}
if ('ukvisajobsMaxJobs' in input) {
const ukvisajobsMaxJobs = input.ukvisajobsMaxJobs ?? null;
await settingsRepo.setSetting('ukvisajobsMaxJobs', ukvisajobsMaxJobs !== null ? String(ukvisajobsMaxJobs) : null);
const val = input.ukvisajobsMaxJobs ?? null;
promises.push(settingsRepo.setSetting('ukvisajobsMaxJobs', val !== null ? String(val) : null));
}
if ('gradcrackerMaxJobsPerTerm' in input) {
const gradcrackerMaxJobsPerTerm = input.gradcrackerMaxJobsPerTerm ?? null;
await settingsRepo.setSetting('gradcrackerMaxJobsPerTerm', gradcrackerMaxJobsPerTerm !== null ? String(gradcrackerMaxJobsPerTerm) : null);
const val = input.gradcrackerMaxJobsPerTerm ?? null;
promises.push(settingsRepo.setSetting('gradcrackerMaxJobsPerTerm', val !== null ? String(val) : null));
}
if ('searchTerms' in input) {
const searchTerms = input.searchTerms ?? null;
await settingsRepo.setSetting('searchTerms', searchTerms !== null ? JSON.stringify(searchTerms) : null);
const val = input.searchTerms ?? null;
promises.push(settingsRepo.setSetting('searchTerms', val !== null ? JSON.stringify(val) : null));
}
if ('jobspyLocation' in input) {
const value = input.jobspyLocation ?? null;
await settingsRepo.setSetting('jobspyLocation', value);
promises.push(settingsRepo.setSetting('jobspyLocation', input.jobspyLocation ?? null));
}
if ('jobspyResultsWanted' in input) {
const value = input.jobspyResultsWanted ?? null;
await settingsRepo.setSetting('jobspyResultsWanted', value !== null ? String(value) : null);
const val = input.jobspyResultsWanted ?? null;
promises.push(settingsRepo.setSetting('jobspyResultsWanted', val !== null ? String(val) : null));
}
if ('jobspyHoursOld' in input) {
const value = input.jobspyHoursOld ?? null;
await settingsRepo.setSetting('jobspyHoursOld', value !== null ? String(value) : null);
const val = input.jobspyHoursOld ?? null;
promises.push(settingsRepo.setSetting('jobspyHoursOld', val !== null ? String(val) : null));
}
if ('jobspyCountryIndeed' in input) {
const value = input.jobspyCountryIndeed ?? null;
await settingsRepo.setSetting('jobspyCountryIndeed', value);
promises.push(settingsRepo.setSetting('jobspyCountryIndeed', input.jobspyCountryIndeed ?? null));
}
if ('jobspySites' in input) {
const value = input.jobspySites ?? null;
await settingsRepo.setSetting('jobspySites', value !== null ? JSON.stringify(value) : null);
const val = input.jobspySites ?? null;
promises.push(settingsRepo.setSetting('jobspySites', val !== null ? JSON.stringify(val) : null));
}
if ('jobspyLinkedinFetchDescription' in input) {
const value = input.jobspyLinkedinFetchDescription ?? null;
await settingsRepo.setSetting('jobspyLinkedinFetchDescription', value !== null ? (value ? '1' : '0') : null);
const val = input.jobspyLinkedinFetchDescription ?? null;
promises.push(settingsRepo.setSetting('jobspyLinkedinFetchDescription', val !== null ? (val ? '1' : '0') : null));
}
if ('showSponsorInfo' in input) {
const value = input.showSponsorInfo ?? null;
await settingsRepo.setSetting('showSponsorInfo', value !== null ? (value ? '1' : '0') : null);
const val = input.showSponsorInfo ?? null;
promises.push(settingsRepo.setSetting('showSponsorInfo', val !== null ? (val ? '1' : '0') : null));
}
if ('openrouterApiKey' in input) {
const value = normalizeEnvInput(input.openrouterApiKey);
await settingsRepo.setSetting('openrouterApiKey', value);
applyEnvValue('OPENROUTER_API_KEY', value);
promises.push(settingsRepo.setSetting('openrouterApiKey', value).then(() => {
applyEnvValue('OPENROUTER_API_KEY', value);
}));
}
if ('rxresumeEmail' in input) {
const value = normalizeEnvInput(input.rxresumeEmail);
await settingsRepo.setSetting('rxresumeEmail', value);
applyEnvValue('RXRESUME_EMAIL', value);
promises.push(settingsRepo.setSetting('rxresumeEmail', value).then(() => {
applyEnvValue('RXRESUME_EMAIL', value);
}));
}
if ('rxresumePassword' in input) {
const value = normalizeEnvInput(input.rxresumePassword);
await settingsRepo.setSetting('rxresumePassword', value);
applyEnvValue('RXRESUME_PASSWORD', value);
promises.push(settingsRepo.setSetting('rxresumePassword', value).then(() => {
applyEnvValue('RXRESUME_PASSWORD', value);
}));
}
if ('basicAuthUser' in input) {
const value = normalizeEnvInput(input.basicAuthUser);
await settingsRepo.setSetting('basicAuthUser', value);
applyEnvValue('BASIC_AUTH_USER', value);
promises.push(settingsRepo.setSetting('basicAuthUser', value).then(() => {
applyEnvValue('BASIC_AUTH_USER', value);
}));
}
if ('basicAuthPassword' in input) {
const value = normalizeEnvInput(input.basicAuthPassword);
await settingsRepo.setSetting('basicAuthPassword', value);
applyEnvValue('BASIC_AUTH_PASSWORD', value);
promises.push(settingsRepo.setSetting('basicAuthPassword', value).then(() => {
applyEnvValue('BASIC_AUTH_PASSWORD', value);
}));
}
if ('ukvisajobsEmail' in input) {
const value = normalizeEnvInput(input.ukvisajobsEmail);
await settingsRepo.setSetting('ukvisajobsEmail', value);
applyEnvValue('UKVISAJOBS_EMAIL', value);
promises.push(settingsRepo.setSetting('ukvisajobsEmail', value).then(() => {
applyEnvValue('UKVISAJOBS_EMAIL', value);
}));
}
if ('ukvisajobsPassword' in input) {
const value = normalizeEnvInput(input.ukvisajobsPassword);
await settingsRepo.setSetting('ukvisajobsPassword', value);
applyEnvValue('UKVISAJOBS_PASSWORD', value);
promises.push(settingsRepo.setSetting('ukvisajobsPassword', value).then(() => {
applyEnvValue('UKVISAJOBS_PASSWORD', value);
}));
}
if ('webhookSecret' in input) {
const value = normalizeEnvInput(input.webhookSecret);
await settingsRepo.setSetting('webhookSecret', value);
applyEnvValue('WEBHOOK_SECRET', value);
promises.push(settingsRepo.setSetting('webhookSecret', value).then(() => {
applyEnvValue('WEBHOOK_SECRET', value);
}));
}
await Promise.all(promises);
const data = await getEffectiveSettings();
res.json({ success: true, data });
} catch (error) {

23
orchestrator/src/server/pipeline/orchestrator.ts
View File

@ -121,8 +121,11 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
const discoveredJobs: CreateJobInput[] = [];
const sourceErrors: string[] = [];
// Read all settings at once to avoid sequential DB calls
const settings = await settingsRepo.getAllSettings();
// Read search terms setting
const searchTermsSetting = await settingsRepo.getSetting('searchTerms');
const searchTermsSetting = settings.searchTerms;
let searchTerms: string[] = [];
if (searchTermsSetting) {
@ -139,7 +142,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
);
// Apply setting override for JobSpy sites
const jobspySitesSettingRaw = await settingsRepo.getSetting('jobspySites');
const jobspySitesSettingRaw = settings.jobspySites;
if (jobspySitesSettingRaw) {
try {
const allowed = JSON.parse(jobspySitesSettingRaw);
@ -157,11 +160,11 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
detail: `JobSpy: scraping ${jobSpySites.join(', ')}...`,
});
const jobspyLocationSetting = await settingsRepo.getSetting('jobspyLocation');
const jobspyResultsWantedSetting = await settingsRepo.getSetting('jobspyResultsWanted');
const jobspyHoursOldSetting = await settingsRepo.getSetting('jobspyHoursOld');
const jobspyCountryIndeedSetting = await settingsRepo.getSetting('jobspyCountryIndeed');
const jobspyLinkedinFetchDescriptionSetting = await settingsRepo.getSetting('jobspyLinkedinFetchDescription');
const jobspyLocationSetting = settings.jobspyLocation;
const jobspyResultsWantedSetting = settings.jobspyResultsWanted;
const jobspyHoursOldSetting = settings.jobspyHoursOld;
const jobspyCountryIndeedSetting = settings.jobspyCountryIndeed;
const jobspyLinkedinFetchDescriptionSetting = settings.jobspyLinkedinFetchDescription;
const jobSpyResult = await runJobSpy({
sites: jobSpySites,
@ -170,7 +173,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
resultsWanted: jobspyResultsWantedSetting ? parseInt(jobspyResultsWantedSetting, 10) : undefined,
hoursOld: jobspyHoursOldSetting ? parseInt(jobspyHoursOldSetting, 10) : undefined,
countryIndeed: jobspyCountryIndeedSetting ?? undefined,
linkedinFetchDescription: jobspyLinkedinFetchDescriptionSetting !== null ? jobspyLinkedinFetchDescriptionSetting === '1' : undefined,
linkedinFetchDescription: jobspyLinkedinFetchDescriptionSetting !== null && jobspyLinkedinFetchDescriptionSetting !== undefined ? jobspyLinkedinFetchDescriptionSetting === '1' : undefined,
});
if (!jobSpyResult.success) {
sourceErrors.push(`jobspy: ${jobSpyResult.error ?? 'unknown error'}`);
@ -189,7 +192,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
// Pass existing URLs to avoid clicking "Apply" on jobs we already have
const existingJobUrls = await jobsRepo.getAllJobUrls();
const gradcrackerMaxJobsSetting = await settingsRepo.getSetting('gradcrackerMaxJobsPerTerm');
const gradcrackerMaxJobsSetting = settings.gradcrackerMaxJobsPerTerm;
const gradcrackerMaxJobs = gradcrackerMaxJobsSetting ? parseInt(gradcrackerMaxJobsSetting, 10) : 50;
const crawlerResult = await runCrawler({
@ -224,7 +227,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
});
// Read max jobs setting from database (default to 50 if not set)
const ukvisajobsMaxJobsSetting = await settingsRepo.getSetting('ukvisajobsMaxJobs');
const ukvisajobsMaxJobsSetting = settings.ukvisajobsMaxJobs;
const ukvisajobsMaxJobs = ukvisajobsMaxJobsSetting ? parseInt(ukvisajobsMaxJobsSetting, 10) : 50;
const ukVisaResult = await runUkVisaJobs({

1
orchestrator/src/server/pipeline/sponsor-matching.test.ts
View File

@ -36,6 +36,7 @@ vi.mock('../repositories/pipeline.js', () => ({
vi.mock('../repositories/settings.js', () => ({
getSetting: vi.fn().mockResolvedValue(null),
getAllSettings: vi.fn().mockResolvedValue({}),
}));
vi.mock('../services/crawler.js', () => ({

16
orchestrator/src/server/services/envSettings.ts
View File

@ -73,34 +73,30 @@ export async function applyStoredEnvOverrides(): Promise<void> {
export async function getEnvSettingsData(
overrides?: Partial<Record<SettingKey, string>>
): Promise<Record<string, string | boolean | number | null>> {
const activeOverrides = overrides || await settingsRepo.getAllSettings();
const readableValues: Record<string, string | boolean | null> = {};
const privateValues: Record<string, string | null> = {};
for (const { settingKey, envKey } of readableStringConfig) {
const override = overrides ? (overrides[settingKey] ?? null) : await settingsRepo.getSetting(settingKey);
const override = activeOverrides[settingKey] ?? null;
const rawValue = override ?? process.env[envKey];
readableValues[settingKey] = normalizeEnvInput(rawValue);
}
for (const { settingKey, envKey, defaultValue } of readableBooleanConfig) {
const override = overrides ? (overrides[settingKey] ?? null) : await settingsRepo.getSetting(settingKey);
const override = activeOverrides[settingKey] ?? null;
const rawValue = override ?? process.env[envKey];
readableValues[settingKey] = parseEnvBoolean(rawValue, defaultValue);
}
for (const { settingKey, envKey, hintKey } of privateStringConfig) {
const override = overrides ? (overrides[settingKey] ?? null) : await settingsRepo.getSetting(settingKey);
const override = activeOverrides[settingKey] ?? null;
const rawValue = override ?? process.env[envKey];
privateValues[hintKey] = rawValue ? rawValue.slice(0, 4) : null;
}
const basicAuthUser = overrides
? (overrides['basicAuthUser'] ?? process.env.BASIC_AUTH_USER)
: (await settingsRepo.getSetting('basicAuthUser')) ?? process.env.BASIC_AUTH_USER;
const basicAuthPassword = overrides
? (overrides['basicAuthPassword'] ?? process.env.BASIC_AUTH_PASSWORD)
: (await settingsRepo.getSetting('basicAuthPassword')) ?? process.env.BASIC_AUTH_PASSWORD;
const basicAuthUser = activeOverrides['basicAuthUser'] ?? process.env.BASIC_AUTH_USER;
const basicAuthPassword = activeOverrides['basicAuthPassword'] ?? process.env.BASIC_AUTH_PASSWORD;
return {
...readableValues,

1
orchestrator/src/server/services/manualJob.test.ts
View File

@ -4,6 +4,7 @@ import { inferManualJobDetails } from "./manualJob.js";
vi.mock("../repositories/settings.js", () => ({
getSetting: vi.fn(),
getAllSettings: vi.fn().mockResolvedValue({}),
}));
const originalEnv = process.env;

1
orchestrator/src/server/services/pdf-skills-validation.test.ts
View File

@ -47,6 +47,7 @@ vi.mock('fs', () => ({
vi.mock('../repositories/settings.js', () => ({
getSetting: vi.fn().mockResolvedValue(null),
getAllSettings: vi.fn().mockResolvedValue({}),
}));
vi.mock('./projectSelection.js', () => ({

1
orchestrator/src/server/services/pdf-tailoring.test.ts
View File

@ -49,6 +49,7 @@ vi.mock('fs', () => ({
vi.mock('../repositories/settings.js', () => ({
getSetting: vi.fn().mockResolvedValue(null),
getAllSettings: vi.fn().mockResolvedValue({}),
}));
vi.mock('./projectSelection.js', () => ({

6
orchestrator/src/server/services/projectSelection.ts
View File

@ -38,8 +38,10 @@ export async function pickProjectIdsForJob(args: {
return fallbackPickProjectIds(args.jobDescription, args.eligibleProjects, desiredCount);
}
const overrideModel = await getSetting('model');
const overrideModelProjectSelection = await getSetting('modelProjectSelection');
const [overrideModel, overrideModelProjectSelection] = await Promise.all([
getSetting('model'),
getSetting('modelProjectSelection'),
]);
// Precedence: Project-specific override > Global override > Env var > Default
const model = overrideModelProjectSelection || overrideModel || process.env.MODEL || 'openai/gpt-4o-mini';

6
orchestrator/src/server/services/scorer.ts
View File

@ -44,8 +44,10 @@ export async function scoreJobSuitability(
return mockScore(job);
}
const overrideModel = await getSetting('model');
const overrideModelScorer = await getSetting('modelScorer');
const [overrideModel, overrideModelScorer] = await Promise.all([
getSetting('model'),
getSetting('modelScorer'),
]);
// Precedence: Scorer-specific override > Global override > Env var > Default
const model = overrideModelScorer || overrideModel || process.env.MODEL || 'openai/gpt-4o-mini';

6
orchestrator/src/server/services/summary.ts
View File

@ -69,8 +69,10 @@ export async function generateTailoring(
return { success: false, error: 'API key not configured' };
}
const overrideModel = await getSetting('model');
const overrideModelTailoring = await getSetting('modelTailoring');
const [overrideModel, overrideModelTailoring] = await Promise.all([
getSetting('model'),
getSetting('modelTailoring'),
]);
// Precedence: Tailoring-specific override > Global override > Env var > Default
const model = overrideModelTailoring || overrideModel || process.env.MODEL || 'openai/gpt-4o-mini';
const prompt = buildTailoringPrompt(profile, jobDescription);