ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rename pipeline webhook to what it is

Browse Source
This commit is contained in:
DaKheera47 2025-12-15 17:23:11 +00:00
parent 8227cabd17
commit 368fe60935
9 changed files with 142 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.env.example
View File

@ -23,6 +23,7 @@ NOTION_DATABASE_ID=
# Optional: Webhook secret for n8n automation
WEBHOOK_SECRET=
PIPELINE_WEBHOOK_URL=
# =============================================================================
# JobSpy (Indeed/LinkedIn scraping) - optional

1
orchestrator/.env.example
View File

@ -11,6 +11,7 @@ NOTION_DATABASE_ID=
# Webhook security (optional)
WEBHOOK_SECRET=
PIPELINE_WEBHOOK_URL=
# Pipeline configuration
PIPELINE_TOP_N=10

5
orchestrator/src/client/api/client.ts
View File

@ -98,7 +98,10 @@ export async function getSettings(): Promise<AppSettings> {
return fetchApi<AppSettings>('/settings');
}
export async function updateSettings(update: { model?: string | null }): Promise<AppSettings> {
export async function updateSettings(update: {
model?: string | null
pipelineWebhookUrl?: string | null
}): Promise<AppSettings> {
return fetchApi<AppSettings>('/settings', {
method: 'PATCH',
body: JSON.stringify(update),

70
orchestrator/src/client/pages/SettingsPage.tsx
View File

@ -15,6 +15,7 @@ import * as api from "../api"
export const SettingsPage: React.FC = () => {
const [settings, setSettings] = useState<AppSettings | null>(null)
const [modelDraft, setModelDraft] = useState("")
const [pipelineWebhookUrlDraft, setPipelineWebhookUrlDraft] = useState("")
const [isSaving, setIsSaving] = useState(false)
const [isLoading, setIsLoading] = useState(true)
@ -27,6 +28,7 @@ export const SettingsPage: React.FC = () => {
if (!isMounted) return
setSettings(data)
setModelDraft(data.overrideModel ?? "")
setPipelineWebhookUrlDraft(data.overridePipelineWebhookUrl ?? "")
})
.catch((error) => {
const message = error instanceof Error ? error.message : "Failed to load settings"
@ -45,22 +47,32 @@ export const SettingsPage: React.FC = () => {
const effectiveModel = settings?.model ?? ""
const defaultModel = settings?.defaultModel ?? ""
const overrideModel = settings?.overrideModel
const effectivePipelineWebhookUrl = settings?.pipelineWebhookUrl ?? ""
const defaultPipelineWebhookUrl = settings?.defaultPipelineWebhookUrl ?? ""
const overridePipelineWebhookUrl = settings?.overridePipelineWebhookUrl
const canSave = useMemo(() => {
if (!settings) return false
const next = modelDraft.trim()
const current = (overrideModel ?? "").trim()
return next !== current
}, [modelDraft, overrideModel, settings])
const nextWebhook = pipelineWebhookUrlDraft.trim()
const currentWebhook = (overridePipelineWebhookUrl ?? "").trim()
return next !== current || nextWebhook !== currentWebhook
}, [modelDraft, overrideModel, settings, pipelineWebhookUrlDraft, overridePipelineWebhookUrl])
const handleSave = async () => {
if (!settings) return
try {
setIsSaving(true)
const trimmed = modelDraft.trim()
const updated = await api.updateSettings({ model: trimmed.length > 0 ? trimmed : null })
const webhookTrimmed = pipelineWebhookUrlDraft.trim()
const updated = await api.updateSettings({
model: trimmed.length > 0 ? trimmed : null,
pipelineWebhookUrl: webhookTrimmed.length > 0 ? webhookTrimmed : null,
})
setSettings(updated)
setModelDraft(updated.overrideModel ?? "")
setPipelineWebhookUrlDraft(updated.overridePipelineWebhookUrl ?? "")
toast.success("Settings saved")
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to save settings"
@ -73,9 +85,10 @@ export const SettingsPage: React.FC = () => {
const handleReset = async () => {
try {
setIsSaving(true)
const updated = await api.updateSettings({ model: null })
const updated = await api.updateSettings({ model: null, pipelineWebhookUrl: null })
setSettings(updated)
setModelDraft("")
setPipelineWebhookUrlDraft("")
toast.success("Reset to default")
} catch (error) {
const message = error instanceof Error ? error.message : "Failed to reset settings"
@ -123,18 +136,51 @@ export const SettingsPage: React.FC = () => {
<div className="break-words font-mono text-xs">{defaultModel || "—"}</div>
</div>
</div>
</CardContent>
</Card>
<div className="flex flex-wrap gap-2">
<Button onClick={handleSave} disabled={isLoading || isSaving || !canSave}>
{isSaving ? "Saving..." : "Save"}
</Button>
<Button variant="outline" onClick={handleReset} disabled={isLoading || isSaving || !settings}>
Reset to default
</Button>
<Card>
<CardHeader>
<CardTitle className="text-base">Pipeline Webhook</CardTitle>
</CardHeader>
<CardContent className="space-y-4">
<div className="space-y-2">
<div className="text-sm font-medium">Pipeline status webhook URL</div>
<Input
value={pipelineWebhookUrlDraft}
onChange={(event) => setPipelineWebhookUrlDraft(event.target.value)}
placeholder={defaultPipelineWebhookUrl || "https://..."}
disabled={isLoading || isSaving}
/>
<div className="text-xs text-muted-foreground">
When set, the server sends a POST on pipeline completion/failure. Leave blank to disable.
</div>
</div>
<Separator />
<div className="grid gap-2 text-sm sm:grid-cols-2">
<div>
<div className="text-xs text-muted-foreground">Effective</div>
<div className="break-words font-mono text-xs">{effectivePipelineWebhookUrl || "—"}</div>
</div>
<div>
<div className="text-xs text-muted-foreground">Default (env)</div>
<div className="break-words font-mono text-xs">{defaultPipelineWebhookUrl || "—"}</div>
</div>
</div>
</CardContent>
</Card>
<div className="flex flex-wrap gap-2">
<Button onClick={handleSave} disabled={isLoading || isSaving || !canSave}>
{isSaving ? "Saving..." : "Save"}
</Button>
<Button variant="outline" onClick={handleReset} disabled={isLoading || isSaving || !settings}>
Reset to default
</Button>
</div>
</main>
)
}

20
orchestrator/src/server/api/routes.ts
View File

@ -184,12 +184,19 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
const defaultModel = process.env.MODEL || 'openai/gpt-4o-mini';
const model = overrideModel || defaultModel;
const overridePipelineWebhookUrl = await settingsRepo.getSetting('pipelineWebhookUrl');
const defaultPipelineWebhookUrl = process.env.PIPELINE_WEBHOOK_URL || process.env.WEBHOOK_URL || '';
const pipelineWebhookUrl = overridePipelineWebhookUrl || defaultPipelineWebhookUrl;
res.json({
success: true,
data: {
model,
defaultModel,
overrideModel,
pipelineWebhookUrl,
defaultPipelineWebhookUrl,
overridePipelineWebhookUrl,
},
});
} catch (error) {
@ -200,6 +207,7 @@ apiRouter.get('/settings', async (_req: Request, res: Response) => {
const updateSettingsSchema = z.object({
model: z.string().trim().min(1).max(200).nullable().optional(),
pipelineWebhookUrl: z.string().trim().min(1).max(2000).nullable().optional(),
});
/**
@ -214,16 +222,28 @@ apiRouter.patch('/settings', async (req: Request, res: Response) => {
await settingsRepo.setSetting('model', model);
}
if ('pipelineWebhookUrl' in input) {
const pipelineWebhookUrl = input.pipelineWebhookUrl ?? null;
await settingsRepo.setSetting('pipelineWebhookUrl', pipelineWebhookUrl);
}
const overrideModel = await settingsRepo.getSetting('model');
const defaultModel = process.env.MODEL || 'openai/gpt-4o-mini';
const model = overrideModel || defaultModel;
const overridePipelineWebhookUrl = await settingsRepo.getSetting('pipelineWebhookUrl');
const defaultPipelineWebhookUrl = process.env.PIPELINE_WEBHOOK_URL || process.env.WEBHOOK_URL || '';
const pipelineWebhookUrl = overridePipelineWebhookUrl || defaultPipelineWebhookUrl;
res.json({
success: true,
data: {
model,
defaultModel,
overrideModel,
pipelineWebhookUrl,
defaultPipelineWebhookUrl,
overridePipelineWebhookUrl,
},
});
} catch (error) {

5
orchestrator/src/server/db/migrate.ts
View File

@ -95,6 +95,11 @@ const migrations = [
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
)`,
// Rename settings key: webhookUrl -> pipelineWebhookUrl (safe to re-run)
`INSERT OR REPLACE INTO settings(key, value, created_at, updated_at)
SELECT 'pipelineWebhookUrl', value, created_at, updated_at FROM settings WHERE key = 'webhookUrl'`,
`DELETE FROM settings WHERE key = 'webhookUrl'`,
// Add source column for existing databases (safe to skip if already present)
`ALTER TABLE jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'gradcracker'`,
`UPDATE jobs SET source = 'gradcracker' WHERE source IS NULL OR source = ''`,

49
orchestrator/src/server/pipeline/orchestrator.ts
View File

@ -17,6 +17,7 @@ import { generateSummary } from '../services/summary.js';
import { generatePdf } from '../services/pdf.js';
import * as jobsRepo from '../repositories/jobs.js';
import * as pipelineRepo from '../repositories/pipeline.js';
import * as settingsRepo from '../repositories/settings.js';
import { progressHelpers, resetProgress, updateProgress } from './progress.js';
import type { CreateJobInput, Job, JobSource, PipelineConfig } from '../../shared/types.js';
@ -34,6 +35,42 @@ const DEFAULT_CONFIG: PipelineConfig = {
// Track if pipeline is currently running
let isPipelineRunning = false;
async function notifyPipelineWebhook(
event: 'pipeline.completed' | 'pipeline.failed',
payload: Record<string, unknown>
) {
const overridePipelineWebhookUrl = await settingsRepo.getSetting('pipelineWebhookUrl')
const pipelineWebhookUrl = (
overridePipelineWebhookUrl ||
process.env.PIPELINE_WEBHOOK_URL ||
process.env.WEBHOOK_URL ||
''
).trim()
if (!pipelineWebhookUrl) return
try {
const headers: Record<string, string> = { 'Content-Type': 'application/json' }
const secret = process.env.WEBHOOK_SECRET
if (secret) headers.Authorization = `Bearer ${secret}`
const response = await fetch(pipelineWebhookUrl, {
method: 'POST',
headers,
body: JSON.stringify({
event,
sentAt: new Date().toISOString(),
...payload,
}),
})
if (!response.ok) {
console.warn(`⚠️ Pipeline webhook POST failed (${response.status}): ${await response.text()}`)
}
} catch (error) {
console.warn('⚠️ Pipeline webhook POST failed:', error)
}
}
/**
* Run the full job discovery and processing pipeline.
*/
@ -196,6 +233,13 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
console.log(' Jobs processed: 0 (manual)');
progressHelpers.complete(created, 0);
await notifyPipelineWebhook('pipeline.completed', {
pipelineRunId: pipelineRun.id,
jobsDiscovered: created,
jobsScored: unprocessedJobs.length,
jobsProcessed: 0,
})
isPipelineRunning = false;
return {
@ -214,6 +258,11 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
});
progressHelpers.failed(message);
await notifyPipelineWebhook('pipeline.failed', {
pipelineRunId: pipelineRun.id,
error: message,
})
isPipelineRunning = false;
console.error('\n❌ Pipeline failed:', message);

2
orchestrator/src/server/repositories/settings.ts
View File

@ -8,6 +8,7 @@ import { db, schema } from '../db/index.js'
const { settings } = schema
export type SettingKey = 'model'
| 'pipelineWebhookUrl'
export async function getSetting(key: SettingKey): Promise<string | null> {
const [row] = await db.select().from(settings).where(eq(settings.key, key))
@ -39,4 +40,3 @@ export async function setSetting(key: SettingKey, value: string | null): Promise
updatedAt: now,
})
}

3
orchestrator/src/shared/types.ts
View File

@ -176,4 +176,7 @@ export interface AppSettings {
model: string;
defaultModel: string;
overrideModel: string | null;
pipelineWebhookUrl: string;
defaultPipelineWebhookUrl: string;
overridePipelineWebhookUrl: string | null;
}