ensure that a "discovered" job must have a score

This commit is contained in:
DaKheera47 2025-12-15 01:19:01 +00:00
parent 77b23317b8
commit 4244c908e5
8 changed files with 96 additions and 251 deletions

View File

@ -26,7 +26,6 @@ export const App: React.FC = () => {
const [isLoading, setIsLoading] = useState(true);
const [isPipelineRunning, setIsPipelineRunning] = useState(false);
const [processingJobId, setProcessingJobId] = useState<string | null>(null);
const [isProcessingAll, setIsProcessingAll] = useState(false);
const [pipelineSources, setPipelineSources] = useState<JobSource[]>(() => {
try {
const raw = localStorage.getItem(PIPELINE_SOURCES_STORAGE_KEY);
@ -159,35 +158,6 @@ export const App: React.FC = () => {
}
};
const handleProcessAll = async () => {
try {
setIsProcessingAll(true);
const result = await api.processAllDiscovered();
toast.message("Processing jobs", { description: `Processing ${result.count} jobs in background...` });
const pollInterval = setInterval(async () => {
try {
const data = await api.getJobs();
setJobs(data.jobs);
setStats(data.byStatus);
const stillDiscovered = data.byStatus.discovered + data.byStatus.processing;
if (stillDiscovered === 0) {
clearInterval(pollInterval);
setIsProcessingAll(false);
toast.success("All jobs processed");
}
} catch {
// Ignore errors
}
}, 3000);
} catch (error) {
setIsProcessingAll(false);
const message = error instanceof Error ? error.message : "Failed to process jobs";
toast.error(message);
}
};
return (
<>
<Header
@ -208,9 +178,7 @@ export const App: React.FC = () => {
onApply={handleApply}
onReject={handleReject}
onProcess={handleProcess}
onProcessAll={handleProcessAll}
processingJobId={processingJobId}
isProcessingAll={isProcessingAll}
/>
</main>

View File

@ -107,15 +107,4 @@ export async function clearDatabase(): Promise<{
});
}
// Bulk operations
export async function processAllDiscovered(): Promise<{
message: string;
count: number;
}> {
return fetchApi<{
message: string;
count: number;
}>('/jobs/process-discovered', {
method: 'POST',
});
}
// Bulk operations (intentionally none - processing is manual)

View File

@ -112,62 +112,60 @@ export const Header: React.FC<HeaderProps> = ({
<span className="hidden sm:inline">Refresh</span>
</Button>
<div className="flex items-center">
<Button
size="sm"
onClick={onRunPipeline}
disabled={isPipelineRunning}
className="rounded-r-none"
>
{isPipelineRunning ? (
<>
<Loader2 className="h-4 w-4 animate-spin" />
Running...
</>
) : (
<>
<Play className="h-4 w-4" />
Run Pipeline
</>
)}
</Button>
<Button
size="sm"
onClick={onRunPipeline}
disabled={isPipelineRunning}
className="rounded-r-none"
>
{isPipelineRunning ? (
<>
<Loader2 className="h-4 w-4 animate-spin" />
Running...
</>
) : (
<>
<Play className="h-4 w-4" />
Run Pipeline
</>
)}
</Button>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
size="sm"
disabled={isPipelineRunning}
className="rounded-l-none border-l border-primary-foreground/20 px-2"
aria-label="Select pipeline sources"
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
size="sm"
disabled={isPipelineRunning}
className="rounded-l-none border-l border-primary-foreground/20 px-2"
aria-label="Select pipeline sources"
>
<ChevronDown className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-56">
<DropdownMenuLabel>Sources</DropdownMenuLabel>
<DropdownMenuSeparator />
{orderedSources.map((source) => (
<DropdownMenuCheckboxItem
key={source}
checked={pipelineSources.includes(source)}
onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
>
<ChevronDown className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="w-56">
<DropdownMenuLabel>Sources</DropdownMenuLabel>
<DropdownMenuSeparator />
{orderedSources.map((source) => (
<DropdownMenuCheckboxItem
key={source}
checked={pipelineSources.includes(source)}
onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
>
{sourceLabel[source]}
</DropdownMenuCheckboxItem>
))}
<DropdownMenuSeparator />
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
All sources
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
Gradcracker only
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
Indeed + LinkedIn only
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
{sourceLabel[source]}
</DropdownMenuCheckboxItem>
))}
<DropdownMenuSeparator />
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
All sources
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
Gradcracker only
</DropdownMenuItem>
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
Indeed + LinkedIn only
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
</header>

View File

@ -3,7 +3,7 @@
*/
import React, { useEffect, useMemo, useState } from "react";
import { LayoutGrid, Loader2, RefreshCcw, Search, Table2 } from "lucide-react";
import { LayoutGrid, Search, Table2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Card, CardContent } from "@/components/ui/card";
@ -19,9 +19,7 @@ interface JobListProps {
onApply: (id: string) => void;
onReject: (id: string) => void;
onProcess: (id: string) => void;
onProcessAll: () => void;
processingJobId: string | null;
isProcessingAll: boolean;
}
type FilterTab = "ready" | "discovered" | "applied" | "all";
@ -156,9 +154,7 @@ export const JobList: React.FC<JobListProps> = ({
onApply,
onReject,
onProcess,
onProcessAll,
processingJobId,
isProcessingAll,
}) => {
const [activeTab, setActiveTab] = useState<FilterTab>("ready");
const [searchQuery, setSearchQuery] = useState("");
@ -252,22 +248,6 @@ export const JobList: React.FC<JobListProps> = ({
</TabsList>
<div className="flex items-center justify-between gap-2 sm:justify-end">
{activeTab === "discovered" && counts.discovered > 0 && (
<Button onClick={onProcessAll} disabled={isProcessingAll} size="sm">
{isProcessingAll ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Processing...
</>
) : (
<>
<RefreshCcw className="mr-2 h-4 w-4" />
Process All ({counts.discovered})
</>
)}
</Button>
)}
<div className="flex items-center rounded-md border bg-muted/20 p-0.5">
<Button
type="button"

View File

@ -170,39 +170,6 @@ apiRouter.post('/jobs/:id/reject', async (req: Request, res: Response) => {
}
});
/**
* POST /api/jobs/process-discovered - Process all discovered jobs (generate PDFs)
*/
apiRouter.post('/jobs/process-discovered', async (req: Request, res: Response) => {
try {
const discoveredJobs = await jobsRepo.getAllJobs(['discovered']);
// Process each job in background
const processInBackground = async () => {
for (const job of discoveredJobs.filter(j => j.status === 'discovered')) {
try {
await processJob(job.id);
} catch (error) {
console.error(`Failed to process job ${job.id}:`, error);
}
}
};
processInBackground().catch(console.error);
res.json({
success: true,
data: {
message: `Processing ${discoveredJobs.length} jobs`,
count: discoveredJobs.length,
}
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
res.status(500).json({ success: false, error: message });
}
});
// ============================================================================
// Pipeline API
// ============================================================================

View File

@ -4,10 +4,7 @@
* Flow:
* 1. Run crawler to discover new jobs
* 2. Score jobs for suitability
* 3. Pick top N jobs
* 4. Generate tailored summaries
* 5. Generate PDF resumes
* 6. Mark as "ready" for user review
* 3. Leave all jobs in "discovered" for manual processing
*/
import { readFile } from 'fs/promises';
@ -15,7 +12,7 @@ import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { runCrawler } from '../services/crawler.js';
import { runJobSpy } from '../services/jobspy.js';
import { scoreAndRankJobs, scoreJobSuitability } from '../services/scorer.js';
import { scoreJobSuitability } from '../services/scorer.js';
import { generateSummary } from '../services/summary.js';
import { generatePdf } from '../services/pdf.js';
import * as jobsRepo from '../repositories/jobs.js';
@ -63,7 +60,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
const pipelineRun = await pipelineRepo.createPipelineRun();
console.log('🚀 Starting job pipeline...');
console.log(` Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore}`);
console.log(` Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore} (manual processing)`);
try {
// Step 1: Load profile
@ -141,9 +138,18 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
jobsDiscovered: created,
});
// Step 4: Get unprocessed jobs and score them
// Step 4: Score all discovered jobs missing a score
console.log('\n🎯 Scoring jobs for suitability...');
const unprocessedJobs = await jobsRepo.getJobsForProcessing(50);
const unprocessedJobs = await jobsRepo.getUnscoredDiscoveredJobs();
updateProgress({
step: 'scoring',
jobsDiscovered: unprocessedJobs.length,
jobsScored: 0,
jobsProcessed: 0,
totalToProcess: 0,
currentJob: undefined,
});
// Score jobs with progress updates
const scoredJobs: Array<Job & { suitabilityScore: number; suitabilityReason: string }> = [];
@ -175,106 +181,27 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
});
}
// Sort by score
scoredJobs.sort((a, b) => b.suitabilityScore - a.suitabilityScore);
// Step 5: Pick top N jobs above threshold
const topJobs = scoredJobs
.filter(j => j.suitabilityScore >= mergedConfig.minSuitabilityScore)
.slice(0, mergedConfig.topN);
progressHelpers.scoringComplete(scoredJobs.length, topJobs.length);
console.log(`\n📊 Selected ${topJobs.length} top jobs for processing:`);
for (const job of topJobs) {
console.log(` - ${job.title} @ ${job.employer} (score: ${job.suitabilityScore})`);
}
// Step 6: Process each top job
let processed = 0;
for (let i = 0; i < topJobs.length; i++) {
const job = topJobs[i];
console.log(`\n📝 Processing: ${job.title} @ ${job.employer}`);
progressHelpers.processingJob(i + 1, topJobs.length, {
id: job.id,
title: job.title,
employer: job.employer,
});
try {
// Mark as processing
await jobsRepo.updateJob(job.id, { status: 'processing' });
// Generate tailored summary
console.log(' Generating summary...');
progressHelpers.generatingSummary({ title: job.title, employer: job.employer });
const summaryResult = await generateSummary(
job.jobDescription || '',
profile
);
if (!summaryResult.success) {
console.warn(` ⚠️ Summary generation failed: ${summaryResult.error}`);
continue;
}
// Update job with summary
await jobsRepo.updateJob(job.id, {
tailoredSummary: summaryResult.summary,
});
// Generate PDF
console.log(' Generating PDF...');
progressHelpers.generatingPdf({ title: job.title, employer: job.employer });
const pdfResult = await generatePdf(
job.id,
summaryResult.summary!,
mergedConfig.profilePath
);
if (!pdfResult.success) {
console.warn(` ⚠️ PDF generation failed: ${pdfResult.error}`);
// Still mark as ready even if PDF failed - user can regenerate
}
// Mark as ready
await jobsRepo.updateJob(job.id, {
status: 'ready',
pdfPath: pdfResult.pdfPath ?? undefined,
});
processed++;
progressHelpers.jobComplete(processed, topJobs.length);
console.log(` ✅ Ready for review!`);
} catch (error) {
console.error(` ❌ Failed to process job: ${error}`);
// Continue with next job
}
}
progressHelpers.scoringComplete(scoredJobs.length);
console.log(`\n📊 Scored ${scoredJobs.length} jobs. Ready for manual processing.`);
// Update pipeline run as completed
await pipelineRepo.updatePipelineRun(pipelineRun.id, {
status: 'completed',
completedAt: new Date().toISOString(),
jobsProcessed: processed,
jobsProcessed: 0,
});
console.log('\n🎉 Pipeline completed!');
console.log(` Jobs discovered: ${created}`);
console.log(` Jobs processed: ${processed}`);
console.log(' Jobs processed: 0 (manual)');
progressHelpers.complete(created, processed);
progressHelpers.complete(created, 0);
isPipelineRunning = false;
return {
success: true,
jobsDiscovered: created,
jobsProcessed: processed,
jobsProcessed: 0,
};
} catch (error) {

View File

@ -215,12 +215,14 @@ export const progressHelpers = {
jobsScored: index,
}),
scoringComplete: (totalScored: number, topN: number) => updateProgress({
step: 'processing',
message: `Scored ${totalScored} jobs. Processing top ${topN}...`,
detail: 'Generating tailored resumes',
scoringComplete: (totalScored: number) => updateProgress({
step: 'scoring',
message: `Scored ${totalScored} jobs.`,
detail: 'Ready for manual processing',
jobsScored: totalScored,
totalToProcess: topN,
totalToProcess: 0,
jobsProcessed: 0,
currentJob: undefined,
}),
processingJob: (index: number, total: number, job: { id: string; title: string; employer: string }) => updateProgress({

View File

@ -2,7 +2,7 @@
* Job repository - data access layer for jobs.
*/
import { eq, desc, sql, and, inArray } from 'drizzle-orm';
import { eq, desc, sql, and, inArray, isNull } from 'drizzle-orm';
import { randomUUID } from 'crypto';
import { db, schema } from '../db/index.js';
import type { Job, CreateJobInput, UpdateJobInput, JobStatus } from '../../shared/types.js';
@ -195,6 +195,20 @@ export async function getJobsForProcessing(limit: number = 10): Promise<Job[]> {
return rows.map(mapRowToJob);
}
/**
* Get discovered jobs missing a suitability score.
*/
export async function getUnscoredDiscoveredJobs(limit?: number): Promise<Job[]> {
const query = db
.select()
.from(jobs)
.where(and(eq(jobs.status, 'discovered'), isNull(jobs.suitabilityScore)))
.orderBy(desc(jobs.discoveredAt));
const rows = typeof limit === 'number' ? await query.limit(limit) : await query;
return rows.map(mapRowToJob);
}
// Helper to map database row to Job type
function mapRowToJob(row: typeof jobs.$inferSelect): Job {
return {