ensure that a "discovered" job must have a score
This commit is contained in:
parent
77b23317b8
commit
4244c908e5
@ -26,7 +26,6 @@ export const App: React.FC = () => {
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [isPipelineRunning, setIsPipelineRunning] = useState(false);
|
||||
const [processingJobId, setProcessingJobId] = useState<string | null>(null);
|
||||
const [isProcessingAll, setIsProcessingAll] = useState(false);
|
||||
const [pipelineSources, setPipelineSources] = useState<JobSource[]>(() => {
|
||||
try {
|
||||
const raw = localStorage.getItem(PIPELINE_SOURCES_STORAGE_KEY);
|
||||
@ -159,35 +158,6 @@ export const App: React.FC = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleProcessAll = async () => {
|
||||
try {
|
||||
setIsProcessingAll(true);
|
||||
const result = await api.processAllDiscovered();
|
||||
toast.message("Processing jobs", { description: `Processing ${result.count} jobs in background...` });
|
||||
|
||||
const pollInterval = setInterval(async () => {
|
||||
try {
|
||||
const data = await api.getJobs();
|
||||
setJobs(data.jobs);
|
||||
setStats(data.byStatus);
|
||||
|
||||
const stillDiscovered = data.byStatus.discovered + data.byStatus.processing;
|
||||
if (stillDiscovered === 0) {
|
||||
clearInterval(pollInterval);
|
||||
setIsProcessingAll(false);
|
||||
toast.success("All jobs processed");
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors
|
||||
}
|
||||
}, 3000);
|
||||
} catch (error) {
|
||||
setIsProcessingAll(false);
|
||||
const message = error instanceof Error ? error.message : "Failed to process jobs";
|
||||
toast.error(message);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Header
|
||||
@ -208,9 +178,7 @@ export const App: React.FC = () => {
|
||||
onApply={handleApply}
|
||||
onReject={handleReject}
|
||||
onProcess={handleProcess}
|
||||
onProcessAll={handleProcessAll}
|
||||
processingJobId={processingJobId}
|
||||
isProcessingAll={isProcessingAll}
|
||||
/>
|
||||
</main>
|
||||
|
||||
|
||||
@ -107,15 +107,4 @@ export async function clearDatabase(): Promise<{
|
||||
});
|
||||
}
|
||||
|
||||
// Bulk operations
|
||||
export async function processAllDiscovered(): Promise<{
|
||||
message: string;
|
||||
count: number;
|
||||
}> {
|
||||
return fetchApi<{
|
||||
message: string;
|
||||
count: number;
|
||||
}>('/jobs/process-discovered', {
|
||||
method: 'POST',
|
||||
});
|
||||
}
|
||||
// Bulk operations (intentionally none - processing is manual)
|
||||
|
||||
@ -112,62 +112,60 @@ export const Header: React.FC<HeaderProps> = ({
|
||||
<span className="hidden sm:inline">Refresh</span>
|
||||
</Button>
|
||||
|
||||
<div className="flex items-center">
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onRunPipeline}
|
||||
disabled={isPipelineRunning}
|
||||
className="rounded-r-none"
|
||||
>
|
||||
{isPipelineRunning ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
Running...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Play className="h-4 w-4" />
|
||||
Run Pipeline
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
<Button
|
||||
size="sm"
|
||||
onClick={onRunPipeline}
|
||||
disabled={isPipelineRunning}
|
||||
className="rounded-r-none"
|
||||
>
|
||||
{isPipelineRunning ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
Running...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Play className="h-4 w-4" />
|
||||
Run Pipeline
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={isPipelineRunning}
|
||||
className="rounded-l-none border-l border-primary-foreground/20 px-2"
|
||||
aria-label="Select pipeline sources"
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
size="sm"
|
||||
disabled={isPipelineRunning}
|
||||
className="rounded-l-none border-l border-primary-foreground/20 px-2"
|
||||
aria-label="Select pipeline sources"
|
||||
>
|
||||
<ChevronDown className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-56">
|
||||
<DropdownMenuLabel>Sources</DropdownMenuLabel>
|
||||
<DropdownMenuSeparator />
|
||||
{orderedSources.map((source) => (
|
||||
<DropdownMenuCheckboxItem
|
||||
key={source}
|
||||
checked={pipelineSources.includes(source)}
|
||||
onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
|
||||
>
|
||||
<ChevronDown className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="w-56">
|
||||
<DropdownMenuLabel>Sources</DropdownMenuLabel>
|
||||
<DropdownMenuSeparator />
|
||||
{orderedSources.map((source) => (
|
||||
<DropdownMenuCheckboxItem
|
||||
key={source}
|
||||
checked={pipelineSources.includes(source)}
|
||||
onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
|
||||
>
|
||||
{sourceLabel[source]}
|
||||
</DropdownMenuCheckboxItem>
|
||||
))}
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
|
||||
All sources
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
|
||||
Gradcracker only
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
|
||||
Indeed + LinkedIn only
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
{sourceLabel[source]}
|
||||
</DropdownMenuCheckboxItem>
|
||||
))}
|
||||
<DropdownMenuSeparator />
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
|
||||
All sources
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
|
||||
Gradcracker only
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
|
||||
Indeed + LinkedIn only
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
*/
|
||||
|
||||
import React, { useEffect, useMemo, useState } from "react";
|
||||
import { LayoutGrid, Loader2, RefreshCcw, Search, Table2 } from "lucide-react";
|
||||
import { LayoutGrid, Search, Table2 } from "lucide-react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent } from "@/components/ui/card";
|
||||
@ -19,9 +19,7 @@ interface JobListProps {
|
||||
onApply: (id: string) => void;
|
||||
onReject: (id: string) => void;
|
||||
onProcess: (id: string) => void;
|
||||
onProcessAll: () => void;
|
||||
processingJobId: string | null;
|
||||
isProcessingAll: boolean;
|
||||
}
|
||||
|
||||
type FilterTab = "ready" | "discovered" | "applied" | "all";
|
||||
@ -156,9 +154,7 @@ export const JobList: React.FC<JobListProps> = ({
|
||||
onApply,
|
||||
onReject,
|
||||
onProcess,
|
||||
onProcessAll,
|
||||
processingJobId,
|
||||
isProcessingAll,
|
||||
}) => {
|
||||
const [activeTab, setActiveTab] = useState<FilterTab>("ready");
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
@ -252,22 +248,6 @@ export const JobList: React.FC<JobListProps> = ({
|
||||
</TabsList>
|
||||
|
||||
<div className="flex items-center justify-between gap-2 sm:justify-end">
|
||||
{activeTab === "discovered" && counts.discovered > 0 && (
|
||||
<Button onClick={onProcessAll} disabled={isProcessingAll} size="sm">
|
||||
{isProcessingAll ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Processing...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<RefreshCcw className="mr-2 h-4 w-4" />
|
||||
Process All ({counts.discovered})
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
)}
|
||||
|
||||
<div className="flex items-center rounded-md border bg-muted/20 p-0.5">
|
||||
<Button
|
||||
type="button"
|
||||
|
||||
@ -170,39 +170,6 @@ apiRouter.post('/jobs/:id/reject', async (req: Request, res: Response) => {
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /api/jobs/process-discovered - Process all discovered jobs (generate PDFs)
|
||||
*/
|
||||
apiRouter.post('/jobs/process-discovered', async (req: Request, res: Response) => {
|
||||
try {
|
||||
const discoveredJobs = await jobsRepo.getAllJobs(['discovered']);
|
||||
|
||||
// Process each job in background
|
||||
const processInBackground = async () => {
|
||||
for (const job of discoveredJobs.filter(j => j.status === 'discovered')) {
|
||||
try {
|
||||
await processJob(job.id);
|
||||
} catch (error) {
|
||||
console.error(`Failed to process job ${job.id}:`, error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
processInBackground().catch(console.error);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
message: `Processing ${discoveredJobs.length} jobs`,
|
||||
count: discoveredJobs.length,
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
res.status(500).json({ success: false, error: message });
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================================
|
||||
// Pipeline API
|
||||
// ============================================================================
|
||||
|
||||
@ -4,10 +4,7 @@
|
||||
* Flow:
|
||||
* 1. Run crawler to discover new jobs
|
||||
* 2. Score jobs for suitability
|
||||
* 3. Pick top N jobs
|
||||
* 4. Generate tailored summaries
|
||||
* 5. Generate PDF resumes
|
||||
* 6. Mark as "ready" for user review
|
||||
* 3. Leave all jobs in "discovered" for manual processing
|
||||
*/
|
||||
|
||||
import { readFile } from 'fs/promises';
|
||||
@ -15,7 +12,7 @@ import { join, dirname } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { runCrawler } from '../services/crawler.js';
|
||||
import { runJobSpy } from '../services/jobspy.js';
|
||||
import { scoreAndRankJobs, scoreJobSuitability } from '../services/scorer.js';
|
||||
import { scoreJobSuitability } from '../services/scorer.js';
|
||||
import { generateSummary } from '../services/summary.js';
|
||||
import { generatePdf } from '../services/pdf.js';
|
||||
import * as jobsRepo from '../repositories/jobs.js';
|
||||
@ -63,7 +60,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
|
||||
const pipelineRun = await pipelineRepo.createPipelineRun();
|
||||
|
||||
console.log('🚀 Starting job pipeline...');
|
||||
console.log(` Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore}`);
|
||||
console.log(` Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore} (manual processing)`);
|
||||
|
||||
try {
|
||||
// Step 1: Load profile
|
||||
@ -141,9 +138,18 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
|
||||
jobsDiscovered: created,
|
||||
});
|
||||
|
||||
// Step 4: Get unprocessed jobs and score them
|
||||
// Step 4: Score all discovered jobs missing a score
|
||||
console.log('\n🎯 Scoring jobs for suitability...');
|
||||
const unprocessedJobs = await jobsRepo.getJobsForProcessing(50);
|
||||
const unprocessedJobs = await jobsRepo.getUnscoredDiscoveredJobs();
|
||||
|
||||
updateProgress({
|
||||
step: 'scoring',
|
||||
jobsDiscovered: unprocessedJobs.length,
|
||||
jobsScored: 0,
|
||||
jobsProcessed: 0,
|
||||
totalToProcess: 0,
|
||||
currentJob: undefined,
|
||||
});
|
||||
|
||||
// Score jobs with progress updates
|
||||
const scoredJobs: Array<Job & { suitabilityScore: number; suitabilityReason: string }> = [];
|
||||
@ -175,106 +181,27 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by score
|
||||
scoredJobs.sort((a, b) => b.suitabilityScore - a.suitabilityScore);
|
||||
|
||||
// Step 5: Pick top N jobs above threshold
|
||||
const topJobs = scoredJobs
|
||||
.filter(j => j.suitabilityScore >= mergedConfig.minSuitabilityScore)
|
||||
.slice(0, mergedConfig.topN);
|
||||
|
||||
progressHelpers.scoringComplete(scoredJobs.length, topJobs.length);
|
||||
|
||||
console.log(`\n📊 Selected ${topJobs.length} top jobs for processing:`);
|
||||
for (const job of topJobs) {
|
||||
console.log(` - ${job.title} @ ${job.employer} (score: ${job.suitabilityScore})`);
|
||||
}
|
||||
|
||||
// Step 6: Process each top job
|
||||
let processed = 0;
|
||||
|
||||
for (let i = 0; i < topJobs.length; i++) {
|
||||
const job = topJobs[i];
|
||||
console.log(`\n📝 Processing: ${job.title} @ ${job.employer}`);
|
||||
|
||||
progressHelpers.processingJob(i + 1, topJobs.length, {
|
||||
id: job.id,
|
||||
title: job.title,
|
||||
employer: job.employer,
|
||||
});
|
||||
|
||||
try {
|
||||
// Mark as processing
|
||||
await jobsRepo.updateJob(job.id, { status: 'processing' });
|
||||
|
||||
// Generate tailored summary
|
||||
console.log(' Generating summary...');
|
||||
progressHelpers.generatingSummary({ title: job.title, employer: job.employer });
|
||||
|
||||
const summaryResult = await generateSummary(
|
||||
job.jobDescription || '',
|
||||
profile
|
||||
);
|
||||
|
||||
if (!summaryResult.success) {
|
||||
console.warn(` ⚠️ Summary generation failed: ${summaryResult.error}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Update job with summary
|
||||
await jobsRepo.updateJob(job.id, {
|
||||
tailoredSummary: summaryResult.summary,
|
||||
});
|
||||
|
||||
// Generate PDF
|
||||
console.log(' Generating PDF...');
|
||||
progressHelpers.generatingPdf({ title: job.title, employer: job.employer });
|
||||
|
||||
const pdfResult = await generatePdf(
|
||||
job.id,
|
||||
summaryResult.summary!,
|
||||
mergedConfig.profilePath
|
||||
);
|
||||
|
||||
if (!pdfResult.success) {
|
||||
console.warn(` ⚠️ PDF generation failed: ${pdfResult.error}`);
|
||||
// Still mark as ready even if PDF failed - user can regenerate
|
||||
}
|
||||
|
||||
// Mark as ready
|
||||
await jobsRepo.updateJob(job.id, {
|
||||
status: 'ready',
|
||||
pdfPath: pdfResult.pdfPath ?? undefined,
|
||||
});
|
||||
|
||||
processed++;
|
||||
progressHelpers.jobComplete(processed, topJobs.length);
|
||||
console.log(` ✅ Ready for review!`);
|
||||
|
||||
} catch (error) {
|
||||
console.error(` ❌ Failed to process job: ${error}`);
|
||||
// Continue with next job
|
||||
}
|
||||
}
|
||||
progressHelpers.scoringComplete(scoredJobs.length);
|
||||
console.log(`\n📊 Scored ${scoredJobs.length} jobs. Ready for manual processing.`);
|
||||
|
||||
// Update pipeline run as completed
|
||||
await pipelineRepo.updatePipelineRun(pipelineRun.id, {
|
||||
status: 'completed',
|
||||
completedAt: new Date().toISOString(),
|
||||
jobsProcessed: processed,
|
||||
jobsProcessed: 0,
|
||||
});
|
||||
|
||||
console.log('\n🎉 Pipeline completed!');
|
||||
console.log(` Jobs discovered: ${created}`);
|
||||
console.log(` Jobs processed: ${processed}`);
|
||||
console.log(' Jobs processed: 0 (manual)');
|
||||
|
||||
progressHelpers.complete(created, processed);
|
||||
progressHelpers.complete(created, 0);
|
||||
isPipelineRunning = false;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
jobsDiscovered: created,
|
||||
jobsProcessed: processed,
|
||||
jobsProcessed: 0,
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
|
||||
@ -215,12 +215,14 @@ export const progressHelpers = {
|
||||
jobsScored: index,
|
||||
}),
|
||||
|
||||
scoringComplete: (totalScored: number, topN: number) => updateProgress({
|
||||
step: 'processing',
|
||||
message: `Scored ${totalScored} jobs. Processing top ${topN}...`,
|
||||
detail: 'Generating tailored resumes',
|
||||
scoringComplete: (totalScored: number) => updateProgress({
|
||||
step: 'scoring',
|
||||
message: `Scored ${totalScored} jobs.`,
|
||||
detail: 'Ready for manual processing',
|
||||
jobsScored: totalScored,
|
||||
totalToProcess: topN,
|
||||
totalToProcess: 0,
|
||||
jobsProcessed: 0,
|
||||
currentJob: undefined,
|
||||
}),
|
||||
|
||||
processingJob: (index: number, total: number, job: { id: string; title: string; employer: string }) => updateProgress({
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
* Job repository - data access layer for jobs.
|
||||
*/
|
||||
|
||||
import { eq, desc, sql, and, inArray } from 'drizzle-orm';
|
||||
import { eq, desc, sql, and, inArray, isNull } from 'drizzle-orm';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { db, schema } from '../db/index.js';
|
||||
import type { Job, CreateJobInput, UpdateJobInput, JobStatus } from '../../shared/types.js';
|
||||
@ -195,6 +195,20 @@ export async function getJobsForProcessing(limit: number = 10): Promise<Job[]> {
|
||||
return rows.map(mapRowToJob);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get discovered jobs missing a suitability score.
|
||||
*/
|
||||
export async function getUnscoredDiscoveredJobs(limit?: number): Promise<Job[]> {
|
||||
const query = db
|
||||
.select()
|
||||
.from(jobs)
|
||||
.where(and(eq(jobs.status, 'discovered'), isNull(jobs.suitabilityScore)))
|
||||
.orderBy(desc(jobs.discoveredAt));
|
||||
|
||||
const rows = typeof limit === 'number' ? await query.limit(limit) : await query;
|
||||
return rows.map(mapRowToJob);
|
||||
}
|
||||
|
||||
// Helper to map database row to Job type
|
||||
function mapRowToJob(row: typeof jobs.$inferSelect): Job {
|
||||
return {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user