From 4244c908e55617e32f6fb8185590203a7307ae74 Mon Sep 17 00:00:00 2001
From: DaKheera47 <shaheergoodboy@gmail.com>
Date: Mon, 15 Dec 2025 01:19:01 +0000
Subject: [PATCH] ensure that a "discovered" job must have a score

---
 orchestrator/src/client/App.tsx               |  32 -----
 orchestrator/src/client/api/client.ts         |  13 +-
 orchestrator/src/client/components/Header.tsx | 106 ++++++++--------
 .../src/client/components/JobList.tsx         |  22 +---
 orchestrator/src/server/api/routes.ts         |  33 -----
 .../src/server/pipeline/orchestrator.ts       | 113 ++++--------------
 orchestrator/src/server/pipeline/progress.ts  |  12 +-
 orchestrator/src/server/repositories/jobs.ts  |  16 ++-
 8 files changed, 96 insertions(+), 251 deletions(-)

diff --git a/orchestrator/src/client/App.tsx b/orchestrator/src/client/App.tsx
index 60a363c..79f686f 100644
--- a/orchestrator/src/client/App.tsx
+++ b/orchestrator/src/client/App.tsx
@@ -26,7 +26,6 @@ export const App: React.FC = () => {
   const [isLoading, setIsLoading] = useState(true);
   const [isPipelineRunning, setIsPipelineRunning] = useState(false);
   const [processingJobId, setProcessingJobId] = useState<string | null>(null);
-  const [isProcessingAll, setIsProcessingAll] = useState(false);
   const [pipelineSources, setPipelineSources] = useState<JobSource[]>(() => {
     try {
       const raw = localStorage.getItem(PIPELINE_SOURCES_STORAGE_KEY);
@@ -159,35 +158,6 @@ export const App: React.FC = () => {
     }
   };
 
-  const handleProcessAll = async () => {
-    try {
-      setIsProcessingAll(true);
-      const result = await api.processAllDiscovered();
-      toast.message("Processing jobs", { description: `Processing ${result.count} jobs in background...` });
-
-      const pollInterval = setInterval(async () => {
-        try {
-          const data = await api.getJobs();
-          setJobs(data.jobs);
-          setStats(data.byStatus);
-
-          const stillDiscovered = data.byStatus.discovered + data.byStatus.processing;
-          if (stillDiscovered === 0) {
-            clearInterval(pollInterval);
-            setIsProcessingAll(false);
-            toast.success("All jobs processed");
-          }
-        } catch {
-          // Ignore errors
-        }
-      }, 3000);
-    } catch (error) {
-      setIsProcessingAll(false);
-      const message = error instanceof Error ? error.message : "Failed to process jobs";
-      toast.error(message);
-    }
-  };
-
   return (
     <>
       <Header
@@ -208,9 +178,7 @@ export const App: React.FC = () => {
           onApply={handleApply}
           onReject={handleReject}
           onProcess={handleProcess}
-          onProcessAll={handleProcessAll}
           processingJobId={processingJobId}
-          isProcessingAll={isProcessingAll}
         />
       </main>
 
diff --git a/orchestrator/src/client/api/client.ts b/orchestrator/src/client/api/client.ts
index 5f26054..6f09f0d 100644
--- a/orchestrator/src/client/api/client.ts
+++ b/orchestrator/src/client/api/client.ts
@@ -107,15 +107,4 @@ export async function clearDatabase(): Promise<{
   });
 }
 
-// Bulk operations
-export async function processAllDiscovered(): Promise<{
-  message: string;
-  count: number;
-}> {
-  return fetchApi<{
-    message: string;
-    count: number;
-  }>('/jobs/process-discovered', {
-    method: 'POST',
-  });
-}
+// Bulk operations (intentionally none - processing is manual)
diff --git a/orchestrator/src/client/components/Header.tsx b/orchestrator/src/client/components/Header.tsx
index 605e3f9..a34b5fd 100644
--- a/orchestrator/src/client/components/Header.tsx
+++ b/orchestrator/src/client/components/Header.tsx
@@ -112,62 +112,60 @@ export const Header: React.FC<HeaderProps> = ({
             <span className="hidden sm:inline">Refresh</span>
           </Button>
 
-          <div className="flex items-center">
-            <Button
-              size="sm"
-              onClick={onRunPipeline}
-              disabled={isPipelineRunning}
-              className="rounded-r-none"
-            >
-              {isPipelineRunning ? (
-                <>
-                  <Loader2 className="h-4 w-4 animate-spin" />
-                  Running...
-                </>
-              ) : (
-                <>
-                  <Play className="h-4 w-4" />
-                  Run Pipeline
-                </>
-              )}
-            </Button>
+          <Button
+            size="sm"
+            onClick={onRunPipeline}
+            disabled={isPipelineRunning}
+            className="rounded-r-none"
+          >
+            {isPipelineRunning ? (
+              <>
+                <Loader2 className="h-4 w-4 animate-spin" />
+                Running...
+              </>
+            ) : (
+              <>
+                <Play className="h-4 w-4" />
+                Run Pipeline
+              </>
+            )}
+          </Button>
 
-            <DropdownMenu>
-              <DropdownMenuTrigger asChild>
-                <Button
-                  size="sm"
-                  disabled={isPipelineRunning}
-                  className="rounded-l-none border-l border-primary-foreground/20 px-2"
-                  aria-label="Select pipeline sources"
+          <DropdownMenu>
+            <DropdownMenuTrigger asChild>
+              <Button
+                size="sm"
+                disabled={isPipelineRunning}
+                className="rounded-l-none border-l border-primary-foreground/20 px-2"
+                aria-label="Select pipeline sources"
+              >
+                <ChevronDown className="h-4 w-4" />
+              </Button>
+            </DropdownMenuTrigger>
+            <DropdownMenuContent align="end" className="w-56">
+              <DropdownMenuLabel>Sources</DropdownMenuLabel>
+              <DropdownMenuSeparator />
+              {orderedSources.map((source) => (
+                <DropdownMenuCheckboxItem
+                  key={source}
+                  checked={pipelineSources.includes(source)}
+                  onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
                 >
-                  <ChevronDown className="h-4 w-4" />
-                </Button>
-              </DropdownMenuTrigger>
-              <DropdownMenuContent align="end" className="w-56">
-                <DropdownMenuLabel>Sources</DropdownMenuLabel>
-                <DropdownMenuSeparator />
-                {orderedSources.map((source) => (
-                  <DropdownMenuCheckboxItem
-                    key={source}
-                    checked={pipelineSources.includes(source)}
-                    onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
-                  >
-                    {sourceLabel[source]}
-                  </DropdownMenuCheckboxItem>
-                ))}
-                <DropdownMenuSeparator />
-                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
-                  All sources
-                </DropdownMenuItem>
-                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
-                  Gradcracker only
-                </DropdownMenuItem>
-                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
-                  Indeed + LinkedIn only
-                </DropdownMenuItem>
-              </DropdownMenuContent>
-            </DropdownMenu>
-          </div>
+                  {sourceLabel[source]}
+                </DropdownMenuCheckboxItem>
+              ))}
+              <DropdownMenuSeparator />
+              <DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
+                All sources
+              </DropdownMenuItem>
+              <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
+                Gradcracker only
+              </DropdownMenuItem>
+              <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
+                Indeed + LinkedIn only
+              </DropdownMenuItem>
+            </DropdownMenuContent>
+          </DropdownMenu>
         </div>
       </div>
     </header>
diff --git a/orchestrator/src/client/components/JobList.tsx b/orchestrator/src/client/components/JobList.tsx
index 07ac840..23b1dac 100644
--- a/orchestrator/src/client/components/JobList.tsx
+++ b/orchestrator/src/client/components/JobList.tsx
@@ -3,7 +3,7 @@
  */
 
 import React, { useEffect, useMemo, useState } from "react";
-import { LayoutGrid, Loader2, RefreshCcw, Search, Table2 } from "lucide-react";
+import { LayoutGrid, Search, Table2 } from "lucide-react";
 
 import { Button } from "@/components/ui/button";
 import { Card, CardContent } from "@/components/ui/card";
@@ -19,9 +19,7 @@ interface JobListProps {
   onApply: (id: string) => void;
   onReject: (id: string) => void;
   onProcess: (id: string) => void;
-  onProcessAll: () => void;
   processingJobId: string | null;
-  isProcessingAll: boolean;
 }
 
 type FilterTab = "ready" | "discovered" | "applied" | "all";
@@ -156,9 +154,7 @@ export const JobList: React.FC<JobListProps> = ({
   onApply,
   onReject,
   onProcess,
-  onProcessAll,
   processingJobId,
-  isProcessingAll,
 }) => {
   const [activeTab, setActiveTab] = useState<FilterTab>("ready");
   const [searchQuery, setSearchQuery] = useState("");
@@ -252,22 +248,6 @@ export const JobList: React.FC<JobListProps> = ({
           </TabsList>
 
           <div className="flex items-center justify-between gap-2 sm:justify-end">
-            {activeTab === "discovered" && counts.discovered > 0 && (
-              <Button onClick={onProcessAll} disabled={isProcessingAll} size="sm">
-                {isProcessingAll ? (
-                  <>
-                    <Loader2 className="mr-2 h-4 w-4 animate-spin" />
-                    Processing...
-                  </>
-                ) : (
-                  <>
-                    <RefreshCcw className="mr-2 h-4 w-4" />
-                    Process All ({counts.discovered})
-                  </>
-                )}
-              </Button>
-            )}
-
             <div className="flex items-center rounded-md border bg-muted/20 p-0.5">
               <Button
                 type="button"
diff --git a/orchestrator/src/server/api/routes.ts b/orchestrator/src/server/api/routes.ts
index 0ca9845..0aaf59d 100644
--- a/orchestrator/src/server/api/routes.ts
+++ b/orchestrator/src/server/api/routes.ts
@@ -170,39 +170,6 @@ apiRouter.post('/jobs/:id/reject', async (req: Request, res: Response) => {
   }
 });
 
-/**
- * POST /api/jobs/process-discovered - Process all discovered jobs (generate PDFs)
- */
-apiRouter.post('/jobs/process-discovered', async (req: Request, res: Response) => {
-  try {
-    const discoveredJobs = await jobsRepo.getAllJobs(['discovered']);
-    
-    // Process each job in background
-    const processInBackground = async () => {
-      for (const job of discoveredJobs.filter(j => j.status === 'discovered')) {
-        try {
-          await processJob(job.id);
-        } catch (error) {
-          console.error(`Failed to process job ${job.id}:`, error);
-        }
-      }
-    };
-    
-    processInBackground().catch(console.error);
-    
-    res.json({ 
-      success: true, 
-      data: { 
-        message: `Processing ${discoveredJobs.length} jobs`,
-        count: discoveredJobs.length,
-      } 
-    });
-  } catch (error) {
-    const message = error instanceof Error ? error.message : 'Unknown error';
-    res.status(500).json({ success: false, error: message });
-  }
-});
-
 // ============================================================================
 // Pipeline API
 // ============================================================================
diff --git a/orchestrator/src/server/pipeline/orchestrator.ts b/orchestrator/src/server/pipeline/orchestrator.ts
index 5ec4f02..f8ac63c 100644
--- a/orchestrator/src/server/pipeline/orchestrator.ts
+++ b/orchestrator/src/server/pipeline/orchestrator.ts
@@ -4,10 +4,7 @@
  * Flow:
  * 1. Run crawler to discover new jobs
  * 2. Score jobs for suitability
- * 3. Pick top N jobs
- * 4. Generate tailored summaries
- * 5. Generate PDF resumes
- * 6. Mark as "ready" for user review
+ * 3. Leave all jobs in "discovered" for manual processing
  */
 
 import { readFile } from 'fs/promises';
@@ -15,7 +12,7 @@ import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import { runCrawler } from '../services/crawler.js';
 import { runJobSpy } from '../services/jobspy.js';
-import { scoreAndRankJobs, scoreJobSuitability } from '../services/scorer.js';
+import { scoreJobSuitability } from '../services/scorer.js';
 import { generateSummary } from '../services/summary.js';
 import { generatePdf } from '../services/pdf.js';
 import * as jobsRepo from '../repositories/jobs.js';
@@ -63,7 +60,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
   const pipelineRun = await pipelineRepo.createPipelineRun();
   
   console.log('🚀 Starting job pipeline...');
-  console.log(`   Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore}`);
+  console.log(`   Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore} (manual processing)`);
   
   try {
     // Step 1: Load profile
@@ -141,9 +138,18 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
       jobsDiscovered: created,
     });
     
-    // Step 4: Get unprocessed jobs and score them
+    // Step 4: Score all discovered jobs missing a score
     console.log('\n🎯 Scoring jobs for suitability...');
-    const unprocessedJobs = await jobsRepo.getJobsForProcessing(50);
+    const unprocessedJobs = await jobsRepo.getUnscoredDiscoveredJobs();
+
+    updateProgress({
+      step: 'scoring',
+      jobsDiscovered: unprocessedJobs.length,
+      jobsScored: 0,
+      jobsProcessed: 0,
+      totalToProcess: 0,
+      currentJob: undefined,
+    });
     
     // Score jobs with progress updates
     const scoredJobs: Array<Job & { suitabilityScore: number; suitabilityReason: string }> = [];
@@ -175,106 +181,27 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
       });
     }
     
-    // Sort by score
-    scoredJobs.sort((a, b) => b.suitabilityScore - a.suitabilityScore);
-    
-    // Step 5: Pick top N jobs above threshold
-    const topJobs = scoredJobs
-      .filter(j => j.suitabilityScore >= mergedConfig.minSuitabilityScore)
-      .slice(0, mergedConfig.topN);
-    
-    progressHelpers.scoringComplete(scoredJobs.length, topJobs.length);
-    
-    console.log(`\n📊 Selected ${topJobs.length} top jobs for processing:`);
-    for (const job of topJobs) {
-      console.log(`   - ${job.title} @ ${job.employer} (score: ${job.suitabilityScore})`);
-    }
-    
-    // Step 6: Process each top job
-    let processed = 0;
-    
-    for (let i = 0; i < topJobs.length; i++) {
-      const job = topJobs[i];
-      console.log(`\n📝 Processing: ${job.title} @ ${job.employer}`);
-      
-      progressHelpers.processingJob(i + 1, topJobs.length, {
-        id: job.id,
-        title: job.title,
-        employer: job.employer,
-      });
-      
-      try {
-        // Mark as processing
-        await jobsRepo.updateJob(job.id, { status: 'processing' });
-        
-        // Generate tailored summary
-        console.log('   Generating summary...');
-        progressHelpers.generatingSummary({ title: job.title, employer: job.employer });
-        
-        const summaryResult = await generateSummary(
-          job.jobDescription || '',
-          profile
-        );
-        
-        if (!summaryResult.success) {
-          console.warn(`   ⚠️ Summary generation failed: ${summaryResult.error}`);
-          continue;
-        }
-        
-        // Update job with summary
-        await jobsRepo.updateJob(job.id, {
-          tailoredSummary: summaryResult.summary,
-        });
-        
-        // Generate PDF
-        console.log('   Generating PDF...');
-        progressHelpers.generatingPdf({ title: job.title, employer: job.employer });
-        
-        const pdfResult = await generatePdf(
-          job.id,
-          summaryResult.summary!,
-          mergedConfig.profilePath
-        );
-        
-        if (!pdfResult.success) {
-          console.warn(`   ⚠️ PDF generation failed: ${pdfResult.error}`);
-          // Still mark as ready even if PDF failed - user can regenerate
-        }
-        
-        // Mark as ready
-        await jobsRepo.updateJob(job.id, {
-          status: 'ready',
-          pdfPath: pdfResult.pdfPath ?? undefined,
-        });
-        
-        processed++;
-        progressHelpers.jobComplete(processed, topJobs.length);
-        console.log(`   ✅ Ready for review!`);
-        
-      } catch (error) {
-        console.error(`   ❌ Failed to process job: ${error}`);
-        // Continue with next job
-      }
-    }
+    progressHelpers.scoringComplete(scoredJobs.length);
+    console.log(`\n📊 Scored ${scoredJobs.length} jobs. Ready for manual processing.`);
     
     // Update pipeline run as completed
     await pipelineRepo.updatePipelineRun(pipelineRun.id, {
       status: 'completed',
       completedAt: new Date().toISOString(),
-      jobsProcessed: processed,
+      jobsProcessed: 0,
     });
     
     console.log('\n🎉 Pipeline completed!');
     console.log(`   Jobs discovered: ${created}`);
-    console.log(`   Jobs processed: ${processed}`);
+    console.log('   Jobs processed: 0 (manual)');
     
-    progressHelpers.complete(created, processed);
+    progressHelpers.complete(created, 0);
     isPipelineRunning = false;
     
     return {
       success: true,
       jobsDiscovered: created,
-      jobsProcessed: processed,
+      jobsProcessed: 0,
     };
     
   } catch (error) {
diff --git a/orchestrator/src/server/pipeline/progress.ts b/orchestrator/src/server/pipeline/progress.ts
index 0815e5d..52aec5f 100644
--- a/orchestrator/src/server/pipeline/progress.ts
+++ b/orchestrator/src/server/pipeline/progress.ts
@@ -215,12 +215,14 @@ export const progressHelpers = {
     jobsScored: index,
   }),
   
-  scoringComplete: (totalScored: number, topN: number) => updateProgress({
-    step: 'processing',
-    message: `Scored ${totalScored} jobs. Processing top ${topN}...`,
-    detail: 'Generating tailored resumes',
+  scoringComplete: (totalScored: number) => updateProgress({
+    step: 'scoring',
+    message: `Scored ${totalScored} jobs.`,
+    detail: 'Ready for manual processing',
     jobsScored: totalScored,
-    totalToProcess: topN,
+    totalToProcess: 0,
+    jobsProcessed: 0,
+    currentJob: undefined,
   }),
   
   processingJob: (index: number, total: number, job: { id: string; title: string; employer: string }) => updateProgress({
diff --git a/orchestrator/src/server/repositories/jobs.ts b/orchestrator/src/server/repositories/jobs.ts
index 98e469f..403d336 100644
--- a/orchestrator/src/server/repositories/jobs.ts
+++ b/orchestrator/src/server/repositories/jobs.ts
@@ -2,7 +2,7 @@
  * Job repository - data access layer for jobs.
  */
 
-import { eq, desc, sql, and, inArray } from 'drizzle-orm';
+import { eq, desc, sql, and, inArray, isNull } from 'drizzle-orm';
 import { randomUUID } from 'crypto';
 import { db, schema } from '../db/index.js';
 import type { Job, CreateJobInput, UpdateJobInput, JobStatus } from '../../shared/types.js';
@@ -195,6 +195,20 @@ export async function getJobsForProcessing(limit: number = 10): Promise<Job[]> {
   return rows.map(mapRowToJob);
 }
 
+/**
+ * Get discovered jobs missing a suitability score.
+ */
+export async function getUnscoredDiscoveredJobs(limit?: number): Promise<Job[]> {
+  const query = db
+    .select()
+    .from(jobs)
+    .where(and(eq(jobs.status, 'discovered'), isNull(jobs.suitabilityScore)))
+    .orderBy(desc(jobs.discoveredAt));
+
+  const rows = typeof limit === 'number' ? await query.limit(limit) : await query;
+  return rows.map(mapRowToJob);
+}
+
 // Helper to map database row to Job type
 function mapRowToJob(row: typeof jobs.$inferSelect): Job {
   return {