job_spy implementation

orchestrator/src/client/App.tsx

@@ -6,10 +6,13 @@ import React, { useCallback, useEffect, useState } from "react";
 import { toast } from "sonner";
 
 import { Toaster } from "@/components/ui/sonner";
-import type { Job, JobStatus } from "../shared/types";
+import type { Job, JobSource, JobStatus } from "../shared/types";
 import { Header, JobList, PipelineProgress, Stats } from "./components";
 import * as api from "./api";
 
+const DEFAULT_PIPELINE_SOURCES: JobSource[] = ["gradcracker", "indeed", "linkedin"];
+const PIPELINE_SOURCES_STORAGE_KEY = "jobops.pipeline.sources";
+
 export const App: React.FC = () => {
   const [jobs, setJobs] = useState<Job[]>([]);
   const [stats, setStats] = useState<Record<JobStatus, number>>({
@@ -24,6 +27,27 @@ export const App: React.FC = () => {
   const [isPipelineRunning, setIsPipelineRunning] = useState(false);
   const [processingJobId, setProcessingJobId] = useState<string | null>(null);
   const [isProcessingAll, setIsProcessingAll] = useState(false);
+  const [pipelineSources, setPipelineSources] = useState<JobSource[]>(() => {
+    try {
+      const raw = localStorage.getItem(PIPELINE_SOURCES_STORAGE_KEY);
+      if (!raw) return DEFAULT_PIPELINE_SOURCES;
+      const parsed = JSON.parse(raw) as unknown;
+      const allowed: JobSource[] = ["gradcracker", "indeed", "linkedin"];
+      if (!Array.isArray(parsed)) return DEFAULT_PIPELINE_SOURCES;
+      const next = parsed.filter((value): value is JobSource => allowed.includes(value));
+      return next.length > 0 ? next : DEFAULT_PIPELINE_SOURCES;
+    } catch {
+      return DEFAULT_PIPELINE_SOURCES;
+    }
+  });
+
+  useEffect(() => {
+    try {
+      localStorage.setItem(PIPELINE_SOURCES_STORAGE_KEY, JSON.stringify(pipelineSources));
+    } catch {
+      // Ignore localStorage errors
+    }
+  }, [pipelineSources]);
 
   const loadJobs = useCallback(async () => {
     try {
@@ -63,8 +87,10 @@ export const App: React.FC = () => {
   const handleRunPipeline = async () => {
     try {
       setIsPipelineRunning(true);
-      await api.runPipeline();
-      toast.message("Pipeline started", { description: "This may take a few minutes." });
+      await api.runPipeline({ sources: pipelineSources });
+      toast.message("Pipeline started", {
+        description: `Sources: ${pipelineSources.join(", ")}. This may take a few minutes.`,
+      });
 
       const pollInterval = setInterval(async () => {
         try {
@@ -170,6 +196,8 @@ export const App: React.FC = () => {
         onClearDatabase={handleClearDatabase}
         isPipelineRunning={isPipelineRunning}
         isLoading={isLoading}
+        pipelineSources={pipelineSources}
+        onPipelineSourcesChange={setPipelineSources}
       />
 
       <main className="container mx-auto max-w-7xl space-y-6 px-4 py-6 pb-12">
@@ -190,4 +218,3 @@ export const App: React.FC = () => {
     </>
   );
 };
-

orchestrator/src/client/api/client.ts

@@ -7,6 +7,7 @@ import type {
   ApiResponse, 
   JobsListResponse, 
   PipelineStatusResponse,
+  JobSource,
   PipelineRun 
 } from '../../shared/types';
 
@@ -83,6 +84,7 @@ export async function getPipelineRuns(): Promise<PipelineRun[]> {
 export async function runPipeline(config?: {
   topN?: number;
   minSuitabilityScore?: number;
+  sources?: JobSource[];
 }): Promise<{ message: string }> {
   return fetchApi<{ message: string }>('/pipeline/run', {
     method: 'POST',

orchestrator/src/client/components/Header.tsx

@@ -3,7 +3,7 @@
  */
 
 import React from "react";
-import { Loader2, Play, RefreshCcw, Rocket, Trash2 } from "lucide-react";
+import { ChevronDown, Loader2, Play, RefreshCcw, Rocket, Trash2 } from "lucide-react";
 
 import { Button } from "@/components/ui/button";
 import {
@@ -17,6 +17,16 @@ import {
   AlertDialogTitle,
   AlertDialogTrigger,
 } from "@/components/ui/alert-dialog";
+import {
+  DropdownMenu,
+  DropdownMenuCheckboxItem,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuLabel,
+  DropdownMenuSeparator,
+  DropdownMenuTrigger,
+} from "@/components/ui/dropdown-menu";
+import type { JobSource } from "../../shared/types";
 
 interface HeaderProps {
   onRunPipeline: () => void;
@@ -24,6 +34,8 @@ interface HeaderProps {
   onClearDatabase: () => void;
   isPipelineRunning: boolean;
   isLoading: boolean;
+  pipelineSources: JobSource[];
+  onPipelineSourcesChange: (sources: JobSource[]) => void;
 }
 
 export const Header: React.FC<HeaderProps> = ({
@@ -32,7 +44,26 @@ export const Header: React.FC<HeaderProps> = ({
   onClearDatabase,
   isPipelineRunning,
   isLoading,
+  pipelineSources,
+  onPipelineSourcesChange,
 }) => {
+  const sourceLabel: Record<JobSource, string> = {
+    gradcracker: "Gradcracker",
+    indeed: "Indeed",
+    linkedin: "LinkedIn",
+  };
+
+  const orderedSources: JobSource[] = ["gradcracker", "indeed", "linkedin"];
+
+  const toggleSource = (source: JobSource, checked: boolean) => {
+    const next = checked
+      ? Array.from(new Set([...pipelineSources, source]))
+      : pipelineSources.filter((s) => s !== source);
+
+    if (next.length === 0) return;
+    onPipelineSourcesChange(next);
+  };
+
   return (
     <header className="sticky top-0 z-40 border-b bg-background/80 backdrop-blur supports-[backdrop-filter]:bg-background/60">
       <div className="container mx-auto flex max-w-7xl items-center justify-between gap-4 px-4 py-4">
@@ -81,19 +112,62 @@ export const Header: React.FC<HeaderProps> = ({
             <span className="hidden sm:inline">Refresh</span>
           </Button>
 
-          <Button size="sm" onClick={onRunPipeline} disabled={isPipelineRunning}>
-            {isPipelineRunning ? (
-              <>
-                <Loader2 className="h-4 w-4 animate-spin" />
-                Running...
-              </>
-            ) : (
-              <>
-                <Play className="h-4 w-4" />
-                Run Pipeline
-              </>
-            )}
-          </Button>
+          <div className="flex items-center">
+            <Button
+              size="sm"
+              onClick={onRunPipeline}
+              disabled={isPipelineRunning}
+              className="rounded-r-none"
+            >
+              {isPipelineRunning ? (
+                <>
+                  <Loader2 className="h-4 w-4 animate-spin" />
+                  Running...
+                </>
+              ) : (
+                <>
+                  <Play className="h-4 w-4" />
+                  Run Pipeline
+                </>
+              )}
+            </Button>
+
+            <DropdownMenu>
+              <DropdownMenuTrigger asChild>
+                <Button
+                  size="sm"
+                  disabled={isPipelineRunning}
+                  className="rounded-l-none border-l border-primary-foreground/20 px-2"
+                  aria-label="Select pipeline sources"
+                >
+                  <ChevronDown className="h-4 w-4" />
+                </Button>
+              </DropdownMenuTrigger>
+              <DropdownMenuContent align="end" className="w-56">
+                <DropdownMenuLabel>Sources</DropdownMenuLabel>
+                <DropdownMenuSeparator />
+                {orderedSources.map((source) => (
+                  <DropdownMenuCheckboxItem
+                    key={source}
+                    checked={pipelineSources.includes(source)}
+                    onCheckedChange={(checked) => toggleSource(source, Boolean(checked))}
+                  >
+                    {sourceLabel[source]}
+                  </DropdownMenuCheckboxItem>
+                ))}
+                <DropdownMenuSeparator />
+                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(orderedSources)}>
+                  All sources
+                </DropdownMenuItem>
+                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["gradcracker"])}>
+                  Gradcracker only
+                </DropdownMenuItem>
+                <DropdownMenuItem onSelect={() => onPipelineSourcesChange(["indeed", "linkedin"])}>
+                  Indeed + LinkedIn only
+                </DropdownMenuItem>
+              </DropdownMenuContent>
+            </DropdownMenu>
+          </div>
         </div>
       </div>
     </header>

orchestrator/src/client/components/JobCard.tsx

@@ -55,6 +55,8 @@ export const JobCard: React.FC<JobCardProps> = ({
 }) => {
   const sourceLabel: Record<Job["source"], string> = {
     gradcracker: "Gradcracker",
+    indeed: "Indeed",
+    linkedin: "LinkedIn",
   };
 
   const hasPdf = !!job.pdfPath;