Live scraping updates in pipeline UI (#100)

* initial commit

* fix clear script

* cancelling pipelines

* formatting

extractors/jobspy/scrape_jobs.py

@@ -1,9 +1,12 @@
 import csv
+import json
 import os
 from pathlib import Path
 
 from jobspy import scrape_jobs
 
+PROGRESS_PREFIX = "JOBOPS_PROGRESS "
+
 
 def _env_str(name: str, default: str) -> str:
     value = os.getenv(name)
@@ -27,6 +30,11 @@ def _env_bool(name: str, default: bool) -> bool:
     return value.strip().lower() in ("1", "true", "yes", "y", "on")
 
 
+def _emit_progress(event: str, payload: dict) -> None:
+    serialized = json.dumps({"event": event, **payload}, ensure_ascii=True)
+    print(f"{PROGRESS_PREFIX}{serialized}", flush=True)
+
+
 def _parse_sites(raw: str) -> list[str]:
     return [s.strip() for s in raw.split(",") if s.strip()]
 
@@ -40,6 +48,8 @@ def main() -> int:
     country_indeed = _env_str("JOBSPY_COUNTRY_INDEED", "UK")
     linkedin_fetch_description = _env_bool("JOBSPY_LINKEDIN_FETCH_DESCRIPTION", True)
     is_remote = _env_bool("JOBSPY_IS_REMOTE", False)
+    term_index = _env_int("JOBSPY_TERM_INDEX", 1)
+    term_total = _env_int("JOBSPY_TERM_TOTAL", 1)
 
     output_csv = Path(_env_str("JOBSPY_OUTPUT_CSV", "jobs.csv"))
     output_json = Path(
@@ -50,6 +60,14 @@ def main() -> int:
     output_json.parent.mkdir(parents=True, exist_ok=True)
 
     print(f"jobspy: Search term: {search_term}")
+    _emit_progress(
+        "term_start",
+        {
+            "termIndex": term_index,
+            "termTotal": term_total,
+            "searchTerm": search_term,
+        },
+    )
     jobs = scrape_jobs(
         site_name=sites,
         search_term=search_term,
@@ -62,6 +80,15 @@ def main() -> int:
     )
 
     print(f"Found {len(jobs)} jobs")
+    _emit_progress(
+        "term_complete",
+        {
+            "termIndex": term_index,
+            "termTotal": term_total,
+            "searchTerm": search_term,
+            "jobsFoundTerm": int(len(jobs)),
+        },
+    )
 
     jobs.to_csv(
         output_csv,

extractors/ukvisajobs/src/main.ts

@@ -32,6 +32,16 @@ const AUTH_CACHE_PATH = join(__dirname, "../storage/ukvisajobs-auth.json");
 const JOBS_PER_PAGE = 15;
 const DEFAULT_MAX_JOBS = 50;
 const MAX_ALLOWED_JOBS = 200;
+const JOBOPS_PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
+
+function emitProgress(
+  event: string,
+  payload: Record<string, unknown> = {},
+): void {
+  if (process.env.JOBOPS_EMIT_PROGRESS !== "1") return;
+  const serialized = JSON.stringify({ event, ...payload });
+  process.stdout.write(`${JOBOPS_PROGRESS_PREFIX}${serialized}\n`);
+}
 
 interface UkVisaJobsApiJob {
   id: string;
@@ -444,6 +454,11 @@ async function main(): Promise<void> {
   if (searchKeyword) {
     console.log(`   Search keyword: ${searchKeyword}`);
   }
+  emitProgress("init", {
+    maxPages,
+    maxJobs,
+    searchKeyword: searchKeyword || "",
+  });
 
   const allJobs: ExtractedJob[] = [];
   const seenIds = new Set<string>();
@@ -481,6 +496,11 @@ async function main(): Promise<void> {
       }
 
       if (response.status !== 1) {
+        emitProgress("error", {
+          pageNo,
+          status: response.status,
+          message: `API returned status ${response.status}`,
+        });
         console.warn(
           `   ⚠️ API returned status ${response.status} on page ${pageNo}`,
         );
@@ -493,6 +513,11 @@ async function main(): Promise<void> {
       }
 
       if (!response.jobs || response.jobs.length === 0) {
+        emitProgress("empty_page", {
+          pageNo,
+          maxPages,
+          totalCollected: allJobs.length,
+        });
         console.log(`   No more jobs on page ${pageNo}`);
         break;
       }
@@ -508,6 +533,14 @@ async function main(): Promise<void> {
         allJobs.push(mapped);
       }
 
+      emitProgress("page_fetched", {
+        pageNo,
+        maxPages,
+        jobsOnPage: response.jobs.length,
+        totalCollected: allJobs.length,
+        totalAvailable,
+      });
+
       // If we got fewer jobs than a full page, we're at the end
       if (response.jobs.length < JOBS_PER_PAGE) {
         break;
@@ -519,6 +552,11 @@ async function main(): Promise<void> {
       await new Promise((resolve) => setTimeout(resolve, 500));
     }
 
+    emitProgress("done", {
+      maxPages,
+      totalCollected: allJobs.length,
+      totalAvailable,
+    });
     console.log(`✅ Scraped ${allJobs.length} jobs`);
 
     // Write output to storage directory (similar to Crawlee dataset structure)
@@ -542,6 +580,7 @@ async function main(): Promise<void> {
     console.log(`   Jobs file: ${outputFile}`);
   } catch (error) {
     const message = error instanceof Error ? error.message : "Unknown error";
+    emitProgress("error", { message });
     console.error(`❌ Error: ${message}`);
     process.exit(1);
   }