Auto-Registering Extractor System (#223)

* initial commit? * Address PR feedback on extractor discovery and startup resilience * Address latest PR review comments * fix city resolution fallback when input parses empty * address PR feedback on extractor registry and pipeline validation * address copilot comments on manifests and registry startup * fix extractor discovery export handling and env isolation in tests * enforce duplicate manifest id failures in strict mode * Fix remaining extractor registry and runtime review comments * docs * docs * test all, logic remains in extractors * Address PR review feedback on extractor registry and validation * Revert extractor moduleResolution to bundler * Enforce shared city filtering across all discovery sources * Deduplicate extractor strict city post-filtering
2026-02-21 17:44:07 +00:00
parent cc7cacd7f5
commit 82e142a8a8
52 changed files with 2207 additions and 1515 deletions
@@ -0,0 +1,94 @@
+import { resolveSearchCities } from "@shared/search-cities.js";
+import type {
+  ExtractorManifest,
+  ExtractorProgressEvent,
+} from "@shared/types/extractors";
+import { runHiringCafe } from "./src/run";
+
+function toProgress(event: {
+  type: string;
+  termIndex: number;
+  termTotal: number;
+  searchTerm: string;
+  pageNo?: number;
+  totalCollected?: number;
+}): ExtractorProgressEvent {
+  if (event.type === "term_start") {
+    return {
+      phase: "list",
+      termsProcessed: Math.max(event.termIndex - 1, 0),
+      termsTotal: event.termTotal,
+      currentUrl: event.searchTerm,
+      detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
+    };
+  }
+
+  if (event.type === "page_fetched") {
+    const pageNo = (event.pageNo ?? 0) + 1;
+    const totalCollected = event.totalCollected ?? 0;
+    return {
+      phase: "list",
+      termsProcessed: Math.max(event.termIndex - 1, 0),
+      termsTotal: event.termTotal,
+      listPagesProcessed: pageNo,
+      jobPagesEnqueued: totalCollected,
+      jobPagesProcessed: totalCollected,
+      currentUrl: `page ${pageNo}`,
+      detail: `Hiring Cafe: term ${event.termIndex}/${event.termTotal}, page ${pageNo} (${totalCollected} collected)`,
+    };
+  }
+
+  return {
+    phase: "list",
+    termsProcessed: event.termIndex,
+    termsTotal: event.termTotal,
+    currentUrl: event.searchTerm,
+    detail: `Hiring Cafe: completed term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
+  };
+}
+
+export const manifest: ExtractorManifest = {
+  id: "hiringcafe",
+  displayName: "Hiring Cafe",
+  providesSources: ["hiringcafe"],
+  async run(context) {
+    if (context.shouldCancel?.()) {
+      return { success: true, jobs: [] };
+    }
+
+    const maxJobsPerTerm = context.settings.jobspyResultsWanted
+      ? parseInt(context.settings.jobspyResultsWanted, 10)
+      : 200;
+
+    const result = await runHiringCafe({
+      country: context.selectedCountry,
+      countryKey: context.selectedCountry,
+      searchTerms: context.searchTerms,
+      locations: resolveSearchCities({
+        single:
+          context.settings.searchCities ?? context.settings.jobspyLocation,
+      }),
+      maxJobsPerTerm,
+      onProgress: (event) => {
+        if (context.shouldCancel?.()) return;
+
+        context.onProgress?.(toProgress(event));
+      },
+    });
+
+    if (!result.success) {
+      return {
+        success: false,
+        jobs: [],
+        error: result.error,
+      };
+    }
+
+    return {
+      success: true,
+      jobs: result.jobs,
+    };
+  },
+};
+
+export default manifest;