Auto-Registering Extractor System (#223)

* initial commit? * Address PR feedback on extractor discovery and startup resilience * Address latest PR review comments * fix city resolution fallback when input parses empty * address PR feedback on extractor registry and pipeline validation * address copilot comments on manifests and registry startup * fix extractor discovery export handling and env isolation in tests * enforce duplicate manifest id failures in strict mode * Fix remaining extractor registry and runtime review comments * docs * docs * test all, logic remains in extractors * Address PR review feedback on extractor registry and validation * Revert extractor moduleResolution to bundler * Enforce shared city filtering across all discovery sources * Deduplicate extractor strict city post-filtering
2026-02-21 17:44:07 +00:00
parent cc7cacd7f5
commit 82e142a8a8
52 changed files with 2207 additions and 1515 deletions
@@ -0,0 +1,74 @@
+import type {
+  ExtractorManifest,
+  ExtractorRuntimeContext,
+} from "@shared/types/extractors";
+import { runJobSpy } from "./src/run";
+
+type JobSpySite = NonNullable<Parameters<typeof runJobSpy>[0]["sites"]>[number];
+
+const JOBSPY_SOURCES = new Set<JobSpySite>(["indeed", "linkedin", "glassdoor"]);
+
+function isJobSpySite(source: string): source is JobSpySite {
+  return JOBSPY_SOURCES.has(source as JobSpySite);
+}
+
+export const manifest: ExtractorManifest = {
+  id: "jobspy",
+  displayName: "JobSpy",
+  providesSources: ["indeed", "linkedin", "glassdoor"],
+  async run(context: ExtractorRuntimeContext) {
+    if (context.shouldCancel?.()) {
+      return { success: true, jobs: [] };
+    }
+
+    const sites = context.selectedSources.filter(isJobSpySite);
+
+    const result = await runJobSpy({
+      sites,
+      searchTerms: context.searchTerms,
+      location:
+        context.settings.searchCities ?? context.settings.jobspyLocation,
+      resultsWanted: context.settings.jobspyResultsWanted
+        ? parseInt(context.settings.jobspyResultsWanted, 10)
+        : undefined,
+      countryIndeed: context.settings.jobspyCountryIndeed,
+      onProgress: (event) => {
+        if (context.shouldCancel?.()) return;
+
+        if (event.type === "term_start") {
+          context.onProgress?.({
+            phase: "list",
+            termsProcessed: Math.max(event.termIndex - 1, 0),
+            termsTotal: event.termTotal,
+            currentUrl: event.searchTerm,
+            detail: `JobSpy: term ${event.termIndex}/${event.termTotal} (${event.searchTerm})`,
+          });
+          return;
+        }
+
+        context.onProgress?.({
+          phase: "list",
+          termsProcessed: event.termIndex,
+          termsTotal: event.termTotal,
+          currentUrl: event.searchTerm,
+          detail: `JobSpy: completed ${event.termIndex}/${event.termTotal} (${event.searchTerm}) with ${event.jobsFoundTerm} jobs`,
+        });
+      },
+    });
+
+    if (!result.success) {
+      return {
+        success: false,
+        jobs: [],
+        error: result.error,
+      };
+    }
+
+    return {
+      success: true,
+      jobs: result.jobs,
+    };
+  },
+};
+
+export default manifest;