fix(hiringcafe): use Next.js SSR endpoint after search API auth broke

Hiring Cafe now returns 401/404 on the legacy /api/search-jobs endpoints.
Switch the extractor to the site's _next/data SSR transport so discovery
runs succeed again without manual cookie injection.

extractors/hiringcafe/src/main.ts

@@ -111,12 +111,6 @@ function parseWorkplaceTypes(
   }
 }
 
-function encodeSearchState(searchState: unknown): string {
-  const json = JSON.stringify(searchState);
-  const urlEncodedJson = encodeURIComponent(json);
-  return Buffer.from(urlEncodedJson, "utf-8").toString("base64");
-}
-
 function asRecord(value: unknown): Record<string, unknown> | null {
   if (!value || typeof value !== "object" || Array.isArray(value)) return null;
   return value as Record<string, unknown>;
@@ -230,7 +224,18 @@ function extractResultsBatch(payload: unknown): RawHiringCafeJob[] {
   }
 
   const payloadRecord = asRecord(payload);
-  const results = payloadRecord?.results;
+  if (!payloadRecord) return [];
+
+  const pageProps = asRecord(payloadRecord.pageProps);
+  const ssrHits = pageProps?.ssrHits ?? payloadRecord.ssrHits;
+  if (Array.isArray(ssrHits)) {
+    return ssrHits.filter(
+      (item): item is RawHiringCafeJob =>
+        Boolean(item) && typeof item === "object" && !Array.isArray(item),
+    );
+  }
+
+  const results = payloadRecord.results;
   if (!Array.isArray(results)) return [];
 
   return results.filter(
@@ -242,7 +247,22 @@ function extractResultsBatch(payload: unknown): RawHiringCafeJob[] {
 function parseTotalCount(payload: unknown): number | null {
   const payloadRecord = asRecord(payload);
   if (!payloadRecord) return null;
-  return toNumberOrNull(payloadRecord.total);
+
+  const pageProps = asRecord(payloadRecord.pageProps);
+  return (
+    toNumberOrNull(pageProps?.ssrTotalCount) ??
+    toNumberOrNull(payloadRecord.ssrTotalCount) ??
+    toNumberOrNull(payloadRecord.total)
+  );
+}
+
+function isLastSsrPage(payload: unknown): boolean {
+  const payloadRecord = asRecord(payload);
+  if (!payloadRecord) return true;
+
+  const pageProps = asRecord(payloadRecord.pageProps);
+  const isLastPage = pageProps?.ssrIsLastPage ?? payloadRecord.ssrIsLastPage;
+  return isLastPage === true;
 }
 
 function buildCityLocationId(input: string): string {
@@ -501,23 +521,37 @@ function createCitySearchState(args: {
   };
 }
 
-async function callHiringCafeApi(
+async function readBuildIdFromPage(page: Page): Promise<string> {
+  const buildId = await page.evaluate(() => {
+    const element = document.querySelector("#__NEXT_DATA__");
+    if (!element?.textContent) return null;
+    const data = JSON.parse(element.textContent) as { buildId?: string };
+    return data.buildId ?? null;
+  });
+  if (!buildId) {
+    throw new Error("Hiring Cafe page did not expose Next.js buildId");
+  }
+  return buildId;
+}
+
+async function fetchSsrSearchPage(
   page: Page,
-  endpoint: string,
+  buildId: string,
-  params: Record<string, string>,
+  searchState: Record<string, unknown>,
+  pageNo: number,
 ): Promise<unknown> {
   const response = await page.evaluate(
-    async ({ endpointArg, paramsArg }) => {
+    async ({ buildIdArg, searchStateArg, pageNoArg }) => {
-      const url = new URL(endpointArg, window.location.origin);
+      const params = new URLSearchParams();
-      for (const [key, value] of Object.entries(paramsArg)) {
+      params.set("searchState", JSON.stringify(searchStateArg));
-        url.searchParams.set(key, value);
+      params.set("page", String(pageNoArg));
-      }
+      const url = `/_next/data/${buildIdArg}/index.json?${params.toString()}`;
 
-      const res = await fetch(url.toString(), {
+      const res = await fetch(url, {
-        method: "GET",
         credentials: "include",
         headers: {
-          Accept: "application/json, text/plain, */*",
+          Accept: "application/json",
+          "x-nextjs-data": "1",
         },
       });
 
@@ -539,7 +573,7 @@ async function callHiringCafeApi(
 
       return output;
     },
-    { endpointArg: endpoint, paramsArg: params },
+    { buildIdArg: buildId, searchStateArg: searchState, pageNoArg: pageNo },
   );
 
   const result = response as BrowserApiResponse;
@@ -547,14 +581,14 @@ async function callHiringCafeApi(
   if (!result.ok) {
     const snippet = result.responseText.slice(0, 250);
     throw new Error(
-      `Hiring Cafe API ${endpoint} failed (${result.status} ${result.statusText}): ${snippet}`,
+      `Hiring Cafe SSR search failed (${result.status} ${result.statusText}): ${snippet}`,
     );
   }
 
   if (result.data === null) {
     const snippet = result.responseText.slice(0, 250);
     throw new Error(
-      `Hiring Cafe API ${endpoint} returned non-JSON response: ${snippet}`,
+      `Hiring Cafe SSR search returned non-JSON response: ${snippet}`,
     );
   }
 
@@ -626,6 +660,8 @@ async function run(): Promise<void> {
       await initializePage();
     }
 
+    const buildId = await readBuildIdFromPage(page);
+
     const countryLocation = resolveHiringCafeCountryLocation(country);
     const countryLong =
       countryLocation?.address_components[0]?.long_name ?? "United Kingdom";
@@ -664,40 +700,23 @@ async function run(): Promise<void> {
             dateFetchedPastNDays,
             workplaceTypes,
           });
-      const encodedSearchState = encodeSearchState(searchState);
+      let termTarget = maxJobsPerTerm;
-
-      let totalAvailable: number | null = null;
-      try {
-        const countPayload = await callHiringCafeApi(
-          page,
-          "/api/search-jobs/get-total-count",
-          {
-            s: encodedSearchState,
-          },
-        );
-        totalAvailable = parseTotalCount(countPayload);
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        console.warn(
-          `Hiring Cafe count request failed for term '${searchTerm}': ${message}`,
-        );
-      }
-
-      const termTarget =
-        totalAvailable !== null
-          ? Math.min(maxJobsPerTerm, totalAvailable)
-          : maxJobsPerTerm;
 
       let pageNo = 0;
       let termCollected = 0;
 
       while (termCollected < termTarget && pageNo < PAGE_LIMIT) {
-        const size = Math.min(1000, termTarget - termCollected);
+        const jobsPayload = await fetchSsrSearchPage(
-        const jobsPayload = await callHiringCafeApi(page, "/api/search-jobs", {
+          page,
-          size: String(size),
+          buildId,
-          page: String(pageNo),
+          searchState,
-          s: encodedSearchState,
+          pageNo,
-        });
+        );
+
+        const totalAvailable = parseTotalCount(jobsPayload);
+        if (totalAvailable !== null) {
+          termTarget = Math.min(maxJobsPerTerm, totalAvailable);
+        }
 
         const batch = extractResultsBatch(jobsPayload);
         if (batch.length === 0) break;
@@ -727,7 +746,7 @@ async function run(): Promise<void> {
           totalCollected: termCollected,
         });
 
-        if (batch.length < size) break;
+        if (isLastSsrPage(jobsPayload)) break;
         pageNo += 1;
       }