fix(qajobsboard): drop expired LinkedIn reposts and resolve hiring location
CI / Linting (Biome) (push) Failing after 41s
CI / Tests (push) Successful in 6m10s
CI / Type Check (adzuna-extractor) (push) Successful in 1m9s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m13s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m9s
CI / Type Check (orchestrator) (push) Failing after 1m16s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m9s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m10s
CI / Documentation (push) Successful in 1m56s

Probe application links for closed listings and feed expires_at; enrich vague Remote/Worldwide rows with real country before blocked-countries filtering.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-05-16 17:42:19 -04:00
co-authored by Cursor
parent 0de7f90278
commit d28a6221e4
5 changed files with 252 additions and 30 deletions
+41 -24
View File
@@ -9,6 +9,11 @@ import type {
ExtractorRunResult,
} from "@shared/types/extractors";
import type { CreateJobInput } from "@shared/types/jobs";
import {
isLinkedInJobUrl,
isPostingExpiredByDate,
probeApplicationLink,
} from "./src/application-link.js";
import {
extractJobLocationFromText,
fetchQaJobDetailEnrichment,
@@ -37,6 +42,7 @@ interface QaJobBoardlyJob {
location?: string;
location_limits?: string[];
published_at?: string;
expires_at?: string;
application_link?: string;
description?: DescriptionBlock;
company?: { name?: string; logo?: string };
@@ -156,8 +162,6 @@ function mapJob(raw: QaJobBoardlyJob): CreateJobInput | null {
};
}
const DETAIL_ENRICH_CONCURRENCY = 4;
function needsDetailEnrichment(location: string | undefined): boolean {
if (!location?.trim()) return true;
return isVagueLocationLabel(location);
@@ -167,31 +171,43 @@ async function enrichJobsFromDetailPages(
jobs: CreateJobInput[],
shouldCancel?: () => boolean,
): Promise<CreateJobInput[]> {
const enriched = jobs.map((job) => ({ ...job }));
const targets = enriched
.map((job, index) => ({ job, index }))
.filter(({ job }) => needsDetailEnrichment(job.location));
const enriched: CreateJobInput[] = [];
for (let offset = 0; offset < targets.length; offset += DETAIL_ENRICH_CONCURRENCY) {
for (const job of jobs) {
if (shouldCancel?.()) break;
const batch = targets.slice(offset, offset + DETAIL_ENRICH_CONCURRENCY);
await Promise.all(
batch.map(async ({ job, index }) => {
try {
const detail = await fetchQaJobDetailEnrichment(job.jobUrl);
if (!detail) return;
enriched[index] = {
...job,
...(detail.location ? { location: detail.location } : {}),
...(detail.jobDescription
? { jobDescription: detail.jobDescription }
: {}),
};
} catch {
// keep feed row when detail page fetch fails
let current = { ...job };
const applicationLink = job.applicationLink?.trim();
if (applicationLink && isLinkedInJobUrl(applicationLink)) {
try {
const probe = await probeApplicationLink(applicationLink);
if (probe?.expired) continue;
if (probe?.location && needsDetailEnrichment(current.location)) {
current = { ...current, location: probe.location };
}
}),
);
} catch {
// keep row when LinkedIn probe fails
}
}
if (needsDetailEnrichment(current.location)) {
try {
const detail = await fetchQaJobDetailEnrichment(current.jobUrl);
if (detail?.expired) continue;
current = {
...current,
...(detail?.location ? { location: detail.location } : {}),
...(detail?.jobDescription
? { jobDescription: detail.jobDescription }
: {}),
};
} catch {
// keep feed row when detail page fetch fails
}
}
enriched.push(current);
}
return enriched;
@@ -238,6 +254,7 @@ export const manifest: ExtractorManifest = {
for (const row of rows as QaJobBoardlyJob[]) {
if (out.length >= cap) break;
if (isPostingExpiredByDate(asString(row.expires_at))) continue;
if (terms.length > 0 && !terms.some((t) => matchesTerm(row, t)))
continue;
const mapped = mapJob(row);