fix(qajobsboard): drop expired LinkedIn reposts and resolve hiring location
CI / Linting (Biome) (push) Failing after 41s
CI / Tests (push) Successful in 6m10s
CI / Type Check (adzuna-extractor) (push) Successful in 1m9s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m13s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m9s
CI / Type Check (orchestrator) (push) Failing after 1m16s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m9s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m10s
CI / Documentation (push) Successful in 1m56s
CI / Linting (Biome) (push) Failing after 41s
CI / Tests (push) Successful in 6m10s
CI / Type Check (adzuna-extractor) (push) Successful in 1m9s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m13s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m9s
CI / Type Check (orchestrator) (push) Failing after 1m16s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m9s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m10s
CI / Documentation (push) Successful in 1m56s
Probe application links for closed listings and feed expires_at; enrich vague Remote/Worldwide rows with real country before blocked-countries filtering. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -9,6 +9,11 @@ import type {
|
||||
ExtractorRunResult,
|
||||
} from "@shared/types/extractors";
|
||||
import type { CreateJobInput } from "@shared/types/jobs";
|
||||
import {
|
||||
isLinkedInJobUrl,
|
||||
isPostingExpiredByDate,
|
||||
probeApplicationLink,
|
||||
} from "./src/application-link.js";
|
||||
import {
|
||||
extractJobLocationFromText,
|
||||
fetchQaJobDetailEnrichment,
|
||||
@@ -37,6 +42,7 @@ interface QaJobBoardlyJob {
|
||||
location?: string;
|
||||
location_limits?: string[];
|
||||
published_at?: string;
|
||||
expires_at?: string;
|
||||
application_link?: string;
|
||||
description?: DescriptionBlock;
|
||||
company?: { name?: string; logo?: string };
|
||||
@@ -156,8 +162,6 @@ function mapJob(raw: QaJobBoardlyJob): CreateJobInput | null {
|
||||
};
|
||||
}
|
||||
|
||||
const DETAIL_ENRICH_CONCURRENCY = 4;
|
||||
|
||||
function needsDetailEnrichment(location: string | undefined): boolean {
|
||||
if (!location?.trim()) return true;
|
||||
return isVagueLocationLabel(location);
|
||||
@@ -167,31 +171,43 @@ async function enrichJobsFromDetailPages(
|
||||
jobs: CreateJobInput[],
|
||||
shouldCancel?: () => boolean,
|
||||
): Promise<CreateJobInput[]> {
|
||||
const enriched = jobs.map((job) => ({ ...job }));
|
||||
const targets = enriched
|
||||
.map((job, index) => ({ job, index }))
|
||||
.filter(({ job }) => needsDetailEnrichment(job.location));
|
||||
const enriched: CreateJobInput[] = [];
|
||||
|
||||
for (let offset = 0; offset < targets.length; offset += DETAIL_ENRICH_CONCURRENCY) {
|
||||
for (const job of jobs) {
|
||||
if (shouldCancel?.()) break;
|
||||
const batch = targets.slice(offset, offset + DETAIL_ENRICH_CONCURRENCY);
|
||||
await Promise.all(
|
||||
batch.map(async ({ job, index }) => {
|
||||
try {
|
||||
const detail = await fetchQaJobDetailEnrichment(job.jobUrl);
|
||||
if (!detail) return;
|
||||
enriched[index] = {
|
||||
...job,
|
||||
...(detail.location ? { location: detail.location } : {}),
|
||||
...(detail.jobDescription
|
||||
? { jobDescription: detail.jobDescription }
|
||||
: {}),
|
||||
};
|
||||
} catch {
|
||||
// keep feed row when detail page fetch fails
|
||||
|
||||
let current = { ...job };
|
||||
|
||||
const applicationLink = job.applicationLink?.trim();
|
||||
if (applicationLink && isLinkedInJobUrl(applicationLink)) {
|
||||
try {
|
||||
const probe = await probeApplicationLink(applicationLink);
|
||||
if (probe?.expired) continue;
|
||||
if (probe?.location && needsDetailEnrichment(current.location)) {
|
||||
current = { ...current, location: probe.location };
|
||||
}
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
// keep row when LinkedIn probe fails
|
||||
}
|
||||
}
|
||||
|
||||
if (needsDetailEnrichment(current.location)) {
|
||||
try {
|
||||
const detail = await fetchQaJobDetailEnrichment(current.jobUrl);
|
||||
if (detail?.expired) continue;
|
||||
current = {
|
||||
...current,
|
||||
...(detail?.location ? { location: detail.location } : {}),
|
||||
...(detail?.jobDescription
|
||||
? { jobDescription: detail.jobDescription }
|
||||
: {}),
|
||||
};
|
||||
} catch {
|
||||
// keep feed row when detail page fetch fails
|
||||
}
|
||||
}
|
||||
|
||||
enriched.push(current);
|
||||
}
|
||||
|
||||
return enriched;
|
||||
@@ -238,6 +254,7 @@ export const manifest: ExtractorManifest = {
|
||||
|
||||
for (const row of rows as QaJobBoardlyJob[]) {
|
||||
if (out.length >= cap) break;
|
||||
if (isPostingExpiredByDate(asString(row.expires_at))) continue;
|
||||
if (terms.length > 0 && !terms.some((t) => matchesTerm(row, t)))
|
||||
continue;
|
||||
const mapped = mapJob(row);
|
||||
|
||||
Reference in New Issue
Block a user