/** * The Muse public jobs API. * * https://www.themuse.com/api/public/jobs?page=0&category=...&location=... * * The endpoint works without auth but is heavily rate-limited; an API key * (THEMUSE_API_KEY / `themuseApiKey` setting) lifts that. We pass each pipeline * search term as a `category` to keep parity with how other extractors iterate * search terms; if your role doesn't map to a Muse category it'll still match * because Muse falls back to generic listings. */ import type { ExtractorManifest, ExtractorRunResult, } from "@shared/types/extractors"; import type { CreateJobInput } from "@shared/types/jobs"; const API_URL = "https://www.themuse.com/api/public/jobs"; interface MuseLocation { name?: string; } interface MuseCompany { name?: string; short_name?: string; } interface MuseRefs { landing_page?: string; } interface MuseJob { id?: number; name?: string; publication_date?: string; type?: string; contents?: string; short_description?: string; locations?: MuseLocation[]; company?: MuseCompany; refs?: MuseRefs; } interface MuseResponse { page?: number; page_count?: number; results?: MuseJob[]; } function asString(value: unknown): string | undefined { if (typeof value !== "string") return undefined; const trimmed = value.trim(); return trimmed ? trimmed : undefined; } function joinLocations( locations: MuseLocation[] | undefined, ): string | undefined { if (!locations || locations.length === 0) return undefined; const cleaned = locations .map((entry) => asString(entry.name)) .filter((name): name is string => Boolean(name)); return cleaned.length > 0 ? cleaned.join("; ") : undefined; } function isRemoteFromLocations( locations: MuseLocation[] | undefined, ): boolean | undefined { if (!locations || locations.length === 0) return undefined; return locations.some((loc) => typeof loc.name === "string" ? /\bflexible|remote\b/i.test(loc.name) : false, ); } // The Muse `category` filter expects an exact, Title-Cased category name (e.g. // "Software Engineer", "Engineering"). User-supplied search terms are commonly // lowercase free-text, which the API silently ignores and returns zero results. // Title-case the term so common values map to real categories; if the term // still doesn't match a category the extractor will fall back to no filter. function toMuseCategory(term: string): string | undefined { const trimmed = term.trim(); if (!trimmed) return undefined; return trimmed .toLowerCase() .split(/\s+/) .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) .join(" "); } function mapJob(raw: MuseJob): CreateJobInput | null { const jobUrl = asString(raw.refs?.landing_page); if (!jobUrl) return null; return { source: "themuse", sourceJobId: raw.id != null ? String(raw.id) : undefined, title: asString(raw.name) ?? "Unknown Title", employer: asString(raw.company?.name) ?? "Unknown Employer", jobUrl, applicationLink: jobUrl, location: joinLocations(raw.locations), isRemote: isRemoteFromLocations(raw.locations), jobType: asString(raw.type), datePosted: asString(raw.publication_date), jobDescription: asString(raw.contents) ?? asString(raw.short_description) ?? undefined, }; } async function fetchPage(args: { apiKey?: string; page: number; category?: string; location?: string; }): Promise { const url = new URL(API_URL); url.searchParams.set("page", String(args.page)); if (args.category) url.searchParams.set("category", args.category); if (args.location) url.searchParams.set("location", args.location); if (args.apiKey) url.searchParams.set("api_key", args.apiKey); const response = await fetch(url.toString(), { headers: { Accept: "application/json" }, }); if (!response.ok) { throw new Error(`The Muse request failed with status ${response.status}`); } return (await response.json()) as MuseResponse; } export const manifest: ExtractorManifest = { id: "themuse", displayName: "The Muse", providesSources: ["themuse"], async run(context): Promise { if (context.shouldCancel?.()) return { success: true, jobs: [] }; const apiKey = context.settings.themuseApiKey?.trim() || undefined; const maxJobsPerTerm = context.settings.themuseMaxJobsPerTerm ? Number.parseInt(context.settings.themuseMaxJobsPerTerm, 10) : 100; const terms = context.searchTerms.length > 0 ? context.searchTerms : [""]; const locationHint = context.settings.searchCities?.split("|")[0]?.trim() || undefined; const seen = new Set(); const out: CreateJobInput[] = []; try { for (let i = 0; i < terms.length; i += 1) { if (context.shouldCancel?.()) break; const term = terms[i].trim(); context.onProgress?.({ phase: "list", termsProcessed: i, termsTotal: terms.length, currentUrl: term || "(all)", detail: `The Muse: term ${i + 1}/${terms.length}`, }); let collected = 0; let page = 0; let pageCount = Number.POSITIVE_INFINITY; // The Muse returns pageCount; cap pages defensively to avoid runaway // loops if the API misbehaves. We try the term as a category first and, // if the very first page is empty, drop the category filter once so an // unknown category doesn't silently nuke the entire term. let categoryToUse: string | undefined = toMuseCategory(term); let droppedCategory = false; while (collected < maxJobsPerTerm && page < pageCount && page < 100) { if (context.shouldCancel?.()) break; const body = await fetchPage({ apiKey, page, category: categoryToUse, location: locationHint, }); if (typeof body.page_count === "number") { pageCount = body.page_count; } const results = Array.isArray(body.results) ? body.results : []; if (results.length === 0) { if (page === 0 && categoryToUse && !droppedCategory) { categoryToUse = undefined; droppedCategory = true; pageCount = Number.POSITIVE_INFINITY; continue; } break; } for (const item of results) { const mapped = mapJob(item); if (!mapped) continue; const key = mapped.sourceJobId || mapped.jobUrl; if (seen.has(key)) continue; seen.add(key); out.push(mapped); collected += 1; if (collected >= maxJobsPerTerm) break; } page += 1; } context.onProgress?.({ phase: "list", termsProcessed: i + 1, termsTotal: terms.length, currentUrl: term || "(all)", jobPagesProcessed: out.length, detail: `The Muse: completed term ${i + 1}/${terms.length} (${collected} found)`, }); } } catch (error) { const message = error instanceof Error ? error.message : "Unknown error"; return { success: false, jobs: out, error: message }; } return { success: true, jobs: out }; }, }; export default manifest;