feat(extractors): add 17 job source extractors and cross-source dedup
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
CI / Linting (Biome) (push) Failing after 36s
CI / Tests (push) Successful in 5m54s
CI / Type Check (adzuna-extractor) (push) Successful in 1m6s
CI / Type Check (gradcracker-extractor) (push) Successful in 1m9s
CI / Type Check (hiringcafe-extractor) (push) Successful in 1m5s
CI / Type Check (orchestrator) (push) Successful in 1m21s
CI / Type Check (startupjobs-extractor) (push) Successful in 1m4s
CI / Type Check (ukvisajobs-extractor) (push) Successful in 1m4s
CI / Documentation (push) Successful in 1m52s
Adds extractor packages: arbeitnow, ashby, careerjet, fourdayweek,
greenhouse, himalayas, jobicy, jooble, lever, reed, remoteok, remotive,
themuse, usajobs, weworkremotely, workday — each with manifest, package
metadata and README.
Pipeline / shared:
- shared/job-fingerprint: stable hash for cross-source dedup, with tests
- discover-jobs: dedup via fingerprint and richer per-source merging
- jobs repository: fingerprint-aware upsert / lookup
- settings-registry, settings types/routes, demo-defaults: knobs for the
new sources
- shared extractors index: register the new manifests
- location-support, profiles route: small fixes for the new sources
Tooling:
- scripts/smoke-extractors.ts to sanity-check each source locally
- scripts/jobber-cron-{cherepaha,dobkin}.env.example: per-host cron
templates (CHANGEME placeholders only)
- .env.example: documented env vars for the new extractors
- .gitignore: ignore extractors/*/storage/ runtime caches (was ukvisajobs only)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -9,6 +9,24 @@ export const EXTRACTOR_SOURCE_IDS = [
|
||||
"adzuna",
|
||||
"hiringcafe",
|
||||
"startupjobs",
|
||||
// --- Public APIs / feeds ---
|
||||
"usajobs",
|
||||
"jobicy",
|
||||
"themuse",
|
||||
"jooble",
|
||||
"careerjet",
|
||||
"reed",
|
||||
"remoteok",
|
||||
"remotive",
|
||||
"arbeitnow",
|
||||
"himalayas",
|
||||
"weworkremotely",
|
||||
"fourdayweek",
|
||||
// --- Public ATS / career-page sources ---
|
||||
"ashby",
|
||||
"lever",
|
||||
"greenhouse",
|
||||
"workday",
|
||||
"manual",
|
||||
] as const;
|
||||
|
||||
@@ -20,6 +38,10 @@ export interface ExtractorSourceMetadata {
|
||||
category: "pipeline" | "manual";
|
||||
requiresCredentials?: boolean;
|
||||
ukOnly?: boolean;
|
||||
/** Country gating: when set, only run/show this source for these country keys. */
|
||||
countryAllowlist?: readonly string[];
|
||||
/** Region tag for grouping / filtering in the UI. */
|
||||
region?: "us" | "uk" | "global" | "remote";
|
||||
}
|
||||
|
||||
export const EXTRACTOR_SOURCE_METADATA: Record<
|
||||
@@ -31,26 +53,157 @@ export const EXTRACTOR_SOURCE_METADATA: Record<
|
||||
order: 10,
|
||||
category: "pipeline",
|
||||
ukOnly: true,
|
||||
region: "uk",
|
||||
},
|
||||
indeed: {
|
||||
label: "Indeed",
|
||||
order: 20,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
linkedin: {
|
||||
label: "LinkedIn",
|
||||
order: 30,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
glassdoor: {
|
||||
label: "Glassdoor",
|
||||
order: 40,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
indeed: { label: "Indeed", order: 20, category: "pipeline" },
|
||||
linkedin: { label: "LinkedIn", order: 30, category: "pipeline" },
|
||||
glassdoor: { label: "Glassdoor", order: 40, category: "pipeline" },
|
||||
ukvisajobs: {
|
||||
label: "UK Visa Jobs",
|
||||
order: 50,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
ukOnly: true,
|
||||
region: "uk",
|
||||
},
|
||||
adzuna: {
|
||||
label: "Adzuna",
|
||||
order: 60,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
region: "global",
|
||||
},
|
||||
hiringcafe: { label: "Hiring Cafe", order: 70, category: "pipeline" },
|
||||
startupjobs: { label: "startup.jobs", order: 80, category: "pipeline" },
|
||||
manual: { label: "Manual", order: 90, category: "manual" },
|
||||
hiringcafe: {
|
||||
label: "Hiring Cafe",
|
||||
order: 70,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
startupjobs: {
|
||||
label: "startup.jobs",
|
||||
order: 80,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
usajobs: {
|
||||
label: "USAJOBS",
|
||||
order: 110,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
countryAllowlist: ["united states", "usa", "us"],
|
||||
region: "us",
|
||||
},
|
||||
jobicy: {
|
||||
label: "Jobicy (Remote)",
|
||||
order: 120,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
themuse: {
|
||||
label: "The Muse",
|
||||
order: 130,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
jooble: {
|
||||
label: "Jooble",
|
||||
order: 140,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
region: "global",
|
||||
},
|
||||
careerjet: {
|
||||
label: "Careerjet",
|
||||
order: 150,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
region: "global",
|
||||
},
|
||||
reed: {
|
||||
label: "Reed",
|
||||
order: 160,
|
||||
category: "pipeline",
|
||||
requiresCredentials: true,
|
||||
ukOnly: true,
|
||||
countryAllowlist: ["united kingdom", "uk", "great britain", "england"],
|
||||
region: "uk",
|
||||
},
|
||||
remoteok: {
|
||||
label: "Remote OK",
|
||||
order: 170,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
remotive: {
|
||||
label: "Remotive",
|
||||
order: 175,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
arbeitnow: {
|
||||
label: "Arbeitnow",
|
||||
order: 180,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
himalayas: {
|
||||
label: "Himalayas",
|
||||
order: 185,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
weworkremotely: {
|
||||
label: "We Work Remotely",
|
||||
order: 190,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
fourdayweek: {
|
||||
label: "4 Day Week",
|
||||
order: 195,
|
||||
category: "pipeline",
|
||||
region: "remote",
|
||||
},
|
||||
ashby: {
|
||||
label: "Ashby (ATS)",
|
||||
order: 210,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
lever: {
|
||||
label: "Lever (ATS)",
|
||||
order: 220,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
greenhouse: {
|
||||
label: "Greenhouse (ATS)",
|
||||
order: 230,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
workday: {
|
||||
label: "Workday (ATS)",
|
||||
order: 240,
|
||||
category: "pipeline",
|
||||
region: "global",
|
||||
},
|
||||
manual: { label: "Manual", order: 900, category: "manual" },
|
||||
};
|
||||
|
||||
export const PIPELINE_EXTRACTOR_SOURCE_IDS = EXTRACTOR_SOURCE_IDS.filter(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
export * from "./extractors";
|
||||
export * from "./job-fingerprint";
|
||||
export * from "./job-url-canonical";
|
||||
export * from "./location-support";
|
||||
export * from "./types";
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildJobContentFingerprint,
|
||||
normalizeEmployerForFingerprint,
|
||||
normalizeTitleForFingerprint,
|
||||
} from "./job-fingerprint";
|
||||
|
||||
describe("buildJobContentFingerprint", () => {
|
||||
it("collapses the same role across sources", () => {
|
||||
const a = buildJobContentFingerprint({
|
||||
employer: "Stripe, Inc.",
|
||||
title: "Senior Software Engineer (Backend) - Toronto, ON",
|
||||
});
|
||||
const b = buildJobContentFingerprint({
|
||||
employer: "stripe inc",
|
||||
title: "Senior Software Engineer (Backend)",
|
||||
});
|
||||
expect(a).toBe(b);
|
||||
expect(a).not.toBeNull();
|
||||
});
|
||||
|
||||
it("ignores trailing location decorations on titles", () => {
|
||||
const a = buildJobContentFingerprint({
|
||||
employer: "Acme",
|
||||
title: "Software Engineer — Remote",
|
||||
});
|
||||
const b = buildJobContentFingerprint({
|
||||
employer: "Acme",
|
||||
title: "Software Engineer",
|
||||
});
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("strips diacritics and punctuation", () => {
|
||||
const a = buildJobContentFingerprint({
|
||||
employer: "Café Münchën",
|
||||
title: "Étudiant – Stage",
|
||||
});
|
||||
const b = buildJobContentFingerprint({
|
||||
employer: "cafe munchen",
|
||||
title: "Etudiant Stage",
|
||||
});
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
it("returns null when employer or title is empty", () => {
|
||||
expect(
|
||||
buildJobContentFingerprint({ employer: "", title: "Engineer" }),
|
||||
).toBeNull();
|
||||
expect(
|
||||
buildJobContentFingerprint({ employer: "Acme", title: "" }),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not collapse different roles at the same employer", () => {
|
||||
const a = buildJobContentFingerprint({
|
||||
employer: "Acme",
|
||||
title: "Software Engineer",
|
||||
});
|
||||
const b = buildJobContentFingerprint({
|
||||
employer: "Acme",
|
||||
title: "Product Designer",
|
||||
});
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
describe("normalizers", () => {
|
||||
it("normalizeEmployerForFingerprint strips legal suffixes", () => {
|
||||
expect(normalizeEmployerForFingerprint("Acme Corporation")).toBe("acme");
|
||||
expect(normalizeEmployerForFingerprint("Acme, LLC")).toBe("acme");
|
||||
expect(normalizeEmployerForFingerprint("Acme GmbH")).toBe("acme");
|
||||
});
|
||||
|
||||
it("normalizeTitleForFingerprint drops leading repost markers", () => {
|
||||
expect(normalizeTitleForFingerprint("[Reposted] Software Engineer")).toBe(
|
||||
"softwareengineer",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Cross-source duplicate detection.
|
||||
*
|
||||
* Two postings from different sources almost always describe the same role
|
||||
* when their employer + title agree once you strip noise (case, punctuation,
|
||||
* tracking suffixes, common decorations like "(Remote)", "- Toronto, ON" etc).
|
||||
* The fingerprint is intentionally coarse so we err on the side of skipping
|
||||
* a duplicate rather than re-showing it from a second source.
|
||||
*/
|
||||
|
||||
const PUNCTUATION_RE = /[\p{P}\p{S}]+/gu;
|
||||
const WHITESPACE_RE = /\s+/g;
|
||||
const LEADING_NOISE_RE = /^(?:re-?post(?:ed)?|new|hot|urgent)\s*[-:]?\s*/i;
|
||||
const PARENS_RE = /\s*[([][^)\]]*[)\]]/g;
|
||||
// Trailing decorations we know are location / arrangement metadata, not role
|
||||
// suffix. Matched after the title body and stripped before fingerprinting.
|
||||
// Examples we want to strip:
|
||||
// "Software Engineer — Remote"
|
||||
// "Senior Engineer - Toronto, ON"
|
||||
// "Designer | Hybrid"
|
||||
// Examples we must NOT strip (otherwise we'd collide unrelated roles):
|
||||
// "Etudiant – Stage" (Stage is a role qualifier in French postings)
|
||||
// "Designer — Senior" (level qualifier)
|
||||
const TRAILING_LOCATION_KEYWORDS_RE =
|
||||
/\s+[-|–—]\s+(?:remote|hybrid|on[\s-]?site|wfh|telework|anywhere)\s*$/i;
|
||||
// Escape the ASCII hyphen explicitly so it doesn't form a character range
|
||||
// with the surrounding delimiters (which would silently swallow letters).
|
||||
const TRAILING_CITY_REGION_RE = /\s+[-|–—]\s+[^,\-|–—]+,\s*[^,\-|–—]+\s*$/;
|
||||
const COMPANY_LEGAL_SUFFIX_RE =
|
||||
/\b(?:inc|inc\.|ltd|ltd\.|llc|gmbh|s\.a\.|s\.r\.l|sa|nv|bv|plc|corp|corporation|co|company|holdings|holding)\b/g;
|
||||
|
||||
function stripDiacritics(input: string): string {
|
||||
return input.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
|
||||
}
|
||||
|
||||
export function normalizeEmployerForFingerprint(
|
||||
employer: string | null | undefined,
|
||||
): string {
|
||||
if (!employer) return "";
|
||||
let value = stripDiacritics(employer.toLowerCase()).trim();
|
||||
value = value.replace(PARENS_RE, " ");
|
||||
value = value.replace(COMPANY_LEGAL_SUFFIX_RE, " ");
|
||||
value = value.replace(PUNCTUATION_RE, " ");
|
||||
value = value.replace(WHITESPACE_RE, "").trim();
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeTitleForFingerprint(
|
||||
title: string | null | undefined,
|
||||
): string {
|
||||
if (!title) return "";
|
||||
let value = stripDiacritics(title.toLowerCase()).trim();
|
||||
value = value.replace(LEADING_NOISE_RE, "");
|
||||
value = value.replace(PARENS_RE, " ");
|
||||
value = value.replace(TRAILING_LOCATION_KEYWORDS_RE, " ");
|
||||
value = value.replace(TRAILING_CITY_REGION_RE, " ");
|
||||
value = value.replace(PUNCTUATION_RE, " ");
|
||||
value = value.replace(WHITESPACE_RE, "").trim();
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a stable, source-agnostic fingerprint for a posting.
|
||||
*
|
||||
* Returns `null` when employer or title is empty after normalization, so
|
||||
* callers fall back to URL/sourceJobId equality and don't accidentally
|
||||
* collapse unrelated rows under the empty key.
|
||||
*/
|
||||
export function buildJobContentFingerprint(args: {
|
||||
employer: string | null | undefined;
|
||||
title: string | null | undefined;
|
||||
}): string | null {
|
||||
const employer = normalizeEmployerForFingerprint(args.employer);
|
||||
const title = normalizeTitleForFingerprint(args.title);
|
||||
if (!employer || !title) return null;
|
||||
return `${employer}::${title}`;
|
||||
}
|
||||
@@ -99,7 +99,12 @@ export const SUPPORTED_COUNTRY_INPUTS = [
|
||||
"worldwide",
|
||||
] as const;
|
||||
|
||||
const UK_ONLY_SOURCES = new Set<JobSource>(["gradcracker", "ukvisajobs"]);
|
||||
const UK_ONLY_SOURCES = new Set<JobSource>([
|
||||
"gradcracker",
|
||||
"ukvisajobs",
|
||||
"reed",
|
||||
]);
|
||||
const US_ONLY_SOURCES = new Set<JobSource>(["usajobs"]);
|
||||
const GLASSDOOR_SUPPORTED_COUNTRIES = new Set(
|
||||
[
|
||||
"australia",
|
||||
@@ -170,6 +175,10 @@ export function isUkCountry(country: string | null | undefined): boolean {
|
||||
return normalizeCountryKey(country) === "united kingdom";
|
||||
}
|
||||
|
||||
export function isUsCountry(country: string | null | undefined): boolean {
|
||||
return normalizeCountryKey(country) === "united states";
|
||||
}
|
||||
|
||||
export function isGlassdoorCountry(
|
||||
country: string | null | undefined,
|
||||
): boolean {
|
||||
@@ -187,6 +196,7 @@ export function isSourceAllowedForCountry(
|
||||
country: string | null | undefined,
|
||||
): boolean {
|
||||
if (UK_ONLY_SOURCES.has(source)) return isUkCountry(country);
|
||||
if (US_ONLY_SOURCES.has(source)) return isUsCountry(country);
|
||||
if (source === "glassdoor") return isGlassdoorCountry(country);
|
||||
if (source === "adzuna") return getAdzunaCountryCode(country) !== null;
|
||||
return true;
|
||||
|
||||
@@ -28,6 +28,24 @@ function parseJsonArrayOrNull(raw: string | undefined): string[] | null {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a delimited list (comma / newline / pipe) into a deduped, trimmed
|
||||
* array. Used for env-backed defaults like LEVER_COMPANIES="acme,stripe".
|
||||
*/
|
||||
function parseCompanyList(raw: string | undefined | null): string[] {
|
||||
if (!raw) return [];
|
||||
const out: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const piece of raw.split(/[\n,;|]+/)) {
|
||||
const value = piece.trim();
|
||||
if (!value) continue;
|
||||
if (seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
out.push(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseBitBoolOrNull(raw: string | undefined): boolean | null {
|
||||
if (!raw) return null;
|
||||
return raw === "true" || raw === "1";
|
||||
@@ -336,6 +354,145 @@ export const settingsRegistry = {
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
// --- New extractor caps & per-source target lists ---
|
||||
usajobsMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number =>
|
||||
parseInt(
|
||||
typeof process !== "undefined"
|
||||
? process.env.USAJOBS_MAX_JOBS_PER_TERM || "100"
|
||||
: "100",
|
||||
10,
|
||||
),
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
jobicyMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
themuseMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
joobleMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
careerjetMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
reedMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
remoteokMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
remotiveMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
arbeitnowMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
himalayasMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
weworkremotelyMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
fourdayweekMaxJobsPerTerm: {
|
||||
kind: "typed" as const,
|
||||
schema: z.number().int().min(1).max(1000),
|
||||
default: (): number => 100,
|
||||
parse: parseIntOrNull,
|
||||
serialize: serializeNullableNumber,
|
||||
},
|
||||
/**
|
||||
* Comma- or newline-separated company slugs to fetch from public ATS feeds.
|
||||
* `lever`, `ashby`, and `greenhouse` each take one entry per company.
|
||||
*/
|
||||
leverCompanies: {
|
||||
kind: "typed" as const,
|
||||
schema: z.array(z.string().trim().min(1).max(100)).max(200),
|
||||
default: (): string[] =>
|
||||
parseCompanyList(
|
||||
typeof process !== "undefined" ? process.env.LEVER_COMPANIES : "",
|
||||
),
|
||||
parse: parseJsonArrayOrNull,
|
||||
serialize: serializeNullableJsonArray,
|
||||
},
|
||||
ashbyCompanies: {
|
||||
kind: "typed" as const,
|
||||
schema: z.array(z.string().trim().min(1).max(100)).max(200),
|
||||
default: (): string[] =>
|
||||
parseCompanyList(
|
||||
typeof process !== "undefined" ? process.env.ASHBY_COMPANIES : "",
|
||||
),
|
||||
parse: parseJsonArrayOrNull,
|
||||
serialize: serializeNullableJsonArray,
|
||||
},
|
||||
greenhouseCompanies: {
|
||||
kind: "typed" as const,
|
||||
schema: z.array(z.string().trim().min(1).max(100)).max(200),
|
||||
default: (): string[] =>
|
||||
parseCompanyList(
|
||||
typeof process !== "undefined" ? process.env.GREENHOUSE_COMPANIES : "",
|
||||
),
|
||||
parse: parseJsonArrayOrNull,
|
||||
serialize: serializeNullableJsonArray,
|
||||
},
|
||||
/**
|
||||
* Workday tenant configurations as JSON, e.g.
|
||||
* `[{"company":"Acme","tenantUrl":"https://acme.wd1.myworkdayjobs.com","sites":["External"]}]`.
|
||||
*/
|
||||
workdayTenants: {
|
||||
kind: "typed" as const,
|
||||
schema: z.array(z.string().trim().min(1).max(2000)).max(50),
|
||||
default: (): string[] =>
|
||||
parseCompanyList(
|
||||
typeof process !== "undefined" ? process.env.WORKDAY_TENANTS : "",
|
||||
),
|
||||
parse: parseJsonArrayOrNull,
|
||||
serialize: serializeNullableJsonArray,
|
||||
},
|
||||
searchTerms: {
|
||||
kind: "typed" as const,
|
||||
schema: z.array(z.string().trim().min(1).max(200)).max(100),
|
||||
@@ -626,6 +783,40 @@ export const settingsRegistry = {
|
||||
envKey: "ADZUNA_APP_ID",
|
||||
schema: z.string().trim().max(200),
|
||||
},
|
||||
// --- New extractor keys / identifiers (non-secret) ---
|
||||
usajobsUserAgent: {
|
||||
kind: "string" as const,
|
||||
envKey: "USAJOBS_USER_AGENT",
|
||||
schema: z.string().trim().max(200),
|
||||
},
|
||||
themuseApiKey: {
|
||||
kind: "string" as const,
|
||||
envKey: "THEMUSE_API_KEY",
|
||||
schema: z.string().trim().max(200),
|
||||
},
|
||||
/** Publisher API key (Basic auth user); Careerjet labels this “API key” in the dashboard. */
|
||||
careerjetAffid: {
|
||||
kind: "string" as const,
|
||||
envKey: "CAREERJET_AFFID",
|
||||
schema: z.string().trim().max(200),
|
||||
},
|
||||
/** Required Referer URL for v4 (your job-search page that triggers API use). */
|
||||
careerjetReferer: {
|
||||
kind: "string" as const,
|
||||
envKey: "CAREERJET_REFERER",
|
||||
schema: z.string().trim().max(500),
|
||||
},
|
||||
/** Must match an IP allowlisted in Careerjet (usually your server egress IP). */
|
||||
careerjetUserIp: {
|
||||
kind: "string" as const,
|
||||
envKey: "CAREERJET_USER_IP",
|
||||
schema: z.string().trim().max(80),
|
||||
},
|
||||
careerjetUserAgent: {
|
||||
kind: "string" as const,
|
||||
envKey: "CAREERJET_USER_AGENT",
|
||||
schema: z.string().trim().max(500),
|
||||
},
|
||||
basicAuthUser: {
|
||||
kind: "string" as const,
|
||||
envKey: "BASIC_AUTH_USER",
|
||||
@@ -658,6 +849,22 @@ export const settingsRegistry = {
|
||||
envKey: "ADZUNA_APP_KEY",
|
||||
schema: z.string().trim().max(2000),
|
||||
},
|
||||
// --- Secrets for new extractors ---
|
||||
usajobsApiKey: {
|
||||
kind: "secret" as const,
|
||||
envKey: "USAJOBS_API_KEY",
|
||||
schema: z.string().trim().max(2000),
|
||||
},
|
||||
joobleApiKey: {
|
||||
kind: "secret" as const,
|
||||
envKey: "JOOBLE_API_KEY",
|
||||
schema: z.string().trim().max(2000),
|
||||
},
|
||||
reedApiKey: {
|
||||
kind: "secret" as const,
|
||||
envKey: "REED_API_KEY",
|
||||
schema: z.string().trim().max(2000),
|
||||
},
|
||||
basicAuthPassword: {
|
||||
kind: "secret" as const,
|
||||
envKey: "BASIC_AUTH_PASSWORD",
|
||||
|
||||
@@ -188,6 +188,22 @@ export const createAppSettings = (
|
||||
adzunaMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
gradcrackerMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
startupjobsMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
usajobsMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
jobicyMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
themuseMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
joobleMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
careerjetMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
reedMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
remoteokMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
remotiveMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
arbeitnowMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
himalayasMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
weworkremotelyMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
fourdayweekMaxJobsPerTerm: { value: 50, default: 50, override: null },
|
||||
leverCompanies: { value: [], default: [], override: null },
|
||||
ashbyCompanies: { value: [], default: [], override: null },
|
||||
greenhouseCompanies: { value: [], default: [], override: null },
|
||||
workdayTenants: { value: [], default: [], override: null },
|
||||
searchTerms: {
|
||||
value: ["Software Engineer"],
|
||||
default: ["Software Engineer"],
|
||||
@@ -256,6 +272,15 @@ export const createAppSettings = (
|
||||
adzunaAppId: null,
|
||||
adzunaAppKeyHint: null,
|
||||
webhookSecretHint: null,
|
||||
usajobsUserAgent: null,
|
||||
themuseApiKey: null,
|
||||
careerjetAffid: null,
|
||||
careerjetReferer: null,
|
||||
careerjetUserIp: null,
|
||||
careerjetUserAgent: null,
|
||||
usajobsApiKeyHint: null,
|
||||
joobleApiKeyHint: null,
|
||||
reedApiKeyHint: null,
|
||||
basicAuthActive: false,
|
||||
localResumeFileConfigured: false,
|
||||
backupEnabled: { value: false, default: false, override: null },
|
||||
|
||||
@@ -201,6 +201,22 @@ export interface AppSettings {
|
||||
adzunaMaxJobsPerTerm: Resolved<number>;
|
||||
gradcrackerMaxJobsPerTerm: Resolved<number>;
|
||||
startupjobsMaxJobsPerTerm: Resolved<number>;
|
||||
usajobsMaxJobsPerTerm: Resolved<number>;
|
||||
jobicyMaxJobsPerTerm: Resolved<number>;
|
||||
themuseMaxJobsPerTerm: Resolved<number>;
|
||||
joobleMaxJobsPerTerm: Resolved<number>;
|
||||
careerjetMaxJobsPerTerm: Resolved<number>;
|
||||
reedMaxJobsPerTerm: Resolved<number>;
|
||||
remoteokMaxJobsPerTerm: Resolved<number>;
|
||||
remotiveMaxJobsPerTerm: Resolved<number>;
|
||||
arbeitnowMaxJobsPerTerm: Resolved<number>;
|
||||
himalayasMaxJobsPerTerm: Resolved<number>;
|
||||
weworkremotelyMaxJobsPerTerm: Resolved<number>;
|
||||
fourdayweekMaxJobsPerTerm: Resolved<number>;
|
||||
leverCompanies: Resolved<string[]>;
|
||||
ashbyCompanies: Resolved<string[]>;
|
||||
greenhouseCompanies: Resolved<string[]>;
|
||||
workdayTenants: Resolved<string[]>;
|
||||
searchTerms: Resolved<string[]>;
|
||||
workplaceTypes: Resolved<Array<"remote" | "hybrid" | "onsite">>;
|
||||
blockedCompanyKeywords: Resolved<string[]>;
|
||||
@@ -241,6 +257,12 @@ export interface AppSettings {
|
||||
ukvisajobsEmail: string | null;
|
||||
adzunaAppId: string | null;
|
||||
basicAuthUser: string | null;
|
||||
usajobsUserAgent: string | null;
|
||||
themuseApiKey: string | null;
|
||||
careerjetAffid: string | null;
|
||||
careerjetReferer: string | null;
|
||||
careerjetUserIp: string | null;
|
||||
careerjetUserAgent: string | null;
|
||||
|
||||
// Secret hints:
|
||||
llmApiKeyHint: string | null;
|
||||
@@ -250,6 +272,9 @@ export interface AppSettings {
|
||||
adzunaAppKeyHint: string | null;
|
||||
basicAuthPasswordHint: string | null;
|
||||
webhookSecretHint: string | null;
|
||||
usajobsApiKeyHint: string | null;
|
||||
joobleApiKeyHint: string | null;
|
||||
reedApiKeyHint: string | null;
|
||||
|
||||
// Computed:
|
||||
basicAuthActive: boolean;
|
||||
|
||||
Reference in New Issue
Block a user