Jobber/extractors/javascriptjobs/manifest.ts

/**
 * JS Remotely / javascript.jobs — JS/TS-focused remote board (server-rendered
 * HTML, no public API or RSS).
 *
 * https://javascript.jobs/remote?keyword=<term>&page=<n>
 *
 * `jsremotely.com` now 301s straight to `javascript.jobs`, so we hit the
 * canonical domain directly. The `/remote` listing supports a `keyword`
 * query param that filters server-side, so we iterate pipeline search terms
 * as separate paginated fetches rather than filtering client-side.
 *
 * Listing cards give us title/employer/badges/salary text; detail pages
 * additionally embed a `JobPosting` (schema.org) block for description,
 * exact date posted, employment type, and structured salary. That block is
 * emitted with unescaped newlines and occasional missing commas, so it is
 * not valid JSON — we pull fields out with targeted regexes instead of
 * `JSON.parse`.
 */

import type {
  ExtractorManifest,
  ExtractorRunResult,
} from "@shared/types/extractors";
import type { CreateJobInput } from "@shared/types/jobs";

const ORIGIN = "https://javascript.jobs";
const USER_AGENT = "JobOps/1.0 (+https://github.com/) job-search pipeline";
/** One job card's markup is well under this; keeps field regexes from
 * bleeding into the next card (or page footer for the last card). */
const CARD_WINDOW = 2500;

interface ListingJob {
  slug: string;
  title: string;
  employer: string;
  jobType?: string;
  isRemoteBadge: boolean;
  salaryText?: string;
  companyLogo?: string;
}

function decodeHtmlEntities(value: string): string {
  return value
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'");
}

function collapseWhitespace(value: string): string {
  return value.replace(/\s+/g, " ").trim();
}

function parseListingJobs(html: string): ListingJob[] {
  const chunks = html.split('<a href="https://javascript.jobs/job/').slice(1);
  const jobs: ListingJob[] = [];

  for (const chunk of chunks) {
    const slug = chunk.split('"')[0]?.trim();
    if (!slug) continue;
    const card = chunk.slice(0, CARD_WINDOW);

    const titleMatch = card.match(
      /tw-text-lg tw-font-medium">\s*([^<]+?)\s*<\/div>/,
    );
    const employerMatch = card.match(/tw-card-title">\s*([^<]+?)\s*<\/span>/);
    if (!titleMatch || !employerMatch) continue;

    const badges = [
      ...card.matchAll(/tw-rounded-\[3px\]">\s*([^<]+?)\s*<\/span>/g),
    ].map((m) => collapseWhitespace(m[1]));
    const isRemoteBadge = badges.some((b) => /^remote$/i.test(b));
    const jobType = badges.find((b) => !/^remote$/i.test(b));

    const salaryMatch = card.match(/Salary:\s*([\s\S]*?)<\/span>/);
    const logoMatch = card.match(/<img[^>]*src="([^"]+)"/);

    jobs.push({
      slug,
      title: decodeHtmlEntities(titleMatch[1].trim()),
      employer: decodeHtmlEntities(employerMatch[1].trim()),
      jobType,
      isRemoteBadge,
      salaryText: salaryMatch
        ? collapseWhitespace(decodeHtmlEntities(salaryMatch[1]))
        : undefined,
      companyLogo: logoMatch?.[1],
    });
  }

  return jobs;
}

interface DetailFields {
  description?: string;
  datePosted?: string;
  employmentType?: string;
  organizationName?: string;
  organizationLogo?: string;
  salaryCurrency?: string;
  salaryMin?: number;
  salaryMax?: number;
  salaryUnit?: string;
}

/** Best-effort string field extraction over the (invalid-JSON) JobPosting
 * block: stops at the next quoted key or a closing brace rather than
 * assuming a specific neighboring key or consistent `key: value` spacing. */
function extractStringField(raw: string, key: string): string | undefined {
  const pattern = new RegExp(
    `"${key}"\\s*:\\s*"([\\s\\S]*?)"\\s*(?=,\\s*\\n?\\s*"[A-Za-z]+"\\s*:|\\n?\\s*})`,
  );
  const match = raw.match(pattern);
  const value = match?.[1]?.trim();
  return value ? value : undefined;
}

/** javascript.jobs emits `minValue`/`maxValue: 0` placeholders even when no
 * salary was entered, so treat 0 the same as "not specified". */
function extractNumberField(raw: string, key: string): number | undefined {
  const match = raw.match(new RegExp(`"${key}"\\s*:\\s*([0-9.]+)`));
  const value = match ? Number.parseFloat(match[1]) : undefined;
  return value != null && Number.isFinite(value) && value > 0
    ? value
    : undefined;
}

function extractObjectBlock(raw: string, key: string): string | undefined {
  const match = raw.match(new RegExp(`"${key}"\\s*:\\s*{([\\s\\S]*?)}`));
  return match?.[1];
}

function parseJobPosting(html: string): DetailFields | null {
  const match = html.match(
    /<script type="application\/ld\+json">([\s\S]*?)<\/script>/,
  );
  const raw = match?.[1];
  if (!raw) return null;

  const orgBlock = extractObjectBlock(raw, "hiringOrganization");
  // baseSalary nests a `value` object inside it; matching keys over the
  // whole ld+json block is simpler than balancing the extra brace pair and
  // these key names don't collide with anything else in this schema.
  const salaryUnit = extractStringField(raw, "unitText");

  return {
    description: extractStringField(raw, "description"),
    datePosted: extractStringField(raw, "datePosted"),
    employmentType: extractStringField(raw, "employmentType"),
    organizationName: orgBlock
      ? extractStringField(orgBlock, "name")
      : undefined,
    organizationLogo: orgBlock
      ? extractStringField(orgBlock, "logo")
      : undefined,
    salaryCurrency: extractStringField(raw, "currency"),
    salaryMin: extractNumberField(raw, "minValue"),
    salaryMax: extractNumberField(raw, "maxValue"),
    salaryUnit,
  };
}

function normalizeJobType(raw: string | undefined): string | undefined {
  if (!raw) return undefined;
  return raw.replace(/_/g, " ").trim() || undefined;
}

function salaryIntervalFromUnit(unit: string | undefined): string | undefined {
  if (!unit) return undefined;
  const lower = unit.toLowerCase();
  if (lower.startsWith("year")) return "yearly";
  if (lower.startsWith("month")) return "monthly";
  if (lower.startsWith("week")) return "weekly";
  if (lower.startsWith("day")) return "daily";
  if (lower.startsWith("hour")) return "hourly";
  return undefined;
}

async function fetchText(url: string): Promise<string> {
  const response = await fetch(url, {
    headers: { Accept: "text/html", "User-Agent": USER_AGENT },
  });
  if (!response.ok) {
    throw new Error(
      `javascript.jobs request failed (${response.status}) for ${url}`,
    );
  }
  return response.text();
}

function listingUrl(term: string, page: number): string {
  const url = new URL(`${ORIGIN}/remote`);
  if (term) url.searchParams.set("keyword", term);
  if (page > 1) url.searchParams.set("page", String(page));
  return url.toString();
}

function readCappedInt(
  raw: string | undefined,
  fallback: number,
  min: number,
  max: number,
): number {
  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
  if (!Number.isFinite(parsed)) return fallback;
  return Math.min(Math.max(parsed, min), max);
}

export const manifest: ExtractorManifest = {
  id: "javascriptjobs",
  displayName: "JS Remotely (javascript.jobs)",
  providesSources: ["javascriptjobs"],
  async run(context): Promise<ExtractorRunResult> {
    if (context.shouldCancel?.()) return { success: true, jobs: [] };

    const cap = readCappedInt(
      context.settings.javascriptjobsMaxJobsPerTerm,
      50,
      1,
      500,
    );
    const maxPages = readCappedInt(
      context.settings.javascriptjobsMaxPages,
      3,
      1,
      20,
    );
    const terms = context.searchTerms.length > 0 ? context.searchTerms : [""];

    const seen = new Set<string>();
    const out: CreateJobInput[] = [];

    try {
      for (let t = 0; t < terms.length; t += 1) {
        if (context.shouldCancel?.()) break;
        const term = terms[t].trim();
        let collectedForTerm = 0;

        for (let page = 1; page <= maxPages; page += 1) {
          if (context.shouldCancel?.()) break;
          if (collectedForTerm >= cap) break;

          const url = listingUrl(term, page);
          context.onProgress?.({
            phase: "list",
            termsProcessed: t,
            termsTotal: terms.length,
            currentUrl: url,
            detail: `JS Remotely: term ${t + 1}/${terms.length}, page ${page}/${maxPages}`,
          });

          const html = await fetchText(url);
          const listings = parseListingJobs(html);
          if (listings.length === 0) break;

          for (const listing of listings) {
            if (collectedForTerm >= cap) break;
            if (seen.has(listing.slug)) continue;

            const jobUrl = `${ORIGIN}/job/${listing.slug}`;
            let jobDescription: string | undefined;
            let datePosted: string | undefined;
            let jobType = listing.jobType;
            let employer = listing.employer;
            let companyLogo = listing.companyLogo;
            let salaryCurrency: string | undefined;
            let salaryMinAmount: number | undefined;
            let salaryMaxAmount: number | undefined;
            let salaryInterval: string | undefined;

            try {
              const detailHtml = await fetchText(jobUrl);
              const detail = parseJobPosting(detailHtml);
              if (detail) {
                jobDescription = detail.description;
                datePosted = detail.datePosted;
                jobType = normalizeJobType(detail.employmentType) ?? jobType;
                employer = detail.organizationName ?? employer;
                companyLogo = detail.organizationLogo ?? companyLogo;
                if (detail.salaryMin != null || detail.salaryMax != null) {
                  salaryCurrency = detail.salaryCurrency;
                  salaryMinAmount = detail.salaryMin;
                  salaryMaxAmount = detail.salaryMax;
                  salaryInterval = salaryIntervalFromUnit(detail.salaryUnit);
                }
              }
            } catch {
              // Fall back to listing-card fields when the detail fetch fails.
            }

            seen.add(listing.slug);
            collectedForTerm += 1;
            out.push({
              source: "javascriptjobs",
              sourceJobId: listing.slug,
              title: listing.title,
              employer,
              jobUrl,
              applicationLink: jobUrl,
              location: "Remote",
              isRemote: true,
              jobType,
              datePosted,
              jobDescription,
              companyLogo,
              salary: listing.salaryText,
              salaryCurrency,
              salaryMinAmount,
              salaryMaxAmount,
              salaryInterval,
            });
          }
        }

        context.onProgress?.({
          phase: "list",
          termsProcessed: t + 1,
          termsTotal: terms.length,
          jobPagesProcessed: out.length,
          detail: `JS Remotely: completed term ${t + 1}/${terms.length} (${collectedForTerm} found)`,
        });
      }

      return { success: true, jobs: out };
    } catch (error) {
      const message = error instanceof Error ? error.message : "Unknown error";
      return { success: false, jobs: out, error: message };
    }
  },
};

export default manifest;