Jobber/extractors/testdevjobs/manifest.ts

/**
 * TestDevJobs — QA-focused board (Gridsome SSR HTML + embedded job state).
 *
 * https://testdevjobs.com/software-testing-jobs/
 */

import type {
  ExtractorManifest,
  ExtractorRunResult,
} from "@shared/types/extractors";
import type { CreateJobInput } from "@shared/types/jobs";

const ORIGIN = "https://testdevjobs.com";
const LIST_PATH = "/software-testing-jobs/";

interface ListingJob {
  path: string;
  title: string;
  employer: string;
  datePosted?: string;
  location?: string;
  jobType?: string;
}

interface TestDevJobState {
  id?: string;
  jobTitle?: string;
  path?: string;
  jobDescription?: string;
  joblocation?: string;
  jobType?: string;
  salary?: string;
  isRemote?: boolean;
  applyLink?: string;
  companyName?: string;
  jobPosted?: string;
}

function asString(value: unknown): string | undefined {
  if (typeof value !== "string") return undefined;
  const trimmed = value.trim();
  return trimmed ? trimmed : undefined;
}

function decodeHtmlEntities(value: string): string {
  return value
    .replace(/&/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'");
}

function stripHtml(html: string): string {
  return decodeHtmlEntities(html)
    .replace(/<[^>]+>/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

function readMaxPages(raw: string | undefined): number {
  const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
  if (!Number.isFinite(parsed)) return 3;
  return Math.min(Math.max(parsed, 1), 20);
}

function parseListingJobs(html: string): ListingJob[] {
  const jobs: ListingJob[] = [];
  const blocks = html.match(
    /<div class="job-tile-wrapper[\s\S]*?<\/div>\s*<\/div>\s*<\/div>/g,
  );

  for (const block of blocks ?? []) {
    const pathMatch = block.match(/href="(\/job\/[^"]+)"/);
    const titleMatch = block.match(/class="ml-5 jobtitle[^"]*"[^>]*>([^<]+)</);
    const employerMatch = block.match(
      /class="mb-0 comptitle[^"]*"[^>]*>([^<]+)</,
    );
    const dateMatch = block.match(/itemprop="datePosted"[^>]*>([^<]+)</);
    const locationMatch = block.match(
      /itemprop="addressLocality"[^>]*>([^<]+)</,
    );
    const typeMatch = block.match(/itemprop="employmentType"[^>]*>([^<]+)</);

    const path = pathMatch?.[1];
    const title = titleMatch?.[1]?.trim();
    if (!path || !title) continue;

    jobs.push({
      path,
      title: decodeHtmlEntities(title),
      employer: decodeHtmlEntities(
        employerMatch?.[1]?.trim() ?? "Unknown Employer",
      ),
      datePosted: dateMatch?.[1]?.trim(),
      location: locationMatch?.[1]?.trim(),
      jobType: typeMatch?.[1]?.trim(),
    });
  }

  return jobs;
}

function parseInitialState(html: string): TestDevJobState | null {
  const match = html.match(/window\.__INITIAL_STATE__=({[\s\S]*?});/);
  if (!match?.[1]) return null;

  try {
    const parsed = JSON.parse(match[1]) as {
      data?: { job?: TestDevJobState };
    };
    return parsed.data?.job ?? null;
  } catch {
    return null;
  }
}

function matchesTerm(values: string[], term: string): boolean {
  const lower = term.toLowerCase();
  return values.some((value) => value.toLowerCase().includes(lower));
}

async function fetchText(url: string): Promise<string> {
  const response = await fetch(url, {
    headers: { Accept: "text/html", "User-Agent": "JobOps/1.0" },
  });
  if (!response.ok) {
    throw new Error(
      `TestDevJobs request failed (${response.status}) for ${url}`,
    );
  }
  return response.text();
}

function listingUrl(page: number): string {
  if (page <= 1) return `${ORIGIN}${LIST_PATH}`;
  return `${ORIGIN}${LIST_PATH}${page}/`;
}

export const manifest: ExtractorManifest = {
  id: "testdevjobs",
  displayName: "TestDevJobs",
  providesSources: ["testdevjobs"],
  async run(context): Promise<ExtractorRunResult> {
    if (context.shouldCancel?.()) return { success: true, jobs: [] };

    const maxJobs = context.settings.testdevjobsMaxJobsPerTerm
      ? Number.parseInt(context.settings.testdevjobsMaxJobsPerTerm, 10)
      : 100;
    const cap = Number.isFinite(maxJobs)
      ? Math.min(Math.max(maxJobs, 1), 500)
      : 100;
    const maxPages = readMaxPages(context.settings.testdevjobsMaxPages);
    const terms = context.searchTerms.length > 0 ? context.searchTerms : [];
    const enrichDetails = true;

    const seen = new Set<string>();
    const out: CreateJobInput[] = [];

    try {
      for (let page = 1; page <= maxPages; page += 1) {
        if (context.shouldCancel?.()) break;
        if (out.length >= cap) break;

        const url = listingUrl(page);
        context.onProgress?.({
          phase: "list",
          termsProcessed: page - 1,
          termsTotal: maxPages,
          currentUrl: url,
          detail: `TestDevJobs: listing page ${page}/${maxPages}`,
        });

        const html = await fetchText(url);
        const listings = parseListingJobs(html);
        if (listings.length === 0) break;

        for (const listing of listings) {
          if (context.shouldCancel?.()) break;
          if (out.length >= cap) break;

          const haystack = [
            listing.title,
            listing.employer,
            listing.location ?? "",
            listing.jobType ?? "",
          ];
          if (
            terms.length > 0 &&
            !terms.some((term) => matchesTerm(haystack, term))
          ) {
            continue;
          }

          let jobUrl = `${ORIGIN}${listing.path}`;
          let applicationLink = jobUrl;
          let jobDescription: string | undefined;
          let location = listing.location;
          let jobType = listing.jobType;
          let datePosted = listing.datePosted;
          let isRemote = listing.location?.toLowerCase().includes("remote");

          if (enrichDetails) {
            try {
              const detailHtml = await fetchText(jobUrl);
              const state = parseInitialState(detailHtml);
              if (state) {
                if (state.path) jobUrl = `${ORIGIN}${state.path}`;
                if (state.applyLink) applicationLink = state.applyLink;
                if (state.jobDescription) jobDescription = state.jobDescription;
                if (state.joblocation) location = state.joblocation;
                if (state.jobType) jobType = state.jobType;
                if (state.jobPosted) datePosted = state.jobPosted;
                if (typeof state.isRemote === "boolean")
                  isRemote = state.isRemote;
              }
            } catch {
              // keep listing row when detail fetch fails
            }
          }

          const key = listing.path;
          if (seen.has(key)) continue;
          seen.add(key);

          out.push({
            source: "testdevjobs",
            sourceJobId: key.split("/").filter(Boolean).pop(),
            title: listing.title,
            employer: listing.employer,
            jobUrl,
            applicationLink,
            location: location ?? "Unknown",
            isRemote,
            datePosted,
            jobDescription,
            jobType,
          });
        }
      }

      context.onProgress?.({
        phase: "list",
        termsProcessed: maxPages,
        termsTotal: maxPages,
        currentUrl: `${ORIGIN}${LIST_PATH}`,
        jobPagesProcessed: out.length,
        detail: `TestDevJobs: ${out.length} jobs`,
      });

      return { success: true, jobs: out };
    } catch (error) {
      const message = error instanceof Error ? error.message : "Unknown error";
      return { success: false, jobs: out, error: message };
    }
  },
};

export default manifest;