rename extractors to their own folder

extractors/gradcracker/.dockerignore

@@ -0,0 +1,8 @@
+# configurations
+.idea
+
+# crawlee storage folder
+storage
+
+# installed files
+node_modules

extractors/gradcracker/.gitignore

@@ -0,0 +1,7 @@
+# This file tells Git which files shouldn't be added to source control
+
+.idea
+dist
+node_modules
+storage
+

extractors/gradcracker/Dockerfile

@@ -0,0 +1,52 @@
+# Specify the base Docker image. You can read more about
+# the available images at https://crawlee.dev/docs/guides/docker-images
+# You can also use any other image from Docker Hub.
+FROM apify/actor-node-playwright-chrome:20-1.50.1 AS builder
+
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
+
+# Install all dependencies. Don't audit to speed up the installation.
+RUN npm install --include=dev --audit=false
+
+# Next, copy the source files using the user set
+# in the base image.
+COPY --chown=myuser . ./
+
+# Install all dependencies and build the project.
+# Don't audit to speed up the installation.
+RUN npm run build
+
+# Create final image
+FROM apify/actor-node-playwright-chrome:20-1.50.1
+
+# Copy only built JS files from builder image
+COPY --from=builder --chown=myuser /home/myuser/dist ./dist
+
+# Copy just package.json and package-lock.json
+# to speed up the build using Docker layer cache.
+COPY --chown=myuser package*.json ./
+
+# Ensure we'll install Camoufox using the npm postinstall script
+ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0
+# Install NPM packages, skip optional and development dependencies to
+# keep the image small. Avoid logging too much and print the dependency
+# tree for debugging
+RUN npm --quiet set progress=false \
+    && npm install --omit=dev \
+    && echo "Installed NPM packages:" \
+    && (npm list --omit=dev --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+# Next, copy the remaining files and directories with the source code.
+# Since we do this after NPM install, quick build will be really fast
+# for most source file changes.
+COPY --chown=myuser . ./
+
+# Run the image. If you know you won't need headful browsers,
+# you can remove the XVFB start script for a micro perf gain.
+CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent

extractors/gradcracker/README.md

@@ -0,0 +1,8 @@
+# Crawlee + PlaywrightCrawler + Camoufox + TypeScript project
+
+This template is a production ready boilerplate for developing with `PlaywrightCrawler`. Use this to bootstrap your projects using the most up-to-date code.
+
+If you're looking for examples or want to learn more visit:
+
+- [Documentation](https://crawlee.dev/js/api/playwright-crawler/class/PlaywrightCrawler)
+- [Examples](https://crawlee.dev/js/docs/examples/playwright-crawler)

extractors/gradcracker/package.json

@@ -0,0 +1,30 @@
+{
+    "name": "job-flow",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of a Crawlee project.",
+    "dependencies": {
+        "camoufox-js": "^0.8.0",
+        "crawlee": "^3.0.0",
+        "playwright": "*"
+    },
+    "devDependencies": {
+        "@apify/tsconfig": "^0.1.0",
+        "@types/fs-extra": "^11",
+        "@types/node": "^24.0.0",
+        "fs-extra": "^11.3.0",
+        "tsx": "^4.4.0",
+        "typescript": "~5.9.0"
+    },
+    "scripts": {
+        "start": "npm run start:dev",
+        "start:prod": "node dist/main.js",
+        "start:dev": "tsx src/main.ts",
+        "build": "tsc",
+        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
+        "get-binaries": "camoufox-js fetch",
+        "postinstall": "npm run get-binaries"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

extractors/gradcracker/src/main.ts

@@ -0,0 +1,66 @@
+// For more information, see https://crawlee.dev/
+import { launchOptions } from "camoufox-js";
+import { PlaywrightCrawler } from "crawlee";
+import { firefox } from "playwright";
+
+import { router } from "./routes.js";
+import { initJobOpsProgress } from "./progress.js";
+
+// locations
+const locations = [
+  "london-and-south-east",
+  "north-west",
+  "yorkshire",
+  "east-midlands",
+  "west-midlands",
+  "south-west",
+];
+
+// roles
+const roles = [
+  "web-development",
+  "software-systems",
+];
+
+// combo of locations and roles
+const gradcrackerUrls = locations.flatMap((location) => {
+  return roles.map((role) => {
+    return `https://www.gradcracker.com/search/computing-technology/${role}-graduate-jobs-in-${location}?order=dateAdded`;
+  });
+});
+
+console.log(`Total gradcracker URLs: ${gradcrackerUrls.length}`)
+
+const startUrls = gradcrackerUrls.map((url) => ({
+  url,
+  userData: { label: "gradcracker-list-page" },
+}));
+
+initJobOpsProgress(startUrls.length);
+
+const crawler = new PlaywrightCrawler({
+  // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
+  requestHandler: router,
+  // Comment this option to scrape the full website.
+  // maxRequestsPerCrawl: 2000,
+  // Add delay between requests to slow down the process
+  minConcurrency: 1,
+  maxConcurrency: 2,
+  navigationTimeoutSecs: 60,
+  // Add delay between requests (in milliseconds)
+  requestHandlerTimeoutSecs: 100,
+  browserPoolOptions: {
+    // Disable the default fingerprint spoofing to avoid conflicts with Camoufox.
+    useFingerprints: false,
+  },
+  launchContext: {
+    launcher: firefox,
+    launchOptions: await launchOptions({
+      headless: true,
+      humanize: true,
+      geoip: true,
+    }),
+  },
+});
+
+await crawler.run(startUrls);

extractors/gradcracker/src/progress.ts

@@ -0,0 +1,83 @@
+type CrawlPhase = "list" | "job";
+
+export interface JobOpsCrawlProgressPayload {
+  phase: CrawlPhase;
+  currentUrl?: string;
+  listPagesProcessed: number;
+  listPagesTotal?: number;
+  jobCardsFound: number;
+  jobPagesEnqueued: number;
+  jobPagesSkipped: number;
+  jobPagesProcessed: number;
+  ts: string;
+}
+
+interface JobOpsCrawlProgressState {
+  listPagesProcessed: number;
+  listPagesTotal?: number;
+  jobCardsFound: number;
+  jobPagesEnqueued: number;
+  jobPagesSkipped: number;
+  jobPagesProcessed: number;
+  currentUrl?: string;
+  phase: CrawlPhase;
+}
+
+const PROGRESS_PREFIX = "JOBOPS_PROGRESS ";
+const isEnabled = () => process.env.JOBOPS_EMIT_PROGRESS === "1";
+
+let state: JobOpsCrawlProgressState = {
+  listPagesProcessed: 0,
+  jobCardsFound: 0,
+  jobPagesEnqueued: 0,
+  jobPagesSkipped: 0,
+  jobPagesProcessed: 0,
+  phase: "list",
+};
+
+function emit(): void {
+  if (!isEnabled()) return;
+
+  const payload: JobOpsCrawlProgressPayload = {
+    phase: state.phase,
+    currentUrl: state.currentUrl,
+    listPagesProcessed: state.listPagesProcessed,
+    listPagesTotal: state.listPagesTotal,
+    jobCardsFound: state.jobCardsFound,
+    jobPagesEnqueued: state.jobPagesEnqueued,
+    jobPagesSkipped: state.jobPagesSkipped,
+    jobPagesProcessed: state.jobPagesProcessed,
+    ts: new Date().toISOString(),
+  };
+
+  process.stdout.write(`${PROGRESS_PREFIX}${JSON.stringify(payload)}\n`);
+}
+
+export function initJobOpsProgress(listPagesTotal: number): void {
+  state.listPagesTotal = listPagesTotal;
+  state.phase = "list";
+  emit();
+}
+
+export function markListPageDone(params: {
+  currentUrl: string;
+  jobCardsFound: number;
+  jobPagesEnqueued: number;
+  jobPagesSkipped: number;
+}): void {
+  state.listPagesProcessed += 1;
+  state.phase = "list";
+  state.currentUrl = params.currentUrl;
+  state.jobCardsFound += params.jobCardsFound;
+  state.jobPagesEnqueued += params.jobPagesEnqueued;
+  state.jobPagesSkipped += params.jobPagesSkipped;
+  emit();
+}
+
+export function markJobPageDone(params: { currentUrl: string }): void {
+  state.jobPagesProcessed += 1;
+  state.phase = "job";
+  state.currentUrl = params.currentUrl;
+  emit();
+}
+

extractors/gradcracker/src/routes.ts

@@ -0,0 +1,317 @@
+import { createPlaywrightRouter, log } from "crawlee";
+import { readFileSync } from "node:fs";
+import { markJobPageDone, markListPageDone } from "./progress.js";
+
+function normalizeUrl(raw: string | null | undefined): string | null {
+  if (!raw) return null;
+  try {
+    const url = new URL(raw);
+    url.hash = "";
+    // Keep search params (some sites encode job IDs there); just normalize trailing slash.
+    const normalized = url.toString().replace(/\/$/, "");
+    return normalized;
+  } catch {
+    return raw.replace(/\/$/, "");
+  }
+}
+
+function getExistingJobUrlSet(): Set<string> {
+  const filePath = process.env.JOBOPS_EXISTING_JOB_URLS_FILE;
+  const raw =
+    filePath
+      ? (() => {
+          try {
+            return readFileSync(filePath, "utf-8");
+          } catch {
+            return null;
+          }
+        })()
+      : process.env.JOBOPS_EXISTING_JOB_URLS;
+
+  if (!raw) return new Set();
+  try {
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) return new Set();
+    const normalized = parsed
+      .map((u) => normalizeUrl(typeof u === "string" ? u : null))
+      .filter((u): u is string => Boolean(u));
+    return new Set(normalized);
+  } catch {
+    return new Set();
+  }
+}
+
+const SKIP_APPLY_FOR_EXISTING = process.env.JOBOPS_SKIP_APPLY_FOR_EXISTING === "1";
+const EXISTING_JOB_URLS = getExistingJobUrlSet();
+
+interface Job {
+  title: string | null;
+  jobUrl: string | null;
+  employer: string | null;
+  employerUrl: string | null;
+  disciplines: string | null;
+  deadline: string | null;
+  salary: string | null;
+  location: string | null;
+  degreeRequired: string | null;
+  starting: string | null;
+}
+
+export const router = createPlaywrightRouter();
+
+router.addHandler(
+  "gradcracker-list-page",
+  async ({ page, request, enqueueLinks }) => {
+    log.info(`Processing: ${request.url}`);
+
+    // Wait until the job cards are rendered
+    await page.waitForSelector("article[wire\\:key]", { timeout: 10000 });
+
+    // Add delay to see the page load
+    await page.waitForTimeout(3000);
+
+    const toAbsolute = (href: string | null) => {
+      if (!href) return null;
+      try {
+        return new URL(href, request.loadedUrl).href;
+      } catch {
+        return href;
+      }
+    };
+
+    const articles = await page.locator("article[wire\\:key]").all();
+    const jobs: Job[] = [];
+    let skippedKnownJobs = 0;
+    let enqueuedJobs = 0;
+
+    console.log(`${articles.length} jobs found`);
+
+    let idx = 1;
+    for (const article of articles) {
+      const titleLocator = article.locator("h2 a");
+      const title = (await titleLocator.textContent())?.trim() ?? null;
+      const jobUrl = toAbsolute(await titleLocator.getAttribute("href"));
+
+      const employerImg = article.locator("figure img");
+      const employer = (await employerImg.getAttribute("alt"))?.trim() ?? null;
+
+      const employerAnchor = article.locator("figure a");
+      const employerUrl = toAbsolute(await employerAnchor.getAttribute("href"));
+
+      let disciplines: string | null = null;
+      try {
+        const disciplinesEl = article.locator("h3");
+        disciplines = (await disciplinesEl.textContent({ timeout: 1000 }))?.trim() ?? null;
+      } catch {
+        // h3 not found or timed out - that's okay, disciplines is optional
+      }
+
+      // Find the "Deadline: ..." pill
+      const deadlineLocator = article
+        .locator("div", { hasText: "Deadline:" })
+        .first();
+      let deadline: string | null = null;
+      if ((await deadlineLocator.count()) > 0) {
+        const deadlineText = (await deadlineLocator.textContent()) ?? "";
+        // Extract deadline and clean up whitespace
+        deadline =
+          deadlineText
+            .replace("Deadline:", "")
+            .split("\n")[0] // Take only first line
+            .trim() || null;
+      }
+
+      const getDdText = async (label: string) => {
+        // Find dt that has the exact label text (ignoring whitespace)
+        const dt = article
+          .locator("dt")
+          .filter({ hasText: new RegExp(`^\\s*${label}\\s*$`) });
+        if ((await dt.count()) === 0) return null;
+
+        // Get the next sibling dd
+        const dd = dt.locator("+ dd");
+        if ((await dd.count()) > 0) {
+          const text = await dd.textContent();
+          if (!text) return null;
+          // Clean up: remove extra whitespace and newlines
+          return text.replace(/\s+/g, " ").trim() || null;
+        }
+        return null;
+      };
+
+      const salary = await getDdText("Salary");
+      const location = await getDdText("Location");
+      const degreeRequired = await getDdText("Degree required");
+      const starting = await getDdText("Starting");
+
+      console.log(`Got job ${idx}/${articles.length}: ${title}`);
+
+      jobs.push({
+        title,
+        jobUrl,
+        employer,
+        employerUrl,
+        disciplines,
+        deadline,
+        salary,
+        location,
+        degreeRequired,
+        starting,
+      });
+
+      idx++;
+
+      // append more links to crawl: single job pages
+      if (jobUrl) {
+        const jobUrlNormalized = normalizeUrl(jobUrl);
+        const isKnownJob =
+          SKIP_APPLY_FOR_EXISTING &&
+          jobUrlNormalized !== null &&
+          EXISTING_JOB_URLS.has(jobUrlNormalized);
+
+        if (isKnownJob) {
+          skippedKnownJobs++;
+        } else {
+          await enqueueLinks({
+            urls: [jobUrl],
+            userData: {
+              ...jobs[jobs.length - 1],
+              label: "gradcracker-single-job-page"
+            },
+          });
+          enqueuedJobs++;
+        }
+      }
+    }
+
+    log.info(`Extracted ${jobs.length} jobs`);
+    if (SKIP_APPLY_FOR_EXISTING && skippedKnownJobs > 0) {
+      log.info(
+        `Skipping ${skippedKnownJobs} already-known job pages; enqueued ${enqueuedJobs} new job pages.`
+      );
+    }
+
+    markListPageDone({
+      currentUrl: request.url,
+      jobCardsFound: jobs.length,
+      jobPagesEnqueued: enqueuedJobs,
+      jobPagesSkipped: skippedKnownJobs,
+    });
+  }
+);
+
+router.addHandler(
+  "gradcracker-single-job-page",
+  async ({ page, request, pushData, log }) => {
+    const { label, ...jobSummary } = request.userData;
+    log.info(`Processing single job page: ${request.url}`);
+
+    // Wait for job content to be present
+    await page.waitForSelector(".body-content", { timeout: 10000 });
+
+    // Optional delay if you want to visually see it while debugging
+    await page.waitForTimeout(2000);
+
+    const jobDescription =
+      (await page.locator(".body-content").textContent())?.trim() || null;
+
+    const applyButton = page.locator('a[dusk="apply-button"]');
+    const hasApplyButton = (await applyButton.count()) > 0;
+
+    const requestUrlNormalized = normalizeUrl(request.url);
+    const isKnownJob =
+      SKIP_APPLY_FOR_EXISTING &&
+      requestUrlNormalized !== null &&
+      EXISTING_JOB_URLS.has(requestUrlNormalized);
+
+    let applicationLink: string | null = null;
+    let spawnedPage: typeof page | null = null;
+
+    if (hasApplyButton && !isKnownJob) {
+      const originalUrl = page.url();
+
+      // Prefer page-scoped popup detection. Using the browser context's "page" event
+      // can accidentally capture unrelated pages created by other concurrent requests.
+      const popupPromise = page.waitForEvent("popup", { timeout: 8000 }).catch(() => null);
+      const navigationPromise = page
+        .waitForNavigation({ timeout: 8000, waitUntil: "domcontentloaded" })
+        .catch(() => null);
+
+      try {
+        // Don't let Playwright auto-wait for navigation; we explicitly handle popup vs same-tab.
+        await applyButton.click();
+
+        // Wait for URL to stabilize (same URL for 3 consecutive checks)
+        const waitForUrlStable = async (targetPage: typeof page, maxWaitMs = 10000, checkIntervalMs = 100, requiredStableChecks = 3) => {
+          let lastUrl = targetPage.url();
+          let stableCount = 0;
+          const startTime = Date.now();
+
+          while (Date.now() - startTime < maxWaitMs) {
+            await targetPage.waitForTimeout(checkIntervalMs);
+            const currentUrl = targetPage.url();
+            if (currentUrl === lastUrl && !currentUrl.includes("gradcracker")) {
+              stableCount++;
+              if (stableCount >= requiredStableChecks) return currentUrl;
+            } else {
+              stableCount = 1;
+              lastUrl = currentUrl;
+            }
+          }
+          return lastUrl;
+        };
+
+        await waitForUrlStable(page);
+
+        const maybePopup = await popupPromise;
+        spawnedPage = maybePopup;
+
+        const targetPage = maybePopup ?? page;
+
+        if (maybePopup) {
+          await maybePopup.waitForLoadState("domcontentloaded", { timeout: 15000 }).catch(() => null);
+          // If the popup initially opens as about:blank, give it a moment to redirect.
+          if (maybePopup.url() === "about:blank") {
+            await maybePopup
+              .waitForURL((u) => u.toString() !== "about:blank", { timeout: 15000 })
+              .catch(() => null);
+          }
+        } else {
+          // Same-tab navigation case.
+          await navigationPromise;
+          await page
+            .waitForURL((u) => u.toString() !== originalUrl, { timeout: 15000 })
+            .catch(() => null);
+        }
+
+        applicationLink = targetPage.url();
+
+        if (applicationLink === originalUrl) {
+          log.info(
+            `Apply click did not change URL (still Gradcracker): ${applicationLink}`
+          );
+        } else {
+          log.info(`Captured application URL: ${applicationLink}`);
+        }
+      } finally {
+        // Ensure we don't leak tabs on retries/errors.
+        if (spawnedPage && spawnedPage !== page) {
+          await spawnedPage.close().catch(() => null);
+        }
+      }
+    } else if (!hasApplyButton) {
+      log.warning(`Apply button not found on page: ${request.url}`);
+    } else {
+      log.info(`Skipping apply click for known job: ${request.url}`);
+    }
+
+    await pushData({
+      ...jobSummary,
+      url: request.url, // Gradcracker job page
+      applicationLink, // External or same-page URL after click
+      jobDescription,
+    });
+
+    markJobPageDone({ currentUrl: request.url });
+  }
+);

extractors/gradcracker/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"extends": "@apify/tsconfig",
+	"compilerOptions": {
+		"module": "NodeNext",
+		"moduleResolution": "NodeNext",
+		"target": "ES2022",
+		"outDir": "dist",
+		"noUnusedLocals": false,
+		"lib": ["DOM"]
+	},
+	"include": ["./src/**/*"]
+}

extractors/jobspy/requirements.txt

@@ -0,0 +1 @@
+python-jobspy

extractors/jobspy/scrape_jobs.py

@@ -0,0 +1,77 @@
+import csv
+import os
+from pathlib import Path
+
+from jobspy import scrape_jobs
+
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value and value.strip() else default
+
+
+def _env_int(name: str, default: int) -> int:
+    value = os.getenv(name)
+    if value is None or value.strip() == "":
+        return default
+    try:
+        return int(value)
+    except ValueError:
+        return default
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    value = os.getenv(name)
+    if value is None or value.strip() == "":
+        return default
+    return value.strip().lower() in ("1", "true", "yes", "y", "on")
+
+
+def _parse_sites(raw: str) -> list[str]:
+    return [s.strip() for s in raw.split(",") if s.strip()]
+
+
+def main() -> int:
+    sites = _parse_sites(_env_str("JOBSPY_SITES", "indeed,linkedin"))
+    search_term = _env_str("JOBSPY_SEARCH_TERM", "web developer")
+    location = _env_str("JOBSPY_LOCATION", "UK")
+    results_wanted = _env_int("JOBSPY_RESULTS_WANTED", 200)
+    hours_old = _env_int("JOBSPY_HOURS_OLD", 72)
+    country_indeed = _env_str("JOBSPY_COUNTRY_INDEED", "UK")
+    linkedin_fetch_description = _env_bool("JOBSPY_LINKEDIN_FETCH_DESCRIPTION", True)
+
+    output_csv = Path(_env_str("JOBSPY_OUTPUT_CSV", "jobs.csv"))
+    output_json = Path(_env_str("JOBSPY_OUTPUT_JSON", str(output_csv.with_suffix(".json"))))
+
+    output_csv.parent.mkdir(parents=True, exist_ok=True)
+    output_json.parent.mkdir(parents=True, exist_ok=True)
+
+    jobs = scrape_jobs(
+        site_name=sites,
+        search_term=search_term,
+        location=location,
+        results_wanted=results_wanted,
+        hours_old=hours_old,
+        country_indeed=country_indeed,
+        linkedin_fetch_description=linkedin_fetch_description,
+    )
+
+    print(f"Found {len(jobs)} jobs")
+
+    jobs.to_csv(
+        output_csv,
+        quoting=csv.QUOTE_NONNUMERIC,
+        escapechar="\\",
+        index=False,
+    )
+
+    jobs.to_json(output_json, orient="records", force_ascii=False)
+
+    print(f"Wrote CSV:  {output_csv}")
+    print(f"Wrote JSON: {output_json}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+