From 40c7cdece327204799a49acc77019979c2c367d0 Mon Sep 17 00:00:00 2001 From: ilia Date: Thu, 9 Jul 2026 15:11:53 -0400 Subject: [PATCH] feat: add Google Jobs extractor and fix scraper reliability gaps Ship a Camoufox-backed Google Jobs source, restore Glassdoor results discarded by a python-jobspy GraphQL quirk, and fix Himalayas/Gradcracker zero-job failures. Also harden prior-skip dismiss matching and multi-profile basic-auth switching for CA/US runs. --- .env.example | 30 +- biome.json | 3 +- docs-site/docs/extractors/eluta.md | 5 +- docs-site/docs/extractors/google-jobs.md | 106 +++ docs-site/docs/extractors/gradcracker.md | 8 + docs-site/docs/extractors/himalayas.md | 45 ++ docs-site/docs/extractors/jobspy.md | 6 +- docs-site/docs/extractors/overview.md | 4 + docs-site/docs/features/duplicate-jobs.md | 12 +- extractors/google-jobs/manifest.ts | 49 ++ extractors/google-jobs/package.json | 22 + extractors/google-jobs/src/main.ts | 616 ++++++++++++++++++ extractors/google-jobs/src/run.ts | 154 +++++ extractors/google-jobs/tsconfig.json | 17 + extractors/gradcracker/src/routes.ts | 31 +- extractors/himalayas/manifest.ts | 6 +- extractors/jobspy/requirements.txt | 7 +- extractors/jobspy/scrape_jobs.py | 69 ++ orchestrator/src/client/api/client.ts | 6 +- orchestrator/src/client/lib/job-dedup.test.ts | 75 ++- orchestrator/src/client/lib/job-dedup.ts | 64 +- .../src/client/pages/OrchestratorPage.tsx | 8 +- .../pages/orchestrator/AutomaticRunTab.tsx | 1 + .../orchestrator/OrchestratorFilters.test.tsx | 2 + .../orchestrator/OrchestratorFilters.tsx | 29 + .../pages/orchestrator/ProfileQuickSwitch.tsx | 32 +- .../pages/orchestrator/automatic-run.ts | 1 + .../orchestrator/useOrchestratorFilters.ts | 22 + .../client/pages/orchestrator/utils.test.ts | 1 + .../src/client/pages/orchestrator/utils.ts | 1 + .../components/JobSourcesSettingsSection.tsx | 4 +- .../src/server/api/routes/profiles.ts | 4 + .../src/server/config/demo-defaults.data.ts | 1 + orchestrator/src/server/db/migrate.ts | 190 +++++- orchestrator/src/server/repositories/jobs.ts | 19 +- .../src/server/repositories/profiles.test.ts | 108 +++ .../src/server/repositories/profiles.ts | 79 ++- orchestrator/src/server/services/settings.ts | 25 +- package-lock.json | 35 + scripts/jobber-cron-dobkin-us.env.example | 30 + scripts/smoke-extractors.ts | 8 + shared/src/extractors/index.ts | 7 + shared/src/job-fingerprint.test.ts | 51 ++ shared/src/job-fingerprint.ts | 67 ++ shared/src/settings-registry.ts | 11 + shared/src/testing/factories.ts | 2 + shared/src/types/settings.ts | 3 + 47 files changed, 1989 insertions(+), 87 deletions(-) create mode 100644 docs-site/docs/extractors/google-jobs.md create mode 100644 docs-site/docs/extractors/himalayas.md create mode 100644 extractors/google-jobs/manifest.ts create mode 100644 extractors/google-jobs/package.json create mode 100644 extractors/google-jobs/src/main.ts create mode 100644 extractors/google-jobs/src/run.ts create mode 100644 extractors/google-jobs/tsconfig.json create mode 100644 orchestrator/src/server/repositories/profiles.test.ts create mode 100644 scripts/jobber-cron-dobkin-us.env.example diff --git a/.env.example b/.env.example index 09bcc7bd..7fb54081 100644 --- a/.env.example +++ b/.env.example @@ -260,7 +260,35 @@ ADZUNA_APP_KEY= # ============================================================================= # Caps via Settings: workingnomadsMaxJobsPerTerm, testdevjobsMaxJobsPerTerm, # testdevjobsMaxPages, builtinMaxJobsPerTerm, builtinMaxPagesPerTerm, -# wellfoundMaxJobsPerTerm (Wellfound uses Camoufox when installed). +# wellfoundMaxJobsPerTerm, googleJobsMaxJobsPerTerm (Wellfound and Google +# Jobs both use Camoufox when installed). + +# ============================================================================= +# Google Jobs — optional (auto-enabled, no credentials, unofficial scrape) +# ============================================================================= +# GOOGLE_JOBS_PROXY=http://user:pass@proxy-host:port +# ^ Most reliable way to avoid Google's CAPTCHA/rate-limit wall (residential or +# mobile proxy recommended; datacenter IPs, including most VPS/cloud hosts, +# are aggressively blocked). Applies to both manual runs and the pipeline. +# +# GOOGLE_JOBS_HEADLESS=false +# ^ Manual/local use ONLY (do not set this in the server .env for the +# unattended pipeline/cron — a headless server has no display for a human +# to see the browser). Opens a real Firefox window so you can solve a +# CAPTCHA by hand; waits up to GOOGLE_JOBS_UNBLOCK_TIMEOUT_MS (default +# 300000 = 5 min) before giving up. Run directly, e.g.: +# GOOGLE_JOBS_HEADLESS=false npm --workspace google-jobs-extractor run start +# Session cookies are saved to extractors/google-jobs/storage/state.json +# and reused by later runs (including headless/automated ones), which can +# reduce how often future runs hit a CAPTCHA. +# +# Caps via Settings: googleJobsMaxJobsPerTerm (default 30, max 150). +# +# Note: there is no country/location override for this extractor — Google +# resolves location from the browser's real (or GOOGLE_JOBS_PROXY's) IP. +# Forcing a mismatched country has been observed to make Google's Jobs +# vertical return "no matches" even though the tab loads correctly. To +# target a different country's jobs, use a proxy with an exit node there. # ============================================================================= # Teamtailor / Huntflow / Factorial / Career pages — optional diff --git a/biome.json b/biome.json index 03034223..9655027c 100644 --- a/biome.json +++ b/biome.json @@ -12,8 +12,9 @@ "!!**/.venv", "!!docs-site/.docusaurus", "!!docs-site/build", - "!!extractors/jobspy/storage", + "!!extractors/*/storage", "!!orchestrator/storage", + "!!storage", "!!data" ] }, diff --git a/docs-site/docs/extractors/eluta.md b/docs-site/docs/extractors/eluta.md index 252d1b21..0e00a945 100644 --- a/docs-site/docs/extractors/eluta.md +++ b/docs-site/docs/extractors/eluta.md @@ -32,10 +32,11 @@ Eluta surfaces Canadian roles indexed directly from employer career sites, often ### Empty feeds - The `location` string may be too broad or spelled differently than Eluta expects. Try a major city plus province (e.g. `Calgary, AB`). +- Eluta's RSS is a small recent slice of *all* jobs for that location (often retail/hospitality-heavy). Pipeline search terms like `software engineer` can legitimately match zero items even when the feed itself is healthy. Broaden terms or add more metro locations. -### RSS HTTP errors +### RSS HTTP errors / "Too Many Requests" -- Eluta may block unusual clients; the extractor sends a conventional User-Agent. Retry later or reduce the number of location feeds per run. +- Eluta rate-limits aggressive or browser-like clients. The extractor uses a dedicated `JobOps/... Eluta RSS consumer` User-Agent; retry later or reduce the number of location feeds per run. ## Related pages diff --git a/docs-site/docs/extractors/google-jobs.md b/docs-site/docs/extractors/google-jobs.md new file mode 100644 index 00000000..29d6487b --- /dev/null +++ b/docs-site/docs/extractors/google-jobs.md @@ -0,0 +1,106 @@ +--- +id: google-jobs +title: Google Jobs Extractor +description: Browser-backed scraping of the Google for Jobs search widget. +sidebar_position: 8 +--- + +## What it is + +Original feature: [Google for Jobs](https://www.google.com/search?q=jobs&udm=8) + +Google Jobs is a browser-backed extractor that opens Google's Jobs vertical (`udm=8`) in Firefox (Camoufox when available) for each pipeline search term, scrolls the results panel to load additional cards, then clicks into each card to resolve a real "Apply on <site>" link and description before mapping rows into the orchestrator `CreateJobInput` shape. + +Implementation split: + +1. `extractors/google-jobs/src/main.ts` builds the search URL, waits for the jobs list, scrolls to lazy-load more cards, extracts card metadata (title/employer/location/"via" source), clicks each card to read the detail pane, and writes dataset JSON. +2. `extractors/google-jobs/src/run.ts` spawns the browser subprocess, applies a hard timeout, reads the dataset, and maps rows to `CreateJobInput`. +3. `extractors/google-jobs/manifest.ts` wires pipeline `searchTerms` into the extractor. + +### A note on the search URL and location + +Google's Jobs vertical is reached with `udm=8` (its current internal vertical id). An older, commonly-documented deep link — `ibp=htl;jobs` — still works but gets silently redirected to `udm=8` anyway, and combining it with extra parameters (a `num=` result-count hint, or a `gl=` country override that doesn't match the browser's real/proxied IP) has been observed to make Google return "no matches" or "can't determine location" even though the tab itself loads correctly. + +Because of that, this extractor does **not** send a `gl` (country) parameter — location is resolved by Google from the browser's real IP (or `GOOGLE_JOBS_PROXY`'s exit IP), which Camoufox's `geoip` option keeps internally consistent. To target a different country's jobs, use a residential proxy with an exit node in that country rather than trying to override the country via a parameter. + +### Dealing with Google's CAPTCHA / bot detection + +Google actively fingerprints and challenges automated browser traffic, especially from datacenter/VPS IPs (which most servers running JobOps' cron pipeline will have). The extractor has layered mitigations, from always-on to opt-in: + +1. **Always on:** Camoufox launches with `humanize: true` and `geoip: true` (human-like cursor movement, IP-consistent locale/geolocation) to reduce — not eliminate — fingerprint-based detection. +2. **Always on:** browser cookies/session are persisted to `extractors/google-jobs/storage/state.json` and reused on the next run. A session that has cleared a CAPTCHA once (including via option 4 below) tends to see fewer challenges for a while afterward. +3. **Opt-in, most reliable:** set `GOOGLE_JOBS_PROXY` (e.g. a residential/mobile proxy URL) to avoid datacenter-IP blocks in the first place. This is the standard real-world fix; unproxied runs from cloud servers should expect to be blocked often. +4. **Opt-in, manual/local only:** set `GOOGLE_JOBS_HEADLESS=false` and run the extractor directly (not via the pipeline) to open a real, visible Firefox window. If Google shows a CAPTCHA, solve it yourself in that window — the script polls and automatically resumes once it clears (up to `GOOGLE_JOBS_UNBLOCK_TIMEOUT_MS`, default 5 minutes). The resulting session cookies are saved and reused by later automated/headless runs. **Do not** set `GOOGLE_JOBS_HEADLESS=false` in the server's always-on `.env` — a headless server has no display for a human to interact with, and the run will just wait until the timeout. + +## Why it exists + +Google for Jobs aggregates postings across the open web — including boards and employer career pages that JobOps does not otherwise scrape. Adding it as a proper source closes some of the gap between "jobs found by manually Googling a role" and "jobs discovered by the pipeline." + +It is **not** a general web crawler and does not replace dedicated extractors: it only surfaces whatever Google's Jobs widget chooses to show for a query, subject to Google's own ranking, freshness, and regional availability. + +## How to use it + +1. Open **Run jobs** and choose **Automatic**. +2. **Google Jobs** is enabled by default in **Sources** (toggle it off if you do not want it for this run). +3. Set your existing automatic run knobs: + - `searchTerms` become the `" jobs"` query sent to Google per term. + - `googleJobsMaxJobsPerTerm` (Settings) caps jobs scraped per run (default `30`, max `150`). + - Location is derived from the browser's real/proxied IP, not from a country setting — see [the note above](#a-note-on-the-search-url-and-location). +4. Start the run and watch progress in the pipeline progress card. + +Local run example (headless, no proxy): + +```bash +GOOGLE_JOBS_SEARCH_TERMS='["automation engineer"]' \ +GOOGLE_JOBS_MAX_JOBS='10' \ +npm --workspace google-jobs-extractor run start +``` + +Local run example (headed, solve the CAPTCHA yourself if one appears): + +```bash +GOOGLE_JOBS_HEADLESS=false \ +GOOGLE_JOBS_SEARCH_TERMS='["automation engineer"]' \ +npm --workspace google-jobs-extractor run start +``` + +Defaults and constraints: + +- No credentials required; this is a direct HTML/DOM scrape of Google's public search results, not an official API (Google does not offer a free Google Jobs API). +- Google's markup and CSS class names change frequently and are obfuscated; the scraper favors known current card classes (e.g. `.EimVGf`) with multiple fallback selectors and a leaf-node text heuristic, but selectors can still rot and return zero results until updated. +- Every card carries Google's own `data-share-url` permalink for that specific listing, which the extractor uses as `jobUrl` — so a job stays unique and reviewable even if the "click into the card for a detail pane" step below fails for it. +- Because resolving a real application link requires clicking each card in a real browser, this extractor is slower per job than API-backed sources — keep `googleJobsMaxJobsPerTerm` modest for frequent runs. +- If no external "Apply on <site>" link is found for a card, the extractor falls back to Google's own permalink for that listing so it's still reviewable, but not directly one-click-applyable. +- Automated, frequent, or high-volume scraping of Google search results may be against Google's Terms of Service; treat this extractor as best-effort and keep run frequency/volume conservative. +- See [Dealing with Google's CAPTCHA / bot detection](#dealing-with-googles-captcha--bot-detection) above for proxy and interactive-solve options. + +## Common problems + +### Google Jobs returns 0 jobs for a term + +The extractor logs one of two distinct messages to help tell these apart: + +- `Google's Jobs tab reported no matches for ""` — Google correctly landed on the Jobs tab but genuinely has no results for that query/locale. Try a broader, more common phrasing (e.g. `"automation engineer"` instead of a very narrow title). +- `Jobs list did not appear for ""` — an unrecognized page state (consent wall, layout change, or rotted selectors). The extractor saves a screenshot and full HTML dump to `extractors/google-jobs/storage/debug-last.png` / `debug-last.html` for inspection; compare it against a fresh manual search to see what changed. + +### Results look sparse or missing company/location + +- The list-view heuristics rely on leaf DOM nodes near the job title; if Google restructures the card markup, company/location extraction can degrade even when titles still work. This is expected best-effort behavior for an unofficial scrape. + +### Run is slow + +- Each job requires a real click + detail-pane render, unlike single-request API extractors. Lower `googleJobsMaxJobsPerTerm` or reduce the number of search terms for faster runs. + +### Blocked, CAPTCHA'd, or consistently empty results + +- Google may rate-limit or challenge automated browser traffic, especially from datacenter/VPS IPs. In order of effort: + 1. Set `GOOGLE_JOBS_PROXY` to a residential/mobile proxy — the most reliable fix for unattended/server runs. + 2. Run locally with `GOOGLE_JOBS_HEADLESS=false` and solve the CAPTCHA yourself once; the saved session (`storage/state.json`) may reduce blocks on subsequent headless runs for a while. + 3. Reduce run frequency, lower the per-term cap, or disable the source for that run and rely on other extractors / Manual Import instead. + +## Related pages + +- [Extractors Overview](/docs/next/extractors/overview) +- [Manual Import](/docs/next/extractors/manual) +- [Pipeline Run](/docs/next/features/pipeline-run) +- [Settings](/docs/next/features/settings) diff --git a/docs-site/docs/extractors/gradcracker.md b/docs-site/docs/extractors/gradcracker.md index b04eec36..d6170cf6 100644 --- a/docs-site/docs/extractors/gradcracker.md +++ b/docs-site/docs/extractors/gradcracker.md @@ -46,3 +46,11 @@ Set `JOBOPS_EMIT_PROGRESS=1` for structured progress lines consumable by orchest - Uses Playwright + Crawlee via Camoufox. - Low concurrency and longer timeouts for stability. +- Employer logo (`figure img` alt text) and employer link are optional. Cards that omit them used to throw a Playwright `getAttribute` timeout and fail the whole region page (0 jobs saved even after cards were counted). Missing employer fields are now skipped per card. + +## Common problems + +### Region pages time out / run returns 0 jobs + +- Gradcracker Livewire markup sometimes omits the employer logo on a card. Older builds aborted the whole page on that timeout. Current builds continue and leave employer blank. +- Anti-bot / Camoufox issues: ensure Camoufox is installed (`npx camoufox-js fetch`) and prefer headed runs when debugging. diff --git a/docs-site/docs/extractors/himalayas.md b/docs-site/docs/extractors/himalayas.md new file mode 100644 index 00000000..23175163 --- /dev/null +++ b/docs-site/docs/extractors/himalayas.md @@ -0,0 +1,45 @@ +--- +id: himalayas +title: Himalayas Extractor +description: Public remote-jobs API pagination and client-side term filtering. +sidebar_position: 16 +--- + +## What it is + +Original site: [himalayas.app](https://himalayas.app) + +The extractor lives in `extractors/himalayas/manifest.ts`. It paginates the public JSON API (`https://himalayas.app/jobs/api?limit=&offset=`), filters rows client-side by pipeline search terms (title + categories), and maps matches into `CreateJobInput`. + +## Why it exists + +Himalayas is a large remote-jobs index with a stable, unauthenticated API — useful for remote-first discovery without browser automation or credentials. + +## How to use it + +1. Enable **Himalayas** in pipeline sources. +2. Optionally set **Himalayas max jobs per term** (`himalayasMaxJobsPerTerm`, default `100`). +3. Pipeline `searchTerms` filter titles/categories client-side (substring match, case-insensitive). +4. Run the pipeline. + +### Defaults and constraints + +- The upstream API **silently caps `limit` at 20** per request. The extractor uses `PAGE_SIZE = 20` and paginates with `offset` (up to 10 pages / 200 rows scanned per run before the per-term cap). +- There is no server-side search — if a term is rare, early pages may not contain matches even though later pages do. Raise the page budget only by changing the extractor constants if you need deeper scans. +- No auth required. + +## Common problems + +### Zero jobs for a term I know exists + +- The term may not appear in the first ~200 API rows (newest-first feed). Try a shorter token (`software`, `engineer`, `SDET`) or accept that Himalayas skews toward whatever is currently at the top of the feed. +- Confirm you are not hitting an old build that requested `limit=50` and stopped after one page (fixed: the API returns at most 20, which used to look like end-of-feed). + +### Rate limits / HTTP errors + +- Retry later; the public API is generally stable but can throttle burst traffic. + +## Related pages + +- [Extractors Overview](/docs/next/extractors/overview) +- [Add an Extractor](/docs/next/workflows/add-an-extractor) diff --git a/docs-site/docs/extractors/jobspy.md b/docs-site/docs/extractors/jobspy.md index 25832c6a..56139b5a 100644 --- a/docs-site/docs/extractors/jobspy.md +++ b/docs-site/docs/extractors/jobspy.md @@ -61,4 +61,8 @@ The service in `orchestrator/src/server/services/jobspy.ts`: - A run returned fewer LinkedIn descriptions than expected. `JOBSPY_LINKEDIN_FETCH_DESCRIPTION=0` disables description fetching to speed up runs. - Different cities need different workplace-type filters. - This is not supported in the current automatic-run flow. JobSpy receives one global workplace-type selection per run/query invocation. \ No newline at end of file + This is not supported in the current automatic-run flow. JobSpy receives one global workplace-type selection per run/query invocation. +- Glassdoor always returns 0 jobs (fixed). + Upstream `python-jobspy` (through at least `1.1.82`) discards an entire page of Glassdoor results whenever Glassdoor's GraphQL response includes *any* `errors` entry — even for an unrelated field. In practice, Glassdoor's backend consistently returns a partial error for `jobsPageSeoData` (`SUBREQUEST_HTTP_ERROR` / "dns error" on Glassdoor's own infra) while the actual job listings still come back fine, so every Glassdoor run silently returned 0 jobs. `extractors/jobspy/scrape_jobs.py` monkeypatches `Glassdoor._fetch_jobs_page` at import time to only treat the response as failed when `data.jobListings` is actually missing, and logs a warning (not an error) when it ignores an unrelated partial error. `requirements.txt` pins `python-jobspy==1.1.82` so an upstream version bump doesn't silently drop this patch — re-verify the patched method still matches before removing the pin. +- A Glassdoor run logs `429` / "Blocked by Glassdoor for too many requests". + This is a real rate limit from running too many Glassdoor requests back-to-back (e.g. repeated manual test runs). Space out runs; the automatic pipeline's normal cadence does not trigger this in practice. \ No newline at end of file diff --git a/docs-site/docs/extractors/overview.md b/docs-site/docs/extractors/overview.md index 18557dca..4f867e9c 100644 --- a/docs-site/docs/extractors/overview.md +++ b/docs-site/docs/extractors/overview.md @@ -25,6 +25,7 @@ Extractor integrations are now registered through manifests and loaded automatic | [Eluta](/docs/next/extractors/eluta) | Canadian listings aggregated from employer career sites (RSS) | Canada-only source (skipped when search geography is not Canada); RSS `location` strings must be set | `ELUTA_RSS_LOCATIONS`, `ELUTA_MAX_JOBS_PER_TERM` | Fetches one or more `eluta.ca` RSS feeds, filters by terms, de-duplicates by guid/URL | | [QAJobsBoard](/docs/next/extractors/qajobsboard) | QA / SDET / automation-heavy board (global JSON feed) | No auth; geography skew is manual/filter downstream | `qajobsboardMaxJobsPerTerm` | Fetches JobBoardly JSON, filters by pipeline terms | | [Arc.dev](/docs/next/extractors/arcdev) | Remote roles from Arc.dev listing pages (tool-tagged paths) | Parses SSR `__NEXT_DATA__`; relies on stable Next payload | `ARC_REMOTE_JOBS_PATHS` (seeds defaults), `arcRemoteJobsPaths`, `arcMaxJobsPerPath` | Merges Arc-managed + external rows; dedupes by URL | +| [Google Jobs](/docs/next/extractors/google-jobs) | Broader open-web coverage via the Google for Jobs widget | Unofficial scrape (no API); CAPTCHA-prone from datacenter IPs; selectors can rot; slower (click-through per job) | `googleJobsMaxJobsPerTerm`, `GOOGLE_JOBS_PROXY`, `GOOGLE_JOBS_HEADLESS` | Browser scrape of `udm=8` Jobs vertical; scrolls list, double-clicks cards, expands description | | [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review | ## Which extractor should I use? @@ -40,6 +41,7 @@ Extractor integrations are now registered through manifests and loaded automatic - Use **BC T-Net** for British Columbia tech RSS listings (runs only when search geography is Canada). - Use **Eluta** for Canadian employer-direct listings via RSS (set metro/province `location` strings). - Use **QAJobsBoard** or **Arc.dev** when you want QA- or remote-stack-focused feeds without extra credentials. +- Use **Google Jobs** when you want broader open-web coverage beyond dedicated job boards (unofficial scrape; best-effort). - Use **Manual Import** when you already have a specific posting and need direct import. Many runs combine sources: broad discovery first, then manual import for high-priority jobs that scraping misses. @@ -82,5 +84,7 @@ JobOps ships **BC T-Net** and **iCIMS tenant HTML** extractors for two cases tha - [Arc.dev](/docs/next/extractors/arcdev) - [Canadian / NA QA contracting firms](/docs/next/extractors/qa-contract-staffing-canada) - [Canadian companies — QA-strong ATS](/docs/next/extractors/canadian-companies-qa-ats) +- [Google Jobs](/docs/next/extractors/google-jobs) +- [Himalayas](/docs/next/extractors/himalayas) - [Manual Import](/docs/next/extractors/manual) - [Add an Extractor](/docs/next/workflows/add-an-extractor) diff --git a/docs-site/docs/features/duplicate-jobs.md b/docs-site/docs/features/duplicate-jobs.md index 8b0a9cec..073e6a03 100644 --- a/docs-site/docs/features/duplicate-jobs.md +++ b/docs-site/docs/features/duplicate-jobs.md @@ -45,9 +45,19 @@ During a pipeline run, if a new posting matches an existing row by URL, source i Open jobs that match a prior skip or apply are **hidden** from Discovered, Ready, and All tabs so the queue stays fresh. Skipped and applied rows themselves remain visible in their statuses. +Use **Filters → Employer keywords → Hide roles you already skipped** to toggle this behavior (on by default). When enabled, JobOps compares **company + job title** only: + +- Same company after normalization (legal suffixes stripped; short names like `CGI` match longer forms like `CGI IT UK Limited`). +- Same title after normalization (including simple plural variants such as `Engineer` vs `Engineers`). +- Prior skip or apply must be within the **last 90 days** (`updatedAt` on the skipped/applied row). + +Uncheck the filter to temporarily review rediscovered rows that match older skips. + ## Defaults and constraints -- Description matching requires at least **80 characters** of normalized text; short or empty descriptions fall back to employer+title only. +- Prior-skip hiding uses employer + title only (not description text). +- The 90-day window applies to both the Jobs list filter and pipeline import suppression for skipped/applied rows. +- Description matching during import still requires at least **80 characters** of normalized text; short or empty descriptions fall back to employer+title only. - Matching is **per profile** (`ownerProfileId`); different login profiles do not share dedup state. - Dedup does **not** delete existing rows retroactively when you change skip list or country filters — run discovery again or skip manually for old data. - Very different titles at the same company (for example `SDET` vs `Product Designer`) are **not** collapsed. diff --git a/extractors/google-jobs/manifest.ts b/extractors/google-jobs/manifest.ts new file mode 100644 index 00000000..c3dc32f8 --- /dev/null +++ b/extractors/google-jobs/manifest.ts @@ -0,0 +1,49 @@ +import type { + ExtractorManifest, + ExtractorRunResult, +} from "@shared/types/extractors"; +import { runGoogleJobs } from "./src/run.js"; + +export const manifest: ExtractorManifest = { + id: "google-jobs", + displayName: "Google Jobs", + providesSources: ["google-jobs"], + async run(context): Promise { + if (context.shouldCancel?.()) return { success: true, jobs: [] }; + + const parsedMax = context.settings.googleJobsMaxJobsPerTerm + ? Number.parseInt(context.settings.googleJobsMaxJobsPerTerm, 10) + : Number.NaN; + const maxJobs = Number.isFinite(parsedMax) ? Math.max(1, parsedMax) : 30; + + context.onProgress?.({ + phase: "list", + termsProcessed: 0, + termsTotal: context.searchTerms.length || 1, + currentUrl: "https://www.google.com/search?udm=8", + detail: "Google Jobs: launching browser scrape", + }); + + const result = await runGoogleJobs({ + searchTerms: context.searchTerms, + maxJobs, + }); + + if (!result.success) { + return { success: false, jobs: [], error: result.error }; + } + + context.onProgress?.({ + phase: "list", + termsProcessed: context.searchTerms.length || 1, + termsTotal: context.searchTerms.length || 1, + currentUrl: "https://www.google.com/search?udm=8", + jobPagesProcessed: result.jobs.length, + detail: `Google Jobs: ${result.jobs.length} jobs`, + }); + + return { success: true, jobs: result.jobs }; + }, +}; + +export default manifest; diff --git a/extractors/google-jobs/package.json b/extractors/google-jobs/package.json new file mode 100644 index 00000000..0d4686df --- /dev/null +++ b/extractors/google-jobs/package.json @@ -0,0 +1,22 @@ +{ + "name": "google-jobs-extractor", + "version": "0.0.1", + "type": "module", + "description": "Google Jobs (Google for Jobs) search widget extractor (browser-backed)", + "main": "manifest.ts", + "dependencies": { + "camoufox-js": "^0.8.0", + "job-ops-shared": "^1.0.0", + "playwright": "^1.57.0", + "tsx": "^4.4.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "~5.9.0" + }, + "scripts": { + "start": "tsx src/main.ts", + "check:types": "tsc --noEmit", + "get-binaries": "camoufox-js fetch" + } +} diff --git a/extractors/google-jobs/src/main.ts b/extractors/google-jobs/src/main.ts new file mode 100644 index 00000000..39c6a0d6 --- /dev/null +++ b/extractors/google-jobs/src/main.ts @@ -0,0 +1,616 @@ +import { access, mkdir, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { launchOptions } from "camoufox-js"; +import { + type Browser, + type BrowserContext, + firefox, + type Page, +} from "playwright"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const OUTPUT_PATH = join(__dirname, "../storage/jobs.json"); +const STATE_PATH = join(__dirname, "../storage/state.json"); +const DEBUG_HTML_PATH = join(__dirname, "../storage/debug-last.html"); +const DEBUG_SCREENSHOT_PATH = join(__dirname, "../storage/debug-last.png"); + +interface ScrapedJob { + title: string; + employer: string; + jobUrl: string; + applicationLink?: string; + location?: string; + jobDescription?: string; + postedVia?: string; + isRemote?: boolean; +} + +interface ScrapedCard { + docId?: string; + title: string; + employer: string; + location?: string; + via?: string; + /** Google's own permalink into this specific listing (from `data-share-url`). Always present when the card matched, so it works as a stable jobUrl even if the detail click fails. */ + shareUrl?: string; +} + +interface ScrapedDetail { + applyLinks: Array<{ href: string; text: string }>; + description?: string; +} + +/** + * Confirmed against Google's live markup (2026). Google renders each job as + * a `.EimVGf` card with a `data-share-url` permalink baked in — no `
    `/ + * `
  • `/`aria-label="Jobs list"` wrapper as older scraping guides assumed. + * Kept as a short list (not a single hardcoded class) since Google rotates + * these obfuscated names periodically. + */ +const CARD_SELECTORS = [".EimVGf", 'li[role="listitem"]', ".iFjolb"] as const; + +function parseTerms(raw: string | undefined): string[] { + if (!raw) return ["software engineer"]; + try { + const parsed = JSON.parse(raw) as unknown; + if (Array.isArray(parsed)) { + return parsed.map((entry) => String(entry).trim()).filter(Boolean); + } + } catch { + // fall through to delimited parsing + } + return raw + .split(/[\n|,]+/) + .map((entry) => entry.trim()) + .filter(Boolean); +} + +/** + * `udm=8` is Google's current internal id for the Jobs vertical (the legacy + * `ibp=htl;jobs` deep link now gets silently redirected to it anyway, but + * only when requested directly — combining the two, or adding `num=`/`gl=` + * overrides that don't match the browser's real/proxied IP, has been + * observed to make the vertical come back "no matches" or "can't determine + * location" even though the widget itself loads correctly). `hl=en` keeps + * copy in English so downstream text parsing (e.g. "via LinkedIn") is + * consistent regardless of where the request is geolocated. Location is + * intentionally NOT set via a `gl` param — it's derived from the real (or + * `GOOGLE_JOBS_PROXY`) IP through Camoufox's `geoip` option instead. + */ +function buildSearchUrl(term: string): string { + const params = new URLSearchParams({ + q: `${term} jobs`, + udm: "8", + hl: "en", + }); + return `https://www.google.com/search?${params.toString()}`; +} + +/** + * Launches Camoufox-hardened Firefox. `humanize`/`geoip` reduce (but do not + * eliminate) Google's automated-traffic detection. `GOOGLE_JOBS_PROXY` (e.g. + * a residential proxy URL) is the most reliable way to avoid datacenter-IP + * blocks entirely, if configured. + */ +async function launchBrowser(headless: boolean): Promise { + const proxy = process.env.GOOGLE_JOBS_PROXY?.trim(); + return firefox.launch( + await launchOptions({ + headless, + humanize: true, + geoip: true, + ...(proxy ? { proxy } : {}), + }), + ); +} + +/** Google shows a cookie/consent wall in some locales before rendering results. */ +async function dismissConsent(page: Page): Promise { + const labels = ["Reject all", "I agree", "Accept all"]; + for (const label of labels) { + try { + const button = page.getByRole("button", { name: label }).first(); + if (await button.isVisible({ timeout: 1500 })) { + await button.click({ timeout: 3000 }); + await page.waitForTimeout(500); + return; + } + } catch { + // selector not present in this render; try the next label + } + } +} + +/** Thrown when Google serves a CAPTCHA / rate-limit challenge instead of results. */ +class GoogleBlockedError extends Error {} + +/** Google serves an interstitial CAPTCHA page for automated / datacenter traffic. */ +async function isBlockedPage(page: Page): Promise { + if (/\/sorry\//.test(page.url())) return true; + try { + return await page.evaluate(() => { + const bodyText = document.body?.textContent ?? ""; + return ( + document.querySelector("#captcha-form") !== null || + document.querySelector(".g-recaptcha") !== null || + /unusual traffic/i.test(bodyText) + ); + }); + } catch { + return false; + } +} + +/** + * In headed/interactive mode, give a human at the keyboard a chance to solve + * the CAPTCHA in the visible browser window instead of failing immediately. + * Polls until the challenge clears (Google's form redirects on success) or + * the timeout elapses. + */ +async function waitForHumanToUnblock( + page: Page, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + console.error( + `[google-jobs] Google is showing a CAPTCHA in the open browser window — please solve it there. Waiting up to ${Math.round(timeoutMs / 1000)}s...`, + ); + while (Date.now() < deadline) { + await page.waitForTimeout(3_000); + if (!(await isBlockedPage(page))) { + console.error("[google-jobs] CAPTCHA cleared, resuming."); + return true; + } + } + return false; +} + +async function waitForJobsList(page: Page): Promise { + for (const selector of CARD_SELECTORS) { + try { + await page.waitForSelector(selector, { timeout: 8_000 }); + return true; + } catch { + // try the next fallback selector + } + } + return false; +} + +/** + * Google sometimes lands correctly on the Jobs vertical/tab but reports no + * matches for the query (distinct from a CAPTCHA block or a selector-rot + * failure) — e.g. "It looks like there aren't any 'Jobs' matches on this topic." + */ +async function isEmptyJobsVertical(page: Page): Promise { + try { + return await page.evaluate(() => { + const bodyText = document.body?.textContent ?? ""; + return /aren.t any .Jobs. matches/i.test(bodyText); + }); + } catch { + return false; + } +} + +function countCardsScript(selectors: readonly string[]): number { + for (const selector of selectors) { + const count = document.querySelectorAll(selector).length; + if (count > 0) return count; + } + return 0; +} + +/** Google lazy-loads additional cards (via a custom `` element) as the page is scrolled. */ +function scrollPageScript(): number { + const before = window.scrollY; + window.scrollTo(0, document.body.scrollHeight); + return window.scrollY - before; +} + +async function growJobsList( + page: Page, + selectors: readonly string[], + targetCount: number, +): Promise { + for (let attempt = 0; attempt < 8; attempt += 1) { + const count = await page.evaluate(countCardsScript, selectors); + if (count >= targetCount) return; + const scrolled = await page.evaluate(scrollPageScript); + if (scrolled <= 0) return; + await page.waitForTimeout(900); + } +} + +function extractCardsScript(selectors: readonly string[]): ScrapedCard[] { + // NOTE: this whole function is serialized via Function.prototype.toString() + // and re-run inside the browser page by Playwright — it cannot reference + // anything from the outer module scope (constants, other functions, etc.). + // All helpers must be declared inside this function body. + function clean(value: string | null | undefined): string { + return (value ?? "").replace(/\s+/g, " ").trim(); + } + + /** Extracts Google's internal `htidocid` from a job's `data-share-url`, matching `data-encoded-docid` in the detail panel's apply-link list. */ + function extractDocId(shareUrl: string | null): string | undefined { + if (!shareUrl) return undefined; + try { + const url = new URL(shareUrl, "https://www.google.com"); + return url.searchParams.get("htidocid") ?? undefined; + } catch { + return undefined; + } + } + + function findCards(): Element[] { + for (const selector of selectors) { + const items = Array.from(document.querySelectorAll(selector)); + if (items.length > 0) return items; + } + return []; + } + + const cards: ScrapedCard[] = []; + + for (const item of findCards()) { + // Known current classes first (title/employer/location+via), falling + // back to a leaf-node text heuristic if Google has rotated them. + const titleEl = item.querySelector(".tNxQIb, h2, [role='heading']"); + const title = clean(titleEl?.textContent); + if (!title) continue; + + const wHYlTdEls = Array.from(item.querySelectorAll(".wHYlTd")); + let employer = clean(wHYlTdEls[0]?.textContent); + let location: string | undefined; + let via: string | undefined; + + const viaBlock = wHYlTdEls.find((el) => + /\bvia\b/i.test(el.textContent ?? ""), + ); + if (viaBlock) { + const text = clean(viaBlock.textContent); + const match = text.match(/^(.*?)\s*(?:•|·)\s*via\s+(.+)$/i); + if (match) { + location = match[1] || undefined; + via = match[2] || undefined; + } else { + via = clean(text.replace(/^via\s*/i, "")); + } + } + + if (!employer || !title || employer === title) { + // Selector rot fallback: scan leaf text nodes structurally. + const leafTexts = Array.from(item.querySelectorAll("div, span")) + .filter((el) => el.children.length === 0) + .map((el) => clean(el.textContent)) + .filter((text) => text && text !== title && !/^[•·|,]+$/.test(text)); + employer = employer || leafTexts[0] || "Unknown Employer"; + if (!via) { + const viaText = leafTexts.find((text) => /^via\b/i.test(text)); + via = viaText ? clean(viaText.replace(/^via\s*/i, "")) : undefined; + location = + location ?? leafTexts.slice(1).find((text) => text !== viaText); + } + } + + const shareUrlRaw = item.getAttribute("data-share-url") ?? undefined; + const docId = extractDocId(shareUrlRaw ?? null); + + cards.push({ + docId, + title, + employer: employer || "Unknown Employer", + location, + via, + shareUrl: shareUrlRaw, + }); + } + + return cards; +} + +/** + * Scopes apply-link extraction to the clicked job's own `data-encoded-docid` + * block. Google's detail viewer can have adjacent/related jobs' apply links + * present in the DOM simultaneously, so a document-wide "first apply link" + * scan can silently pick up the wrong job's link. + */ +function extractDetailScript(docId: string | undefined): ScrapedDetail { + function clean(value: string | null | undefined): string { + return (value ?? "").replace(/\s+/g, " ").trim(); + } + + function collectApplyLinks( + root: ParentNode, + ): Array<{ href: string; text: string }> { + const links: Array<{ href: string; text: string }> = []; + for (const anchor of Array.from(root.querySelectorAll("a[href]"))) { + const href = (anchor as HTMLAnchorElement).href; + const text = + clean(anchor.textContent) || clean(anchor.getAttribute("aria-label")); + if (!href) continue; + if ( + !/apply/i.test(text) && + !/apply/i.test(anchor.getAttribute("aria-label") ?? "") + ) { + continue; + } + if ( + href.includes("google.com/search") || + href.includes("accounts.google.com") + ) { + continue; + } + links.push({ href, text }); + } + return links; + } + + let applyLinks: Array<{ href: string; text: string }> = []; + if (docId) { + const scoped = document.querySelector(`[data-encoded-docid="${docId}"]`); + if (scoped) applyLinks = collectApplyLinks(scoped); + } + if (applyLinks.length === 0) { + applyLinks = collectApplyLinks(document); + } + + // `.textContent` also pulls in text from nested