From 2b2af06bb8bdff74b5205e4b1930c56eb5746c02 Mon Sep 17 00:00:00 2001 From: DaKheera47 Date: Wed, 7 Jan 2026 23:53:01 +0000 Subject: [PATCH] autologin for ukvisajobs --- .env.example | 9 +- .gitignore | 5 +- docker-compose.yml | 7 +- extractors/ukvisajobs/README.md | 39 +- extractors/ukvisajobs/package-lock.json | 1110 +++++++++++++++++ extractors/ukvisajobs/package.json | 11 +- extractors/ukvisajobs/src/main.ts | 269 +++- .../src/server/services/ukvisajobs.ts | 38 +- 8 files changed, 1417 insertions(+), 71 deletions(-) diff --git a/.env.example b/.env.example index 6e4c965..1a10850 100644 --- a/.env.example +++ b/.env.example @@ -40,9 +40,8 @@ JOBSPY_LINKEDIN_FETCH_DESCRIPTION=1 # ============================================================================= # UKVisaJobs (UK visa sponsorship jobs) - optional # ============================================================================= -# Get these tokens from browser dev tools after logging into my.ukvisajobs.com +# Provide email/password for automatic login and token refresh. # See extractors/ukvisajobs/README.md for detailed instructions. -UKVISAJOBS_TOKEN= -UKVISAJOBS_AUTH_TOKEN= -UKVISAJOBS_CSRF_TOKEN= -UKVISAJOBS_CI_SESSION= +UKVISAJOBS_EMAIL= +UKVISAJOBS_PASSWORD= +UKVISAJOBS_HEADLESS=true diff --git a/.gitignore b/.gitignore index b3c4c83..2bd4547 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ # Data directory (bind mount in Docker) data/ +# Extractor storage outputs and cached auth +extractors/ukvisajobs/storage/ + # OS files .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db diff --git a/docker-compose.yml b/docker-compose.yml index 3bb1f61..5aec690 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,10 +51,9 @@ services: - WEBHOOK_SECRET=${WEBHOOK_SECRET:-} # UKVisaJobs (UK visa sponsorship jobs) - optional - - UKVISAJOBS_TOKEN=${UKVISAJOBS_TOKEN:-} - - UKVISAJOBS_AUTH_TOKEN=${UKVISAJOBS_AUTH_TOKEN:-} - - UKVISAJOBS_CSRF_TOKEN=${UKVISAJOBS_CSRF_TOKEN:-} - - UKVISAJOBS_CI_SESSION=${UKVISAJOBS_CI_SESSION:-} + - UKVISAJOBS_EMAIL=${UKVISAJOBS_EMAIL:-} + - UKVISAJOBS_PASSWORD=${UKVISAJOBS_PASSWORD:-} + - UKVISAJOBS_HEADLESS=${UKVISAJOBS_HEADLESS:-true} - UKVISAJOBS_SEARCH_KEYWORD=${UKVISAJOBS_SEARCH_KEYWORD:-} # Python path (uses system python in container) diff --git a/extractors/ukvisajobs/README.md b/extractors/ukvisajobs/README.md index 4412b83..3c2663d 100644 --- a/extractors/ukvisajobs/README.md +++ b/extractors/ukvisajobs/README.md @@ -1,4 +1,4 @@ -# UK Visa Jobs Extractor +# UK Visa Jobs Extractor Fetches job listings from [my.ukvisajobs.com](https://my.ukvisajobs.com) that may sponsor work visas. @@ -8,28 +8,38 @@ Fetches job listings from [my.ukvisajobs.com](https://my.ukvisajobs.com) that ma npm install ``` +If Playwright browsers are skipped in your environment, install Firefox: + +```bash +npx playwright install firefox +``` + +If Camoufox assets are missing, fetch them: + +```bash +npx camoufox-js fetch +``` + ## Configuration -Set the following environment variables (you can get these from your browser's dev tools after logging in): +Set the following environment variables: | Variable | Description | |----------|-------------| -| `UKVISAJOBS_TOKEN` | JWT token from the request body (required) | -| `UKVISAJOBS_AUTH_TOKEN` | Auth cookie token (defaults to UKVISAJOBS_TOKEN) | -| `UKVISAJOBS_CSRF_TOKEN` | CSRF token from cookies | -| `UKVISAJOBS_CI_SESSION` | CI session ID from cookies | +| `UKVISAJOBS_EMAIL` | Login email for automatic token refresh | +| `UKVISAJOBS_PASSWORD` | Login password for automatic token refresh | +| `UKVISAJOBS_HEADLESS` | Set to `false` to show the browser (default: true) | | `UKVISAJOBS_MAX_JOBS` | Maximum jobs to fetch (default: 50, max: 200) | | `UKVISAJOBS_SEARCH_KEYWORD` | Optional search filter | -## How to get tokens +## Automatic login & cache -1. Log into `my.ukvisajobs.com` in your browser -2. Open Developer Tools → Network tab -3. Navigate to the jobs page -4. Find the `fetch-jobs-data` POST request -5. Copy values: - - From **Request Body**: copy the `token` field → `UKVISAJOBS_TOKEN` - - From **Cookies**: copy `authToken`, `csrf_token`, `ci_session` +The extractor will: + +1. Launch a Camoufox (Playwright Firefox) browser and sign in +2. Navigate to the open jobs page and capture the token/cookies +3. Cache the session to `storage/ukvisajobs-auth.json` +4. Reuse the cached values until the API reports an expired token, then refresh ## Running @@ -38,3 +48,4 @@ npm start ``` Output is written to `storage/datasets/default/` as JSON files. + diff --git a/extractors/ukvisajobs/package-lock.json b/extractors/ukvisajobs/package-lock.json index 71e9a57..165f27f 100644 --- a/extractors/ukvisajobs/package-lock.json +++ b/extractors/ukvisajobs/package-lock.json @@ -8,6 +8,10 @@ "name": "ukvisajobs-extractor", "version": "0.0.1", "license": "ISC", + "dependencies": { + "camoufox-js": "^0.8.0", + "playwright": "^1.57.0" + }, "devDependencies": { "@apify/tsconfig": "^0.1.0", "@types/node": "^24.0.0", @@ -464,6 +468,36 @@ "node": ">=18" } }, + "node_modules/@isaacs/balanced-match": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/balanced-match/-/balanced-match-4.0.1.tgz", + "integrity": "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@isaacs/brace-expansion": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/brace-expansion/-/brace-expansion-5.0.0.tgz", + "integrity": "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==", + "dependencies": { + "@isaacs/balanced-match": "^4.0.1" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, "node_modules/@types/node": { "version": "24.10.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz", @@ -474,6 +508,270 @@ "undici-types": "~7.16.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/baseline-browser-mapping": { + "version": "2.9.12", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.12.tgz", + "integrity": "sha512-Mij6Lij93pTAIsSYy5cyBQ975Qh9uLEc5rwGTpomiZeXZL9yIS6uORJakb3ScHgfs0serMMfIbXzokPMuEiRyw==", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/better-sqlite3": { + "version": "12.5.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.5.0.tgz", + "integrity": "sha512-WwCZ/5Diz7rsF29o27o0Gcc1Du+l7Zsv7SYtVPG0X3G/uUI1LqdxrQI7c9Hs2FWpqXXERjW9hp6g3/tH7DlVKg==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/browserslist": { + "version": "4.28.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", + "integrity": "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "baseline-browser-mapping": "^2.9.0", + "caniuse-lite": "^1.0.30001759", + "electron-to-chromium": "^1.5.263", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.2.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/camoufox-js": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/camoufox-js/-/camoufox-js-0.8.4.tgz", + "integrity": "sha512-0isRdXX2NnLZjkvBG94H2eQlx2PVA4LmMTlVTkY/vtytRNiAa/bp6yHHJoSNfyIx62HsEruiyKe7OPtEqUWkyA==", + "dependencies": { + "adm-zip": "^0.5.16", + "better-sqlite3": "^12.2.0", + "commander": "^14.0.0", + "fingerprint-generator": "^2.1.66", + "glob": "^13.0.0", + "impit": "^0.7.0", + "language-tags": "^2.0.1", + "maxmind": "^5.0.0", + "progress": "^2.0.3", + "ua-parser-js": "^2.0.2", + "xml2js": "^0.6.2" + }, + "bin": { + "camoufox-js": "dist/__main__.js" + }, + "engines": { + "node": ">= 20" + }, + "peerDependencies": { + "playwright-core": "*" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001762", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001762.tgz", + "integrity": "sha512-PxZwGNvH7Ak8WX5iXzoK1KPZttBXNPuaOvI2ZYU7NrlM+d9Ov+TUvlLOBNGzVXAntMSMMlJPd+jY6ovrVjSmUw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ] + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "engines": { + "node": ">=20" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-europe-js": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/detect-europe-js/-/detect-europe-js-0.1.2.tgz", + "integrity": "sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ] + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.267", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.267.tgz", + "integrity": "sha512-0Drusm6MVRXSOJpGbaSVgcQsuB4hEkMpHXaVstcPmhu5LIedxs1xNK/nIxmQIU/RPC0+1/o0AVZfBTkTNJOdUw==" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/esbuild": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.2.tgz", @@ -516,6 +814,45 @@ "@esbuild/win32-x64": "0.27.2" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, + "node_modules/fingerprint-generator": { + "version": "2.1.79", + "resolved": "https://registry.npmjs.org/fingerprint-generator/-/fingerprint-generator-2.1.79.tgz", + "integrity": "sha512-0dr3kTgvRYHleRPp6OBDcPb8amJmOyFr9aOuwnpN6ooWJ5XyT+/aL/SZ6CU4ZrEtzV26EyJ2Lg7PT32a0NdrRA==", + "dependencies": { + "generative-bayesian-network": "^2.1.79", + "header-generator": "^2.1.79", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -531,6 +868,15 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/generative-bayesian-network": { + "version": "2.1.79", + "resolved": "https://registry.npmjs.org/generative-bayesian-network/-/generative-bayesian-network-2.1.79.tgz", + "integrity": "sha512-aPH+V2wO+HE0BUX1LbsM8Ak99gmV43lgh+D7GDteM0zgnPqiAwcK9JZPxMPZa3aJUleFtFaL1lAei8g9zNrDIA==", + "dependencies": { + "adm-zip": "^0.5.9", + "tslib": "^2.4.0" + } + }, "node_modules/get-tsconfig": { "version": "4.13.0", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz", @@ -544,6 +890,510 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, + "node_modules/glob": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.0.tgz", + "integrity": "sha512-tvZgpqk6fz4BaNZ66ZsRaZnbHvP/jG3uKJvAZOwEVUL4RTA5nJeeLYfyN9/VA8NX/V3IBG+hkeuGpKjvELkVhA==", + "dependencies": { + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "path-scurry": "^2.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/header-generator": { + "version": "2.1.79", + "resolved": "https://registry.npmjs.org/header-generator/-/header-generator-2.1.79.tgz", + "integrity": "sha512-YvHx8teq4QmV5mz7wdPMsj9n1OZBPnZxA4QE+EOrtx7xbmGvd1gBvDNKCb5XqS4GR/TL75MU5hqMqqqANdILRg==", + "dependencies": { + "browserslist": "^4.21.1", + "generative-bayesian-network": "^2.1.79", + "ow": "^0.28.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/impit": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit/-/impit-0.7.6.tgz", + "integrity": "sha512-AkS6Gv63+E6GMvBrcRhMmOREKpq5oJ0J5m3xwfkHiEs97UIsbpEqFmW3sFw/sdyOTDGRF5q4EjaLxtb922Ta8g==", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "impit-darwin-arm64": "0.7.6", + "impit-darwin-x64": "0.7.6", + "impit-linux-arm64-gnu": "0.7.6", + "impit-linux-arm64-musl": "0.7.6", + "impit-linux-x64-gnu": "0.7.6", + "impit-linux-x64-musl": "0.7.6", + "impit-win32-arm64-msvc": "0.7.6", + "impit-win32-x64-msvc": "0.7.6" + } + }, + "node_modules/impit-darwin-arm64": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-darwin-arm64/-/impit-darwin-arm64-0.7.6.tgz", + "integrity": "sha512-M7NQXkttyzqilWfzVkNCp7hApT69m0etyJkVpHze4bR5z1kJnHhdsb8BSdDv2dzvZL4u1JyqZNxq+qoMn84eUw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-darwin-x64": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-darwin-x64/-/impit-darwin-x64-0.7.6.tgz", + "integrity": "sha512-kikTesWirAwJp9JPxzGLoGVc+heBlEabWS5AhTkQedACU153vmuL90OBQikVr3ul2N0LPImvnuB+51wV0zDE6g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-gnu": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-gnu/-/impit-linux-arm64-gnu-0.7.6.tgz", + "integrity": "sha512-H6GHjVr/0lG9VEJr6IHF8YLq+YkSIOF4k7Dfue2ygzUAj1+jZ5ZwnouhG/XrZHYW6EWsZmEAjjRfWE56Q0wDRQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-arm64-musl": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-linux-arm64-musl/-/impit-linux-arm64-musl-0.7.6.tgz", + "integrity": "sha512-1sCB/UBVXLZTpGJsXRdNNSvhN9xmmQcYLMWAAB4Itb7w684RHX1pLoCb6ichv7bfAf6tgaupcFIFZNBp3ghmQA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-gnu": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-linux-x64-gnu/-/impit-linux-x64-gnu-0.7.6.tgz", + "integrity": "sha512-yYhlRnZ4fhKt8kuGe0JK2WSHc8TkR6BEH0wn+guevmu8EOn9Xu43OuRvkeOyVAkRqvFnlZtMyySUo/GuSLz9Gw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-linux-x64-musl": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-linux-x64-musl/-/impit-linux-x64-musl-0.7.6.tgz", + "integrity": "sha512-sdGWyu+PCLmaOXy7Mzo4WP61ZLl5qpZ1L+VeXW+Ycazgu0e7ox0NZLdiLRunIrEzD+h0S+e4CyzNwaiP3yIolg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-arm64-msvc": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-win32-arm64-msvc/-/impit-win32-arm64-msvc-0.7.6.tgz", + "integrity": "sha512-sM5deBqo0EuXg5GACBUMKEua9jIau/i34bwNlfrf/Amnw1n0GB4/RkuUh+sKiUcbNAntrRq+YhCq8qDP8IW19w==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/impit-win32-x64-msvc": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/impit-win32-x64-msvc/-/impit-win32-x64-msvc-0.7.6.tgz", + "integrity": "sha512-ry63ADGLCB/PU/vNB1VioRt2V+klDJ34frJUXUZBEv1kA96HEAg9AxUk+604o+UHS3ttGH2rkLmrbwHOdAct5Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-standalone-pwa": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-standalone-pwa/-/is-standalone-pwa-0.1.1.tgz", + "integrity": "sha512-9Cbovsa52vNQCjdXOzeQq5CnCbAcRk05aU62K20WO372NrTv0NxibLFCK6lQ4/iZEFdEA3p3t2VNOn8AJ53F5g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ] + }, + "node_modules/language-subtag-registry": { + "version": "0.3.23", + "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz", + "integrity": "sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==" + }, + "node_modules/language-tags": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-2.1.0.tgz", + "integrity": "sha512-D4CgpyCt+61f6z2jHjJS1OmZPviAWM57iJ9OKdFFWSNgS7Udj9QVWqyGs/cveVNF57XpZmhSvMdVIV5mjLA7Vg==", + "dependencies": { + "language-subtag-registry": "^0.3.20" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead." + }, + "node_modules/lru-cache": { + "version": "11.2.4", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.4.tgz", + "integrity": "sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/maxmind": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-5.0.3.tgz", + "integrity": "sha512-oMtZwLrsp0LcZehfYKIirtwKMBycMMqMA1/Dc9/BlUqIEtXO75mIzMJ3PYCV1Ji+BpoUCk+lTzRfh9c+ptGdyQ==", + "dependencies": { + "mmdb-lib": "3.0.1", + "tiny-lru": "11.4.5" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.1.tgz", + "integrity": "sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==", + "dependencies": { + "@isaacs/brace-expansion": "^5.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/mmdb-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-3.0.1.tgz", + "integrity": "sha512-dyAyMR+cRykZd1mw5altC9f4vKpCsuywPwo8l/L5fKqDay2zmqT0mF/BvUoXnQiqGn+nceO914rkPKJoyFnGxA==", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==" + }, + "node_modules/node-abi": { + "version": "3.85.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.85.0.tgz", + "integrity": "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/ow": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/ow/-/ow-0.28.2.tgz", + "integrity": "sha512-dD4UpyBh/9m4X2NVjA+73/ZPBRF+uF4zIMFvvQsabMiEK8x41L3rQ8EENOi35kyyoaJwNxEeJcP6Fj1H4U409Q==", + "dependencies": { + "@sindresorhus/is": "^4.2.0", + "callsites": "^3.1.0", + "dot-prop": "^6.0.1", + "lodash.isequal": "^4.5.0", + "vali-date": "^1.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-scurry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", + "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -554,6 +1404,139 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/sax": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.3.tgz", + "integrity": "sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ==" + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tiny-lru": { + "version": "11.4.5", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.4.5.tgz", + "integrity": "sha512-hkcz3FjNJfKXjV4mjQ1OrXSLAehg8Hw+cEZclOVT+5c/cWQWImQ9wolzTjth+dmmDe++p3bme3fTxz6Q4Etsqw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -574,6 +1557,17 @@ "fsevents": "~2.3.3" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -588,12 +1582,128 @@ "node": ">=14.17" } }, + "node_modules/ua-is-frozen": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ua-is-frozen/-/ua-is-frozen-0.1.2.tgz", + "integrity": "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ] + }, + "node_modules/ua-parser-js": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.7.tgz", + "integrity": "sha512-CFdHVHr+6YfbktNZegH3qbYvYgC7nRNEUm2tk7nSFXSODUu4tDBpaFpP1jdXBUOKKwapVlWRfTtS8bCPzsQ47w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "dependencies": { + "detect-europe-js": "^0.1.2", + "is-standalone-pwa": "^0.1.1", + "ua-is-frozen": "^0.1.2" + }, + "bin": { + "ua-parser-js": "script/cli.js" + }, + "engines": { + "node": "*" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", "dev": true, "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", + "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/vali-date": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/vali-date/-/vali-date-1.0.0.tgz", + "integrity": "sha512-sgECfZthyaCKW10N0fm27cg8HYTFK5qMWgypqkXMQ4Wbl/zZKx7xZICgcoxIIE+WFAP/MBL2EFwC/YvLxw3Zeg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } } } } diff --git a/extractors/ukvisajobs/package.json b/extractors/ukvisajobs/package.json index e72e9c4..09535d5 100644 --- a/extractors/ukvisajobs/package.json +++ b/extractors/ukvisajobs/package.json @@ -4,7 +4,10 @@ "type": "module", "description": "UK Visa Jobs extractor - fetches job listings that may sponsor work visas", "main": "dist/main.js", - "dependencies": {}, + "dependencies": { + "camoufox-js": "^0.8.0", + "playwright": "^1.57.0" + }, "devDependencies": { "@apify/tsconfig": "^0.1.0", "@types/node": "^24.0.0", @@ -15,8 +18,10 @@ "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "tsx src/main.ts", - "build": "tsc" + "build": "tsc", + "get-binaries": "camoufox-js fetch", + "postinstall": "npm run get-binaries" }, "author": "", "license": "ISC" -} \ No newline at end of file +} diff --git a/extractors/ukvisajobs/src/main.ts b/extractors/ukvisajobs/src/main.ts index 471cd37..d5de320 100644 --- a/extractors/ukvisajobs/src/main.ts +++ b/extractors/ukvisajobs/src/main.ts @@ -1,25 +1,28 @@ -/** +/** * UK Visa Jobs Extractor * * Fetches job listings from my.ukvisajobs.com that may sponsor work visas. * Outputs JSON to stdout for the orchestrator to consume. * * Environment variables: - * UKVISAJOBS_TOKEN - JWT token (required) - * UKVISAJOBS_AUTH_TOKEN - Auth cookie token (defaults to UKVISAJOBS_TOKEN) - * UKVISAJOBS_CSRF_TOKEN - CSRF token cookie - * UKVISAJOBS_CI_SESSION - CI session cookie + * UKVISAJOBS_EMAIL - Login email for auto-refresh + * UKVISAJOBS_PASSWORD - Login password for auto-refresh + * UKVISAJOBS_HEADLESS - Set to "false" to show the browser (default: true) * UKVISAJOBS_MAX_JOBS - Maximum jobs to fetch (default: 50, max: 200) - Set via UI Settings * UKVISAJOBS_SEARCH_KEYWORD - Optional search filter */ -import { mkdir, writeFile } from 'fs/promises'; +import { mkdir, writeFile, readFile } from 'fs/promises'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; +import type { Request } from 'playwright'; const __dirname = dirname(fileURLToPath(import.meta.url)); const API_URL = 'https://my.ukvisajobs.com/ukvisa-api/api/fetch-jobs-data'; +const SIGNIN_URL = 'https://my.ukvisajobs.com/signin'; +const OPEN_JOBS_URL = 'https://my.ukvisajobs.com/open-jobs/1?is_global=0&sortBy=desc&visaAcceptance=false&applicants_outside_uk=false&pageNo=1'; +const AUTH_CACHE_PATH = join(__dirname, '../storage/ukvisajobs-auth.json'); const JOBS_PER_PAGE = 15; const DEFAULT_MAX_JOBS = 50; const MAX_ALLOWED_JOBS = 200; @@ -77,6 +80,27 @@ interface ExtractedJob { jobLevel?: string; } +interface UkVisaJobsAuthSession { + token: string; + authToken: string; + csrfToken: string; + ciSession: string; + fetchedAt: string; + source: 'cache' | 'browser'; +} + +class UkVisaJobsAuthError extends Error { + status: number; + responseText: string; + + constructor(message: string, status: number, responseText: string) { + super(message); + this.name = 'UkVisaJobsAuthError'; + this.status = status; + this.responseText = responseText; + } +} + function toStringOrNull(value: unknown): string | null { if (value === null || value === undefined) return null; if (typeof value === 'string') { @@ -101,8 +125,7 @@ function toNumberOrNull(value: unknown): number | null { async function fetchPage( pageNo: number, - token: string, - cookies: string, + session: UkVisaJobsAuthSession, options: { searchKeyword?: string } = {} ): Promise { // Use native FormData API (Node.js 18+) @@ -113,7 +136,9 @@ async function fetchPage( formData.append('visaAcceptance', 'false'); formData.append('applicants_outside_uk', 'false'); formData.append('searchKeyword', options.searchKeyword || 'null'); - formData.append('token', token); + formData.append('token', session.token); + + const cookies = buildCookieHeader(session); const response = await fetch(API_URL, { method: 'POST', @@ -130,6 +155,13 @@ async function fetchPage( if (!response.ok) { const text = await response.text(); + if (isAuthErrorResponse(response.status, text)) { + throw new UkVisaJobsAuthError( + `UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`, + response.status, + text + ); + } throw new Error(`UKVisaJobs API returned ${response.status}: ${response.statusText} - ${text}`); } @@ -143,12 +175,12 @@ function mapJob(raw: UkVisaJobsApiJob): ExtractedJob { const maxSalary = toNumberOrNull(raw.max_salary); if (minSalary !== null && minSalary > 0 && maxSalary !== null && maxSalary > 0) { - salary = `£${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}`; + salary = `£${minSalary.toLocaleString()}-${maxSalary.toLocaleString()}`; if (raw.salary_interval) { salary += ` / ${raw.salary_interval}`; } } else if (maxSalary !== null && maxSalary > 0) { - salary = `£${maxSalary.toLocaleString()}`; + salary = `£${maxSalary.toLocaleString()}`; if (raw.salary_interval) { salary += ` / ${raw.salary_interval}`; } @@ -188,30 +220,181 @@ function mapJob(raw: UkVisaJobsApiJob): ExtractedJob { }; } -async function main(): Promise { - console.log('🇬🇧 UK Visa Jobs Extractor starting...'); +function buildCookieHeader(session: UkVisaJobsAuthSession): string { + const cookieParts: string[] = []; + if (session.csrfToken) cookieParts.push(`csrf_token=${session.csrfToken}`); + if (session.ciSession) cookieParts.push(`ci_session=${session.ciSession}`); + if (session.authToken) cookieParts.push(`authToken=${session.authToken}`); + return cookieParts.join('; '); +} - // Get credentials from environment - const token = process.env.UKVISAJOBS_TOKEN; - const authToken = process.env.UKVISAJOBS_AUTH_TOKEN || token; - const csrfToken = process.env.UKVISAJOBS_CSRF_TOKEN || ''; - const ciSession = process.env.UKVISAJOBS_CI_SESSION || ''; +function getLoginCredentials(): { email: string; password: string } | null { + const email = process.env.UKVISAJOBS_EMAIL; + const password = process.env.UKVISAJOBS_PASSWORD; + if (!email || !password) return null; + return { email, password }; +} + +async function loadCachedAuthSession(): Promise { + try { + const data = await readFile(AUTH_CACHE_PATH, 'utf8'); + const parsed = JSON.parse(data) as UkVisaJobsAuthSession; + if (!parsed?.token) return null; + return { + token: parsed.token, + authToken: parsed.authToken || parsed.token, + csrfToken: parsed.csrfToken || '', + ciSession: parsed.ciSession || '', + fetchedAt: parsed.fetchedAt || new Date().toISOString(), + source: 'cache', + }; + } catch (error) { + return null; + } +} + +async function saveCachedAuthSession(session: UkVisaJobsAuthSession): Promise { + const payload = { + token: session.token, + authToken: session.authToken, + csrfToken: session.csrfToken, + ciSession: session.ciSession, + fetchedAt: session.fetchedAt, + source: session.source, + }; + await mkdir(dirname(AUTH_CACHE_PATH), { recursive: true }); + await writeFile(AUTH_CACHE_PATH, JSON.stringify(payload, null, 2)); +} + +function extractMultipartField(body: string, field: string): string | null { + const nameToken = `name="${field}"`; + const index = body.indexOf(nameToken); + if (index === -1) return null; + + const afterName = body.slice(index + nameToken.length); + let separatorIndex = afterName.indexOf('\r\n\r\n'); + let separatorLength = 4; + if (separatorIndex === -1) { + separatorIndex = afterName.indexOf('\n\n'); + separatorLength = 2; + } + if (separatorIndex === -1) return null; + + const valueStart = index + nameToken.length + separatorIndex + separatorLength; + const remainder = body.slice(valueStart); + const endIndex = remainder.indexOf('\r\n'); + if (endIndex === -1) return remainder.trim(); + return remainder.slice(0, endIndex).trim(); +} + +function extractTokenFromRequest(request: Request): string | null { + const postData = request.postData(); + if (!postData) return null; + const multipartToken = extractMultipartField(postData, 'token'); + if (multipartToken) return multipartToken; + try { + const params = new URLSearchParams(postData); + const token = params.get('token'); + return token || null; + } catch (error) { + return null; + } +} + +function isAuthErrorResponse(status: number, bodyText: string): boolean { + if (status === 401 || status === 403) return true; + if (status !== 400) return false; + try { + const parsed = JSON.parse(bodyText) as { errorType?: string; message?: string }; + if (parsed?.errorType === 'expired') return true; + if (parsed?.message && parsed.message.toLowerCase().includes('expired')) return true; + } catch (error) { + // ignore JSON parse failures + } + return bodyText.toLowerCase().includes('expired'); +} + +async function loginWithBrowser(email: string, password: string): Promise { + const [{ launchOptions }, { firefox }] = await Promise.all([ + import('camoufox-js'), + import('playwright'), + ]); + const headless = process.env.UKVISAJOBS_HEADLESS !== 'false'; + const browser = await firefox.launch(await launchOptions({ + headless, + humanize: true, + geoip: true, + })); + const context = await browser.newContext(); + const page = await context.newPage(); + + try { + await page.goto(SIGNIN_URL, { waitUntil: 'domcontentloaded' }); + await page.waitForSelector('#email', { timeout: 15000 }); + await page.fill('#email', email); + await page.fill('#password', password); + await page.keyboard.press('Enter'); + await page.waitForTimeout(7000); + + const requestPromise = page.waitForRequest( + (request) => request.url().includes('/ukvisa-api/api/fetch-jobs-data') && request.method() === 'POST', + { timeout: 30000 } + ); + + await page.goto(OPEN_JOBS_URL, { waitUntil: 'networkidle' }); + await page.waitForTimeout(5000); + + let fetchRequest: Request | null = null; + try { + fetchRequest = await requestPromise; + } catch (error) { + fetchRequest = null; + } + + const cookies = await context.cookies('https://my.ukvisajobs.com'); + const csrfToken = cookies.find((cookie) => cookie.name === 'csrf_token')?.value || ''; + const ciSession = cookies.find((cookie) => cookie.name === 'ci_session')?.value || ''; + const authToken = cookies.find((cookie) => cookie.name === 'authToken')?.value || ''; + const token = fetchRequest ? extractTokenFromRequest(fetchRequest) : authToken; + + if (!token) { + throw new Error('Failed to locate auth token from browser session.'); + } + + return { + token, + authToken: authToken || token, + csrfToken, + ciSession, + fetchedAt: new Date().toISOString(), + source: 'browser', + }; + } finally { + await browser.close(); + } +} + +async function main(): Promise { + console.log('🇬🇧 UK Visa Jobs Extractor starting...'); + const credentials = getLoginCredentials(); const searchKeyword = process.env.UKVISAJOBS_SEARCH_KEYWORD || undefined; - if (!token) { - console.error('❌ UKVISAJOBS_TOKEN environment variable is not set'); - process.exit(1); + let authSession = await loadCachedAuthSession(); + + if (!authSession) { + if (!credentials) { + console.error('ERROR: UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD must be set'); + process.exit(1); + } + console.log(' No cached session found. Logging in to refresh tokens...'); + authSession = await loginWithBrowser(credentials.email, credentials.password); + await saveCachedAuthSession(authSession); } - // Build cookies string - const cookieParts: string[] = []; - if (csrfToken) cookieParts.push(`csrf_token=${csrfToken}`); - if (ciSession) cookieParts.push(`ci_session=${ciSession}`); - if (authToken) cookieParts.push(`authToken=${authToken}`); - const cookies = cookieParts.join('; '); - - console.log(` Cookies configured: ${cookieParts.length > 0 ? 'Yes' : 'No'}`); - console.log(` Token length: ${token.length}`); + const cookies = buildCookieHeader(authSession); + console.log(` Auth source: ${authSession.source}`); + console.log(` Cookies configured: ${cookies ? 'Yes' : 'No'}`); + console.log(` Token length: ${authSession.token.length}`); // Get max jobs from environment const maxJobsEnv = toNumberOrNull(process.env.UKVISAJOBS_MAX_JOBS); @@ -232,10 +415,25 @@ async function main(): Promise { while (pageNo <= maxPages && allJobs.length < maxJobs) { console.log(` Fetching page ${pageNo}/${maxPages}...`); - const response = await fetchPage(pageNo, token, cookies, { searchKeyword }); + let response: UkVisaJobsApiResponse; + try { + response = await fetchPage(pageNo, authSession, { searchKeyword }); + } catch (error) { + if (error instanceof UkVisaJobsAuthError) { + if (!credentials) { + throw new Error('UKVisaJobs auth expired. Set UKVISAJOBS_EMAIL and UKVISAJOBS_PASSWORD to refresh.'); + } + console.log(' Auth expired. Refreshing tokens...'); + authSession = await loginWithBrowser(credentials.email, credentials.password); + await saveCachedAuthSession(authSession); + response = await fetchPage(pageNo, authSession, { searchKeyword }); + } else { + throw error; + } + } if (response.status !== 1) { - console.warn(` ⚠️ API returned status ${response.status} on page ${pageNo}`); + console.warn(` ⚠️ API returned status ${response.status} on page ${pageNo}`); break; } @@ -271,7 +469,7 @@ async function main(): Promise { await new Promise((resolve) => setTimeout(resolve, 500)); } - console.log(`✅ Scraped ${allJobs.length} jobs`); + console.log(`✅ Scraped ${allJobs.length} jobs`); // Write output to storage directory (similar to Crawlee dataset structure) const storageDir = join(__dirname, '../storage/datasets/default'); @@ -292,7 +490,7 @@ async function main(): Promise { } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; - console.error(`❌ Error: ${message}`); + console.error(`❌ Error: ${message}`); process.exit(1); } } @@ -301,3 +499,6 @@ main().catch((error) => { console.error('Fatal error:', error); process.exit(1); }); + + + diff --git a/orchestrator/src/server/services/ukvisajobs.ts b/orchestrator/src/server/services/ukvisajobs.ts index 802a52c..1e727c5 100644 --- a/orchestrator/src/server/services/ukvisajobs.ts +++ b/orchestrator/src/server/services/ukvisajobs.ts @@ -1,4 +1,4 @@ -/** +/** * Service for running the UK Visa Jobs extractor (extractors/ukvisajobs). * * Spawns the extractor as a child process and reads its output dataset. @@ -13,6 +13,14 @@ import type { CreateJobInput } from '../../shared/types.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const UKVISAJOBS_DIR = join(__dirname, '../../../../extractors/ukvisajobs'); const STORAGE_DIR = join(UKVISAJOBS_DIR, 'storage/datasets/default'); +const AUTH_CACHE_PATH = join(UKVISAJOBS_DIR, 'storage/ukvisajobs-auth.json'); + +interface UkVisaJobsAuthSession { + token?: string; + authToken?: string; + csrfToken?: string; + ciSession?: string; +} export interface RunUkVisaJobsOptions { /** Maximum number of jobs to fetch per search term. Defaults to 50, max 200. */ @@ -73,11 +81,11 @@ async function fetchJobDescription(url: string): Promise { try { console.log(` Fetching description from ${url}...`); - // Build cookies if present in env (similar to extractor) + const authSession = await loadCachedAuthSession(); const cookieParts: string[] = []; - if (process.env.UKVISAJOBS_CSRF_TOKEN) cookieParts.push(`csrf_token=${process.env.UKVISAJOBS_CSRF_TOKEN}`); - if (process.env.UKVISAJOBS_CI_SESSION) cookieParts.push(`ci_session=${process.env.UKVISAJOBS_CI_SESSION}`); - const token = process.env.UKVISAJOBS_AUTH_TOKEN || process.env.UKVISAJOBS_TOKEN; + if (authSession?.csrfToken) cookieParts.push(`csrf_token=${authSession.csrfToken}`); + if (authSession?.ciSession) cookieParts.push(`ci_session=${authSession.ciSession}`); + const token = authSession?.authToken || authSession?.token; if (token) cookieParts.push(`authToken=${token}`); const headers: Record = { @@ -101,7 +109,16 @@ async function fetchJobDescription(url: string): Promise { // If we only got a tiny bit of text, it might have failed return cleaned.length > 100 ? cleaned : null; } catch (error) { - console.warn(` ⚠️ Failed to fetch description: ${error instanceof Error ? error.message : 'Unknown error'}`); + console.warn(` ⚠️ Failed to fetch description: ${error instanceof Error ? error.message : 'Unknown error'}`); + return null; + } +} + +async function loadCachedAuthSession(): Promise { + try { + const data = await readFile(AUTH_CACHE_PATH, 'utf-8'); + return JSON.parse(data) as UkVisaJobsAuthSession; + } catch { return null; } } @@ -118,7 +135,7 @@ async function clearStorageDataset(): Promise { } export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise { - console.log('🇬🇧 Running UK Visa Jobs extractor...'); + console.log('🇬🇧 Running UK Visa Jobs extractor...'); // Determine terms to run const terms: string[] = []; @@ -192,11 +209,11 @@ export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise } } - console.log(` ✅ Fetched ${runJobs.length} jobs for ${termLabel} (${newCount} new unique)`); + console.log(` ✅ Fetched ${runJobs.length} jobs for ${termLabel} (${newCount} new unique)`); } catch (error) { const message = error instanceof Error ? error.message : 'Unknown error'; - console.error(`❌ UK Visa Jobs failed for ${termLabel}: ${message}`); + console.error(`❌ UK Visa Jobs failed for ${termLabel}: ${message}`); // Continue to next term instead of failing completely } @@ -207,7 +224,7 @@ export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise } } - console.log(`✅ UK Visa Jobs: imported total ${allJobs.length} unique jobs`); + console.log(`✅ UK Visa Jobs: imported total ${allJobs.length} unique jobs`); return { success: true, jobs: allJobs }; } @@ -254,3 +271,4 @@ async function readDataset(): Promise { return jobs; } +