diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2afa9bf..d59439d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: - adzuna-extractor - hiringcafe-extractor - gradcracker-extractor + - startupjobs-extractor - ukvisajobs-extractor steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cd191c..ad392fb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,9 +7,14 @@ on: description: "Next release version (x.y.z)" required: true type: string + release_title: + description: "Optional release title shown on GitHub (defaults to vX.Y.Z)" + required: false + type: string permissions: contents: write + packages: write concurrency: group: release-${{ inputs.version }} @@ -83,8 +88,50 @@ jobs: git tag "v$RELEASE_VERSION" git push origin "v$RELEASE_VERSION" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta (tags/labels) + id: docker-meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/job-ops + tags: | + type=raw,value=v${{ inputs.version }} + type=raw,value=latest + type=sha + + - name: Build and push GHCR image + uses: docker/build-push-action@v6 + with: + context: . + file: ./Dockerfile + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Create GitHub release env: GH_TOKEN: ${{ github.token }} RELEASE_VERSION: ${{ inputs.version }} - run: gh release create "v$RELEASE_VERSION" --title "v$RELEASE_VERSION" --generate-notes + INPUT_RELEASE_TITLE: ${{ inputs.release_title }} + run: | + RELEASE_TITLE="$(printf '%s' "$INPUT_RELEASE_TITLE" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -z "$RELEASE_TITLE" ]; then + RELEASE_TITLE="v$RELEASE_VERSION" + fi + + gh release create "v$RELEASE_VERSION" --title "$RELEASE_TITLE" --generate-notes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bba3cc8..560984a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,8 @@ Releases are driven from GitHub Actions. 1. Open the `release` workflow in GitHub Actions. 2. Enter the next version as `x.y.z` (for example `0.1.30`). -3. Run the workflow. +3. Optionally enter a separate release title for GitHub (for example `Google Dorks!`). +4. Run the workflow. The workflow will: @@ -68,9 +69,10 @@ The workflow will: - update `package-lock.json` - commit the version bump to `main` - create and push tag `vX.Y.Z` -- create the GitHub release +- publish the `ghcr.io/.../job-ops` image for that release +- create the GitHub release using either the custom title or `vX.Y.Z` -The app version shown in the UI is sourced from `orchestrator/package.json`, so the release version, tag, and displayed app version stay aligned. +The app version shown in the UI is sourced from `orchestrator/package.json`, so the release version, tag, and displayed app version stay aligned even when the GitHub release title is customized separately. ## Validation Before PR (CI-Parity Checks) diff --git a/Dockerfile b/Dockerfile index 32677c6..c1c6917 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ COPY orchestrator/package*.json ./orchestrator/ COPY extractors/adzuna/package*.json ./extractors/adzuna/ COPY extractors/hiringcafe/package*.json ./extractors/hiringcafe/ COPY extractors/gradcracker/package*.json ./extractors/gradcracker/ +COPY extractors/startupjobs/package*.json ./extractors/startupjobs/ COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/ # Install Node dependencies with npm cache (dev deps needed for build) @@ -59,6 +60,7 @@ COPY extractors/adzuna ./extractors/adzuna COPY extractors/hiringcafe ./extractors/hiringcafe COPY extractors/gradcracker ./extractors/gradcracker COPY extractors/jobspy ./extractors/jobspy +COPY extractors/startupjobs ./extractors/startupjobs COPY extractors/ukvisajobs ./extractors/ukvisajobs # Build documentation site bundle @@ -105,6 +107,7 @@ COPY orchestrator/package*.json ./orchestrator/ COPY extractors/adzuna/package*.json ./extractors/adzuna/ COPY extractors/hiringcafe/package*.json ./extractors/hiringcafe/ COPY extractors/gradcracker/package*.json ./extractors/gradcracker/ +COPY extractors/startupjobs/package*.json ./extractors/startupjobs/ COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/ # Install production Node dependencies only @@ -122,6 +125,7 @@ COPY extractors/adzuna ./extractors/adzuna COPY extractors/hiringcafe ./extractors/hiringcafe COPY extractors/gradcracker ./extractors/gradcracker COPY extractors/jobspy ./extractors/jobspy +COPY extractors/startupjobs ./extractors/startupjobs COPY extractors/ukvisajobs ./extractors/ukvisajobs # Reuse Camoufox binaries from builder instead of fetching again diff --git a/docs-site/docs/extractors/overview.md b/docs-site/docs/extractors/overview.md index 74eb009..cd7655c 100644 --- a/docs-site/docs/extractors/overview.md +++ b/docs-site/docs/extractors/overview.md @@ -17,6 +17,7 @@ Extractor integrations are now registered through manifests and loaded automatic | [JobSpy](/docs/next/extractors/jobspy) | Multi-source discovery (Indeed, LinkedIn, Glassdoor) | Requires Python wrapper execution per term; source availability and quality vary by site/location | `JOBSPY_SITES`, `JOBSPY_SEARCH_TERMS`, `JOBSPY_RESULTS_WANTED`, `JOBSPY_HOURS_OLD`, `JOBSPY_LINKEDIN_FETCH_DESCRIPTION` | Produces JSON per term, then orchestrator normalizes and de-duplicates by `jobUrl` | | [Adzuna](/docs/next/extractors/adzuna) | API-based multi-country discovery with low scraping overhead | Requires valid App ID/App Key; country must be in Adzuna-supported list | `ADZUNA_APP_ID`, `ADZUNA_APP_KEY`, `ADZUNA_MAX_JOBS_PER_TERM` | API pagination to dataset output; orchestrator maps progress and de-duplicates by `sourceJobId`/`jobUrl` | | [Hiring Cafe](/docs/next/extractors/hiring-cafe) | Browser-backed discovery using Hiring Cafe search APIs | Subject to upstream anti-bot checks; uses browser context and encoded search-state payloads | `HIRING_CAFE_SEARCH_TERMS`, `HIRING_CAFE_COUNTRY`, `HIRING_CAFE_MAX_JOBS_PER_TERM`, `HIRING_CAFE_DATE_FETCHED_PAST_N_DAYS` | Uses existing pipeline term/country/budget knobs and maps directly to normalized jobs | +| [startup.jobs](/docs/next/extractors/startup-jobs) | Startup-focused discovery through the published `startup-jobs-scraper` package | No credentials required; detail enrichment depends on Playwright browser binaries being installed | existing pipeline `searchTerms`, selected country/cities, `jobspyResultsWanted`; `npx playwright install` for fresh environments | Algolia-backed search plus detail-page enrichment via package import; orchestrator maps normalized records and de-duplicates by `jobUrl` | | [UKVisaJobs](/docs/next/extractors/ukvisajobs) | UK visa sponsorship-focused roles | Requires authenticated session and periodic token/cookie refresh | `UKVISAJOBS_EMAIL`, `UKVISAJOBS_PASSWORD`, `UKVISAJOBS_MAX_JOBS`, `UKVISAJOBS_SEARCH_KEYWORD` | API pagination + dataset output; orchestrator de-dupes and may fetch missing descriptions | | [Manual Import](/docs/next/extractors/manual) | One-off jobs not covered by scrapers | Inference quality depends on model/provider and input quality; some URLs cannot be fetched reliably | App/API endpoints (`/api/manual-jobs/infer`, `/api/manual-jobs/import`) | Accepts text/HTML/URL, runs inference, then saves and scores job after review | @@ -25,6 +26,7 @@ Extractor integrations are now registered through manifests and loaded automatic - Use **JobSpy** for broad first-pass sourcing across common boards. - Use **Adzuna** when you want API-first discovery in supported non-UK markets. - Use **Hiring Cafe** when you want another term/country-driven source without adding credentials. +- Use **startup.jobs** when you want startup-heavy listings without maintaining another scraper locally. - Use **Gradcracker** when targeting graduate pipelines in the UK. - Use **UKVisaJobs** for sponsorship-specific UK searches. - Use **Manual Import** when you already have a specific posting and need direct import. @@ -37,6 +39,7 @@ Many runs combine sources: broad discovery first, then manual import for high-pr - [JobSpy](/docs/next/extractors/jobspy) - [Adzuna](/docs/next/extractors/adzuna) - [Hiring Cafe](/docs/next/extractors/hiring-cafe) +- [startup.jobs](/docs/next/extractors/startup-jobs) - [UKVisaJobs](/docs/next/extractors/ukvisajobs) - [Manual Import](/docs/next/extractors/manual) - [Add an Extractor](/docs/next/workflows/add-an-extractor) diff --git a/docs-site/docs/extractors/startup-jobs.md b/docs-site/docs/extractors/startup-jobs.md new file mode 100644 index 0000000..03bc925 --- /dev/null +++ b/docs-site/docs/extractors/startup-jobs.md @@ -0,0 +1,64 @@ +--- +id: startup-jobs +title: startup.jobs Extractor +description: startup.jobs extraction integrated through the startup-jobs-scraper package. +sidebar_position: 8 +--- + +## What it is + +Original website: [startup.jobs](https://startup.jobs) + +This extractor wraps the published [`startup-jobs-scraper`](https://www.npmjs.com/package/startup-jobs-scraper) package and feeds normalized startup.jobs listings into the existing pipeline. + +Implementation split: + +1. `extractors/startupjobs/src/run.ts` calls `scrapeStartupJobsViaAlgolia` and maps package records into `CreateJobInput`. +2. `extractors/startupjobs/src/manifest.ts` adapts pipeline settings, emits progress updates, and registers the source for runtime discovery. + +## Why it exists + +startup.jobs adds a startup-focused board to job-ops without introducing another bespoke scraper in this repository. + +Using the published package also keeps the integration small and makes it easier to evolve the scraping logic independently from the app. + +## How to use it + +1. Open **Run jobs** and choose **Automatic**. +2. Leave **startup.jobs** enabled in **Sources** or toggle it on. +3. Set your usual automatic run controls: + - `searchTerms` are sent as `query`. + - country or city filters are reused as the package `location` option. + - run budget path (`jobspyResultsWanted`) is reused as `requestedCount` per term. +4. Start the run and monitor progress in the pipeline progress card. + +Defaults and constraints: + +- No new credentials are required. +- The integration runs with `enrichDetails: true`, so it opens job detail pages for richer records. +- Browser binaries are not downloaded automatically with the package. Install them with `npx playwright install` before using this extractor in a fresh environment. +- When **Search cities** is set, the extractor runs once per city and once per search term. +- Without explicit cities, the selected country is used as the location filter except for broad modes such as `worldwide` and `usa/ca`. + +## Common problems + +### startup.jobs does not appear in sources + +- Check that the app is running a build that includes the new extractor manifest. +- This source does not require credentials, so it should appear as soon as the updated build is loaded. + +### Results are broader than expected + +- If no city is configured, the extractor uses the selected country when possible and otherwise falls back to a broad search. +- Add **Search cities** when you want tighter geographic filtering. + +### Job descriptions are missing + +- Detail enrichment depends on Playwright browser binaries being installed locally. +- Run `npx playwright install` and retry if the extractor cannot open job detail pages. + +## Related pages + +- [Extractors Overview](/docs/next/extractors/overview) +- [Pipeline Run](/docs/next/features/pipeline-run) +- [Add an Extractor](/docs/next/workflows/add-an-extractor) diff --git a/docs-site/sidebars.ts b/docs-site/sidebars.ts index 3d3f089..664c602 100644 --- a/docs-site/sidebars.ts +++ b/docs-site/sidebars.ts @@ -49,6 +49,7 @@ const sidebars: SidebarsConfig = { "extractors/jobspy", "extractors/adzuna", "extractors/hiring-cafe", + "extractors/startup-jobs", "extractors/manual", "extractors/ukvisajobs", ], diff --git a/extractors/startupjobs/README.md b/extractors/startupjobs/README.md new file mode 100644 index 0000000..bf6783a --- /dev/null +++ b/extractors/startupjobs/README.md @@ -0,0 +1,10 @@ +# startup.jobs Extractor + +Extractor wrapper around the published `startup-jobs-scraper` package. + +## Notes + +- Uses `scrapeStartupJobsViaAlgolia` directly from `startup-jobs-scraper`. +- Runs with `enrichDetails: true` so job descriptions and other detail-page fields are fetched during pipeline runs. +- Browser binaries are not downloaded automatically. Install them with `npx playwright install` or `npm --workspace startupjobs-extractor run get-binaries`. +- Reuses the pipeline's existing search terms, country, city, and budget controls. diff --git a/extractors/startupjobs/package.json b/extractors/startupjobs/package.json new file mode 100644 index 0000000..19a9ab4 --- /dev/null +++ b/extractors/startupjobs/package.json @@ -0,0 +1,17 @@ +{ + "name": "startupjobs-extractor", + "version": "0.0.1", + "type": "module", + "description": "startup.jobs extractor backed by the startup-jobs-scraper package", + "dependencies": { + "startup-jobs-scraper": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "~5.9.0" + }, + "scripts": { + "check:types": "tsc --noEmit", + "get-binaries": "npx playwright install" + } +} diff --git a/extractors/startupjobs/src/manifest.ts b/extractors/startupjobs/src/manifest.ts new file mode 100644 index 0000000..16b9328 --- /dev/null +++ b/extractors/startupjobs/src/manifest.ts @@ -0,0 +1,89 @@ +import { resolveSearchCities } from "@shared/search-cities.js"; +import type { + ExtractorManifest, + ExtractorProgressEvent, +} from "@shared/types/extractors"; +import { runStartupJobs } from "./run"; + +function toProgress(event: { + type: string; + termIndex: number; + termTotal: number; + searchTerm: string; + location?: string; + jobsFoundTerm?: number; +}): ExtractorProgressEvent { + const scope = event.location + ? `${event.searchTerm} @ ${event.location}` + : event.searchTerm; + + if (event.type === "term_start") { + return { + phase: "list", + termsProcessed: Math.max(event.termIndex - 1, 0), + termsTotal: event.termTotal, + currentUrl: scope, + detail: `startup.jobs: term ${event.termIndex}/${event.termTotal} (${scope})`, + }; + } + + return { + phase: "list", + termsProcessed: event.termIndex, + termsTotal: event.termTotal, + currentUrl: scope, + jobPagesProcessed: event.jobsFoundTerm ?? 0, + jobPagesEnqueued: event.jobsFoundTerm ?? 0, + detail: `startup.jobs: completed ${event.termIndex}/${event.termTotal} (${scope}) with ${event.jobsFoundTerm ?? 0} jobs`, + }; +} + +export const manifest: ExtractorManifest = { + id: "startupjobs", + displayName: "startup.jobs", + providesSources: ["startupjobs"], + async run(context) { + if (context.shouldCancel?.()) { + return { success: true, jobs: [] }; + } + + const parsedMaxJobsPerTerm = context.settings.startupjobsMaxJobsPerTerm + ? Number.parseInt(context.settings.startupjobsMaxJobsPerTerm, 10) + : context.settings.jobspyResultsWanted + ? Number.parseInt(context.settings.jobspyResultsWanted, 10) + : Number.NaN; + const maxJobsPerTerm = Number.isFinite(parsedMaxJobsPerTerm) + ? Math.max(1, parsedMaxJobsPerTerm) + : 50; + + const result = await runStartupJobs({ + selectedCountry: context.selectedCountry, + searchTerms: context.searchTerms, + locations: resolveSearchCities({ + single: + context.settings.searchCities ?? context.settings.jobspyLocation, + }), + maxJobsPerTerm, + shouldCancel: context.shouldCancel, + onProgress: (event) => { + if (context.shouldCancel?.()) return; + context.onProgress?.(toProgress(event)); + }, + }); + + if (!result.success) { + return { + success: false, + jobs: [], + error: result.error, + }; + } + + return { + success: true, + jobs: result.jobs, + }; + }, +}; + +export default manifest; diff --git a/extractors/startupjobs/src/run.ts b/extractors/startupjobs/src/run.ts new file mode 100644 index 0000000..6be97e0 --- /dev/null +++ b/extractors/startupjobs/src/run.ts @@ -0,0 +1,198 @@ +import { + formatCountryLabel, + normalizeCountryKey, +} from "@shared/location-support.js"; +import { resolveSearchCities } from "@shared/search-cities.js"; +import type { CreateJobInput } from "@shared/types/jobs"; +import { + type StartupJobRecord, + scrapeStartupJobsViaAlgolia, +} from "startup-jobs-scraper"; + +export type StartupJobsProgressEvent = + | { + type: "term_start"; + termIndex: number; + termTotal: number; + searchTerm: string; + location?: string; + } + | { + type: "term_complete"; + termIndex: number; + termTotal: number; + searchTerm: string; + location?: string; + jobsFoundTerm: number; + }; + +export interface RunStartupJobsOptions { + searchTerms?: string[]; + selectedCountry?: string; + locations?: string[]; + maxJobsPerTerm?: number; + onProgress?: (event: StartupJobsProgressEvent) => void; + shouldCancel?: () => boolean; +} + +export interface StartupJobsResult { + success: boolean; + jobs: CreateJobInput[]; + error?: string; +} + +function toPositiveIntOrFallback( + value: number | string | undefined, + fallback: number, +): number { + const parsed = + typeof value === "number" + ? value + : typeof value === "string" + ? Number.parseInt(value, 10) + : Number.NaN; + + if (!Number.isFinite(parsed)) return fallback; + return Math.max(1, Math.floor(parsed)); +} + +function inferJobType(disciplines: string | undefined): string | undefined { + if (!disciplines) return undefined; + const segments = disciplines + .split("|") + .map((value) => value.trim()) + .filter(Boolean); + return segments.length > 1 ? segments[segments.length - 1] : undefined; +} + +function mapStartupJob(row: StartupJobRecord): CreateJobInput | null { + if (!row.jobUrl) return null; + + return { + source: "startupjobs", + title: row.title || "Unknown Title", + employer: row.employer || "Unknown Employer", + employerUrl: row.employerUrl || undefined, + jobUrl: row.jobUrl, + applicationLink: row.applicationLink || row.jobUrl, + disciplines: row.disciplines || undefined, + deadline: row.deadline || undefined, + salary: row.salary || undefined, + location: row.location || undefined, + degreeRequired: row.degreeRequired || undefined, + starting: row.starting || undefined, + jobDescription: row.jobDescription || undefined, + jobType: inferJobType(row.disciplines), + isRemote: row.location?.toLowerCase().includes("remote") ?? undefined, + }; +} + +function resolveRunLocations(args: { + selectedCountry?: string; + locations?: string[]; +}): Array { + const locations = resolveSearchCities({ + list: args.locations, + }); + + const normalizedLocations = locations + .map((location) => normalizeCountryKey(location)) + .filter((location) => location !== "worldwide" && location !== "usa/ca"); + + if (normalizedLocations.length > 0) { + return normalizedLocations.map((location) => formatCountryLabel(location)); + } + + const countryKey = normalizeCountryKey(args.selectedCountry); + if (!countryKey || countryKey === "worldwide" || countryKey === "usa/ca") { + return [null]; + } + + return [formatCountryLabel(countryKey)]; +} + +export async function runStartupJobs( + options: RunStartupJobsOptions = {}, +): Promise { + const searchTerms = + options.searchTerms && options.searchTerms.length > 0 + ? options.searchTerms + : ["software engineer"]; + const runLocations = resolveRunLocations({ + selectedCountry: options.selectedCountry, + locations: options.locations, + }); + const maxJobsPerTerm = toPositiveIntOrFallback(options.maxJobsPerTerm, 50); + const termTotal = searchTerms.length * runLocations.length; + const jobs: CreateJobInput[] = []; + const seen = new Set(); + let runIndex = 0; + + try { + for (const location of runLocations) { + for (const searchTerm of searchTerms) { + runIndex += 1; + if (options.shouldCancel?.()) { + return { success: true, jobs }; + } + + options.onProgress?.({ + type: "term_start", + termIndex: runIndex, + termTotal, + searchTerm, + location: location ?? undefined, + }); + + const records = await scrapeStartupJobsViaAlgolia({ + query: searchTerm, + requestedCount: maxJobsPerTerm, + enrichDetails: true, + location: location ?? undefined, + }); + + let jobsFoundTerm = 0; + for (const record of records) { + const mapped = mapStartupJob(record); + if (!mapped) continue; + const dedupeKey = mapped.jobUrl; + if (seen.has(dedupeKey)) continue; + seen.add(dedupeKey); + jobs.push(mapped); + jobsFoundTerm += 1; + } + + options.onProgress?.({ + type: "term_complete", + termIndex: runIndex, + termTotal, + searchTerm, + location: location ?? undefined, + jobsFoundTerm, + }); + } + } + + return { + success: true, + jobs, + }; + } catch (error) { + const message = + error instanceof Error + ? error.message + : typeof error === "string" + ? error + : "Unexpected error while running startup.jobs extractor."; + const missingBrowser = + /playwright|browser|executable/i.test(message) && + /install/i.test(message); + return { + success: false, + jobs: [], + error: missingBrowser + ? `${message}. Install browser binaries with 'npx playwright install'.` + : message, + }; + } +} diff --git a/extractors/startupjobs/tests/manifest.test.ts b/extractors/startupjobs/tests/manifest.test.ts new file mode 100644 index 0000000..77ad5ed --- /dev/null +++ b/extractors/startupjobs/tests/manifest.test.ts @@ -0,0 +1,38 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../src/run", () => ({ + runStartupJobs: vi.fn(), +})); + +describe("startupjobs manifest", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("prefers startupjobsMaxJobsPerTerm when provided", async () => { + const { manifest } = await import("../src/manifest"); + const { runStartupJobs } = await import("../src/run"); + const runStartupJobsMock = vi.mocked(runStartupJobs); + runStartupJobsMock.mockResolvedValue({ + success: true, + jobs: [], + }); + + await manifest.run({ + source: "startupjobs", + selectedSources: ["startupjobs"], + settings: { + startupjobsMaxJobsPerTerm: "70", + jobspyResultsWanted: "30", + }, + searchTerms: ["software engineer"], + selectedCountry: "united kingdom", + }); + + expect(runStartupJobsMock).toHaveBeenCalledWith( + expect.objectContaining({ + maxJobsPerTerm: 70, + }), + ); + }); +}); diff --git a/extractors/startupjobs/tests/run.test.ts b/extractors/startupjobs/tests/run.test.ts new file mode 100644 index 0000000..3595d6a --- /dev/null +++ b/extractors/startupjobs/tests/run.test.ts @@ -0,0 +1,75 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("startup-jobs-scraper", () => ({ + scrapeStartupJobsViaAlgolia: vi.fn(), +})); + +describe("runStartupJobs", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("falls back to the default max jobs per term when options.maxJobsPerTerm is NaN", async () => { + const { scrapeStartupJobsViaAlgolia } = await import( + "startup-jobs-scraper" + ); + const scrapeMock = vi.mocked(scrapeStartupJobsViaAlgolia); + scrapeMock.mockResolvedValueOnce([]); + + const { runStartupJobs } = await import("../src/run"); + + await runStartupJobs({ + searchTerms: ["backend engineer"], + maxJobsPerTerm: Number.NaN, + }); + + expect(scrapeMock).toHaveBeenCalledWith( + expect.objectContaining({ + requestedCount: 50, + }), + ); + }); + + it("drops broad location sentinels and falls back to selectedCountry behavior", async () => { + const { scrapeStartupJobsViaAlgolia } = await import( + "startup-jobs-scraper" + ); + const scrapeMock = vi.mocked(scrapeStartupJobsViaAlgolia); + scrapeMock.mockResolvedValueOnce([]); + + const { runStartupJobs } = await import("../src/run"); + + await runStartupJobs({ + searchTerms: ["platform engineer"], + selectedCountry: "worldwide", + locations: ["Worldwide"], + }); + + expect(scrapeMock).toHaveBeenCalledWith( + expect.objectContaining({ + location: undefined, + }), + ); + }); + + it("normalizes explicit city-country aliases before passing location to the scraper", async () => { + const { scrapeStartupJobsViaAlgolia } = await import( + "startup-jobs-scraper" + ); + const scrapeMock = vi.mocked(scrapeStartupJobsViaAlgolia); + scrapeMock.mockResolvedValueOnce([]); + + const { runStartupJobs } = await import("../src/run"); + + await runStartupJobs({ + searchTerms: ["software engineer"], + locations: ["UK"], + }); + + expect(scrapeMock).toHaveBeenCalledWith( + expect.objectContaining({ + location: "United Kingdom", + }), + ); + }); +}); diff --git a/extractors/startupjobs/tsconfig.json b/extractors/startupjobs/tsconfig.json new file mode 100644 index 0000000..8ee7793 --- /dev/null +++ b/extractors/startupjobs/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "bundler", + "target": "ES2022", + "strict": true, + "noUnusedLocals": false, + "lib": ["ES2022", "DOM"], + "types": ["node"], + "baseUrl": ".", + "paths": { + "@shared/*": ["../../shared/src/*"] + } + }, + "include": ["./src/**/*"] +} diff --git a/orchestrator/src/client/pages/OrchestratorPage.test.tsx b/orchestrator/src/client/pages/OrchestratorPage.test.tsx index e514c82..84e0014 100644 --- a/orchestrator/src/client/pages/OrchestratorPage.test.tsx +++ b/orchestrator/src/client/pages/OrchestratorPage.test.tsx @@ -753,6 +753,7 @@ describe("OrchestratorPage", () => { gradcrackerMaxJobsPerTerm: 150, ukvisajobsMaxJobs: 150, adzunaMaxJobsPerTerm: 150, + startupjobsMaxJobsPerTerm: 150, jobspyCountryIndeed: "united kingdom", searchCities: "United Kingdom", }); diff --git a/orchestrator/src/client/pages/orchestrator/AutomaticRunTab.tsx b/orchestrator/src/client/pages/orchestrator/AutomaticRunTab.tsx index 6dd82a1..f352e1b 100644 --- a/orchestrator/src/client/pages/orchestrator/AutomaticRunTab.tsx +++ b/orchestrator/src/client/pages/orchestrator/AutomaticRunTab.tsx @@ -182,6 +182,7 @@ export const AutomaticRunTab: React.FC = ({ const rememberedRunBudget = settings?.jobspyResultsWanted?.value ?? + settings?.startupjobsMaxJobsPerTerm?.value ?? settings?.adzunaMaxJobsPerTerm?.value ?? settings?.gradcrackerMaxJobsPerTerm?.value ?? settings?.ukvisajobsMaxJobs?.value ?? diff --git a/orchestrator/src/client/pages/orchestrator/automatic-run.test.ts b/orchestrator/src/client/pages/orchestrator/automatic-run.test.ts index 7109e17..4a787a8 100644 --- a/orchestrator/src/client/pages/orchestrator/automatic-run.test.ts +++ b/orchestrator/src/client/pages/orchestrator/automatic-run.test.ts @@ -52,6 +52,17 @@ describe("automatic-run utilities", () => { expect(cap).toBeLessThanOrEqual(750); }); + it("assigns a dedicated startupjobs max-jobs limit", () => { + const limits = deriveExtractorLimits({ + budget: 120, + searchTerms: ["backend", "platform"], + sources: ["startupjobs"], + }); + + expect(limits.startupjobsMaxJobsPerTerm).toBeGreaterThan(0); + expect(limits.startupjobsMaxJobsPerTerm).toBeLessThanOrEqual(120); + }); + it("returns zero estimate when no search terms are provided", () => { const estimate = calculateAutomaticEstimate({ values: { @@ -112,4 +123,21 @@ describe("automatic-run utilities", () => { expect(estimate.discovered.cap).toBeGreaterThan(0); expect(estimate.discovered.cap).toBeLessThanOrEqual(120); }); + + it("includes startupjobs in estimate caps using the shared term budget", () => { + const estimate = calculateAutomaticEstimate({ + values: { + topN: 10, + minSuitabilityScore: 50, + searchTerms: ["backend", "platform"], + runBudget: 120, + country: "united kingdom", + cityLocations: [], + }, + sources: ["startupjobs"], + }); + + expect(estimate.discovered.cap).toBeGreaterThan(0); + expect(estimate.discovered.cap).toBeLessThanOrEqual(120); + }); }); diff --git a/orchestrator/src/client/pages/orchestrator/automatic-run.ts b/orchestrator/src/client/pages/orchestrator/automatic-run.ts index 22d6edc..9686ccd 100644 --- a/orchestrator/src/client/pages/orchestrator/automatic-run.ts +++ b/orchestrator/src/client/pages/orchestrator/automatic-run.ts @@ -66,6 +66,7 @@ export interface ExtractorLimits { gradcrackerMaxJobsPerTerm: number; ukvisajobsMaxJobs: number; adzunaMaxJobsPerTerm: number; + startupjobsMaxJobsPerTerm: number; } export function deriveExtractorLimits(args: { @@ -82,6 +83,7 @@ export function deriveExtractorLimits(args: { const includesUkVisaJobs = args.sources.includes("ukvisajobs"); const includesAdzuna = args.sources.includes("adzuna"); const includesHiringCafe = args.sources.includes("hiringcafe"); + const includesStartupJobs = args.sources.includes("startupjobs"); const weightedContributors = (includesIndeed ? termCount : 0) + @@ -90,7 +92,8 @@ export function deriveExtractorLimits(args: { (includesGradcracker ? termCount : 0) + (includesUkVisaJobs ? 1 : 0) + (includesAdzuna ? termCount : 0) + - (includesHiringCafe ? termCount : 0); + (includesHiringCafe ? termCount : 0) + + (includesStartupJobs ? termCount : 0); if (weightedContributors <= 0) { return { @@ -98,6 +101,7 @@ export function deriveExtractorLimits(args: { gradcrackerMaxJobsPerTerm: budget, ukvisajobsMaxJobs: budget, adzunaMaxJobsPerTerm: budget, + startupjobsMaxJobsPerTerm: budget, }; } @@ -109,6 +113,7 @@ export function deriveExtractorLimits(args: { gradcrackerMaxJobsPerTerm: perUnit, ukvisajobsMaxJobs: Math.min(budget, perUnit + remainder), adzunaMaxJobsPerTerm: perUnit, + startupjobsMaxJobsPerTerm: perUnit, }; } @@ -173,6 +178,7 @@ export function calculateAutomaticEstimate(args: { const hasGlassdoor = sources.includes("glassdoor"); const hasAdzuna = sources.includes("adzuna"); const hasHiringCafe = sources.includes("hiringcafe"); + const hasStartupJobs = sources.includes("startupjobs"); const limits = deriveExtractorLimits({ budget: values.runBudget, searchTerms: values.searchTerms, @@ -191,9 +197,17 @@ export function calculateAutomaticEstimate(args: { const hiringCafeCap = hasHiringCafe ? limits.jobspyResultsWanted * termCount : 0; + const startupJobsCap = hasStartupJobs + ? limits.startupjobsMaxJobsPerTerm * termCount + : 0; const discoveredCap = - jobspyCap + gradcrackerCap + ukvisaCap + adzunaCap + hiringCafeCap; + jobspyCap + + gradcrackerCap + + ukvisaCap + + adzunaCap + + hiringCafeCap + + startupJobsCap; const discoveredMin = Math.round(discoveredCap * 0.35); const discoveredMax = Math.round(discoveredCap * 0.75); const processedMin = Math.min(values.topN, discoveredMin); diff --git a/orchestrator/src/client/pages/orchestrator/usePipelineControls.ts b/orchestrator/src/client/pages/orchestrator/usePipelineControls.ts index b621f45..d87684c 100644 --- a/orchestrator/src/client/pages/orchestrator/usePipelineControls.ts +++ b/orchestrator/src/client/pages/orchestrator/usePipelineControls.ts @@ -181,11 +181,13 @@ export function usePipelineControls( ); const hasAdzuna = compatibleSources.includes("adzuna"); const hasHiringCafe = compatibleSources.includes("hiringcafe"); + const hasStartupJobs = compatibleSources.includes("startupjobs"); const serializedCities = serializeCityLocationsSetting( values.cityLocations, ); const searchCities = - (hasJobSpySite || hasAdzuna || hasHiringCafe) && serializedCities + (hasJobSpySite || hasAdzuna || hasHiringCafe || hasStartupJobs) && + serializedCities ? serializedCities : formatCountryLabel(values.country); await api.updateSettings({ @@ -194,6 +196,7 @@ export function usePipelineControls( gradcrackerMaxJobsPerTerm: limits.gradcrackerMaxJobsPerTerm, ukvisajobsMaxJobs: limits.ukvisajobsMaxJobs, adzunaMaxJobsPerTerm: limits.adzunaMaxJobsPerTerm, + startupjobsMaxJobsPerTerm: limits.startupjobsMaxJobsPerTerm, jobspyCountryIndeed: values.country, searchCities, }); diff --git a/orchestrator/src/client/pages/orchestrator/utils.test.ts b/orchestrator/src/client/pages/orchestrator/utils.test.ts index 81b1cf4..f5ddd91 100644 --- a/orchestrator/src/client/pages/orchestrator/utils.test.ts +++ b/orchestrator/src/client/pages/orchestrator/utils.test.ts @@ -17,6 +17,10 @@ describe("orchestrator utils", () => { expect(getEnabledSources(withoutKey)).not.toContain("adzuna"); }); + it("enables startupjobs without credentials", () => { + expect(getEnabledSources(createAppSettings())).toContain("startupjobs"); + }); + it("counts processing jobs in ready and discovered tabs", () => { const jobs = [ createJob({ id: "ready", status: "ready", closedAt: null }), diff --git a/orchestrator/src/client/pages/orchestrator/utils.ts b/orchestrator/src/client/pages/orchestrator/utils.ts index 11caf7e..10428aa 100644 --- a/orchestrator/src/client/pages/orchestrator/utils.ts +++ b/orchestrator/src/client/pages/orchestrator/utils.ts @@ -195,6 +195,10 @@ export const getEnabledSources = ( enabled.push(source); continue; } + if (source === "startupjobs") { + enabled.push(source); + continue; + } if ( source === "indeed" || source === "linkedin" || diff --git a/orchestrator/src/server/config/demo-defaults.data.ts b/orchestrator/src/server/config/demo-defaults.data.ts index 19388a2..1752337 100644 --- a/orchestrator/src/server/config/demo-defaults.data.ts +++ b/orchestrator/src/server/config/demo-defaults.data.ts @@ -254,6 +254,7 @@ export const DEMO_SOURCE_BASE_URLS: Record = { ukvisajobs: "https://www.ukvisajobs.com", adzuna: "https://www.adzuna.com", hiringcafe: "https://hiring.cafe", + startupjobs: "https://startup.jobs", manual: "https://example.com", }; diff --git a/package-lock.json b/package-lock.json index 123b49e..9cfb966 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,7 +44,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -106,7 +105,7 @@ "extractors/gradcracker": { "name": "gradcracker-extractor", "version": "0.0.1", - "license": "ISC", + "license": "SEE LICENSE IN ../../LICENSE", "dependencies": { "camoufox-js": "^0.8.0", "crawlee": "^3.0.0", @@ -181,10 +180,31 @@ "undici-types": "~7.16.0" } }, + "extractors/startupjobs": { + "name": "startupjobs-extractor", + "version": "0.0.1", + "dependencies": { + "startup-jobs-scraper": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "typescript": "~5.9.0" + } + }, + "extractors/startupjobs/node_modules/@types/node": { + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, "extractors/ukvisajobs": { "name": "ukvisajobs-extractor", "version": "0.0.1", - "license": "ISC", + "license": "SEE LICENSE IN ../../LICENSE", "dependencies": { "camoufox-js": "^0.8.0", "job-ops-shared": "^1.0.0", @@ -349,7 +369,6 @@ "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.48.1.tgz", "integrity": "sha512-4Fu7dnzQyQmMFknYwTiN/HxPbH4DyxvQ1m+IxpPp5oslOgz8m6PG5qhiGbqJzH4HiT1I58ecDiCAC716UyVA8Q==", "license": "MIT", - "peer": true, "dependencies": { "@algolia/client-common": "5.48.1", "@algolia/requester-browser-xhr": "5.48.1", @@ -448,9 +467,9 @@ } }, "node_modules/@apify/consts": { - "version": "2.50.0", - "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.50.0.tgz", - "integrity": "sha512-a7SqvlOcxSfUb3bnj6nW2nv296llM5hto49xC+VT511ppX4Fo3i9FEKURFi8WiTImEIQL1sJfqsJr9zzjpXlIg==", + "version": "2.51.1", + "resolved": "https://registry.npmjs.org/@apify/consts/-/consts-2.51.1.tgz", + "integrity": "sha512-QV16f41BjmE7uYQgB+JeS5bhbEdFvP8eF1R5LiKlvGkERckSlMl1JIIaW1b/XwJdp3bEBKBGPtNlvYa06wyhwg==", "license": "Apache-2.0" }, "node_modules/@apify/datastructures": { @@ -459,13 +478,24 @@ "integrity": "sha512-E6yQyc/XZDqJopbaGmhzZXMJqwGf96ELtDANZa0t68jcOAJZS+pF7YUfQOLszXq6JQAdnRvTH2caotL6urX7HA==", "license": "Apache-2.0" }, - "node_modules/@apify/log": { - "version": "2.5.31", - "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.31.tgz", - "integrity": "sha512-Gzs0BDGybdL5sbJpYZqziQrmUGhKM+Pdf34fHXKEW7P+SlE7sRPjoKKVmDbjdHiWEaC5RHMTM7Q7ibTvYtFCRQ==", + "node_modules/@apify/input_secrets": { + "version": "1.2.26", + "resolved": "https://registry.npmjs.org/@apify/input_secrets/-/input_secrets-1.2.26.tgz", + "integrity": "sha512-mm1xDPHrUuYG//jeAbqXeoaGDOTKQvP7b+504yRPusvZqGeNDcRvwQbEdysWhIn+AaTg8xlwUKw4Qmzca0PPPA==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.50.0", + "@apify/log": "^2.5.33", + "@apify/utilities": "^2.25.5", + "ow": "^0.28.2" + } + }, + "node_modules/@apify/log": { + "version": "2.5.33", + "resolved": "https://registry.npmjs.org/@apify/log/-/log-2.5.33.tgz", + "integrity": "sha512-rD+RY/Lvgy2ZAQD6QHbzoGHKvqILSXHZggTv2PN80ZZl7JMVQ22pYpoysYITHl4eGuievCiwrhkvdbNqTHqoPQ==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.51.1", "ansi-colors": "^4.1.1" } }, @@ -500,13 +530,13 @@ "license": "Apache-2.0" }, "node_modules/@apify/utilities": { - "version": "2.25.3", - "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.25.3.tgz", - "integrity": "sha512-A2M3kXitwDYS3Hb2fgAEUEv44dnfjLsf6UTzm1RGjURZIN9EHiph1v0EeJG1Pit6MPFW/QHSuU0Sj2JeHZysag==", + "version": "2.25.5", + "resolved": "https://registry.npmjs.org/@apify/utilities/-/utilities-2.25.5.tgz", + "integrity": "sha512-I53XgSbNw2mYHPbPTIM7CjooHBHapWzvW6eKxpzt5IO9zB3OIzWOk2xRCodi1pAt3+A+BGiJJyddF/cQYGJenA==", "license": "Apache-2.0", "dependencies": { - "@apify/consts": "^2.50.0", - "@apify/log": "^2.5.31" + "@apify/consts": "^2.51.1", + "@apify/log": "^2.5.33" } }, "node_modules/@asamuzakjp/css-color": { @@ -550,7 +580,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2804,7 +2833,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -2827,7 +2855,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -2937,7 +2964,6 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.1.tgz", "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -3359,7 +3385,6 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.1.tgz", "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -4349,7 +4374,6 @@ "resolved": "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.9.2.tgz", "integrity": "sha512-C5wZsGuKTY8jEYsqdxhhFOe1ZDjH0uIYJ9T/jebHwkyxqnr4wW0jTkB72OMqNjsoQRcb0JN3PcSeTwFlVgzCZg==", "license": "MIT", - "peer": true, "dependencies": { "@docusaurus/core": "3.9.2", "@docusaurus/logger": "3.9.2", @@ -5923,7 +5947,6 @@ "resolved": "https://registry.npmjs.org/@mdx-js/react/-/react-3.1.1.tgz", "integrity": "sha512-f++rKLQgUVYDAtECQ6fn/is15GkEH9+nZPM3MS0RcxVqoTfawHvDlSCH7JbMhAM6uJ32v3eXLvLmLvjGu7PTQw==", "license": "MIT", - "peer": true, "dependencies": { "@types/mdx": "^2.0.0" }, @@ -7825,7 +7848,6 @@ "resolved": "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz", "integrity": "sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/core": "^7.21.3", "@svgr/babel-preset": "8.1.0", @@ -8018,6 +8040,12 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "license": "MIT" }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@trysound/sax": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", @@ -8278,7 +8306,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.3.tgz", "integrity": "sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==", "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -8312,7 +8339,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.0.2" @@ -8324,7 +8350,6 @@ "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/react": "*" } @@ -8661,7 +8686,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8751,7 +8775,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -8797,7 +8820,6 @@ "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-5.48.1.tgz", "integrity": "sha512-Rf7xmeuIo7nb6S4mp4abW2faW8DauZyE2faBIKFaUfP3wnpOvNSbiI5AwVhqBNj0jPgBWEvhyCu0sLjN2q77Rg==", "license": "MIT", - "peer": true, "dependencies": { "@algolia/abtesting": "1.14.1", "@algolia/client-abtesting": "5.48.1", @@ -8936,6 +8958,51 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/apify": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/apify/-/apify-3.7.0.tgz", + "integrity": "sha512-2WeMeI6BIH2ql5dtnsreKcWxE+OGqsj9LKEe1c0b7kaQpqlEHXN63OqPfDmX1EFjuGo5Z1I3uVYBx3BTQZ9+Ag==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.51.0", + "@apify/input_secrets": "^1.2.0", + "@apify/log": "^2.4.3", + "@apify/timeout": "^0.3.0", + "@apify/utilities": "^2.13.0", + "@crawlee/core": "^3.14.1", + "@crawlee/types": "^3.14.1", + "@crawlee/utils": "^3.14.1", + "apify-client": "^2.17.0", + "fs-extra": "^11.2.0", + "ow": "^0.28.2", + "semver": "^7.5.4", + "tslib": "^2.6.2", + "ws": "^8.18.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/apify-client": { + "version": "2.22.2", + "resolved": "https://registry.npmjs.org/apify-client/-/apify-client-2.22.2.tgz", + "integrity": "sha512-fqBi84kmbpsU7Rcmr3Oq0dxAcQls0sBnDeVCX5nWohvK0ELRk1pcaLjqp1CGtwyUGyQ9ZpWjEoi+QXxGY/92tw==", + "license": "Apache-2.0", + "dependencies": { + "@apify/consts": "^2.50.0", + "@apify/log": "^2.2.6", + "@apify/utilities": "^2.23.2", + "@crawlee/types": "^3.3.0", + "ansi-colors": "^4.1.1", + "async-retry": "^1.3.3", + "axios": "^1.6.7", + "content-type": "^1.0.5", + "ow": "^0.28.2", + "proxy-agent": "^6.5.0", + "tslib": "^2.5.0", + "type-fest": "^4.0.0" + } + }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -8989,6 +9056,18 @@ "node": ">=12.0.0" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/astring": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/astring/-/astring-1.9.0.tgz", @@ -8998,6 +9077,30 @@ "astring": "bin/astring" } }, + "node_modules/async-retry": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", + "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", + "license": "MIT", + "dependencies": { + "retry": "0.13.1" + } + }, + "node_modules/async-retry/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/autoprefixer": { "version": "10.4.24", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.24.tgz", @@ -9034,6 +9137,17 @@ "postcss": "^8.1.0" } }, + "node_modules/axios": { + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.6.tgz", + "integrity": "sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/babel-loader": { "version": "9.2.1", "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.2.1.tgz", @@ -9153,6 +9267,15 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/basic-ftp": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.2.0.tgz", + "integrity": "sha512-VoMINM2rqJwJgfdHq6RiUudKt2BV+FY5ZFezP/ypmwayk68+NzzAQy4XXLlqsGD4MCzq3DrmNFD/uUmBJuGoXw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -9450,7 +9573,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -10115,6 +10237,18 @@ "node": ">=10" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -10605,7 +10739,6 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.1.tgz", "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -10945,6 +11078,15 @@ "integrity": "sha512-YW32lKOmIBgbxtu3g5SaiqWNwa/9ISQt2EcgOq0+RAIFufFp9is6tqNnKahqE5kuKvrnYAzs28r+s6pXJR8Vcw==", "license": "MIT" }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/data-urls": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", @@ -11125,6 +11267,29 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -11425,6 +11590,31 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/encoding-sniffer/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -11508,6 +11698,21 @@ "node": ">= 0.4" } }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esast-util-from-estree": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz", @@ -11617,6 +11822,46 @@ "node": ">=0.8.0" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/eslint-scope": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", @@ -12171,7 +12416,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -12479,6 +12723,22 @@ } } }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/form-data-encoder": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-4.1.0.tgz", @@ -12727,6 +12987,20 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/github-from-package": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", @@ -13109,6 +13383,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-yarn": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-3.0.0.tgz", @@ -17879,6 +18168,15 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/no-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz", @@ -18010,7 +18308,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -18412,6 +18709,38 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-json": { "version": "8.1.1", "resolved": "https://registry.npmjs.org/package-json/-/package-json-8.1.1.tgz", @@ -18670,6 +18999,18 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/parse5/node_modules/entities": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", @@ -18819,7 +19160,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -18873,7 +19213,6 @@ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.1.tgz", "integrity": "sha512-+2uTZHxSCcxjvGc5C891LrS1/NlxglGxzrC4seZiVjcYVQfUa87wBL6rTDqzGjuoWNjnBzRqKmF6zRYGMvQUaQ==", "license": "Apache-2.0", - "peer": true, "dependencies": { "playwright-core": "1.58.1" }, @@ -18892,7 +19231,6 @@ "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.1.tgz", "integrity": "sha512-bcWzOaTxcW+VOOGBCQgnaKToLJ65d6AqfLVKEWvexyS3AS6rbXl+xdpYRMGSRBClPvyj44njOWoxjNdL/H9UNg==", "license": "Apache-2.0", - "peer": true, "bin": { "playwright-core": "cli.js" }, @@ -18933,7 +19271,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -19846,7 +20183,6 @@ "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.1.tgz", "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", "license": "MIT", - "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -20543,6 +20879,34 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/proxy-chain": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/proxy-chain/-/proxy-chain-2.7.1.tgz", @@ -20557,6 +20921,12 @@ "node": ">=14" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pump": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", @@ -20730,7 +21100,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -20743,7 +21112,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -20800,7 +21168,6 @@ "resolved": "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-6.0.0.tgz", "integrity": "sha512-YMMxTUQV/QFSnbgrP3tjDzLHRg7vsbMn8e9HAa8o/1iXoiomo48b7sk/kkmWEuWNDPJVlKSJRB6Y2fHqdJk+SQ==", "license": "MIT", - "peer": true, "dependencies": { "@types/react": "*" }, @@ -20876,7 +21243,6 @@ "resolved": "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz", "integrity": "sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.12.13", "history": "^4.9.0", @@ -22495,6 +22861,82 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/startup-jobs-scraper": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/startup-jobs-scraper/-/startup-jobs-scraper-0.1.0.tgz", + "integrity": "sha512-XXs/JxrcwP46+zrTX2Dpg7g5cnTqKeflAAbfnYo89EchRVcmvyi+x/lUrlrG8qaKBPKdyvALNJWRCkT+yiMA+Q==", + "license": "ISC", + "dependencies": { + "apify": "^3.7.0", + "cheerio": "^1.2.0", + "crawlee": "^3.0.0", + "playwright": "*", + "wreq-js": "^2.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/startup-jobs-scraper/node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/startup-jobs-scraper/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/startup-jobs-scraper/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/startupjobs-extractor": { + "resolved": "extractors/startupjobs", + "link": true + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -22808,7 +23250,6 @@ "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.0.tgz", "integrity": "sha512-jTwoImyr/QbOWFFso3YoU3ik0jBBDJ6JTOQiy/J2YxVJdZCc+5u7skhNwiOR3FQIygFqVUPHl7qbbxtjW2K3Qg==", "license": "BSD-2-Clause", - "peer": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", "acorn": "^8.15.0", @@ -23091,15 +23532,13 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "peer": true + "license": "0BSD" }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -23184,7 +23623,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -23266,6 +23704,15 @@ "resolved": "extractors/ukvisajobs", "link": true }, + "node_modules/undici": { + "version": "7.24.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.24.4.tgz", + "integrity": "sha512-BM/JzwwaRXxrLdElV2Uo6cTLEjhSb3WXboncJamZ15NgUURmvlXvxa6xkwIOILIjPNo9i8ku136ZvWV0Uly8+w==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -23690,7 +24137,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -23960,7 +24406,6 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.2.tgz", "integrity": "sha512-dRXm0a2qcHPUBEzVk8uph0xWSjV/xZxenQQbLwnwP7caQCYpqG1qddwlyEkIDkYn0K8tvmcrZ+bOrzoQ3HxCDw==", "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -24479,6 +24924,24 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/wreq-js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/wreq-js/-/wreq-js-2.2.0.tgz", + "integrity": "sha512-lXW1/bvdPTpFMdfBftkJIp6OzxkAqAON4dlrKrmaFNT86eu60VCEVmEdK3nWY1ZyiEZ6IXQPRrc1uXG394BoBA==", + "cpu": [ + "x64", + "arm64" + ], + "license": "MIT", + "os": [ + "darwin", + "linux", + "win32" + ], + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", @@ -24779,7 +25242,6 @@ "@radix-ui/react-tooltip": "^1.2.8", "@tailwindcss/vite": "^4.1.18", "@tanstack/react-query": "^5.90.21", - "@types/canvas-confetti": "^1.9.0", "better-sqlite3": "^11.6.0", "canvas-confetti": "^1.9.4", "class-variance-authority": "^0.7.1", @@ -24790,7 +25252,6 @@ "drizzle-orm": "^0.38.2", "express": "^4.18.2", "framer-motion": "^12.34.3", - "get-tsconfig": "^4.10.0", "html-to-text": "^9.0.5", "jsdom": "^25.0.1", "lucide-react": "^0.561.0", @@ -24814,6 +25275,7 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.1", "@types/better-sqlite3": "^7.6.8", + "@types/canvas-confetti": "^1.9.0", "@types/cors": "^2.8.17", "@types/express": "^4.17.21", "@types/html-to-text": "^9.0.4", @@ -26061,7 +26523,8 @@ "orchestrator/node_modules/@types/aria-query": { "version": "5.0.4", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "orchestrator/node_modules/@types/babel__core": { "version": "7.20.5", @@ -26104,13 +26567,13 @@ "version": "7.6.13", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*" } }, "orchestrator/node_modules/@types/canvas-confetti": { "version": "1.9.0", + "dev": true, "license": "MIT" }, "orchestrator/node_modules/@types/chai": { @@ -26338,15 +26801,10 @@ "node": ">=12" } }, - "orchestrator/node_modules/asynckit": { - "version": "0.4.0", - "license": "MIT" - }, "orchestrator/node_modules/better-sqlite3": { "version": "11.10.0", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" @@ -26378,16 +26836,6 @@ "url": "https://polar.sh/cva" } }, - "orchestrator/node_modules/combined-stream": { - "version": "1.0.8", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "orchestrator/node_modules/concurrently": { "version": "9.2.1", "dev": true, @@ -26530,17 +26978,11 @@ "version": "2.5.1", "license": "MIT" }, - "orchestrator/node_modules/delayed-stream": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "orchestrator/node_modules/dom-accessibility-api": { "version": "0.5.16", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "orchestrator/node_modules/dom-helpers": { "version": "5.2.1", @@ -26714,19 +27156,6 @@ "dev": true, "license": "MIT" }, - "orchestrator/node_modules/es-set-tostringtag": { - "version": "2.1.0", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "orchestrator/node_modules/esbuild": { "version": "0.19.12", "dev": true, @@ -27162,20 +27591,6 @@ } } }, - "orchestrator/node_modules/form-data": { - "version": "4.0.5", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "orchestrator/node_modules/gel": { "version": "2.2.0", "dev": true, @@ -27206,19 +27621,6 @@ "node": ">=10" } }, - "orchestrator/node_modules/has-tostringtag": { - "version": "1.0.2", - "license": "MIT", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "orchestrator/node_modules/html-url-attributes": { "version": "3.0.1", "license": "MIT", @@ -27245,7 +27647,6 @@ "orchestrator/node_modules/jsdom": { "version": "25.0.1", "license": "MIT", - "peer": true, "dependencies": { "cssstyle": "^4.1.0", "data-urls": "^5.0.0", @@ -27357,6 +27758,7 @@ "version": "1.5.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "lz-string": "bin/bin.js" } @@ -27402,6 +27804,7 @@ "version": "27.5.1", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", @@ -27415,6 +27818,7 @@ "version": "5.2.0", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -27425,7 +27829,6 @@ "orchestrator/node_modules/react-hook-form": { "version": "7.71.1", "license": "MIT", - "peer": true, "engines": { "node": ">=18.0.0" }, @@ -27440,7 +27843,8 @@ "orchestrator/node_modules/react-is": { "version": "17.0.2", "dev": true, - "license": "MIT" + "license": "MIT", + "peer": true }, "orchestrator/node_modules/react-markdown": { "version": "10.1.0", @@ -27695,8 +28099,7 @@ }, "orchestrator/node_modules/tailwindcss": { "version": "4.1.18", - "license": "MIT", - "peer": true + "license": "MIT" }, "orchestrator/node_modules/tailwindcss-animate": { "version": "1.0.7", @@ -27819,7 +28222,6 @@ "orchestrator/node_modules/vite": { "version": "6.4.1", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", diff --git a/shared/src/extractors/index.ts b/shared/src/extractors/index.ts index ad9ee1e..261dc4c 100644 --- a/shared/src/extractors/index.ts +++ b/shared/src/extractors/index.ts @@ -8,6 +8,7 @@ export const EXTRACTOR_SOURCE_IDS = [ "ukvisajobs", "adzuna", "hiringcafe", + "startupjobs", "manual", ] as const; @@ -48,6 +49,7 @@ export const EXTRACTOR_SOURCE_METADATA: Record< requiresCredentials: true, }, hiringcafe: { label: "Hiring Cafe", order: 70, category: "pipeline" }, + startupjobs: { label: "startup.jobs", order: 80, category: "pipeline" }, manual: { label: "Manual", order: 90, category: "manual" }, }; diff --git a/shared/src/location-support.test.ts b/shared/src/location-support.test.ts index 73b2cb0..0bfb823 100644 --- a/shared/src/location-support.test.ts +++ b/shared/src/location-support.test.ts @@ -55,6 +55,10 @@ describe("location-support", () => { expect(isSourceAllowedForCountry("glassdoor", "japan")).toBe(false); expect(isSourceAllowedForCountry("adzuna", "united states")).toBe(true); expect(isSourceAllowedForCountry("adzuna", "japan")).toBe(false); + expect(isSourceAllowedForCountry("startupjobs", "united states")).toBe( + true, + ); + expect(isSourceAllowedForCountry("startupjobs", "worldwide")).toBe(true); }); it("filters incompatible sources while preserving compatible order", () => { @@ -66,11 +70,12 @@ describe("location-support", () => { "glassdoor", "ukvisajobs", "adzuna", + "startupjobs", "linkedin", ], "united states", ), - ).toEqual(["indeed", "glassdoor", "adzuna", "linkedin"]); + ).toEqual(["indeed", "glassdoor", "adzuna", "startupjobs", "linkedin"]); }); it("supports glassdoor only in explicitly supported countries", () => { diff --git a/shared/src/settings-registry.ts b/shared/src/settings-registry.ts index 4bafe2d..2a79a47 100644 --- a/shared/src/settings-registry.ts +++ b/shared/src/settings-registry.ts @@ -217,6 +217,19 @@ export const settingsRegistry = { parse: parseIntOrNull, serialize: serializeNullableNumber, }, + startupjobsMaxJobsPerTerm: { + kind: "typed" as const, + schema: z.number().int().min(1).max(1000), + default: (): number => + parseInt( + typeof process !== "undefined" + ? process.env.STARTUPJOBS_MAX_RESULTS || "50" + : "50", + 10, + ), + parse: parseIntOrNull, + serialize: serializeNullableNumber, + }, searchTerms: { kind: "typed" as const, schema: z.array(z.string().trim().min(1).max(200)).max(100), diff --git a/shared/src/testing/factories.ts b/shared/src/testing/factories.ts index 900facd..87edeff 100644 --- a/shared/src/testing/factories.ts +++ b/shared/src/testing/factories.ts @@ -153,6 +153,7 @@ export const createAppSettings = ( ukvisajobsMaxJobs: { value: 50, default: 50, override: null }, adzunaMaxJobsPerTerm: { value: 50, default: 50, override: null }, gradcrackerMaxJobsPerTerm: { value: 50, default: 50, override: null }, + startupjobsMaxJobsPerTerm: { value: 50, default: 50, override: null }, searchTerms: { value: ["Software Engineer"], default: ["Software Engineer"], diff --git a/shared/src/types/settings.ts b/shared/src/types/settings.ts index a884a14..1cb4629 100644 --- a/shared/src/types/settings.ts +++ b/shared/src/types/settings.ts @@ -152,6 +152,7 @@ export interface AppSettings { ukvisajobsMaxJobs: Resolved; adzunaMaxJobsPerTerm: Resolved; gradcrackerMaxJobsPerTerm: Resolved; + startupjobsMaxJobsPerTerm: Resolved; searchTerms: Resolved; blockedCompanyKeywords: Resolved; scoringInstructions: Resolved;