From 67508d56eab262266363dfdb10843d1cff5fafc4 Mon Sep 17 00:00:00 2001 From: ilia Date: Tue, 12 May 2026 20:36:27 -0400 Subject: [PATCH] fix(docker): copy full extractors tree into image for runtime manifests The Dockerfile only copied a fixed list of extractor dirs; new sources were listed in shared but their manifest.ts files were absent from the container, so discovery logged missingManifest in production. Copy extractors/ once before npm install in builder and production, and skip redundant per-extractor COPY lines. Add extractors/*/storage/ to .dockerignore to avoid baking local cache into the build context. Co-authored-by: Cursor --- .dockerignore | 3 +++ Dockerfile | 27 +++++---------------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/.dockerignore b/.dockerignore index aa1326f..5ac7382 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,9 @@ # Data (mounted as volume) data/ +# Extractor runtime caches (optional local dirs; never needed in image) +extractors/*/storage/ + # Environment files (passed via docker-compose) .env **/.env diff --git a/Dockerfile b/Dockerfile index 6f62599..07e0f9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,11 +35,8 @@ COPY package*.json ./ COPY docs-site/package*.json ./docs-site/ COPY shared/package*.json ./shared/ COPY orchestrator/package*.json ./orchestrator/ -COPY extractors/adzuna/package*.json ./extractors/adzuna/ -COPY extractors/hiringcafe/package*.json ./extractors/hiringcafe/ -COPY extractors/gradcracker/package*.json ./extractors/gradcracker/ -COPY extractors/startupjobs/package*.json ./extractors/startupjobs/ -COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/ +# All npm workspaces under extractors/* (manifests + package.json per extractor) +COPY extractors ./extractors # Install Node dependencies with npm cache (dev deps needed for build) RUN --mount=type=cache,target=/root/.npm \ @@ -56,12 +53,7 @@ COPY shared ./shared COPY docs-site ./docs-site COPY orchestrator ./orchestrator COPY visa-sponsor-providers ./visa-sponsor-providers -COPY extractors/adzuna ./extractors/adzuna -COPY extractors/hiringcafe ./extractors/hiringcafe -COPY extractors/gradcracker ./extractors/gradcracker -COPY extractors/jobspy ./extractors/jobspy -COPY extractors/startupjobs ./extractors/startupjobs -COPY extractors/ukvisajobs ./extractors/ukvisajobs +# extractors/ already copied before npm install (full tree for manifests at runtime) # Build documentation site bundle WORKDIR /app/docs-site @@ -107,11 +99,7 @@ COPY package*.json ./ COPY docs-site/package*.json ./docs-site/ COPY shared/package*.json ./shared/ COPY orchestrator/package*.json ./orchestrator/ -COPY extractors/adzuna/package*.json ./extractors/adzuna/ -COPY extractors/hiringcafe/package*.json ./extractors/hiringcafe/ -COPY extractors/gradcracker/package*.json ./extractors/gradcracker/ -COPY extractors/startupjobs/package*.json ./extractors/startupjobs/ -COPY extractors/ukvisajobs/package*.json ./extractors/ukvisajobs/ +COPY extractors ./extractors # Install production Node dependencies only RUN --mount=type=cache,target=/root/.npm \ @@ -124,12 +112,7 @@ COPY --from=builder /app/docs-site/build ./orchestrator/dist/docs COPY shared ./shared COPY orchestrator ./orchestrator COPY visa-sponsor-providers ./visa-sponsor-providers -COPY extractors/adzuna ./extractors/adzuna -COPY extractors/hiringcafe ./extractors/hiringcafe -COPY extractors/gradcracker ./extractors/gradcracker -COPY extractors/jobspy ./extractors/jobspy -COPY extractors/startupjobs ./extractors/startupjobs -COPY extractors/ukvisajobs ./extractors/ukvisajobs +# extractors/ already copied before npm install # Reuse Camoufox binaries from builder instead of fetching again COPY --from=builder /root/.cache/camoufox /root/.cache/camoufox