load data from data directory correctly, honour env first

orchestrator/src/server/app.ts

@@ -8,6 +8,7 @@ import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import { readFile } from 'fs/promises';
 import { apiRouter } from './api/index.js';
+import { getDataDir } from './config/dataDir.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -84,9 +85,7 @@ export function createApp() {
   app.use('/api', apiRouter);
 
   // Serve static files for generated PDFs
-  const pdfDir = process.env.DATA_DIR
-    ? join(process.env.DATA_DIR, 'pdfs')
-    : join(__dirname, '../../data/pdfs');
+  const pdfDir = join(getDataDir(), 'pdfs');
   app.use('/pdfs', express.static(pdfDir));
 
   // Health check

orchestrator/src/server/config/dataDir.ts

@@ -0,0 +1,33 @@
+import { existsSync } from 'fs';
+import { basename, join, resolve } from 'path';
+
+let cachedDir: string | null = null;
+
+export function getDataDir(): string {
+  const fromEnv = (process.env.DATA_DIR || '').trim();
+  if (fromEnv) return fromEnv;
+
+  if (cachedDir) return cachedDir;
+
+  const cwd = process.cwd();
+  const cwdBase = basename(cwd);
+  const parentDir = join(cwd, '..');
+  const parentLooksLikeRoot = [
+    join(parentDir, 'docker-compose.yml'),
+    join(parentDir, 'Dockerfile'),
+    join(parentDir, '.env'),
+  ].some((marker) => existsSync(marker));
+  const candidates = cwdBase === 'orchestrator' && parentLooksLikeRoot
+    ? [join(parentDir, 'data'), join(cwd, 'data')]
+    : [join(cwd, 'data'), join(parentDir, 'data')];
+
+  for (const candidate of candidates) {
+    if (existsSync(candidate)) {
+      cachedDir = resolve(candidate);
+      return cachedDir;
+    }
+  }
+
+  cachedDir = resolve(join(cwd, 'data'));
+  return cachedDir;
+}

orchestrator/src/server/config/env.ts

@@ -0,0 +1,15 @@
+import { config } from 'dotenv';
+import { existsSync } from 'fs';
+import { join } from 'path';
+
+const candidates = [
+  join(process.cwd(), '.env'),
+  join(process.cwd(), '..', '.env'),
+];
+
+for (const envPath of candidates) {
+  if (existsSync(envPath)) {
+    config({ path: envPath });
+    break;
+  }
+}

orchestrator/src/server/db/clear.ts

@@ -3,15 +3,11 @@
  */
 
 import Database from 'better-sqlite3';
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
+import { join } from 'path';
+import { getDataDir } from '../config/dataDir.js';
 
 // Database path - can be overridden via env for Docker
-const DB_PATH = process.env.DATA_DIR
-  ? join(process.env.DATA_DIR, 'jobs.db')
-  : join(__dirname, '../../../data/jobs.db');
+const DB_PATH = join(getDataDir(), 'jobs.db');
 
 /**
  * Clear all data from the database (keeps the schema intact).

orchestrator/src/server/db/index.ts

@@ -5,16 +5,12 @@
 import Database from 'better-sqlite3';
 import { drizzle } from 'drizzle-orm/better-sqlite3';
 import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
 import { existsSync, mkdirSync } from 'fs';
 import * as schema from './schema.js';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
+import { getDataDir } from '../config/dataDir.js';
 
 // Database path - can be overridden via env for Docker
-const DB_PATH = process.env.DATA_DIR
-  ? join(process.env.DATA_DIR, 'jobs.db')
-  : join(__dirname, '../../../data/jobs.db');
+const DB_PATH = join(getDataDir(), 'jobs.db');
 
 // Ensure data directory exists
 const dataDir = dirname(DB_PATH);

orchestrator/src/server/db/migrate.ts

@@ -4,15 +4,11 @@
 
 import Database from 'better-sqlite3';
 import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
 import { existsSync, mkdirSync } from 'fs';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
+import { getDataDir } from '../config/dataDir.js';
 
 // Database path - can be overridden via env for Docker
-const DB_PATH = process.env.DATA_DIR
-  ? join(process.env.DATA_DIR, 'jobs.db')
-  : join(__dirname, '../../../data/jobs.db');
+const DB_PATH = join(getDataDir(), 'jobs.db');
 
 // Ensure data directory exists
 const dataDir = dirname(DB_PATH);

orchestrator/src/server/index.ts

@@ -2,16 +2,10 @@
  * Express server entry point.
  */
 
-import { join, dirname } from 'path';
-import { fileURLToPath } from 'url';
-import { config } from 'dotenv';
+import './config/env.js';
 import { createApp } from './app.js';
 import { initialize as initializeVisaSponsors } from './services/visa-sponsors/index.js';
 
-// Load environment variables from orchestrator root
-const __dirname = dirname(fileURLToPath(import.meta.url));
-config({ path: join(__dirname, '../../.env') });
-
 const app = createApp();
 const PORT = process.env.PORT || 3001;
 

orchestrator/src/server/pipeline/orchestrator.ts

@@ -24,6 +24,7 @@ import * as pipelineRepo from '../repositories/pipeline.js';
 import * as settingsRepo from '../repositories/settings.js';
 import { progressHelpers, resetProgress, updateProgress } from './progress.js';
 import type { CreateJobInput, Job, JobSource, PipelineConfig } from '../../shared/types.js';
+import { getDataDir } from '../config/dataDir.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DEFAULT_PROFILE_PATH = join(__dirname, '../../../../resume-generator/base.json');
@@ -33,7 +34,11 @@ const DEFAULT_CONFIG: PipelineConfig = {
   minSuitabilityScore: 50,
   sources: ['gradcracker', 'indeed', 'linkedin', 'ukvisajobs'],
   profilePath: DEFAULT_PROFILE_PATH,
-  outputDir: join(__dirname, '../../../data/pdfs'),
+  outputDir: join(getDataDir(), 'pdfs'),
+  enableCrawling: true,
+  enableScoring: true,
+  enableImporting: true,
+  enableAutoTailoring: true,
 };
 
 // Track if pipeline is currently running

orchestrator/src/server/pipeline/run.ts

@@ -5,13 +5,10 @@
  * Usage: npm run pipeline:run
  */
 
-import { config } from 'dotenv';
+import '../config/env.js';
 import { runPipeline } from './orchestrator.js';
 import { closeDb } from '../db/index.js';
 
-// Load environment variables
-config();
-
 async function main() {
   console.log('='.repeat(60));
   console.log('🚀 Job Pipeline Runner');

orchestrator/src/server/services/jobspy.ts

@@ -9,6 +9,7 @@ import { readFile, mkdir, unlink } from 'fs/promises';
 import { join, dirname } from 'path';
 import { fileURLToPath } from 'url';
 import type { CreateJobInput, JobSource } from '../../shared/types.js';
+import { getDataDir } from '../config/dataDir.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const JOBSPY_DIR = join(__dirname, '../../../../extractors/jobspy');
@@ -19,11 +20,6 @@ function getPythonPath(): string {
   return process.platform === 'win32' ? 'python' : 'python3';
 }
 
-function getDataDir(): string {
-  if (process.env.DATA_DIR) return process.env.DATA_DIR;
-  return join(__dirname, '../../../data');
-}
-
 function toStringOrNull(value: unknown): string | null {
   if (value === null || value === undefined) return null;
   if (typeof value === 'string') {

orchestrator/src/server/services/pdf.ts

@@ -12,14 +12,13 @@ import { existsSync } from 'fs';
 import { getSetting } from '../repositories/settings.js';
 import { pickProjectIdsForJob } from './projectSelection.js';
 import { extractProjectsFromProfile, resolveResumeProjectsSettings } from './resumeProjects.js';
+import { getDataDir } from '../config/dataDir.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
 // Paths - can be overridden via env for Docker
 const RESUME_GEN_DIR = process.env.RESUME_GEN_DIR || join(__dirname, '../../../../resume-generator');
-const OUTPUT_DIR = process.env.DATA_DIR 
-  ? join(process.env.DATA_DIR, 'pdfs')
-  : join(__dirname, '../../../data/pdfs');
+const OUTPUT_DIR = join(getDataDir(), 'pdfs');
 
 export interface PdfResult {
   success: boolean;

orchestrator/src/server/services/visa-sponsors/index.ts

@@ -6,10 +6,9 @@
 
 import fs from 'fs';
 import path from 'path';
-import { fileURLToPath } from 'url';
+import { getDataDir } from '../../config/dataDir.js';
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const DATA_DIR = path.join(__dirname, '../../../../data/visa-sponsors');
+const DATA_DIR = path.join(getDataDir(), 'visa-sponsors');
 
 // Ensure data directory exists
 if (!fs.existsSync(DATA_DIR)) {