rename extractors to their own folder

This commit is contained in:
DaKheera47
2025-12-14 22:44:37 +00:00
parent cefb75a9ec
commit d24f71ab3d
16 changed files with 16 additions and 15 deletions
+4 -4
View File
@@ -1,5 +1,5 @@
/**
* Service for running the job crawler (job-extractor).
* Service for running the Gradcracker crawler (extractors/gradcracker).
* Wraps the existing Crawlee-based crawler.
*/
@@ -11,7 +11,7 @@ import { createInterface } from 'readline';
import type { CreateJobInput } from '../../shared/types.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const CRAWLER_DIR = join(__dirname, '../../../../job-extractor');
const CRAWLER_DIR = join(__dirname, '../../../../extractors/gradcracker');
const STORAGE_DIR = join(CRAWLER_DIR, 'storage/datasets/default');
const JOBOPS_STORAGE_DIR = join(CRAWLER_DIR, 'storage/jobops');
@@ -29,7 +29,7 @@ export interface RunCrawlerOptions {
existingJobUrls?: string[];
/**
* Optional callback for live crawl progress emitted by job-extractor.
* Optional callback for live crawl progress emitted by the Gradcracker extractor.
*/
onProgress?: (update: JobExtractorProgress) => void;
}
@@ -57,7 +57,7 @@ async function writeExistingJobUrlsFile(existingJobUrls: string[] | undefined):
}
/**
* Run the job-extractor crawler and return discovered jobs.
* Run the Gradcracker crawler and return discovered jobs.
*/
export async function runCrawler(options: RunCrawlerOptions = {}): Promise<CrawlerResult> {
console.log('🕷️ Starting job crawler...');
+1 -1
View File
@@ -11,7 +11,7 @@ import { fileURLToPath } from 'url';
import type { CreateJobInput, JobSource } from '../../shared/types.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const JOBSPY_DIR = join(__dirname, '../../../../jobspy-extractor');
const JOBSPY_DIR = join(__dirname, '../../../../extractors/jobspy');
const JOBSPY_SCRIPT = join(JOBSPY_DIR, 'scrape_jobs.py');
function getPythonPath(): string {