ilia/Jobber
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial implementation

Browse Source
This commit is contained in:
DaKheera47
2025-12-26 20:17:05 +00:00
parent 8a236c0be9
commit 0f36d9b8a6
18 changed files with 1249 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orchestrator/.env.example
+9
View File
@@ -32,3 +32,12 @@ JOBSPY_RESULTS_WANTED=200
JOBSPY_HOURS_OLD=72
JOBSPY_COUNTRY_INDEED=UK
JOBSPY_LINKEDIN_FETCH_DESCRIPTION=1
# =============================================================================
# UKVisaJobs (UK visa sponsorship job scraping) - optional
# =============================================================================
# Get these tokens from browser dev tools after logging into my.ukvisajobs.com
UKVISAJOBS_TOKEN=
UKVISAJOBS_AUTH_TOKEN=
UKVISAJOBS_CSRF_TOKEN=
UKVISAJOBS_CI_SESSION=
orchestrator/src/client/App.tsx
+2 -2
View File
@@ -12,7 +12,7 @@ import { Header, JobList, PipelineProgress, Stats } from "./components";
import * as api from "./api";
import { SettingsPage } from "./pages/SettingsPage";
const DEFAULT_PIPELINE_SOURCES: JobSource[] = ["gradcracker", "indeed", "linkedin"];
const DEFAULT_PIPELINE_SOURCES: JobSource[] = ["gradcracker", "indeed", "linkedin", "ukvisajobs"];
const PIPELINE_SOURCES_STORAGE_KEY = "jobops.pipeline.sources";
export const App: React.FC = () => {
@@ -33,7 +33,7 @@ export const App: React.FC = () => {
const raw = localStorage.getItem(PIPELINE_SOURCES_STORAGE_KEY);
if (!raw) return DEFAULT_PIPELINE_SOURCES;
const parsed = JSON.parse(raw) as unknown;
const allowed: JobSource[] = ["gradcracker", "indeed", "linkedin"];
const allowed: JobSource[] = ["gradcracker", "indeed", "linkedin", "ukvisajobs"];
if (!Array.isArray(parsed)) return DEFAULT_PIPELINE_SOURCES;
const next = parsed.filter((value): value is JobSource => allowed.includes(value));
return next.length > 0 ? next : DEFAULT_PIPELINE_SOURCES;
orchestrator/src/client/components/Header.tsx
+2 -1
View File
@@ -60,9 +60,10 @@ export const Header: React.FC<HeaderProps> = ({
gradcracker: "Gradcracker",
indeed: "Indeed",
linkedin: "LinkedIn",
ukvisajobs: "UK Visa Jobs",
};
const orderedSources: JobSource[] = ["gradcracker", "indeed", "linkedin"];
const orderedSources: JobSource[] = ["gradcracker", "indeed", "linkedin", "ukvisajobs"];
const toggleSource = (source: JobSource, checked: boolean) => {
const next = checked
orchestrator/src/server/api/routes.ts
+40 -40
View File
@@ -60,10 +60,10 @@ apiRouter.get('/jobs', async (req: Request, res: Response) => {
try {
const statusFilter = req.query.status as string | undefined;
const statuses = statusFilter?.split(',').filter(Boolean) as JobStatus[] | undefined;
const jobs = await jobsRepo.getAllJobs(statuses);
const stats = await jobsRepo.getJobStats();
const response: ApiResponse<JobsListResponse> = {
success: true,
data: {
@@ -72,7 +72,7 @@ apiRouter.get('/jobs', async (req: Request, res: Response) => {
byStatus: stats,
},
};
res.json(response);
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -86,11 +86,11 @@ apiRouter.get('/jobs', async (req: Request, res: Response) => {
apiRouter.get('/jobs/:id', async (req: Request, res: Response) => {
try {
const job = await jobsRepo.getJobById(req.params.id);
if (!job) {
return res.status(404).json({ success: false, error: 'Job not found' });
}
res.json({ success: true, data: job });
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -113,11 +113,11 @@ apiRouter.patch('/jobs/:id', async (req: Request, res: Response) => {
try {
const input = updateJobSchema.parse(req.body);
const job = await jobsRepo.updateJob(req.params.id, input);
if (!job) {
return res.status(404).json({ success: false, error: 'Job not found' });
}
res.json({ success: true, data: job });
} catch (error) {
if (error instanceof z.ZodError) {
@@ -137,11 +137,11 @@ apiRouter.post('/jobs/:id/process', async (req: Request, res: Response) => {
const force = forceRaw === '1' || forceRaw === 'true';
const result = await processJob(req.params.id, { force });
if (!result.success) {
return res.status(400).json({ success: false, error: result.error });
}
const job = await jobsRepo.getJobById(req.params.id);
res.json({ success: true, data: job });
} catch (error) {
@@ -156,13 +156,13 @@ apiRouter.post('/jobs/:id/process', async (req: Request, res: Response) => {
apiRouter.post('/jobs/:id/apply', async (req: Request, res: Response) => {
try {
const job = await jobsRepo.getJobById(req.params.id);
if (!job) {
return res.status(404).json({ success: false, error: 'Job not found' });
}
const appliedAt = new Date().toISOString();
// Sync to Notion
const notionResult = await createNotionEntry({
id: job.id,
@@ -175,7 +175,7 @@ apiRouter.post('/jobs/:id/apply', async (req: Request, res: Response) => {
pdfPath: job.pdfPath,
appliedAt,
});
// Update job status
const updatedJob = await jobsRepo.updateJob(job.id, {
status: 'applied',
@@ -186,7 +186,7 @@ apiRouter.post('/jobs/:id/apply', async (req: Request, res: Response) => {
if (updatedJob) {
notifyJobCompleteWebhook(updatedJob).catch(console.warn)
}
res.json({ success: true, data: updatedJob });
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -200,11 +200,11 @@ apiRouter.post('/jobs/:id/apply', async (req: Request, res: Response) => {
apiRouter.post('/jobs/:id/reject', async (req: Request, res: Response) => {
try {
const job = await jobsRepo.updateJob(req.params.id, { status: 'rejected' });
if (!job) {
return res.status(404).json({ success: false, error: 'Job not found' });
}
res.json({ success: true, data: job });
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -351,7 +351,7 @@ apiRouter.get('/pipeline/status', async (req: Request, res: Response) => {
try {
const { isRunning } = getPipelineStatus();
const lastRun = await pipelineRepo.getLatestPipelineRun();
const response: ApiResponse<PipelineStatusResponse> = {
success: true,
data: {
@@ -360,7 +360,7 @@ apiRouter.get('/pipeline/status', async (req: Request, res: Response) => {
nextScheduledRun: null, // Would come from n8n
},
};
res.json(response);
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -377,20 +377,20 @@ apiRouter.get('/pipeline/progress', (req: Request, res: Response) => {
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Accel-Buffering', 'no'); // Disable Nginx buffering
// Send initial progress
const sendProgress = (data: unknown) => {
res.write(`data: ${JSON.stringify(data)}\n\n`);
};
// Subscribe to progress updates
const unsubscribe = subscribeToProgress(sendProgress);
// Send heartbeat every 30 seconds to keep connection alive
const heartbeat = setInterval(() => {
res.write(': heartbeat\n\n');
}, 30000);
// Cleanup on close
req.on('close', () => {
clearInterval(heartbeat);
@@ -417,19 +417,19 @@ apiRouter.get('/pipeline/runs', async (req: Request, res: Response) => {
const runPipelineSchema = z.object({
topN: z.number().min(1).max(50).optional(),
minSuitabilityScore: z.number().min(0).max(100).optional(),
sources: z.array(z.enum(['gradcracker', 'indeed', 'linkedin'])).min(1).optional(),
sources: z.array(z.enum(['gradcracker', 'indeed', 'linkedin', 'ukvisajobs'])).min(1).optional(),
});
apiRouter.post('/pipeline/run', async (req: Request, res: Response) => {
try {
const config = runPipelineSchema.parse(req.body);
// Start pipeline in background
runPipeline(config).catch(console.error);
res.json({
success: true,
data: { message: 'Pipeline started' }
res.json({
success: true,
data: { message: 'Pipeline started' }
});
} catch (error) {
if (error instanceof z.ZodError) {
@@ -451,21 +451,21 @@ apiRouter.post('/webhook/trigger', async (req: Request, res: Response) => {
// Optional: Add authentication check
const authHeader = req.headers.authorization;
const expectedToken = process.env.WEBHOOK_SECRET;
if (expectedToken && authHeader !== `Bearer ${expectedToken}`) {
return res.status(401).json({ success: false, error: 'Unauthorized' });
}
try {
// Start pipeline in background
runPipeline().catch(console.error);
res.json({
success: true,
data: {
res.json({
success: true,
data: {
message: 'Pipeline triggered',
triggeredAt: new Date().toISOString(),
}
}
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
@@ -483,14 +483,14 @@ apiRouter.post('/webhook/trigger', async (req: Request, res: Response) => {
apiRouter.delete('/database', async (req: Request, res: Response) => {
try {
const result = clearDatabase();
res.json({
success: true,
data: {
res.json({
success: true,
data: {
message: 'Database cleared',
jobsDeleted: result.jobsDeleted,
runsDeleted: result.runsDeleted,
}
}
});
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
orchestrator/src/server/db/schema.ts
+6 -6
View File
@@ -7,9 +7,9 @@ import { sql } from 'drizzle-orm';
export const jobs = sqliteTable('jobs', {
id: text('id').primaryKey(),
// From crawler
source: text('source', { enum: ['gradcracker', 'indeed', 'linkedin'] }).notNull().default('gradcracker'),
source: text('source', { enum: ['gradcracker', 'indeed', 'linkedin', 'ukvisajobs'] }).notNull().default('gradcracker'),
sourceJobId: text('source_job_id'),
jobUrlDirect: text('job_url_direct'),
datePosted: text('date_posted'),
@@ -51,17 +51,17 @@ export const jobs = sqliteTable('jobs', {
companyReviewsCount: integer('company_reviews_count'),
vacancyCount: integer('vacancy_count'),
workFromHomeType: text('work_from_home_type'),
// Orchestrator enrichments
status: text('status', {
enum: ['discovered', 'processing', 'ready', 'applied', 'rejected', 'expired']
status: text('status', {
enum: ['discovered', 'processing', 'ready', 'applied', 'rejected', 'expired']
}).notNull().default('discovered'),
suitabilityScore: real('suitability_score'),
suitabilityReason: text('suitability_reason'),
tailoredSummary: text('tailored_summary'),
pdfPath: text('pdf_path'),
notionPageId: text('notion_page_id'),
// Timestamps
discoveredAt: text('discovered_at').notNull().default(sql`(datetime('now'))`),
processedAt: text('processed_at'),
orchestrator/src/server/pipeline/orchestrator.ts
+46 -30
View File
@@ -12,6 +12,7 @@ import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import { runCrawler } from '../services/crawler.js';
import { runJobSpy } from '../services/jobspy.js';
import { runUkVisaJobs } from '../services/ukvisajobs.js';
import { scoreJobSuitability } from '../services/scorer.js';
import { generateSummary } from '../services/summary.js';
import { generatePdf } from '../services/pdf.js';
@@ -27,7 +28,7 @@ const DEFAULT_PROFILE_PATH = join(__dirname, '../../../../resume-generator/base.
const DEFAULT_CONFIG: PipelineConfig = {
topN: 10,
minSuitabilityScore: 50,
sources: ['gradcracker', 'indeed', 'linkedin'],
sources: ['gradcracker', 'indeed', 'linkedin', 'ukvisajobs'],
profilePath: DEFAULT_PROFILE_PATH,
outputDir: join(__dirname, '../../../data/pdfs'),
};
@@ -88,22 +89,22 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
error: 'Pipeline is already running',
};
}
isPipelineRunning = true;
resetProgress();
const mergedConfig = { ...DEFAULT_CONFIG, ...config };
// Create pipeline run record
const pipelineRun = await pipelineRepo.createPipelineRun();
console.log('🚀 Starting job pipeline...');
console.log(` Config: topN=${mergedConfig.topN}, minScore=${mergedConfig.minSuitabilityScore} (manual processing)`);
try {
// Step 1: Load profile
console.log('\n📋 Loading profile...');
const profile = await loadProfile(mergedConfig.profilePath);
// Step 2: Run crawler
console.log('\n🕷️ Running crawler...');
progressHelpers.startCrawling();
@@ -154,6 +155,21 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
}
}
// Run UKVisaJobs extractor if selected
if (mergedConfig.sources.includes('ukvisajobs')) {
updateProgress({
step: 'crawling',
detail: 'UKVisaJobs: scraping visa-sponsoring jobs...',
});
const ukVisaResult = await runUkVisaJobs({ maxJobs: 50 });
if (!ukVisaResult.success) {
sourceErrors.push(`ukvisajobs: ${ukVisaResult.error ?? 'unknown error'}`);
} else {
discoveredJobs.push(...ukVisaResult.jobs);
}
}
if (discoveredJobs.length === 0 && sourceErrors.length > 0) {
throw new Error(`All sources failed: ${sourceErrors.join('; ')}`);
}
@@ -163,18 +179,18 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
}
progressHelpers.crawlingComplete(discoveredJobs.length);
// Step 3: Import discovered jobs
console.log('\n💾 Importing jobs to database...');
const { created, skipped } = await jobsRepo.bulkCreateJobs(discoveredJobs);
console.log(` Created: ${created}, Skipped (duplicates): ${skipped}`);
progressHelpers.importComplete(created, skipped);
await pipelineRepo.updatePipelineRun(pipelineRun.id, {
jobsDiscovered: created,
});
// Step 4: Score all discovered jobs missing a score
console.log('\n🎯 Scoring jobs for suitability...');
const unprocessedJobs = await jobsRepo.getUnscoredDiscoveredJobs();
@@ -187,7 +203,7 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
totalToProcess: 0,
currentJob: undefined,
});
// Score jobs with progress updates
const scoredJobs: Array<Job & { suitabilityScore: number; suitabilityReason: string }> = [];
for (let i = 0; i < unprocessedJobs.length; i++) {
@@ -217,21 +233,21 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
suitabilityReason: reason,
});
}
progressHelpers.scoringComplete(scoredJobs.length);
console.log(`\n📊 Scored ${scoredJobs.length} jobs. Ready for manual processing.`);
// Update pipeline run as completed
await pipelineRepo.updatePipelineRun(pipelineRun.id, {
status: 'completed',
completedAt: new Date().toISOString(),
jobsProcessed: 0,
});
console.log('\n🎉 Pipeline completed!');
console.log(` Jobs discovered: ${created}`);
console.log(' Jobs processed: 0 (manual)');
progressHelpers.complete(created, 0);
await notifyPipelineWebhook('pipeline.completed', {
@@ -241,22 +257,22 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
jobsProcessed: 0,
})
isPipelineRunning = false;
return {
success: true,
jobsDiscovered: created,
jobsProcessed: 0,
};
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
await pipelineRepo.updatePipelineRun(pipelineRun.id, {
status: 'failed',
completedAt: new Date().toISOString(),
errorMessage: message,
});
progressHelpers.failed(message);
await notifyPipelineWebhook('pipeline.failed', {
@@ -264,9 +280,9 @@ export async function runPipeline(config: Partial<PipelineConfig> = {}): Promise
error: message,
})
isPipelineRunning = false;
console.error('\n❌ Pipeline failed:', message);
return {
success: false,
jobsDiscovered: 0,
@@ -287,7 +303,7 @@ export async function processJob(
error?: string;
}> {
console.log(`📝 Processing job ${jobId}...`);
try {
const job = await jobsRepo.getJobById(jobId);
if (!job) {
@@ -297,9 +313,9 @@ export async function processJob(
if (job.status !== 'discovered' && job.status !== 'ready') {
return { success: false, error: `Job cannot be processed from status: ${job.status}` };
}
const profile = await loadProfile(DEFAULT_PROFILE_PATH);
// Mark as processing
await jobsRepo.updateJob(job.id, { status: 'processing' });
@@ -314,7 +330,7 @@ export async function processJob(
job.suitabilityScore = suitability.score;
job.suitabilityReason = suitability.reason;
}
// Generate summary (AI)
// If forcing, always recompute; otherwise compute if missing.
if (options?.force || !job.tailoredSummary) {
@@ -323,7 +339,7 @@ export async function processJob(
job.jobDescription || '',
profile
);
if (summaryResult.success) {
await jobsRepo.updateJob(job.id, {
tailoredSummary: summaryResult.summary,
@@ -331,7 +347,7 @@ export async function processJob(
job.tailoredSummary = summaryResult.summary ?? null;
}
}
// Generate PDF
console.log(' Generating PDF...');
const pdfResult = await generatePdf(
@@ -340,16 +356,16 @@ export async function processJob(
job.jobDescription || '',
DEFAULT_PROFILE_PATH
);
// Mark as ready
await jobsRepo.updateJob(job.id, {
status: 'ready',
pdfPath: pdfResult.pdfPath ?? undefined,
});
console.log(' ✅ Done!');
return { success: true };
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
return { success: false, error: message };
orchestrator/src/server/services/ukvisajobs.ts
+125
View File
@@ -0,0 +1,125 @@
/**
* Service for running the UK Visa Jobs extractor (extractors/ukvisajobs).
*
* Spawns the extractor as a child process and reads its output dataset.
*/
import { spawn } from 'child_process';
import { readdir, readFile, rm, mkdir } from 'fs/promises';
import { join, dirname } from 'path';
import { fileURLToPath } from 'url';
import type { CreateJobInput } from '../../shared/types.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const UKVISAJOBS_DIR = join(__dirname, '../../../../extractors/ukvisajobs');
const STORAGE_DIR = join(UKVISAJOBS_DIR, 'storage/datasets/default');
export interface RunUkVisaJobsOptions {
/** Maximum number of jobs to fetch. Defaults to 50, max 200. */
maxJobs?: number;
/** Search keyword filter (optional) */
searchKeyword?: string;
}
export interface UkVisaJobsResult {
success: boolean;
jobs: CreateJobInput[];
error?: string;
}
/**
* Clear previous extraction results.
*/
async function clearStorageDataset(): Promise<void> {
try {
await rm(STORAGE_DIR, { recursive: true, force: true });
} catch {
// Ignore if directory doesn't exist
}
}
/**
* Run the UK Visa Jobs extractor.
*/
export async function runUkVisaJobs(options: RunUkVisaJobsOptions = {}): Promise<UkVisaJobsResult> {
console.log('🇬🇧 Running UK Visa Jobs extractor...');
try {
// Clear previous results
await clearStorageDataset();
await mkdir(STORAGE_DIR, { recursive: true });
// Run the extractor using npx tsx directly (more reliable in Docker/different environments)
await new Promise<void>((resolve, reject) => {
const child = spawn('npx', ['tsx', 'src/main.ts'], {
cwd: UKVISAJOBS_DIR,
stdio: 'inherit',
env: {
...process.env,
UKVISAJOBS_MAX_JOBS: String(options.maxJobs ?? 50),
UKVISAJOBS_SEARCH_KEYWORD: options.searchKeyword ?? '',
},
});
child.on('close', (code) => {
if (code === 0) resolve();
else reject(new Error(`UK Visa Jobs extractor exited with code ${code}`));
});
child.on('error', reject);
});
// Read the output dataset
const jobs = await readDataset();
console.log(`✅ UK Visa Jobs: imported ${jobs.length} jobs`);
return { success: true, jobs };
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown error';
console.error(`❌ UK Visa Jobs failed: ${message}`);
return { success: false, jobs: [], error: message };
}
}
/**
* Read jobs from the extractor's output dataset.
*/
async function readDataset(): Promise<CreateJobInput[]> {
const jobs: CreateJobInput[] = [];
try {
const files = await readdir(STORAGE_DIR);
const jsonFiles = files.filter((f) => f.endsWith('.json') && f !== 'jobs.json');
for (const file of jsonFiles.sort()) {
try {
const content = await readFile(join(STORAGE_DIR, file), 'utf-8');
const job = JSON.parse(content);
// Map to CreateJobInput format
jobs.push({
source: 'ukvisajobs',
sourceJobId: job.sourceJobId,
title: job.title || 'Unknown Title',
employer: job.employer || 'Unknown Employer',
employerUrl: job.employerUrl,
jobUrl: job.jobUrl,
applicationLink: job.applicationLink || job.jobUrl,
location: job.location,
deadline: job.deadline,
salary: job.salary,
jobDescription: job.jobDescription,
datePosted: job.datePosted,
degreeRequired: job.degreeRequired,
jobType: job.jobType,
jobLevel: job.jobLevel,
});
} catch {
// Skip invalid files
}
}
} catch {
// Dataset directory doesn't exist yet
}
return jobs;
}
orchestrator/src/shared/types.ts
+6 -5
View File
@@ -2,7 +2,7 @@
* Shared types for the job-ops orchestrator.
*/
export type JobStatus =
export type JobStatus =
| 'discovered' // Crawled but not processed
| 'processing' // Currently generating resume
| 'ready' // PDF generated, waiting for user to apply
@@ -13,11 +13,12 @@ export type JobStatus =
export type JobSource =
| 'gradcracker'
| 'indeed'
| 'linkedin';
| 'linkedin'
| 'ukvisajobs';
export interface Job {
id: string;
// Source / provenance
source: JobSource;
sourceJobId: string | null; // External ID (if provided)
@@ -37,7 +38,7 @@ export interface Job {
degreeRequired: string | null;
starting: string | null;
jobDescription: string | null;
// Orchestrator enrichments
status: JobStatus;
suitabilityScore: number | null; // 0-100 AI-generated score
@@ -71,7 +72,7 @@ export interface Job {
companyReviewsCount: number | null;
vacancyCount: number | null;
workFromHomeType: string | null;
// Timestamps
discoveredAt: string;
processedAt: string | null;