tanyar09
8de65bc04c
- Created core modules: `ai-analyzer`, `core-parser`, and `job-search-parser`. - Implemented LinkedIn and job search parsers with integrated AI analysis. - Added CLI tools for AI analysis and job parsing. - Included comprehensive README files for each module detailing usage and features. - Established a `.gitignore` file to exclude unnecessary files. - Introduced sample data for testing and demonstration purposes. - Set up package.json files for dependency management across modules. - Implemented logging and error handling utilities for better debugging and user feedback.
413 lines
13 KiB
JavaScript
413 lines
13 KiB
JavaScript
/**
|
||
* LinkedIn Parser Demo
|
||
*
|
||
* Demonstrates the LinkedIn Parser's capabilities for scraping LinkedIn content
|
||
* with keyword-based searching, location filtering, and AI analysis.
|
||
*
|
||
* This demo uses simulated data for safety and demonstration purposes.
|
||
*/
|
||
|
||
const { logger } = require("../ai-analyzer");
|
||
const fs = require("fs");
|
||
const path = require("path");
|
||
|
||
// Terminal colors for demo output
|
||
const colors = {
|
||
reset: "\x1b[0m",
|
||
bright: "\x1b[1m",
|
||
cyan: "\x1b[36m",
|
||
green: "\x1b[32m",
|
||
yellow: "\x1b[33m",
|
||
blue: "\x1b[34m",
|
||
magenta: "\x1b[35m",
|
||
red: "\x1b[31m",
|
||
};
|
||
|
||
const demo = {
|
||
title: (text) =>
|
||
console.log(`\n${colors.bright}${colors.cyan}${text}${colors.reset}`),
|
||
section: (text) =>
|
||
console.log(`\n${colors.bright}${colors.magenta}${text}${colors.reset}`),
|
||
success: (text) => console.log(`${colors.green}✅ ${text}${colors.reset}`),
|
||
info: (text) => console.log(`${colors.blue}ℹ️ ${text}${colors.reset}`),
|
||
warning: (text) => console.log(`${colors.yellow}⚠️ ${text}${colors.reset}`),
|
||
error: (text) => console.log(`${colors.red}❌ ${text}${colors.reset}`),
|
||
code: (text) => console.log(`${colors.cyan}${text}${colors.reset}`),
|
||
};
|
||
|
||
// Mock data for demonstration
|
||
const mockPosts = [
|
||
{
|
||
id: "post_1",
|
||
content:
|
||
"Just got laid off from my software engineering role at TechCorp. Looking for new opportunities in Toronto. This is really tough but I'm staying positive!",
|
||
original_content:
|
||
"Just got #laidoff from my software engineering role at TechCorp! Looking for new opportunities in #Toronto. This is really tough but I'm staying positive! 🚀",
|
||
author: {
|
||
name: "John Doe",
|
||
title: "Software Engineer",
|
||
company: "TechCorp",
|
||
location: "Toronto, Ontario, Canada",
|
||
profile_url: "https://linkedin.com/in/johndoe",
|
||
},
|
||
engagement: { likes: 45, comments: 12, shares: 3 },
|
||
metadata: {
|
||
post_date: "2024-01-10T14:30:00Z",
|
||
scraped_at: "2024-01-15T10:30:00Z",
|
||
search_keyword: "layoff",
|
||
location_validated: true,
|
||
},
|
||
},
|
||
{
|
||
id: "post_2",
|
||
content:
|
||
"Our company is downsizing and I'm affected. This is really tough news but I'm grateful for the time I had here.",
|
||
original_content:
|
||
"Our company is #downsizing and I'm affected. This is really tough news but I'm grateful for the time I had here. #RIF #layoff",
|
||
author: {
|
||
name: "Jane Smith",
|
||
title: "Product Manager",
|
||
company: "StartupXYZ",
|
||
location: "Vancouver, British Columbia, Canada",
|
||
profile_url: "https://linkedin.com/in/janesmith",
|
||
},
|
||
engagement: { likes: 23, comments: 8, shares: 1 },
|
||
metadata: {
|
||
post_date: "2024-01-09T16:45:00Z",
|
||
scraped_at: "2024-01-15T10:30:00Z",
|
||
search_keyword: "downsizing",
|
||
location_validated: true,
|
||
},
|
||
},
|
||
{
|
||
id: "post_3",
|
||
content:
|
||
"Open to work! Looking for new opportunities in software development. I have 5 years of experience in React, Node.js, and cloud technologies.",
|
||
original_content:
|
||
"Open to work! Looking for new opportunities in software development. I have 5 years of experience in #React, #NodeJS, and #cloud technologies. #opentowork #jobsearch",
|
||
author: {
|
||
name: "Bob Wilson",
|
||
title: "Full Stack Developer",
|
||
company: "Freelance",
|
||
location: "Calgary, Alberta, Canada",
|
||
profile_url: "https://linkedin.com/in/bobwilson",
|
||
},
|
||
engagement: { likes: 67, comments: 15, shares: 8 },
|
||
metadata: {
|
||
post_date: "2024-01-08T11:20:00Z",
|
||
scraped_at: "2024-01-15T10:30:00Z",
|
||
search_keyword: "open to work",
|
||
location_validated: true,
|
||
},
|
||
},
|
||
];
|
||
|
||
async function runDemo() {
|
||
demo.title("=== LinkedIn Parser Demo ===");
|
||
demo.info(
|
||
"This demo showcases the LinkedIn Parser's capabilities for scraping LinkedIn content."
|
||
);
|
||
demo.info("All data shown is simulated for demonstration purposes.");
|
||
demo.info("Press Enter to continue through each section...\n");
|
||
|
||
await waitForEnter();
|
||
|
||
// 1. Configuration Demo
|
||
await demonstrateConfiguration();
|
||
|
||
// 2. Keyword Loading Demo
|
||
await demonstrateKeywordLoading();
|
||
|
||
// 3. Search Process Demo
|
||
await demonstrateSearchProcess();
|
||
|
||
// 4. Location Filtering Demo
|
||
await demonstrateLocationFiltering();
|
||
|
||
// 5. AI Analysis Demo
|
||
await demonstrateAIAnalysis();
|
||
|
||
// 6. Output Generation Demo
|
||
await demonstrateOutputGeneration();
|
||
|
||
demo.title("=== Demo Complete ===");
|
||
demo.success("LinkedIn Parser demo completed successfully!");
|
||
demo.info("Check the README.md for detailed usage instructions.");
|
||
}
|
||
|
||
async function demonstrateConfiguration() {
|
||
demo.section("1. Configuration Setup");
|
||
demo.info(
|
||
"The LinkedIn Parser uses environment variables and command-line options for configuration."
|
||
);
|
||
|
||
demo.code("// Environment Variables (.env file)");
|
||
demo.info("LINKEDIN_USERNAME=your_email@example.com");
|
||
demo.info("LINKEDIN_PASSWORD=your_password");
|
||
demo.info("CITY=Toronto");
|
||
demo.info("DATE_POSTED=past-week");
|
||
demo.info("SORT_BY=date_posted");
|
||
demo.info("WHEELS=5");
|
||
demo.info("LOCATION_FILTER=Ontario,Manitoba");
|
||
demo.info("ENABLE_LOCATION_CHECK=true");
|
||
demo.info("ENABLE_LOCAL_AI=true");
|
||
demo.info('AI_CONTEXT="job layoffs and workforce reduction"');
|
||
demo.info("OLLAMA_MODEL=mistral");
|
||
|
||
demo.code("// Command Line Options");
|
||
demo.info('node index.js --keyword="layoff,downsizing" --city="Vancouver"');
|
||
demo.info("node index.js --no-location --no-ai");
|
||
demo.info("node index.js --output=results/my-results.json");
|
||
demo.info("node index.js --ai-after");
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
async function demonstrateKeywordLoading() {
|
||
demo.section("2. Keyword Loading");
|
||
demo.info(
|
||
"Keywords can be loaded from CSV files or specified via command line."
|
||
);
|
||
|
||
// Simulate loading keywords from CSV
|
||
demo.code("// Loading keywords from CSV file");
|
||
logger.step("Loading keywords from keywords/linkedin-keywords.csv");
|
||
|
||
const keywords = [
|
||
"layoff",
|
||
"downsizing",
|
||
"reduction in force",
|
||
"RIF",
|
||
"termination",
|
||
"job loss",
|
||
"workforce reduction",
|
||
"open to work",
|
||
"actively seeking",
|
||
"job search",
|
||
];
|
||
|
||
demo.success(`Loaded ${keywords.length} keywords from CSV file`);
|
||
demo.info("Keywords: " + keywords.slice(0, 5).join(", ") + "...");
|
||
|
||
demo.code("// Command line keyword override");
|
||
demo.info('node index.js --keyword="layoff,downsizing"');
|
||
demo.info('node index.js --add-keyword="hiring freeze"');
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
async function demonstrateSearchProcess() {
|
||
demo.section("3. Search Process Simulation");
|
||
demo.info(
|
||
"The parser performs automated LinkedIn searches for each keyword."
|
||
);
|
||
|
||
const keywords = ["layoff", "downsizing", "open to work"];
|
||
|
||
for (const keyword of keywords) {
|
||
demo.code(`// Searching for keyword: "${keyword}"`);
|
||
logger.search(`Searching for "${keyword}" in Toronto`);
|
||
|
||
// Simulate search process
|
||
await simulateSearch();
|
||
|
||
const foundCount = Math.floor(Math.random() * 50) + 10;
|
||
const acceptedCount = Math.floor(foundCount * 0.3);
|
||
|
||
logger.info(`Found ${foundCount} posts, checking profiles for location...`);
|
||
logger.success(`Accepted ${acceptedCount} posts after location validation`);
|
||
|
||
console.log();
|
||
}
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
async function demonstrateLocationFiltering() {
|
||
demo.section("4. Location Filtering");
|
||
demo.info(
|
||
"Posts are filtered based on author location using geographic validation."
|
||
);
|
||
|
||
demo.code("// Location filter configuration");
|
||
demo.info("LOCATION_FILTER=Ontario,Manitoba");
|
||
demo.info("ENABLE_LOCATION_CHECK=true");
|
||
|
||
demo.code("// Location validation examples");
|
||
const testLocations = [
|
||
{ location: "Toronto, Ontario, Canada", valid: true },
|
||
{ location: "Vancouver, British Columbia, Canada", valid: false },
|
||
{ location: "Calgary, Alberta, Canada", valid: false },
|
||
{ location: "Winnipeg, Manitoba, Canada", valid: true },
|
||
{ location: "New York, NY, USA", valid: false },
|
||
];
|
||
|
||
testLocations.forEach(({ location, valid }) => {
|
||
logger.location(`Checking location: ${location}`);
|
||
if (valid) {
|
||
logger.success(`✅ Location valid - post accepted`);
|
||
} else {
|
||
logger.warning(`❌ Location invalid - post rejected`);
|
||
}
|
||
});
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
async function demonstrateAIAnalysis() {
|
||
demo.section("5. AI Analysis");
|
||
demo.info(
|
||
"Posts can be analyzed using local Ollama or OpenAI for relevance scoring."
|
||
);
|
||
|
||
demo.code("// AI analysis configuration");
|
||
demo.info("ENABLE_LOCAL_AI=true");
|
||
demo.info('AI_CONTEXT="job layoffs and workforce reduction"');
|
||
demo.info("OLLAMA_MODEL=mistral");
|
||
|
||
demo.code("// Analyzing posts with AI");
|
||
logger.ai("Starting AI analysis of accepted posts...");
|
||
|
||
for (let i = 0; i < mockPosts.length; i++) {
|
||
const post = mockPosts[i];
|
||
logger.info(`Analyzing post ${i + 1}: ${post.content.substring(0, 50)}...`);
|
||
|
||
// Simulate AI analysis
|
||
await simulateProcessing();
|
||
|
||
const relevanceScore = 0.7 + Math.random() * 0.3;
|
||
const confidence = 0.8 + Math.random() * 0.2;
|
||
|
||
logger.success(
|
||
`Relevance: ${relevanceScore.toFixed(
|
||
2
|
||
)}, Confidence: ${confidence.toFixed(2)}`
|
||
);
|
||
|
||
// Add AI analysis to post
|
||
post.ai_analysis = {
|
||
relevance_score: relevanceScore,
|
||
confidence: confidence,
|
||
context_match: relevanceScore > 0.7,
|
||
analysis_text: `This post discusses ${post.metadata.search_keyword} and is relevant to the search context.`,
|
||
};
|
||
}
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
async function demonstrateOutputGeneration() {
|
||
demo.section("6. Output Generation");
|
||
demo.info("Results are saved to JSON files with comprehensive metadata.");
|
||
|
||
demo.code("// Generating output file");
|
||
logger.file("Saving results to JSON file...");
|
||
|
||
const outputData = {
|
||
metadata: {
|
||
timestamp: new Date().toISOString(),
|
||
keywords: ["layoff", "downsizing", "open to work"],
|
||
city: "Toronto",
|
||
date_posted: "past-week",
|
||
sort_by: "date_posted",
|
||
total_posts_found: 150,
|
||
accepted_posts: mockPosts.length,
|
||
rejected_posts: 147,
|
||
processing_time_seconds: 180,
|
||
},
|
||
posts: mockPosts,
|
||
};
|
||
|
||
// Save to demo file
|
||
const outputPath = path.join(__dirname, "demo-results.json");
|
||
fs.writeFileSync(outputPath, JSON.stringify(outputData, null, 2));
|
||
|
||
demo.success(`Results saved to: ${outputPath}`);
|
||
demo.info(`Total posts processed: ${outputData.metadata.total_posts_found}`);
|
||
demo.info(`Posts accepted: ${outputData.metadata.accepted_posts}`);
|
||
demo.info(`Posts rejected: ${outputData.metadata.rejected_posts}`);
|
||
|
||
demo.code("// Output file structure");
|
||
demo.info("📁 demo-results.json");
|
||
demo.info(" ├── metadata");
|
||
demo.info(" │ ├── timestamp");
|
||
demo.info(" │ ├── keywords");
|
||
demo.info(" │ ├── city");
|
||
demo.info(" │ ├── total_posts_found");
|
||
demo.info(" │ ├── accepted_posts");
|
||
demo.info(" │ └── processing_time_seconds");
|
||
demo.info(" └── posts[]");
|
||
demo.info(" ├── id");
|
||
demo.info(" ├── content");
|
||
demo.info(" ├── author");
|
||
demo.info(" ├── engagement");
|
||
demo.info(" ├── ai_analysis");
|
||
demo.info(" └── metadata");
|
||
|
||
await waitForEnter();
|
||
}
|
||
|
||
// Helper functions
|
||
function waitForEnter() {
|
||
return new Promise((resolve) => {
|
||
const readline = require("readline");
|
||
const rl = readline.createInterface({
|
||
input: process.stdin,
|
||
output: process.stdout,
|
||
});
|
||
|
||
rl.question("\nPress Enter to continue...", () => {
|
||
rl.close();
|
||
resolve();
|
||
});
|
||
});
|
||
}
|
||
|
||
async function simulateSearch() {
|
||
return new Promise((resolve) => {
|
||
const steps = [
|
||
"Launching browser",
|
||
"Logging in",
|
||
"Navigating to search",
|
||
"Loading results",
|
||
];
|
||
let i = 0;
|
||
const interval = setInterval(() => {
|
||
if (i < steps.length) {
|
||
logger.info(steps[i]);
|
||
i++;
|
||
} else {
|
||
clearInterval(interval);
|
||
resolve();
|
||
}
|
||
}, 800);
|
||
});
|
||
}
|
||
|
||
async function simulateProcessing() {
|
||
return new Promise((resolve) => {
|
||
const dots = [".", "..", "..."];
|
||
let i = 0;
|
||
const interval = setInterval(() => {
|
||
process.stdout.write(`\rProcessing${dots[i]}`);
|
||
i = (i + 1) % dots.length;
|
||
}, 500);
|
||
|
||
setTimeout(() => {
|
||
clearInterval(interval);
|
||
process.stdout.write("\r");
|
||
resolve();
|
||
}, 1500);
|
||
});
|
||
}
|
||
|
||
// Run the demo if this file is executed directly
|
||
if (require.main === module) {
|
||
runDemo().catch((error) => {
|
||
demo.error(`Demo failed: ${error.message}`);
|
||
process.exit(1);
|
||
});
|
||
}
|
||
|
||
module.exports = { runDemo };
|