update CoreParser to increase default timeout and change navigation waitUntil option to networkidle

core-parser/index.js

@@ -0,0 +1,63 @@
+const playwright = require('playwright');
+const AuthManager = require('./auth-manager');
+const NavigationManager = require('./navigation');
+
+class CoreParser {
+  constructor(config = {}) {
+    this.config = {
+      headless: true,
+      timeout: 60000,  // Increased default timeout
+      ...config
+    };
+    this.browser = null;
+    this.context = null;
+    this.pages = {};
+    this.authManager = new AuthManager(this);
+    this.navigationManager = new NavigationManager(this);
+  }
+
+  async init() {
+    this.browser = await playwright.chromium.launch({
+      headless: this.config.headless
+    });
+    this.context = await this.browser.newContext();
+  }
+
+  async createPage(id) {
+    if (!this.browser) await this.init();
+    const page = await this.context.newPage();
+    this.pages[id] = page;
+    return page;
+  }
+
+  getPage(id) {
+    return this.pages[id];
+  }
+
+  async authenticate(site, credentials, pageId) {
+    return this.authManager.authenticate(site, credentials, pageId);
+  }
+
+  async navigateTo(url, options = {}) {
+    const {
+      pageId = "default",
+      waitUntil = "networkidle",  // Changed default to networkidle
+      retries = 1,
+      retryDelay = 2000,
+      timeout = this.config.timeout,
+    } = options;
+
+    return this.navigationManager.navigateTo(url, options);
+  }
+
+  async cleanup() {
+    if (this.browser) {
+      await this.browser.close();
+      this.browser = null;
+      this.context = null;
+      this.pages = {};
+    }
+  }
+}
+
+module.exports = CoreParser;

core-parser/package.json

@@ -1,27 +1,7 @@
 {
   "name": "core-parser",
   "version": "1.0.0",
-  "description": "Core browser automation and parsing engine for all parsers",
   "main": "index.js",
-  "scripts": {
-    "test": "jest",
-    "install:browsers": "npx playwright install chromium"
-  },
-  "keywords": [
-    "parser",
-    "playwright",
-    "browser",
-    "automation",
-    "core"
-  ],
-  "author": "Job Market Intelligence Team",
-  "license": "ISC",
-  "type": "commonjs",
-  "dependencies": {
-    "playwright": "^1.53.2",
-    "dotenv": "^17.0.0"
-  },
-  "devDependencies": {
-    "jest": "^29.7.0"
-  }
+  "description": "Core parser utilities for browser management",
+  "dependencies": {}
 }