WebSocket is not open when 'maxrequestreached' event is reached

[LeMoussel]

• Version: [email protected]
• Platform / OS version: Windows 10
• Node.js version: v10.15.0
• Link: http://f4b1.com/ (Rem: same error for other sites)

f4b1.com

Login - f4b1.com Contact - f4b1.com Les différents paramètres APN opérateur : Free Mobile, Orange, SFR, Bouygues et MVNO Maximum request reached! Crawl Terminated

(node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED) at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19) at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21) at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47) at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26) at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33) at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19) (node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 3) (node:14984) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code. (node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED) at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19) at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21) at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47) at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26) at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33) at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19) (node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 7) (node:14984) UnhandledPromiseRejectionWarning: Error: WebSocket is not open: readyState 3 (CLOSED) at WebSocket.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\ws\lib\websocket.js:314:19) at Connection.send (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Connection.js:84:21) at Browser._createPageInContext (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:168:47) at BrowserContext.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:260:26) at Browser.newPage (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\node_modules\puppeteer\lib\Browser.js:160:33) at HCCrawler._newCrawler (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:554:38) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:336:32) at HCCrawler._request (D:\Developpement\NodeJS\headless-chrome-crawler\node_modules\headless-chrome-crawler\lib\hccrawler.js:358:19) at process._tickCallback (internal/process/next_tick.js:68:7) (node:14984) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). (rejection id: 9)

Example code:

const HCCrawler = require('headless-chrome-crawler');
const findChrome = require('chrome-finder'); // https://github.com/gwuhaolin/chrome-finder

(async () => {
  const crawler = await HCCrawler.launch({
    maxRequest: 5,
    maxDepth: 2,
    maxConcurrency: 4,
    obeyRobotsTxt: true,
    executablePath: findChrome(),  // use local Chrome browser
    headless: false,

    evaluatePage: (() => ({
      title: $('title').text(),
    })),

    onSuccess: (result => {
      console.log(result.result.title);
    }),
  });

  crawler.on('maxrequestreached', () => {
    console.log('Maximum request reached!');
  });

  await crawler.queue('http://f4b1.com/');

  await crawler.onIdle();
  await crawler.close();
})().then(() => { console.log('Crawl Terminated'); });

[popstas]

@LeMoussel, try my quick fix - https://github.com/popstas/headless-chrome-crawler/commit/43a7c14240d8045acafb0be34e2b1523d8aca66e