From bfd01dbb7fbdaefaf3e862b3052eab8952626452 Mon Sep 17 00:00:00 2001 From: JakubDrobnik Date: Fri, 14 Dec 2018 09:12:12 +0100 Subject: [PATCH] Fixed: close consent dialog if we found it --- src/places_crawler.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/places_crawler.js b/src/places_crawler.js index d4c18af..a437f4e 100644 --- a/src/places_crawler.js +++ b/src/places_crawler.js @@ -68,18 +68,28 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) => const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim(); return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null; }); + // If we find consent dialog, close it! + if (await page.$('.widget-consent-dialog')) { + await page.click('.widget-consent-dialog .widget-consent-button-later'); + } // Get all reviews + await page.waitForSelector('button.section-reviewchart-numreviews') await page.click('button.section-reviewchart-numreviews'); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await sleep(5000); // Sort reviews by newest, one click sometimes didn't work :) - await page.click('.section-tab-info-stats-button-flex'); - await sleep(1000); - await page.click('.section-tab-info-stats-button-flex'); - await sleep(1000); - await page.click('.section-tab-info-stats-button-flex'); - await sleep(5000); - await page.click('.context-menu-entry[data-index="1"]'); + try { + await page.click('.section-tab-info-stats-button-flex'); + await sleep(1000); + await page.click('.section-tab-info-stats-button-flex'); + await sleep(1000); + await page.click('.section-tab-info-stats-button-flex'); + await sleep(5000); + await page.click('.context-menu-entry[data-index="1"]'); + } catch (err) { + // It can happen, it is not big issue + console.log('Cannot select reviews by newest!'); + } await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); const reviewEls = await page.$$('div.section-review'); for (const reviewEl of reviewEls) {