From 2c931b5b16addacd4c791455acf692284017b5e1 Mon Sep 17 00:00:00 2001 From: JakubDrobnik Date: Thu, 22 Nov 2018 11:07:56 +0100 Subject: [PATCH] :tada: --- src/enqueue_places_crawler.js | 5 +++++ src/places_crawler.js | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/enqueue_places_crawler.js b/src/enqueue_places_crawler.js index 8ca87e1..b477747 100644 --- a/src/enqueue_places_crawler.js +++ b/src/enqueue_places_crawler.js @@ -50,6 +50,11 @@ const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppe await sleep(5000); await waitForGoogleMapLoader(page); // In case there is no listing, put just detail page to queue + try { + await page.waitForSelector('h1.section-hero-header-title'); + } catch (e) { + // It can happen, doesn't matter + } const maybeDetailPlace = await page.$('h1.section-hero-header-title'); if (maybeDetailPlace) { const url = page.url(); diff --git a/src/places_crawler.js b/src/places_crawler.js index dadc3ca..56c871a 100644 --- a/src/places_crawler.js +++ b/src/places_crawler.js @@ -49,7 +49,8 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => { await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await sleep(2000); // Sort reviews by newest - await page.click('.section-tab-info-stats-button-flex'); + await page.waitForSelector('.section-tab-info-stats-button-flex'); + await page.click('.section-tab-info-stats-button-flex .maps-sprite-reviews-expand-more'); await page.waitForSelector('.context-menu-entry[data-index="1"]'); await page.click('.context-menu-entry[data-index="1"]'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');