Several updates, readme updated, new selectores fixed

src/enqueue_places_crawler.js

@@ -9,6 +9,7 @@ const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelec
 const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom, maxPlacesPerCrawl) => {
     let results = await page.$$('.section-result');
     const resultsCount = results.length;
+
     for (let resultIndex = 0; resultIndex < resultsCount; resultIndex++) {
         // Need to get results again, pupptr lost context..
         await page.waitForSelector('.searchbox', { timeout: DEFAULT_TIMEOUT });
@@ -21,6 +22,7 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom,
         await link.click();
         await waitForGoogleMapLoader(page);
         await page.waitForSelector('.section-back-to-list-button', { timeout: DEFAULT_TIMEOUT });
+        // After redirection to detail page, save the URL to Request queue to process it later
         const url = page.url();
         await requestQueue.addRequest({ url, userData: { label: 'detail' } });
         log.info(`Added to queue ${url}`);
@@ -28,20 +30,23 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom,
             log.info(`Reach max places per crawl ${maxPlacesPerCrawl}, stopped enqueuing new places.`);
             break;
         }
+
         await page.click('.section-back-to-list-button');
     }
 };
 
 /**
- * Crawler add all place detail from listing to queue
+ * Adds all places from listing to queue
  * @param page
  * @param searchString
- * @param launchPuppeteerOptions
  * @param requestQueue
- * @param listingPagination
  * @param maxPlacesPerCrawl
  */
-const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteerOptions, requestQueue, listingPagination, maxPlacesPerCrawl) => {
+const enqueueAllPlaceDetails = async (page, searchString, requestQueue, maxPlacesPerCrawl) => {
+    // Save state of listing pagination
+    // NOTE: If pageFunction failed crawler skipped already scraped pagination
+    const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
+
     await page.type('#searchboxinput', searchString);
     await sleep(5000);
     await page.click('#searchbox-searchbutton');
@@ -50,8 +55,9 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
     try {
         await page.waitForSelector('h1.section-hero-header-title');
     } catch (e) {
-        // It can happen, doesn't matter :)
+        // It can happen, if there are listing, not just detail page
     }
+
     // In case there is no listing, put just detail page to queue
     const maybeDetailPlace = await page.$('h1.section-hero-header-title');
     if (maybeDetailPlace) {
@@ -59,6 +65,8 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
         await requestQueue.addRequest({ url, userData: { label: 'detail' } });
         return;
     }
+
+    // In case there is listing, go through all details, limits with maxPlacesPerCrawl
     const nextButtonSelector = '[jsaction="pane.paginationSection.nextPage"]';
     while (true) {
         await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
@@ -71,7 +79,8 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
         } else {
             log.debug(`Added links from pagination ${from} - ${to}`);
             await enqueueAllUrlsFromPagination(page, requestQueue, from, maxPlacesPerCrawl);
-            listingPagination = { from, to };
+            listingPagination.from = from;
+            listingPagination.to = to;
             await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
         }
         await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
@@ -87,6 +96,9 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
             await waitForGoogleMapLoader(page);
         }
     }
+
+    listingPagination.isFinish = true;
+    await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
 };
 
-module.exports = { run: enqueueAllPlaceDetailsCrawler };
+module.exports = { enqueueAllPlaceDetails };

src/main.js

@@ -26,7 +26,7 @@ Apify.main(async () => {
     const launchPuppeteerOptions = {};
     if (proxyConfig) Object.assign(launchPuppeteerOptions, proxyConfig);
 
-    // Scrape all place detail links
+    // Create and run crawler
     const crawler = placesCrawler.setUpCrawler(launchPuppeteerOptions, requestQueue, maxCrawledPlaces);
     await crawler.run();
 

src/places_crawler.js

@@ -1,11 +1,143 @@
 const Apify = require('apify');
 
 const { sleep, log } = Apify.utils;
-const infiniteScroll = require('./infinite_scroll');
-
 const { injectJQuery } = Apify.utils.puppeteer;
-const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
+const infiniteScroll = require('./infinite_scroll');
-const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
+const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts');
+const { enqueueAllPlaceDetails } = require('./enqueue_places_crawler');
+
+/**
+ * This is the worst part - parsing data from place detail
+ * @param page
+ */
+const extractPlaceDetail = async (page) => {
+    // Extracts basic information
+    const titleSel = 'h1.section-hero-header-title';
+    await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
+    const detail = await page.evaluate(() => {
+        return {
+            title: $('h1.section-hero-header-title').text().trim(),
+            totalScore: $('span.section-star-display').eq(0).text().trim(),
+            categoryName: $('[jsaction="pane.rating.category"]').text().trim(),
+            address: $('[data-section-id="ad"] .widget-pane-link').text().trim(),
+            plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(),
+        };
+    });
+
+    // Extracty histogram for popular times
+    const histogramSel = '.section-popular-times';
+    if (await page.$(histogramSel)) {
+        detail.popularTimesHistogram = await page.evaluate(() => {
+            const graphs = {};
+            const days = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'];
+            // Extract all days graphs
+            $('.section-popular-times-graph').each(function(i) {
+                const day = days[i];
+                graphs[day] = [];
+                let graphStartFromHour;
+                // Finds where x axis starts
+                $(this).find('.section-popular-times-label').each(function(labelIndex) {
+                    if (graphStartFromHour) return;
+                    const hourText = $(this).text().trim();
+                    graphStartFromHour = hourText.includes('p')
+                        ? 12 + (parseInt(hourText) - labelIndex)
+                        : parseInt(hourText) - labelIndex;
+                });
+                // Finds values from y axis
+                $(this).find('.section-popular-times-bar').each(function (barIndex) {
+                    const occupancyMatch = $(this).attr('aria-label').match(/\d+\s+?%/);
+                    if (occupancyMatch) {
+                        const maybeHour = graphStartFromHour + barIndex;
+                        graphs[day].push({
+                            hour: maybeHour > 24 ? maybeHour - 24 : maybeHour,
+                            occupancyPercent: parseInt(occupancyMatch[0]),
+                        });
+                    }
+                });
+            });
+            return graphs;
+        });
+    }
+
+    // Extracts reviews
+    detail.reviews = [];
+    const reviewsButtonSel = 'button[jsaction="pane.reviewChart.moreReviews"]';
+    if (detail.totalScore) {
+        detail.totalScore = parseFloat(detail.totalScore.replace(',', '.'));
+        detail.reviewsCount = await page.evaluate((selector) => {
+            const numberReviewsText = $(selector).text().trim();
+            return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
+        }, reviewsButtonSel);
+        // If we find consent dialog, close it!
+        if (await page.$('.widget-consent-dialog')) {
+            await page.click('.widget-consent-dialog .widget-consent-button-later');
+        }
+        // Get all reviews
+        await page.waitForSelector(reviewsButtonSel);
+        await page.click(reviewsButtonSel);
+        await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
+        await sleep(5000);
+        // Sort reviews by newest, one click sometimes didn't work :)
+        try {
+            const sortButtonEl = '.section-tab-info-stats-button-flex';
+            for (let i = 0; i < 3; i++) {
+                await page.click(sortButtonEl);
+                await sleep(1000);
+            }
+            await page.click('.context-menu-entry[data-index="1"]');
+        } catch (err) {
+            // It can happen, it is not big issue :)
+            log.debug('Cannot select reviews by newest!');
+        }
+        await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
+        const reviewEls = await page.$$('div.section-review');
+        for (const reviewEl of reviewEls) {
+            const moreButton = await reviewEl.$('.section-expand-review');
+            if (moreButton) {
+                await moreButton.click();
+                await sleep(2000);
+            }
+            const review = await page.evaluate((reviewEl) => {
+                const $review = $(reviewEl);
+                const reviewData = {
+                    name: $review.find('.section-review-title').text().trim(),
+                    text: $review.find('.section-review-review-content .section-review-text').text(),
+                    stars: $review.find('.section-review-stars').attr('aria-label').trim(),
+                    publishAt: $review.find('.section-review-publish-date').text().trim(),
+                    likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
+                };
+                const $response = $review.find('.section-review-owner-response');
+                if ($response) {
+                    reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
+                }
+                return reviewData;
+            }, reviewEl);
+            detail.reviews.push(review);
+        }
+        await page.click('button.section-header-back-button');
+    }
+
+    // Extracts place images
+    await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
+    const imagesButtonSel = '[jsaction="pane.imagepack.button"]';
+    if (await page.$(imagesButtonSel)) {
+        await page.click(imagesButtonSel);
+        await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
+        detail.imageUrls = await page.evaluate(() => {
+            const urls = [];
+            $('.gallery-image-high-res').each(function () {
+                const urlMatch = $(this).attr('style').match(/url\("(.*)"\)/);
+                if (!urlMatch) return;
+                let imageUrl = urlMatch[1];
+                if (imageUrl[0] === '/') imageUrl = `https:${imageUrl}`;
+                urls.push(imageUrl);
+            });
+            return urls;
+        });
+    }
+    return detail;
+};
+
 /**
  * Method to set up crawler to get all place details and save them to default dataset
  * @param launchPuppeteerOptions
@@ -19,13 +151,15 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) =>
         requestQueue,
         maxRequestRetries: MAX_PAGE_RETRIES,
         retireInstanceAfterRequestCount: 10,
-        handlePageTimeoutSecs: 2 * 3600, // Two hours because startUrl crawler
+        handlePageTimeoutSecs: 15 * 60, // 15 min because startUrl enqueueing
-        maxOpenPagesPerInstance: 1, // Because startUrl crawler crashes if we mixed tabs with details scraping
+        maxOpenPagesPerInstance: 1, // Because startUrl enqueueing crashes if we mixed tabs with details scraping
-        // maxConcurrency: 1,
     };
     if (maxCrawledPlaces) {
         crawlerOpts.maxRequestsPerCrawl = maxCrawledPlaces + 1; // The first one is startUrl
     }
+    if (!Apify.isAtHome()) {
+        crawlerOpts.maxConcurrency = 2;
+    }
     return new Apify.PuppeteerCrawler({
         ...crawlerOpts,
         gotoFunction: async ({ request, page }) => {
@@ -34,142 +168,22 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) =>
         },
         handlePageFunction: async ({ request, page }) => {
             const { label, searchString } = request.userData;
+
             log.info(`Open ${request.url} with label: ${label}`);
             await injectJQuery(page);
+
             if (label === 'startUrl') {
-                // enqueue all places
+                log.info(`Start enqueuing places details for search: ${searchString}`);
-                log.info(`Start enqueuing place details for search: ${searchString}`);
+                await enqueueAllPlaceDetails(page, searchString, requestQueue, maxCrawledPlaces);
-                // Store state of listing pagination
+                log.info('Enqueuing places finished!');
-                // NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
-                const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
-                await enqueueAllPlaceDetailsCrawler.run(page, searchString, launchPuppeteerOptions, requestQueue, listingPagination, maxCrawledPlaces);
-                listingPagination.isFinish = true;
-                await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
             } else {
-                // Timeout because timeout for handle page is 2 hours
+                // Get data for place and save it to dataset
-                setTimeout(() => {
+                log.info(`Extracting details from place url ${request.url}`);
-                    throw new Error('HandlePagefunction timed out!');
+                const placeDetail = await extractPlaceDetail(page);
-                }, 600000);
-                // Get data from review
-                const titleSel = 'h1.section-hero-header-title';
-                await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
-                const placeDetail = await page.evaluate(() => {
-                    return {
-                        title: $('h1.section-hero-header-title').text().trim(),
-                        totalScore: $('span.section-star-display').eq(0).text().trim(),
-                        categoryName: $('[jsaction="pane.rating.category"]').text().trim(),
-                        address: $('[data-section-id="ad"] .widget-pane-link').text().trim(),
-                        plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(),
-                    };
-                });
                 placeDetail.url = request.url;
-                const histogramSel = '.section-popular-times';
-                if (await page.$(histogramSel)) {
-                    placeDetail.popularTimesHistogram = await page.evaluate(() => {
-                        const graphs = {};
-                        const days = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'];
-                        // Days graphs
-                        $('.section-popular-times-graph').each(function(i) {
-                            const day = days[i];
-                            graphs[day] = [];
-                            let graphStartFromHour;
-                            $(this).find('.section-popular-times-label').each(function(labelIndex) {
-                                if (graphStartFromHour) return;
-                                const hourText = $(this).text().trim();
-                                graphStartFromHour = hourText.includes('p')
-                                    ? 12 + (parseInt(hourText) - labelIndex)
-                                    : parseInt(hourText) - labelIndex;
-                            });
-                            $(this).find('.section-popular-times-bar').each(function (barIndex) {
-                                const occupancy = $(this).attr('aria-label').match(/\d+\s{1,}%/)[0];
-                                const maybeHour = graphStartFromHour + barIndex;
-                                graphs[day].push({
-                                    hour: maybeHour > 24 ? maybeHour - 24 : maybeHour,
-                                    occupancy,
-                                });
-                            });
-                        });
-                        return graphs;
-                    });
-                }
-                placeDetail.reviews = [];
-                const reviewsButtonSel = 'button[jsaction="pane.reviewChart.moreReviews"]';
-                if (placeDetail.totalScore) {
-                    placeDetail.reviewsCount = await page.evaluate((selector) => {
-                        const numberReviewsText = $(selector).text().trim();
-                        return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
-                    }, reviewsButtonSel);
-                    // If we find consent dialog, close it!
-                    if (await page.$('.widget-consent-dialog')) {
-                        await page.click('.widget-consent-dialog .widget-consent-button-later');
-                    }
-                    // Get all reviews
-                    await page.waitForSelector(reviewsButtonSel);
-                    await page.click(reviewsButtonSel);
-                    await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
-                    await sleep(5000);
-                    // Sort reviews by newest, one click sometimes didn't work :)
-                    try {
-                        const sortButtonEl = '.section-tab-info-stats-button-flex';
-                        await page.click(sortButtonEl);
-                        await sleep(1000);
-                        await page.click(sortButtonEl);
-                        await sleep(1000);
-                        await page.click(sortButtonEl);
-                        await sleep(5000);
-                        await page.click('.context-menu-entry[data-index="1"]');
-                    } catch (err) {
-                        // It can happen, it is not big issue :)
-                        log.debug('Cannot select reviews by newest!');
-                    }
-                    await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
-                    const reviewEls = await page.$$('div.section-review');
-                    for (const reviewEl of reviewEls) {
-                        const moreButton = await reviewEl.$('.section-expand-review');
-                        if (moreButton) {
-                            await moreButton.click();
-                            await sleep(2000);
-                        }
-                        const review = await page.evaluate((reviewEl) => {
-                            const $review = $(reviewEl);
-                            const reviewData = {
-                                name: $review.find('.section-review-title').text().trim(),
-                                text: $review.find('.section-review-review-content .section-review-text').text(),
-                                stars: $review.find('.section-review-stars').attr('aria-label').trim(),
-                                publishAt: $review.find('.section-review-publish-date').text().trim(),
-                                likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
-                            };
-                            const $response = $review.find('.section-review-owner-response');
-                            if ($response) {
-                                reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
-                            }
-                            return reviewData;
-                        }, reviewEl);
-                        placeDetail.reviews.push(review);
-                    }
-                    await page.click('button.section-header-back-button');
-                }
-                await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
-                const imagesButtonSel = '[jsaction="pane.imagepack.button"]';
-                console.log(imagesButtonSel);
-                if (await page.$(imagesButtonSel)) {
-                    await page.click(imagesButtonSel);
-                    await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
-                    placeDetail.imageUrls = await page.evaluate(() => {
-                        const urls = [];
-                        $('.gallery-image-high-res').each(function () {
-                            const urlMatch = $(this).attr('style').match(/url\("(.*)"\)/);
-                            if (!urlMatch) return;
-                            let imageUrl = urlMatch[1];
-                            if (imageUrl[0] === '/') imageUrl = `https:${imageUrl}`;
-                            urls.push(imageUrl);
-                        });
-                        return urls;
-                    });
-                }
                 await Apify.pushData(placeDetail);
+                log.info(`Finished place url ${request.url}`);
             }
-            log.info('Finished', request.url);
         },
         handleFailedRequestFunction: async ({ request }) => {
             // This function is called when crawling of a request failed too many time