Moved crawler for enqueue places to main crawler

apify.json

@@ -1,11 +1,6 @@
 {
 	"name": "crawler-google-places",
-	"actId": null,
+	"version": "0.1",
-	"version": {
-		"versionNumber": "0.1",
 	"buildTag": "latest",
-		"envVars": [],
+	"env": null
-		"sourceType": "TARBALL",
-		"tarballUrl": null
-	}
 }

src/enqueue_places_crawler.js

@@ -1,9 +1,11 @@
 const Apify = require('apify');
-const { sleep } = Apify.utils;
-const { injectJQuery } = Apify.utils.puppeteer;
-const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
 
-const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelector('#searchbox').classList.contains('loading'), { timeout: DEFAULT_TIMEOUT });
+const { sleep } = Apify.utils;
+const { DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
+
+const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelector('#searchbox')
+    .classList
+    .contains('loading'), { timeout: DEFAULT_TIMEOUT });
 
 const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
     let results = await page.$$('.section-result');
@@ -36,14 +38,7 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
  * @param listingPagination
  * @param retries
  */
-const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination, retries = 0) => {
+const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteerOptions, requestQueue, listingPagination) => {
-    let browser;
-    try {
-        browser = await Apify.launchPuppeteer(launchPuppeteerOptions);
-        const page = await browser.newPage();
-        await page._client.send('Emulation.clearDeviceMetricsOverride');
-        await page.goto(startUrl);
-        await injectJQuery(page);
     await page.type('#searchboxinput', searchString);
     await sleep(5000);
     await page.click('#searchbox-searchbutton');
@@ -61,10 +56,10 @@ const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppe
         await requestQueue.addRequest({ url, userData: { label: 'detail' } });
         return;
     }
-        const nextButtonSelector = '#section-pagination-button-next';
+    const nextButtonSelector = '[jsaction="pane.paginationSection.nextPage"]';
     while (true) {
         await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
-            const paginationText = await page.$eval('.section-pagination-right', (el) => el.innerText);
+        const paginationText = await page.$eval('.n7lv7yjyC35__right', (el) => el.innerText);
         const [fromString, toString] = paginationText.match(/\d+/g);
         const from = parseInt(fromString);
         const to = parseInt(toString);
@@ -78,29 +73,19 @@ const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppe
         }
         await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
         const isNextPaginationDisabled = await page.evaluate((nextButtonSelector) => {
-                return !!$(nextButtonSelector).attr('disabled');
+            return !!$(nextButtonSelector)
+                .attr('disabled');
         }, nextButtonSelector);
         const noResultsEl = await page.$('.section-no-result-title');
         if (isNextPaginationDisabled || noResultsEl) {
             break;
         } else {
             // NOTE: puppeteer API click() didn't work :(
-                await page.evaluate((sel) => $(sel).click(), nextButtonSelector);
+            await page.evaluate((sel) => $(sel)
+                .click(), nextButtonSelector);
             await waitForGoogleMapLoader(page);
         }
     }
-    } catch (err) {
-        if (retries < MAX_PAGE_RETRIES) {
-            ++retries;
-            console.log(`Retiring enqueueAllPlaceDetails for ${retries} time, error:`);
-            console.error(err);
-            await browser.close();
-            await enqueueAllPlaceDetailsCrawler(startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination, ++retries);
-        }
-        throw err;
-    } finally {
-        if (browser) await browser.close();
-    }
 };
 
 module.exports = { run: enqueueAllPlaceDetailsCrawler };

src/main.js

@@ -1,7 +1,5 @@
 const Apify = require('apify');
 const placesCrawler = require('./places_crawler');
-const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
-const { LISTING_PAGINATION_KEY } = require('./consts');
 
 Apify.main(async () => {
     const input = await Apify.getValue('INPUT');
@@ -22,21 +20,11 @@ Apify.main(async () => {
 
     console.log('Start url is', startUrl);
     const requestQueue = await Apify.openRequestQueue();
+    await requestQueue.addRequest({ url: startUrl, userData: { label: 'startUrl', searchString } });
 
-    // Store state of listing pagination
-    // NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
-    const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
     const launchPuppeteerOptions = {};
     if (proxyConfig) Object.assign(launchPuppeteerOptions, proxyConfig);
 
-    // Enqueue all links to scrape from listings
-    if (!listingPagination.isFinish) {
-        console.log(`Start enqueuing place details for search: ${searchString}`);
-        await enqueueAllPlaceDetailsCrawler.run(startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination);
-        listingPagination.isFinish = true;
-        await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
-    }
-
     // Scrape all place detail links
     const crawler = placesCrawler.setUpCrawler(launchPuppeteerOptions, requestQueue);
     await crawler.run();

src/places_crawler.js

@@ -4,7 +4,8 @@ const { sleep } = Apify.utils;
 const infiniteScroll = require('./infinite_scroll');
 
 const { injectJQuery } = Apify.utils.puppeteer;
-const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts');
+const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
+const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
 
 /**
  * Method to set up crawler to get all place details and save them to default dataset
@@ -18,17 +19,32 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
         requestQueue,
         maxRequestRetries: MAX_PAGE_RETRIES,
         retireInstanceAfterRequestCount: 10,
-        handlePageTimeoutSecs: 600,
+        handlePageTimeoutSecs: 2 * 3600, // Two hours because startUrl crawler
+        maxOpenPagesPerInstance: 1, // Because startUrl crawler crashes if we mixed it with details scraping
         // maxConcurrency: 1,
         gotoFunction: async ({ request, page }) => {
             await page._client.send('Emulation.clearDeviceMetricsOverride');
             await page.goto(request.url, { timeout: 60000 });
         },
         handlePageFunction: async ({ request, page }) => {
-            const { label } = request.userData;
+            const { label, searchString } = request.userData;
             console.log(`Open ${request.url} with label: ${label}`);
-            // Get data from review
             await injectJQuery(page);
+            if (label === 'startUrl') {
+                // enqueue all places
+                console.log(`Start enqueuing place details for search: ${searchString}`);
+                // Store state of listing pagination
+                // NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
+                const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
+                await enqueueAllPlaceDetailsCrawler.run(page, searchString, launchPuppeteerOptions, requestQueue, listingPagination);
+                listingPagination.isFinish = true;
+                await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
+            } else {
+                // Timeout because timeout for handle page is 2 hours
+                setTimeout(() => {
+                    throw new Error('HandlePagefunction timed out!');
+                }, 600000);
+                // Get data from review
                 await page.waitForSelector('h1.section-hero-header-title', { timeout: DEFAULT_TIMEOUT });
                 const placeDetail = await page.evaluate(() => {
                     return {
@@ -85,6 +101,7 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
                     }
                 }
                 await Apify.pushData(placeDetail);
+            }
 
             console.log(request.url, 'Done');
         },