JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed waiting

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-22 11:20:04 +01:00
parent 5d41a1b998
commit f88b0b0e3c
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/places_crawler.js
View File

@ -1,6 +1,8 @@
const Apify = require('apify'); const Apify = require('apify');
const { sleep } = Apify.utils; const { sleep } = Apify.utils;
const infiniteScroll = require('./infinite_scroll'); const infiniteScroll = require('./infinite_scroll');
const { injectJQuery } = Apify.utils.puppeteer; const { injectJQuery } = Apify.utils.puppeteer;
const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts'); const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts');
@ -17,7 +19,7 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
maxRequestRetries: MAX_PAGE_RETRIES, maxRequestRetries: MAX_PAGE_RETRIES,
retireInstanceAfterRequestCount: 10, retireInstanceAfterRequestCount: 10,
handlePageTimeoutSecs: 600, handlePageTimeoutSecs: 600,
maxConcurrency: 1, // maxConcurrency: 1,
gotoFunction: async ({ request, page }) => { gotoFunction: async ({ request, page }) => {
await page._client.send('Emulation.clearDeviceMetricsOverride'); await page._client.send('Emulation.clearDeviceMetricsOverride');
await page.goto(request.url, { timeout: 60000 }); await page.goto(request.url, { timeout: 60000 });
@ -49,10 +51,14 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(2000); await sleep(2000);
// Sort reviews by newest // Sort reviews by newest
await page.waitForSelector('.section-tab-info-stats-button-flex', { timeout: DEFAULT_TIMEOUT }); try {
await page.click('.section-tab-info-stats-button-flex .maps-sprite-reviews-expand-more'); await page.waitForSelector('.section-tab-info-stats-button-flex', { timeout: DEFAULT_TIMEOUT });
await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT }); await page.click('.section-tab-info-stats-button-flex');
await page.click('.context-menu-entry[data-index="1"]'); await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT });
await page.click('.context-menu-entry[data-index="1"]');
} catch (err) {
console.log(`For ${request.url} we can not change sorting of reviews.`);
}
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
const reviewEls = await page.$$('div.section-review'); const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) { for (const reviewEl of reviewEls) {