JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added longer waiting timeout + live view

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-19 18:17:17 +01:00
parent 6666f0c8a8
commit 9cc3e856f5
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.js
View File

@ -6,6 +6,8 @@ const Apify = require('apify');
const { sleep } = Apify.utils; const { sleep } = Apify.utils;
const { injectJQuery } = Apify.utils.puppeteer; const { injectJQuery } = Apify.utils.puppeteer;
const DEFAULT_TIMEOUT = 60 * 1000; // 60 sec
const sleepPromised = ms => new Promise(resolve => setTimeout(resolve, ms)); const sleepPromised = ms => new Promise(resolve => setTimeout(resolve, ms));
const logError = (msg, e) => { const logError = (msg, e) => {
@ -151,6 +153,7 @@ Apify.main(async () => {
launchPuppeteerOptions: { launchPuppeteerOptions: {
useApifyProxy: true, useApifyProxy: true,
apifyProxyGroups: ['CZECH_LUMINATI'], apifyProxyGroups: ['CZECH_LUMINATI'],
liveView: true,
}, },
requestQueue, requestQueue,
handlePageTimeoutSecs: 1200, handlePageTimeoutSecs: 1200,
@ -163,7 +166,7 @@ Apify.main(async () => {
await page.click('#searchbox-searchbutton'); await page.click('#searchbox-searchbutton');
await sleep(5000); await sleep(5000);
while(true) { while(true) {
await page.waitForSelector('#section-pagination-button-next'); await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT });
await enqueueAllUrlsFromPagination(page, requestQueue); await enqueueAllUrlsFromPagination(page, requestQueue);
const nextButton = await page.$('#section-pagination-button-next'); const nextButton = await page.$('#section-pagination-button-next');
const isNextPagination = (await nextButton.getProperty('disabled') === 'true'); const isNextPagination = (await nextButton.getProperty('disabled') === 'true');
@ -178,7 +181,7 @@ Apify.main(async () => {
} else { } else {
// Get data from review // Get data from review
await injectJQuery(page); await injectJQuery(page);
await page.waitForSelector('h1.section-hero-header-title'); await page.waitForSelector('h1.section-hero-header-title', { timeout: DEFAULT_TIMEOUT });
const placeDetail = await page.evaluate(() => { const placeDetail = await page.evaluate(() => {
return { return {
title: $('h1.section-hero-header-title').text().trim(), title: $('h1.section-hero-header-title').text().trim(),