JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Several stability improvements

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-20 17:35:01 +01:00
parent ec3dff3800
commit 3e823db515
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/infinite_scroll.js
View File

@ -43,7 +43,7 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
`scrollHeight=${scrollInfo.scrollHeight}, ` + `scrollHeight=${scrollInfo.scrollHeight}, ` +
`maxHeight=${maxHeight}`; `maxHeight=${maxHeight}`;
}; };
const defaultScrollDelay = 2000; const defaultScrollDelay = 3000;
// Catch and count all pages request for resources // Catch and count all pages request for resources
const resourcesStats = { const resourcesStats = {

12
src/main.js
View File

@ -69,6 +69,7 @@ Apify.main(async () => {
const browser = await Apify.launchPuppeteer(launchPuppeteerOptions); const browser = await Apify.launchPuppeteer(launchPuppeteerOptions);
const page = await browser.newPage(); const page = await browser.newPage();
await page.goto(startUrl); await page.goto(startUrl);
await injectJQuery(page);
await page.type('#searchboxinput', searchString); await page.type('#searchboxinput', searchString);
await sleep(5000); await sleep(5000);
await page.click('#searchbox-searchbutton'); await page.click('#searchbox-searchbutton');
@ -79,21 +80,24 @@ Apify.main(async () => {
const [fromString, toString] = paginationText.match(/\d+/g); const [fromString, toString] = paginationText.match(/\d+/g);
const from = parseInt(fromString); const from = parseInt(fromString);
const to = parseInt(toString); const to = parseInt(toString);
if (listingPagination.to && to <= listingPagination.to) { if (listingPagination.from && from <= listingPagination.from) {
console.log(`Skiped pagination ${from} - ${to}, already done!`); console.log(`Skiped pagination ${from} - ${to}, already done!`);
} else { } else {
console.log(`Added links from pagination ${from} - ${to}`); console.log(`Added links from pagination ${from} - ${to}`);
await enqueueAllUrlsFromPagination(page, requestQueue); await enqueueAllUrlsFromPagination(page, requestQueue);
listingPagination = { from, to };
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
} }
listingPagination = { from, to };
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT });
const nextButton = await page.$('#section-pagination-button-next'); const nextButton = await page.$('#section-pagination-button-next');
const isNextPaginationDisabled = (await nextButton.getProperty('disabled') === 'true'); const isNextPaginationDisabled = await page.evaluate(() => {
return !!$('#section-pagination-button-next').attr('disabled');
});
if (isNextPaginationDisabled) { if (isNextPaginationDisabled) {
break; break;
} else { } else {
await nextButton.click(); await nextButton.click();
await page.waitFor(() => !document.querySelector('#searchbox').classList.contains('loading'));
} }
} }
listingPagination.isFinish = true; listingPagination.isFinish = true;