JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed infinite scroll

Browse Source
This commit is contained in:
drobnikj 2019-01-08 13:01:31 +01:00
parent 84b4040a0b
commit 3b87790d07
1 changed files with 38 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/infinite_scroll.js
View File

@ -35,10 +35,10 @@ const getPageScrollInfo = (page, elementToScroll) => page.evaluate((elementToScr
module.exports = async (page, maxHeight, elementToScroll = 'body') => { module.exports = async (page, maxHeight, elementToScroll = 'body') => {
const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other']; const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
const stringifyScrollInfo = (scrollInfo) => { const stringifyScrollInfo = (scrollInfo) => {
return `scrollTop=${scrollInfo.scrollTop}, ` + return `scrollTop=${scrollInfo.scrollTop}, `
`clientHeight=${scrollInfo.clientHeight}, ` + + `clientHeight=${scrollInfo.clientHeight}, `
`scrollHeight=${scrollInfo.scrollHeight}, ` + + `scrollHeight=${scrollInfo.scrollHeight}, `
`maxHeight=${maxHeight}`; + `maxHeight=${maxHeight}`;
}; };
const defaultScrollDelay = 3000; const defaultScrollDelay = 3000;
const defaultElementTimeout = 60000; const defaultElementTimeout = 60000;
@ -72,7 +72,6 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
} }
}); });
try {
await page.waitForSelector(elementToScroll, { timeout: defaultElementTimeout }); await page.waitForSelector(elementToScroll, { timeout: defaultElementTimeout });
let scrollInfo = await getPageScrollInfo(page, elementToScroll); let scrollInfo = await getPageScrollInfo(page, elementToScroll);
logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`); logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`);
@ -121,8 +120,4 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
} }
page.removeAllListeners('request'); page.removeAllListeners('request');
logInfo(`Infinite scroll finished (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)})`); logInfo(`Infinite scroll finished (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)})`);
} catch (err) {
// Infinite scroll should not break whole crawler
logError('An exception thrown in infiniteScroll()', err);
}
}; };