JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed waiting

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-21 15:28:39 +01:00
parent af631cb93d
commit 7dfa99bcec
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/infinite_scroll.js
View File

@ -80,6 +80,7 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
let scrollInfo = await getPageScrollInfo(page, elementToScroll);
logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`);
let previosReviewsCount = 0;
while (true) {
scrollInfo = await getPageScrollInfo(page, elementToScroll);
@ -97,13 +98,25 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`);
const pendingRequestsCount = resourcesStats.requested - (resourcesStats.finished + resourcesStats.failed + resourcesStats.forgotten);
if (pendingRequestsCount === 0) {
// If the page is scrolled to the very bottom or beyond maximum height, we are done
if (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight)) break;
const isLoaderOnPage = await page.evaluate(() => {
const loader = $('.section-loading-spinner');
if (loader) {
return loader.parent().attr('style') !== 'display: none;';
}
});
const reviewsCount = await page.evaluate(() => $('div.section-review').length);
// console.log(reviewsCount, previosReviewsCount, isLoaderOnPage);
if (reviewsCount === previosReviewsCount
&& (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight))
&& !isLoaderOnPage
) break;
previosReviewsCount = reviewsCount;
// Otherwise we try to scroll down
await scrollTo(page, elementToScroll, maxHeight);
}
await sleep(defaultScrollDelay);
}
// Scroll back up, otherwise the screenshot of the browser would only show the bottom of

8
src/main.js
View File

@ -142,7 +142,7 @@ Apify.main(async () => {
handlePageTimeoutSecs: 600,
gotoFunction: async ({ request, page }) => {
await page._client.send('Emulation.clearDeviceMetricsOverride');
await page.goto(request.url, { timeout: 60000 })
await page.goto(request.url, { timeout: 60000 });
},
handlePageFunction: async ({ request, page }) => {
const { label } = request.userData;
@ -169,14 +169,14 @@ Apify.main(async () => {
// Get all reviews
await page.click('button.section-reviewchart-numreviews');
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await infiniteScroll(page, 99999999999, '.section-scrollbox');
sleep(2000);
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
await sleep(2000);
const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) {
const moreButton = await reviewEl.$('.section-expand-review');
if (moreButton) {
await moreButton.click();
sleep(1000);
await sleep(1000);
}
const review = await page.evaluate((reviewEl) => {
const $review = $(reviewEl);