JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed waiting

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-21 15:28:39 +01:00
parent af631cb93d
commit 7dfa99bcec
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/infinite_scroll.js
View File

@ -80,6 +80,7 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
let scrollInfo = await getPageScrollInfo(page, elementToScroll); let scrollInfo = await getPageScrollInfo(page, elementToScroll);
logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`); logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`);
let previosReviewsCount = 0;
while (true) { while (true) {
scrollInfo = await getPageScrollInfo(page, elementToScroll); scrollInfo = await getPageScrollInfo(page, elementToScroll);
@ -97,13 +98,25 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`); logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`);
const pendingRequestsCount = resourcesStats.requested - (resourcesStats.finished + resourcesStats.failed + resourcesStats.forgotten); const pendingRequestsCount = resourcesStats.requested - (resourcesStats.finished + resourcesStats.failed + resourcesStats.forgotten);
if (pendingRequestsCount === 0) { if (pendingRequestsCount === 0) {
// If the page is scrolled to the very bottom or beyond maximum height, we are done // If the page is scrolled to the very bottom or beyond maximum height, we are done
if (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight)) break; const isLoaderOnPage = await page.evaluate(() => {
const loader = $('.section-loading-spinner');
if (loader) {
return loader.parent().attr('style') !== 'display: none;';
}
});
const reviewsCount = await page.evaluate(() => $('div.section-review').length);
// console.log(reviewsCount, previosReviewsCount, isLoaderOnPage);
if (reviewsCount === previosReviewsCount
&& (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight))
&& !isLoaderOnPage
) break;
previosReviewsCount = reviewsCount;
// Otherwise we try to scroll down // Otherwise we try to scroll down
await scrollTo(page, elementToScroll, maxHeight); await scrollTo(page, elementToScroll, maxHeight);
} }
await sleep(defaultScrollDelay); await sleep(defaultScrollDelay);
} }
// Scroll back up, otherwise the screenshot of the browser would only show the bottom of // Scroll back up, otherwise the screenshot of the browser would only show the bottom of

8
src/main.js
View File

@ -142,7 +142,7 @@ Apify.main(async () => {
handlePageTimeoutSecs: 600, handlePageTimeoutSecs: 600,
gotoFunction: async ({ request, page }) => { gotoFunction: async ({ request, page }) => {
await page._client.send('Emulation.clearDeviceMetricsOverride'); await page._client.send('Emulation.clearDeviceMetricsOverride');
await page.goto(request.url, { timeout: 60000 }) await page.goto(request.url, { timeout: 60000 });
}, },
handlePageFunction: async ({ request, page }) => { handlePageFunction: async ({ request, page }) => {
const { label } = request.userData; const { label } = request.userData;
@ -169,14 +169,14 @@ Apify.main(async () => {
// Get all reviews // Get all reviews
await page.click('button.section-reviewchart-numreviews'); await page.click('button.section-reviewchart-numreviews');
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await infiniteScroll(page, 99999999999, '.section-scrollbox'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
sleep(2000); await sleep(2000);
const reviewEls = await page.$$('div.section-review'); const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) { for (const reviewEl of reviewEls) {
const moreButton = await reviewEl.$('.section-expand-review'); const moreButton = await reviewEl.$('.section-expand-review');
if (moreButton) { if (moreButton) {
await moreButton.click(); await moreButton.click();
sleep(1000); await sleep(1000);
} }
const review = await page.evaluate((reviewEl) => { const review = await page.evaluate((reviewEl) => {
const $review = $(reviewEl); const $review = $(reviewEl);