From 7dfa99bcec89202a168a1a2b50c43d78bda2190a Mon Sep 17 00:00:00 2001
From: JakubDrobnik <drobnik.j@gmail.com>
Date: Wed, 21 Nov 2018 15:28:39 +0100
Subject: [PATCH] Fixed waiting

---
 src/infinite_scroll.js | 17 +++++++++++++++--
 src/main.js            |  8 ++++----
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/infinite_scroll.js b/src/infinite_scroll.js
index 394a631..277a7d9 100644
--- a/src/infinite_scroll.js
+++ b/src/infinite_scroll.js
@@ -80,6 +80,7 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
         let scrollInfo = await getPageScrollInfo(page, elementToScroll);
         logInfo(`Infinite scroll started (${stringifyScrollInfo(scrollInfo)}).`);
 
+        let previosReviewsCount = 0;
         while (true) {
             scrollInfo = await getPageScrollInfo(page, elementToScroll);
 
@@ -97,13 +98,25 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
             logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`);
 
             const pendingRequestsCount = resourcesStats.requested - (resourcesStats.finished + resourcesStats.failed + resourcesStats.forgotten);
+
             if (pendingRequestsCount === 0) {
                 // If the page is scrolled to the very bottom or beyond maximum height, we are done
-                if (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight)) break;
+                const isLoaderOnPage = await page.evaluate(() => {
+                    const loader = $('.section-loading-spinner');
+                    if (loader) {
+                        return loader.parent().attr('style') !== 'display: none;';
+                    }
+                });
+                const reviewsCount = await page.evaluate(() => $('div.section-review').length);
+                // console.log(reviewsCount, previosReviewsCount, isLoaderOnPage);
+                if (reviewsCount === previosReviewsCount
+                    && (scrollInfo.scrollTop + scrollInfo.clientHeight >= Math.min(scrollInfo.scrollHeight, maxHeight))
+                    && !isLoaderOnPage
+                ) break;
+                previosReviewsCount = reviewsCount;
                 // Otherwise we try to scroll down
                 await scrollTo(page, elementToScroll, maxHeight);
             }
-
             await sleep(defaultScrollDelay);
         }
         // Scroll back up, otherwise the screenshot of the browser would only show the bottom of
diff --git a/src/main.js b/src/main.js
index 6a7df5d..fa994c7 100644
--- a/src/main.js
+++ b/src/main.js
@@ -142,7 +142,7 @@ Apify.main(async () => {
         handlePageTimeoutSecs: 600,
         gotoFunction: async ({ request, page }) => {
             await page._client.send('Emulation.clearDeviceMetricsOverride');
-            await page.goto(request.url, { timeout: 60000 })
+            await page.goto(request.url, { timeout: 60000 });
         },
         handlePageFunction: async ({ request, page }) => {
             const { label } = request.userData;
@@ -169,14 +169,14 @@ Apify.main(async () => {
                 // Get all reviews
                 await page.click('button.section-reviewchart-numreviews');
                 await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
-                await infiniteScroll(page, 99999999999, '.section-scrollbox');
-                sleep(2000);
+                await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
+                await sleep(2000);
                 const reviewEls = await page.$$('div.section-review');
                 for (const reviewEl of reviewEls) {
                     const moreButton = await reviewEl.$('.section-expand-review');
                     if (moreButton) {
                         await moreButton.click();
-                        sleep(1000);
+                        await sleep(1000);
                     }
                     const review = await page.evaluate((reviewEl) => {
                         const $review = $(reviewEl);