diff --git a/src/infinite_scroll.js b/src/infinite_scroll.js index 2213cd6..9488703 100644 --- a/src/infinite_scroll.js +++ b/src/infinite_scroll.js @@ -43,7 +43,7 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => { `scrollHeight=${scrollInfo.scrollHeight}, ` + `maxHeight=${maxHeight}`; }; - const defaultScrollDelay = 2000; + const defaultScrollDelay = 3000; // Catch and count all pages request for resources const resourcesStats = { diff --git a/src/main.js b/src/main.js index e3d4968..0f5573a 100644 --- a/src/main.js +++ b/src/main.js @@ -69,6 +69,7 @@ Apify.main(async () => { const browser = await Apify.launchPuppeteer(launchPuppeteerOptions); const page = await browser.newPage(); await page.goto(startUrl); + await injectJQuery(page); await page.type('#searchboxinput', searchString); await sleep(5000); await page.click('#searchbox-searchbutton'); @@ -79,21 +80,24 @@ Apify.main(async () => { const [fromString, toString] = paginationText.match(/\d+/g); const from = parseInt(fromString); const to = parseInt(toString); - if (listingPagination.to && to <= listingPagination.to) { + if (listingPagination.from && from <= listingPagination.from) { console.log(`Skiped pagination ${from} - ${to}, already done!`); } else { console.log(`Added links from pagination ${from} - ${to}`); await enqueueAllUrlsFromPagination(page, requestQueue); + listingPagination = { from, to }; + await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination); } - listingPagination = { from, to }; - await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination); await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT }); const nextButton = await page.$('#section-pagination-button-next'); - const isNextPaginationDisabled = (await nextButton.getProperty('disabled') === 'true'); + const isNextPaginationDisabled = await page.evaluate(() => { + return !!$('#section-pagination-button-next').attr('disabled'); + }); if (isNextPaginationDisabled) { break; } else { await nextButton.click(); + await page.waitFor(() => !document.querySelector('#searchbox').classList.contains('loading')); } } listingPagination.isFinish = true;