JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Addad waitting for element

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-20 16:47:32 +01:00
parent d4bd1748a1
commit ec3dff3800
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main.js
View File

@ -16,7 +16,8 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
const resultsCount = results.length; const resultsCount = results.length;
for (let resultIndex = 0; resultIndex < resultsCount; resultIndex++) { for (let resultIndex = 0; resultIndex < resultsCount; resultIndex++) {
// Need to get results again, pupptr lost context.. // Need to get results again, pupptr lost context..
await page.waitForSelector('.section-result'); await page.waitForSelector('.searchbox');
await page.waitFor(() => !document.querySelector('#searchbox').classList.contains('loading'));
results = await page.$$('.section-result'); results = await page.$$('.section-result');
const link = await results[resultIndex].$('h3'); const link = await results[resultIndex].$('h3');
await link.click(); await link.click();
@ -25,7 +26,6 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
await requestQueue.addRequest({ url, userData: { label: 'detail' } }); await requestQueue.addRequest({ url, userData: { label: 'detail' } });
console.log(`Added to queue ${url}`); console.log(`Added to queue ${url}`);
await page.click('.section-back-to-list-button'); await page.click('.section-back-to-list-button');
await sleep(5000);
} }
return detailLinks; return detailLinks;
}; };
@ -87,6 +87,7 @@ Apify.main(async () => {
} }
listingPagination = { from, to }; listingPagination = { from, to };
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination); await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT });
const nextButton = await page.$('#section-pagination-button-next'); const nextButton = await page.$('#section-pagination-button-next');
const isNextPaginationDisabled = (await nextButton.getProperty('disabled') === 'true'); const isNextPaginationDisabled = (await nextButton.getProperty('disabled') === 'true');
if (isNextPaginationDisabled) { if (isNextPaginationDisabled) {
@ -94,7 +95,6 @@ Apify.main(async () => {
} else { } else {
await nextButton.click(); await nextButton.click();
} }
await sleep(5000);
} }
listingPagination.isFinish = true; listingPagination.isFinish = true;
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination); await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);