JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fixed: close consent dialog if we found it

Browse Source
This commit is contained in:
JakubDrobnik 2018-12-14 09:12:12 +01:00
parent 457b500987
commit bfd01dbb7f
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/places_crawler.js
View File

@ -68,18 +68,28 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) =>
const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim(); const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim();
return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null; return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
}); });
// If we find consent dialog, close it!
if (await page.$('.widget-consent-dialog')) {
await page.click('.widget-consent-dialog .widget-consent-button-later');
}
// Get all reviews // Get all reviews
await page.waitForSelector('button.section-reviewchart-numreviews')
await page.click('button.section-reviewchart-numreviews'); await page.click('button.section-reviewchart-numreviews');
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(5000); await sleep(5000);
// Sort reviews by newest, one click sometimes didn't work :) // Sort reviews by newest, one click sometimes didn't work :)
await page.click('.section-tab-info-stats-button-flex'); try {
await sleep(1000); await page.click('.section-tab-info-stats-button-flex');
await page.click('.section-tab-info-stats-button-flex'); await sleep(1000);
await sleep(1000); await page.click('.section-tab-info-stats-button-flex');
await page.click('.section-tab-info-stats-button-flex'); await sleep(1000);
await sleep(5000); await page.click('.section-tab-info-stats-button-flex');
await page.click('.context-menu-entry[data-index="1"]'); await sleep(5000);
await page.click('.context-menu-entry[data-index="1"]');
} catch (err) {
// It can happen, it is not big issue
console.log('Cannot select reviews by newest!');
}
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
const reviewEls = await page.$$('div.section-review'); const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) { for (const reviewEl of reviewEls) {