JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

🎉

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-22 11:07:56 +01:00
parent c1203af221
commit 2c931b5b16
2 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/enqueue_places_crawler.js
View File

@ -50,6 +50,11 @@ const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppe
await sleep(5000); await sleep(5000);
await waitForGoogleMapLoader(page); await waitForGoogleMapLoader(page);
// In case there is no listing, put just detail page to queue // In case there is no listing, put just detail page to queue
try {
await page.waitForSelector('h1.section-hero-header-title');
} catch (e) {
// It can happen, doesn't matter
}
const maybeDetailPlace = await page.$('h1.section-hero-header-title'); const maybeDetailPlace = await page.$('h1.section-hero-header-title');
if (maybeDetailPlace) { if (maybeDetailPlace) {
const url = page.url(); const url = page.url();

3
src/places_crawler.js
View File

@ -49,7 +49,8 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(2000); await sleep(2000);
// Sort reviews by newest // Sort reviews by newest
await page.click('.section-tab-info-stats-button-flex'); await page.waitForSelector('.section-tab-info-stats-button-flex');
await page.click('.section-tab-info-stats-button-flex .maps-sprite-reviews-expand-more');
await page.waitForSelector('.context-menu-entry[data-index="1"]'); await page.waitForSelector('.context-menu-entry[data-index="1"]');
await page.click('.context-menu-entry[data-index="1"]'); await page.click('.context-menu-entry[data-index="1"]');
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');