JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Omit owner responses from review text

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-23 12:23:59 +01:00
parent f88b0b0e3c
commit 5d824b5b28
1 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/places_crawler.js
View File

@ -49,33 +49,34 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
// Get all reviews // Get all reviews
await page.click('button.section-reviewchart-numreviews'); await page.click('button.section-reviewchart-numreviews');
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(2000); await sleep(5000);
// Sort reviews by newest // Sort reviews by newest
try { await page.click('.section-tab-info-stats-button-flex');
await page.waitForSelector('.section-tab-info-stats-button-flex', { timeout: DEFAULT_TIMEOUT }); await sleep(5000);
await page.click('.section-tab-info-stats-button-flex'); await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT });
await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT }); await page.click('.context-menu-entry[data-index="1"]');
await page.click('.context-menu-entry[data-index="1"]');
} catch (err) {
console.log(`For ${request.url} we can not change sorting of reviews.`);
}
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
const reviewEls = await page.$$('div.section-review'); const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) { for (const reviewEl of reviewEls) {
const moreButton = await reviewEl.$('.section-expand-review'); const moreButton = await reviewEl.$('.section-expand-review');
if (moreButton) { if (moreButton) {
await moreButton.click(); await moreButton.click();
await sleep(1000); await sleep(2000);
} }
const review = await page.evaluate((reviewEl) => { const review = await page.evaluate((reviewEl) => {
const $review = $(reviewEl); const $review = $(reviewEl);
return { const reviewData = {
name: $review.find('.section-review-title').text().trim(), name: $review.find('.section-review-title').text().trim(),
text: $review.find('.section-review-text').text(), text: $review.find('.section-review-review-content .section-review-text').text(),
stars: $review.find('.section-review-stars').attr('aria-label').trim(), stars: $review.find('.section-review-stars').attr('aria-label').trim(),
publishAt: $review.find('.section-review-publish-date').text().trim(), publishAt: $review.find('.section-review-publish-date').text().trim(),
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(), likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
}; };
const $response = $review.find('.section-review-owner-response');
if ($response) {
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
}
return reviewData;
}, reviewEl); }, reviewEl);
placeDetail.reviews.push(review); placeDetail.reviews.push(review);
} }