JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Omit owner responses from review text

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-23 12:23:59 +01:00
parent f88b0b0e3c
commit 5d824b5b28
1 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/places_crawler.js
View File

@ -49,33 +49,34 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
// Get all reviews
await page.click('button.section-reviewchart-numreviews');
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(2000);
await sleep(5000);
// Sort reviews by newest
try {
await page.waitForSelector('.section-tab-info-stats-button-flex', { timeout: DEFAULT_TIMEOUT });
await page.click('.section-tab-info-stats-button-flex');
await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT });
await page.click('.context-menu-entry[data-index="1"]');
} catch (err) {
console.log(`For ${request.url} we can not change sorting of reviews.`);
}
await page.click('.section-tab-info-stats-button-flex');
await sleep(5000);
await page.waitForSelector('.context-menu-entry[data-index="1"]', { timeout: DEFAULT_TIMEOUT });
await page.click('.context-menu-entry[data-index="1"]');
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
const reviewEls = await page.$$('div.section-review');
for (const reviewEl of reviewEls) {
const moreButton = await reviewEl.$('.section-expand-review');
if (moreButton) {
await moreButton.click();
await sleep(1000);
await sleep(2000);
}
const review = await page.evaluate((reviewEl) => {
const $review = $(reviewEl);
return {
const reviewData = {
name: $review.find('.section-review-title').text().trim(),
text: $review.find('.section-review-text').text(),
text: $review.find('.section-review-review-content .section-review-text').text(),
stars: $review.find('.section-review-stars').attr('aria-label').trim(),
publishAt: $review.find('.section-review-publish-date').text().trim(),
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
};
const $response = $review.find('.section-review-owner-response');
if ($response) {
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
}
return reviewData;
}, reviewEl);
placeDetail.reviews.push(review);
}