mirror of
https://github.com/davidjohnbarton/crawler-google-places.git
synced 2025-12-12 16:38:45 +00:00
Several updates, readme updated, new selectores fixed
This commit is contained in:
drobnikj
parent
76fdde6dbd
commit
9f1ae7fca0
|
|
@ -9,6 +9,7 @@ const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelec
|
|||
const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom, maxPlacesPerCrawl) => {
|
||||
let results = await page.$$('.section-result');
|
||||
const resultsCount = results.length;
|
||||
|
||||
for (let resultIndex = 0; resultIndex < resultsCount; resultIndex++) {
|
||||
// Need to get results again, pupptr lost context..
|
||||
await page.waitForSelector('.searchbox', { timeout: DEFAULT_TIMEOUT });
|
||||
|
|
@ -21,6 +22,7 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom,
|
|||
await link.click();
|
||||
await waitForGoogleMapLoader(page);
|
||||
await page.waitForSelector('.section-back-to-list-button', { timeout: DEFAULT_TIMEOUT });
|
||||
// After redirection to detail page, save the URL to Request queue to process it later
|
||||
const url = page.url();
|
||||
await requestQueue.addRequest({ url, userData: { label: 'detail' } });
|
||||
log.info(`Added to queue ${url}`);
|
||||
|
|
@ -28,20 +30,23 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue, paginationFrom,
|
|||
log.info(`Reach max places per crawl ${maxPlacesPerCrawl}, stopped enqueuing new places.`);
|
||||
break;
|
||||
}
|
||||
|
||||
await page.click('.section-back-to-list-button');
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Crawler add all place detail from listing to queue
|
||||
* Adds all places from listing to queue
|
||||
* @param page
|
||||
* @param searchString
|
||||
* @param launchPuppeteerOptions
|
||||
* @param requestQueue
|
||||
* @param listingPagination
|
||||
* @param maxPlacesPerCrawl
|
||||
*/
|
||||
const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteerOptions, requestQueue, listingPagination, maxPlacesPerCrawl) => {
|
||||
const enqueueAllPlaceDetails = async (page, searchString, requestQueue, maxPlacesPerCrawl) => {
|
||||
// Save state of listing pagination
|
||||
// NOTE: If pageFunction failed crawler skipped already scraped pagination
|
||||
const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
|
||||
|
||||
await page.type('#searchboxinput', searchString);
|
||||
await sleep(5000);
|
||||
await page.click('#searchbox-searchbutton');
|
||||
|
|
@ -50,8 +55,9 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
|
|||
try {
|
||||
await page.waitForSelector('h1.section-hero-header-title');
|
||||
} catch (e) {
|
||||
// It can happen, doesn't matter :)
|
||||
// It can happen, if there are listing, not just detail page
|
||||
}
|
||||
|
||||
// In case there is no listing, put just detail page to queue
|
||||
const maybeDetailPlace = await page.$('h1.section-hero-header-title');
|
||||
if (maybeDetailPlace) {
|
||||
|
|
@ -59,6 +65,8 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
|
|||
await requestQueue.addRequest({ url, userData: { label: 'detail' } });
|
||||
return;
|
||||
}
|
||||
|
||||
// In case there is listing, go through all details, limits with maxPlacesPerCrawl
|
||||
const nextButtonSelector = '[jsaction="pane.paginationSection.nextPage"]';
|
||||
while (true) {
|
||||
await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
|
||||
|
|
@ -71,7 +79,8 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
|
|||
} else {
|
||||
log.debug(`Added links from pagination ${from} - ${to}`);
|
||||
await enqueueAllUrlsFromPagination(page, requestQueue, from, maxPlacesPerCrawl);
|
||||
listingPagination = { from, to };
|
||||
listingPagination.from = from;
|
||||
listingPagination.to = to;
|
||||
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
|
||||
}
|
||||
await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
|
||||
|
|
@ -87,6 +96,9 @@ const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteer
|
|||
await waitForGoogleMapLoader(page);
|
||||
}
|
||||
}
|
||||
|
||||
listingPagination.isFinish = true;
|
||||
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
|
||||
};
|
||||
|
||||
module.exports = { run: enqueueAllPlaceDetailsCrawler };
|
||||
module.exports = { enqueueAllPlaceDetails };
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ Apify.main(async () => {
|
|||
const launchPuppeteerOptions = {};
|
||||
if (proxyConfig) Object.assign(launchPuppeteerOptions, proxyConfig);
|
||||
|
||||
// Scrape all place detail links
|
||||
// Create and run crawler
|
||||
const crawler = placesCrawler.setUpCrawler(launchPuppeteerOptions, requestQueue, maxCrawledPlaces);
|
||||
await crawler.run();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,143 @@
|
|||
const Apify = require('apify');
|
||||
|
||||
const { sleep, log } = Apify.utils;
|
||||
const infiniteScroll = require('./infinite_scroll');
|
||||
|
||||
const { injectJQuery } = Apify.utils.puppeteer;
|
||||
const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
|
||||
const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
|
||||
const infiniteScroll = require('./infinite_scroll');
|
||||
const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts');
|
||||
const { enqueueAllPlaceDetails } = require('./enqueue_places_crawler');
|
||||
|
||||
/**
|
||||
* This is the worst part - parsing data from place detail
|
||||
* @param page
|
||||
*/
|
||||
const extractPlaceDetail = async (page) => {
|
||||
// Extracts basic information
|
||||
const titleSel = 'h1.section-hero-header-title';
|
||||
await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
|
||||
const detail = await page.evaluate(() => {
|
||||
return {
|
||||
title: $('h1.section-hero-header-title').text().trim(),
|
||||
totalScore: $('span.section-star-display').eq(0).text().trim(),
|
||||
categoryName: $('[jsaction="pane.rating.category"]').text().trim(),
|
||||
address: $('[data-section-id="ad"] .widget-pane-link').text().trim(),
|
||||
plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(),
|
||||
};
|
||||
});
|
||||
|
||||
// Extracty histogram for popular times
|
||||
const histogramSel = '.section-popular-times';
|
||||
if (await page.$(histogramSel)) {
|
||||
detail.popularTimesHistogram = await page.evaluate(() => {
|
||||
const graphs = {};
|
||||
const days = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'];
|
||||
// Extract all days graphs
|
||||
$('.section-popular-times-graph').each(function(i) {
|
||||
const day = days[i];
|
||||
graphs[day] = [];
|
||||
let graphStartFromHour;
|
||||
// Finds where x axis starts
|
||||
$(this).find('.section-popular-times-label').each(function(labelIndex) {
|
||||
if (graphStartFromHour) return;
|
||||
const hourText = $(this).text().trim();
|
||||
graphStartFromHour = hourText.includes('p')
|
||||
? 12 + (parseInt(hourText) - labelIndex)
|
||||
: parseInt(hourText) - labelIndex;
|
||||
});
|
||||
// Finds values from y axis
|
||||
$(this).find('.section-popular-times-bar').each(function (barIndex) {
|
||||
const occupancyMatch = $(this).attr('aria-label').match(/\d+\s+?%/);
|
||||
if (occupancyMatch) {
|
||||
const maybeHour = graphStartFromHour + barIndex;
|
||||
graphs[day].push({
|
||||
hour: maybeHour > 24 ? maybeHour - 24 : maybeHour,
|
||||
occupancyPercent: parseInt(occupancyMatch[0]),
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return graphs;
|
||||
});
|
||||
}
|
||||
|
||||
// Extracts reviews
|
||||
detail.reviews = [];
|
||||
const reviewsButtonSel = 'button[jsaction="pane.reviewChart.moreReviews"]';
|
||||
if (detail.totalScore) {
|
||||
detail.totalScore = parseFloat(detail.totalScore.replace(',', '.'));
|
||||
detail.reviewsCount = await page.evaluate((selector) => {
|
||||
const numberReviewsText = $(selector).text().trim();
|
||||
return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
|
||||
}, reviewsButtonSel);
|
||||
// If we find consent dialog, close it!
|
||||
if (await page.$('.widget-consent-dialog')) {
|
||||
await page.click('.widget-consent-dialog .widget-consent-button-later');
|
||||
}
|
||||
// Get all reviews
|
||||
await page.waitForSelector(reviewsButtonSel);
|
||||
await page.click(reviewsButtonSel);
|
||||
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
|
||||
await sleep(5000);
|
||||
// Sort reviews by newest, one click sometimes didn't work :)
|
||||
try {
|
||||
const sortButtonEl = '.section-tab-info-stats-button-flex';
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await page.click(sortButtonEl);
|
||||
await sleep(1000);
|
||||
}
|
||||
await page.click('.context-menu-entry[data-index="1"]');
|
||||
} catch (err) {
|
||||
// It can happen, it is not big issue :)
|
||||
log.debug('Cannot select reviews by newest!');
|
||||
}
|
||||
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
|
||||
const reviewEls = await page.$$('div.section-review');
|
||||
for (const reviewEl of reviewEls) {
|
||||
const moreButton = await reviewEl.$('.section-expand-review');
|
||||
if (moreButton) {
|
||||
await moreButton.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
const review = await page.evaluate((reviewEl) => {
|
||||
const $review = $(reviewEl);
|
||||
const reviewData = {
|
||||
name: $review.find('.section-review-title').text().trim(),
|
||||
text: $review.find('.section-review-review-content .section-review-text').text(),
|
||||
stars: $review.find('.section-review-stars').attr('aria-label').trim(),
|
||||
publishAt: $review.find('.section-review-publish-date').text().trim(),
|
||||
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
|
||||
};
|
||||
const $response = $review.find('.section-review-owner-response');
|
||||
if ($response) {
|
||||
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
|
||||
}
|
||||
return reviewData;
|
||||
}, reviewEl);
|
||||
detail.reviews.push(review);
|
||||
}
|
||||
await page.click('button.section-header-back-button');
|
||||
}
|
||||
|
||||
// Extracts place images
|
||||
await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
|
||||
const imagesButtonSel = '[jsaction="pane.imagepack.button"]';
|
||||
if (await page.$(imagesButtonSel)) {
|
||||
await page.click(imagesButtonSel);
|
||||
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
|
||||
detail.imageUrls = await page.evaluate(() => {
|
||||
const urls = [];
|
||||
$('.gallery-image-high-res').each(function () {
|
||||
const urlMatch = $(this).attr('style').match(/url\("(.*)"\)/);
|
||||
if (!urlMatch) return;
|
||||
let imageUrl = urlMatch[1];
|
||||
if (imageUrl[0] === '/') imageUrl = `https:${imageUrl}`;
|
||||
urls.push(imageUrl);
|
||||
});
|
||||
return urls;
|
||||
});
|
||||
}
|
||||
return detail;
|
||||
};
|
||||
|
||||
/**
|
||||
* Method to set up crawler to get all place details and save them to default dataset
|
||||
* @param launchPuppeteerOptions
|
||||
|
|
@ -19,13 +151,15 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) =>
|
|||
requestQueue,
|
||||
maxRequestRetries: MAX_PAGE_RETRIES,
|
||||
retireInstanceAfterRequestCount: 10,
|
||||
handlePageTimeoutSecs: 2 * 3600, // Two hours because startUrl crawler
|
||||
maxOpenPagesPerInstance: 1, // Because startUrl crawler crashes if we mixed tabs with details scraping
|
||||
// maxConcurrency: 1,
|
||||
handlePageTimeoutSecs: 15 * 60, // 15 min because startUrl enqueueing
|
||||
maxOpenPagesPerInstance: 1, // Because startUrl enqueueing crashes if we mixed tabs with details scraping
|
||||
};
|
||||
if (maxCrawledPlaces) {
|
||||
crawlerOpts.maxRequestsPerCrawl = maxCrawledPlaces + 1; // The first one is startUrl
|
||||
}
|
||||
if (!Apify.isAtHome()) {
|
||||
crawlerOpts.maxConcurrency = 2;
|
||||
}
|
||||
return new Apify.PuppeteerCrawler({
|
||||
...crawlerOpts,
|
||||
gotoFunction: async ({ request, page }) => {
|
||||
|
|
@ -34,142 +168,22 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue, maxCrawledPlaces) =>
|
|||
},
|
||||
handlePageFunction: async ({ request, page }) => {
|
||||
const { label, searchString } = request.userData;
|
||||
|
||||
log.info(`Open ${request.url} with label: ${label}`);
|
||||
await injectJQuery(page);
|
||||
|
||||
if (label === 'startUrl') {
|
||||
// enqueue all places
|
||||
log.info(`Start enqueuing place details for search: ${searchString}`);
|
||||
// Store state of listing pagination
|
||||
// NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
|
||||
const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
|
||||
await enqueueAllPlaceDetailsCrawler.run(page, searchString, launchPuppeteerOptions, requestQueue, listingPagination, maxCrawledPlaces);
|
||||
listingPagination.isFinish = true;
|
||||
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
|
||||
log.info(`Start enqueuing places details for search: ${searchString}`);
|
||||
await enqueueAllPlaceDetails(page, searchString, requestQueue, maxCrawledPlaces);
|
||||
log.info('Enqueuing places finished!');
|
||||
} else {
|
||||
// Timeout because timeout for handle page is 2 hours
|
||||
setTimeout(() => {
|
||||
throw new Error('HandlePagefunction timed out!');
|
||||
}, 600000);
|
||||
// Get data from review
|
||||
const titleSel = 'h1.section-hero-header-title';
|
||||
await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
|
||||
const placeDetail = await page.evaluate(() => {
|
||||
return {
|
||||
title: $('h1.section-hero-header-title').text().trim(),
|
||||
totalScore: $('span.section-star-display').eq(0).text().trim(),
|
||||
categoryName: $('[jsaction="pane.rating.category"]').text().trim(),
|
||||
address: $('[data-section-id="ad"] .widget-pane-link').text().trim(),
|
||||
plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(),
|
||||
};
|
||||
});
|
||||
// Get data for place and save it to dataset
|
||||
log.info(`Extracting details from place url ${request.url}`);
|
||||
const placeDetail = await extractPlaceDetail(page);
|
||||
placeDetail.url = request.url;
|
||||
const histogramSel = '.section-popular-times';
|
||||
if (await page.$(histogramSel)) {
|
||||
placeDetail.popularTimesHistogram = await page.evaluate(() => {
|
||||
const graphs = {};
|
||||
const days = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'];
|
||||
// Days graphs
|
||||
$('.section-popular-times-graph').each(function(i) {
|
||||
const day = days[i];
|
||||
graphs[day] = [];
|
||||
let graphStartFromHour;
|
||||
$(this).find('.section-popular-times-label').each(function(labelIndex) {
|
||||
if (graphStartFromHour) return;
|
||||
const hourText = $(this).text().trim();
|
||||
graphStartFromHour = hourText.includes('p')
|
||||
? 12 + (parseInt(hourText) - labelIndex)
|
||||
: parseInt(hourText) - labelIndex;
|
||||
});
|
||||
$(this).find('.section-popular-times-bar').each(function (barIndex) {
|
||||
const occupancy = $(this).attr('aria-label').match(/\d+\s{1,}%/)[0];
|
||||
const maybeHour = graphStartFromHour + barIndex;
|
||||
graphs[day].push({
|
||||
hour: maybeHour > 24 ? maybeHour - 24 : maybeHour,
|
||||
occupancy,
|
||||
});
|
||||
});
|
||||
});
|
||||
return graphs;
|
||||
});
|
||||
}
|
||||
placeDetail.reviews = [];
|
||||
const reviewsButtonSel = 'button[jsaction="pane.reviewChart.moreReviews"]';
|
||||
if (placeDetail.totalScore) {
|
||||
placeDetail.reviewsCount = await page.evaluate((selector) => {
|
||||
const numberReviewsText = $(selector).text().trim();
|
||||
return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
|
||||
}, reviewsButtonSel);
|
||||
// If we find consent dialog, close it!
|
||||
if (await page.$('.widget-consent-dialog')) {
|
||||
await page.click('.widget-consent-dialog .widget-consent-button-later');
|
||||
}
|
||||
// Get all reviews
|
||||
await page.waitForSelector(reviewsButtonSel);
|
||||
await page.click(reviewsButtonSel);
|
||||
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
|
||||
await sleep(5000);
|
||||
// Sort reviews by newest, one click sometimes didn't work :)
|
||||
try {
|
||||
const sortButtonEl = '.section-tab-info-stats-button-flex';
|
||||
await page.click(sortButtonEl);
|
||||
await sleep(1000);
|
||||
await page.click(sortButtonEl);
|
||||
await sleep(1000);
|
||||
await page.click(sortButtonEl);
|
||||
await sleep(5000);
|
||||
await page.click('.context-menu-entry[data-index="1"]');
|
||||
} catch (err) {
|
||||
// It can happen, it is not big issue :)
|
||||
log.debug('Cannot select reviews by newest!');
|
||||
}
|
||||
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
|
||||
const reviewEls = await page.$$('div.section-review');
|
||||
for (const reviewEl of reviewEls) {
|
||||
const moreButton = await reviewEl.$('.section-expand-review');
|
||||
if (moreButton) {
|
||||
await moreButton.click();
|
||||
await sleep(2000);
|
||||
}
|
||||
const review = await page.evaluate((reviewEl) => {
|
||||
const $review = $(reviewEl);
|
||||
const reviewData = {
|
||||
name: $review.find('.section-review-title').text().trim(),
|
||||
text: $review.find('.section-review-review-content .section-review-text').text(),
|
||||
stars: $review.find('.section-review-stars').attr('aria-label').trim(),
|
||||
publishAt: $review.find('.section-review-publish-date').text().trim(),
|
||||
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
|
||||
};
|
||||
const $response = $review.find('.section-review-owner-response');
|
||||
if ($response) {
|
||||
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
|
||||
}
|
||||
return reviewData;
|
||||
}, reviewEl);
|
||||
placeDetail.reviews.push(review);
|
||||
}
|
||||
await page.click('button.section-header-back-button');
|
||||
}
|
||||
await page.waitForSelector(titleSel, { timeout: DEFAULT_TIMEOUT });
|
||||
const imagesButtonSel = '[jsaction="pane.imagepack.button"]';
|
||||
console.log(imagesButtonSel);
|
||||
if (await page.$(imagesButtonSel)) {
|
||||
await page.click(imagesButtonSel);
|
||||
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
|
||||
placeDetail.imageUrls = await page.evaluate(() => {
|
||||
const urls = [];
|
||||
$('.gallery-image-high-res').each(function () {
|
||||
const urlMatch = $(this).attr('style').match(/url\("(.*)"\)/);
|
||||
if (!urlMatch) return;
|
||||
let imageUrl = urlMatch[1];
|
||||
if (imageUrl[0] === '/') imageUrl = `https:${imageUrl}`;
|
||||
urls.push(imageUrl);
|
||||
});
|
||||
return urls;
|
||||
});
|
||||
}
|
||||
await Apify.pushData(placeDetail);
|
||||
log.info(`Finished place url ${request.url}`);
|
||||
}
|
||||
log.info('Finished', request.url);
|
||||
},
|
||||
handleFailedRequestFunction: async ({ request }) => {
|
||||
// This function is called when crawling of a request failed too many time
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user