JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 08:28:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Moved crawler for enqueue places to main crawler

Browse Source
This commit is contained in:
JakubDrobnik 2018-12-10 15:27:34 +01:00
parent e02ccfe8c5
commit 0689be833c
4 changed files with 127 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
apify.json
View File

@ -1,11 +1,6 @@
{ {
"name": "crawler-google-places", "name": "crawler-google-places",
"actId": null, "version": "0.1",
"version": { "buildTag": "latest",
"versionNumber": "0.1", "env": null
"buildTag": "latest",
"envVars": [],
"sourceType": "TARBALL",
"tarballUrl": null
}
} }

115
src/enqueue_places_crawler.js
View File

@ -1,9 +1,11 @@
const Apify = require('apify'); const Apify = require('apify');
const { sleep } = Apify.utils;
const { injectJQuery } = Apify.utils.puppeteer;
const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelector('#searchbox').classList.contains('loading'), { timeout: DEFAULT_TIMEOUT }); const { sleep } = Apify.utils;
const { DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
const waitForGoogleMapLoader = (page) => page.waitFor(() => !document.querySelector('#searchbox')
.classList
.contains('loading'), { timeout: DEFAULT_TIMEOUT });
const enqueueAllUrlsFromPagination = async (page, requestQueue) => { const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
let results = await page.$$('.section-result'); let results = await page.$$('.section-result');
@ -36,70 +38,53 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
* @param listingPagination * @param listingPagination
* @param retries * @param retries
*/ */
const enqueueAllPlaceDetailsCrawler = async (startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination, retries = 0) => { const enqueueAllPlaceDetailsCrawler = async (page, searchString, launchPuppeteerOptions, requestQueue, listingPagination) => {
let browser; await page.type('#searchboxinput', searchString);
await sleep(5000);
await page.click('#searchbox-searchbutton');
await sleep(5000);
await waitForGoogleMapLoader(page);
// In case there is no listing, put just detail page to queue
try { try {
browser = await Apify.launchPuppeteer(launchPuppeteerOptions); await page.waitForSelector('h1.section-hero-header-title');
const page = await browser.newPage(); } catch (e) {
await page._client.send('Emulation.clearDeviceMetricsOverride'); // It can happen, doesn't matter
await page.goto(startUrl); }
await injectJQuery(page); const maybeDetailPlace = await page.$('h1.section-hero-header-title');
await page.type('#searchboxinput', searchString); if (maybeDetailPlace) {
await sleep(5000); const url = page.url();
await page.click('#searchbox-searchbutton'); await requestQueue.addRequest({ url, userData: { label: 'detail' } });
await sleep(5000); return;
await waitForGoogleMapLoader(page); }
// In case there is no listing, put just detail page to queue const nextButtonSelector = '[jsaction="pane.paginationSection.nextPage"]';
try { while (true) {
await page.waitForSelector('h1.section-hero-header-title'); await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
} catch (e) { const paginationText = await page.$eval('.n7lv7yjyC35__right', (el) => el.innerText);
// It can happen, doesn't matter const [fromString, toString] = paginationText.match(/\d+/g);
const from = parseInt(fromString);
const to = parseInt(toString);
if (listingPagination.from && from <= listingPagination.from) {
console.log(`Skiped pagination ${from} - ${to}, already done!`);
} else {
console.log(`Added links from pagination ${from} - ${to}`);
await enqueueAllUrlsFromPagination(page, requestQueue);
listingPagination = { from, to };
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
} }
const maybeDetailPlace = await page.$('h1.section-hero-header-title'); await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
if (maybeDetailPlace) { const isNextPaginationDisabled = await page.evaluate((nextButtonSelector) => {
const url = page.url(); return !!$(nextButtonSelector)
await requestQueue.addRequest({ url, userData: { label: 'detail' } }); .attr('disabled');
return; }, nextButtonSelector);
const noResultsEl = await page.$('.section-no-result-title');
if (isNextPaginationDisabled || noResultsEl) {
break;
} else {
// NOTE: puppeteer API click() didn't work :(
await page.evaluate((sel) => $(sel)
.click(), nextButtonSelector);
await waitForGoogleMapLoader(page);
} }
const nextButtonSelector = '#section-pagination-button-next';
while (true) {
await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
const paginationText = await page.$eval('.section-pagination-right', (el) => el.innerText);
const [fromString, toString] = paginationText.match(/\d+/g);
const from = parseInt(fromString);
const to = parseInt(toString);
if (listingPagination.from && from <= listingPagination.from) {
console.log(`Skiped pagination ${from} - ${to}, already done!`);
} else {
console.log(`Added links from pagination ${from} - ${to}`);
await enqueueAllUrlsFromPagination(page, requestQueue);
listingPagination = { from, to };
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
}
await page.waitForSelector(nextButtonSelector, { timeout: DEFAULT_TIMEOUT });
const isNextPaginationDisabled = await page.evaluate((nextButtonSelector) => {
return !!$(nextButtonSelector).attr('disabled');
}, nextButtonSelector);
const noResultsEl = await page.$('.section-no-result-title');
if (isNextPaginationDisabled || noResultsEl) {
break;
} else {
// NOTE: puppeteer API click() didn't work :(
await page.evaluate((sel) => $(sel).click(), nextButtonSelector);
await waitForGoogleMapLoader(page);
}
}
} catch (err) {
if (retries < MAX_PAGE_RETRIES) {
++retries;
console.log(`Retiring enqueueAllPlaceDetails for ${retries} time, error:`);
console.error(err);
await browser.close();
await enqueueAllPlaceDetailsCrawler(startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination, ++retries);
}
throw err;
} finally {
if (browser) await browser.close();
} }
}; };

14
src/main.js
View File

@ -1,7 +1,5 @@
const Apify = require('apify'); const Apify = require('apify');
const placesCrawler = require('./places_crawler'); const placesCrawler = require('./places_crawler');
const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
const { LISTING_PAGINATION_KEY } = require('./consts');
Apify.main(async () => { Apify.main(async () => {
const input = await Apify.getValue('INPUT'); const input = await Apify.getValue('INPUT');
@ -22,21 +20,11 @@ Apify.main(async () => {
console.log('Start url is', startUrl); console.log('Start url is', startUrl);
const requestQueue = await Apify.openRequestQueue(); const requestQueue = await Apify.openRequestQueue();
await requestQueue.addRequest({ url: startUrl, userData: { label: 'startUrl', searchString } });
// Store state of listing pagination
// NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
const launchPuppeteerOptions = {}; const launchPuppeteerOptions = {};
if (proxyConfig) Object.assign(launchPuppeteerOptions, proxyConfig); if (proxyConfig) Object.assign(launchPuppeteerOptions, proxyConfig);
// Enqueue all links to scrape from listings
if (!listingPagination.isFinish) {
console.log(`Start enqueuing place details for search: ${searchString}`);
await enqueueAllPlaceDetailsCrawler.run(startUrl, searchString, launchPuppeteerOptions, requestQueue, listingPagination);
listingPagination.isFinish = true;
await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
}
// Scrape all place detail links // Scrape all place detail links
const crawler = placesCrawler.setUpCrawler(launchPuppeteerOptions, requestQueue); const crawler = placesCrawler.setUpCrawler(launchPuppeteerOptions, requestQueue);
await crawler.run(); await crawler.run();

129
src/places_crawler.js
View File

@ -4,7 +4,8 @@ const { sleep } = Apify.utils;
const infiniteScroll = require('./infinite_scroll'); const infiniteScroll = require('./infinite_scroll');
const { injectJQuery } = Apify.utils.puppeteer; const { injectJQuery } = Apify.utils.puppeteer;
const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT } = require('./consts'); const { MAX_PAGE_RETRIES, DEFAULT_TIMEOUT, LISTING_PAGINATION_KEY } = require('./consts');
const enqueueAllPlaceDetailsCrawler = require('./enqueue_places_crawler');
/** /**
* Method to set up crawler to get all place details and save them to default dataset * Method to set up crawler to get all place details and save them to default dataset
@ -18,73 +19,89 @@ const setUpCrawler = (launchPuppeteerOptions, requestQueue) => {
requestQueue, requestQueue,
maxRequestRetries: MAX_PAGE_RETRIES, maxRequestRetries: MAX_PAGE_RETRIES,
retireInstanceAfterRequestCount: 10, retireInstanceAfterRequestCount: 10,
handlePageTimeoutSecs: 600, handlePageTimeoutSecs: 2 * 3600, // Two hours because startUrl crawler
maxOpenPagesPerInstance: 1, // Because startUrl crawler crashes if we mixed it with details scraping
// maxConcurrency: 1, // maxConcurrency: 1,
gotoFunction: async ({ request, page }) => { gotoFunction: async ({ request, page }) => {
await page._client.send('Emulation.clearDeviceMetricsOverride'); await page._client.send('Emulation.clearDeviceMetricsOverride');
await page.goto(request.url, { timeout: 60000 }); await page.goto(request.url, { timeout: 60000 });
}, },
handlePageFunction: async ({ request, page }) => { handlePageFunction: async ({ request, page }) => {
const { label } = request.userData; const { label, searchString } = request.userData;
console.log(`Open ${request.url} with label: ${label}`); console.log(`Open ${request.url} with label: ${label}`);
// Get data from review
await injectJQuery(page); await injectJQuery(page);
await page.waitForSelector('h1.section-hero-header-title', { timeout: DEFAULT_TIMEOUT }); if (label === 'startUrl') {
const placeDetail = await page.evaluate(() => { // enqueue all places
return { console.log(`Start enqueuing place details for search: ${searchString}`);
title: $('h1.section-hero-header-title').text().trim(), // Store state of listing pagination
totalScore: $('span.section-star-display').eq(0).text().trim(), // NOTE: Ensured - If pageFunction failed crawler skipped already scraped pagination
categoryName: $('[jsaction="pane.rating.category"]').text().trim(), const listingPagination = await Apify.getValue(LISTING_PAGINATION_KEY) || {};
address: $('[data-section-id="ad"] .widget-pane-link').text().trim(), await enqueueAllPlaceDetailsCrawler.run(page, searchString, launchPuppeteerOptions, requestQueue, listingPagination);
plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(), listingPagination.isFinish = true;
}; await Apify.setValue(LISTING_PAGINATION_KEY, listingPagination);
}); } else {
placeDetail.url = request.url; // Timeout because timeout for handle page is 2 hours
placeDetail.reviews = []; setTimeout(() => {
if (placeDetail.totalScore) { throw new Error('HandlePagefunction timed out!');
placeDetail.reviewsCount = await page.evaluate(() => { }, 600000);
const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim(); // Get data from review
return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null; await page.waitForSelector('h1.section-hero-header-title', { timeout: DEFAULT_TIMEOUT });
const placeDetail = await page.evaluate(() => {
return {
title: $('h1.section-hero-header-title').text().trim(),
totalScore: $('span.section-star-display').eq(0).text().trim(),
categoryName: $('[jsaction="pane.rating.category"]').text().trim(),
address: $('[data-section-id="ad"] .widget-pane-link').text().trim(),
plusCode: $('[data-section-id="ol"] .widget-pane-link').text().trim(),
};
}); });
// Get all reviews placeDetail.url = request.url;
await page.click('button.section-reviewchart-numreviews'); placeDetail.reviews = [];
await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT }); if (placeDetail.totalScore) {
await sleep(5000); placeDetail.reviewsCount = await page.evaluate(() => {
// Sort reviews by newest, one click sometimes didn't work :) const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim();
await page.click('.section-tab-info-stats-button-flex'); return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
await sleep(1000); });
await page.click('.section-tab-info-stats-button-flex'); // Get all reviews
await sleep(1000); await page.click('button.section-reviewchart-numreviews');
await page.click('.section-tab-info-stats-button-flex'); await page.waitForSelector('.section-star-display', { timeout: DEFAULT_TIMEOUT });
await sleep(5000); await sleep(5000);
await page.click('.context-menu-entry[data-index="1"]'); // Sort reviews by newest, one click sometimes didn't work :)
await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox'); await page.click('.section-tab-info-stats-button-flex');
const reviewEls = await page.$$('div.section-review'); await sleep(1000);
for (const reviewEl of reviewEls) { await page.click('.section-tab-info-stats-button-flex');
const moreButton = await reviewEl.$('.section-expand-review'); await sleep(1000);
if (moreButton) { await page.click('.section-tab-info-stats-button-flex');
await moreButton.click(); await sleep(5000);
await sleep(2000); await page.click('.context-menu-entry[data-index="1"]');
} await infiniteScroll(page, 99999999999, '.section-scrollbox.section-listbox');
const review = await page.evaluate((reviewEl) => { const reviewEls = await page.$$('div.section-review');
const $review = $(reviewEl); for (const reviewEl of reviewEls) {
const reviewData = { const moreButton = await reviewEl.$('.section-expand-review');
name: $review.find('.section-review-title').text().trim(), if (moreButton) {
text: $review.find('.section-review-review-content .section-review-text').text(), await moreButton.click();
stars: $review.find('.section-review-stars').attr('aria-label').trim(), await sleep(2000);
publishAt: $review.find('.section-review-publish-date').text().trim(),
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
};
const $response = $review.find('.section-review-owner-response');
if ($response) {
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
} }
return reviewData; const review = await page.evaluate((reviewEl) => {
}, reviewEl); const $review = $(reviewEl);
placeDetail.reviews.push(review); const reviewData = {
name: $review.find('.section-review-title').text().trim(),
text: $review.find('.section-review-review-content .section-review-text').text(),
stars: $review.find('.section-review-stars').attr('aria-label').trim(),
publishAt: $review.find('.section-review-publish-date').text().trim(),
likesCount: $review.find('.section-review-thumbs-up-count').text().trim(),
};
const $response = $review.find('.section-review-owner-response');
if ($response) {
reviewData.responseFromOwnerText = $response.find('.section-review-text').text().trim();
}
return reviewData;
}, reviewEl);
placeDetail.reviews.push(review);
}
} }
await Apify.pushData(placeDetail);
} }
await Apify.pushData(placeDetail);
console.log(request.url, 'Done'); console.log(request.url, 'Done');
}, },