JoBo/crawler-google-places
1
0
Fork 0
mirror of https://github.com/davidjohnbarton/crawler-google-places.git synced 2025-12-12 16:38:45 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ESLinting

Browse Source
This commit is contained in:
JakubDrobnik 2018-11-20 15:41:20 +01:00
parent 389c5aa786
commit 633b16c480
3 changed files with 53 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
.eslintrc Normal file
View File

@ -0,0 +1,40 @@
{
"extends": ["airbnb-base"],
"plugins": [
"import",
"promise"
],
"parserOptions": {
"ecmaVersion": 8,
"ecmaFeatures": {
"jsx": true
}
},
"globals": {
"Npm": true
},
"rules": {
"indent": ["error", 4, { "SwitchCase": 1 }],
"no-underscore-dangle": [2, {
"allow": ["_id", "_outputSeqNo", "_queueOrderNo", "_skipOutput", "_retryCount", "_crashesCount", "_pageFailed", "_pageCrashed"],
"allowAfterThis": true
}],
"no-param-reassign": 0,
"consistent-return": 0,
"array-callback-return": 0,
"no-plusplus": 0,
"max-len": ["error", 150],
"func-names": 0,
"import/no-extraneous-dependencies": ["error", { "devDependencies": true }],
"import/prefer-default-export": 0,
"import/no-unresolved": 0,
"class-methods-use-this": 0,
"strict": 0,
"object-curly-newline": 0,
"no-await-in-loop": 0,
"arrow-body-style": 0,
"arrow-parens": ["error", "always"],
"no-restricted-syntax": 0,
"no-console": 0
}
}

15
src/infinite_scroll.js
View File

@ -1,6 +1,6 @@
const Apify = require('apify'); const Apify = require('apify');
const sleepPromised = ms => new Promise(resolve => setTimeout(resolve, ms)); const sleepPromised = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const logError = (msg, e) => { const logError = (msg, e) => {
console.log(`ERROR: ${msg}`); console.log(`ERROR: ${msg}`);
@ -65,7 +65,6 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
delete pendingRequests[msg._requestId]; delete pendingRequests[msg._requestId];
resourcesStats.failed++; resourcesStats.failed++;
} }
} }
}); });
page.on('requestfinished', (msg) => { page.on('requestfinished', (msg) => {
@ -89,12 +88,12 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => {
const now = Date.now(); const now = Date.now();
const timeout = 30000; // TODO: use resourceTimeout const timeout = 30000; // TODO: use resourceTimeout
Object.keys(pendingRequests) Object.keys(pendingRequests)
.forEach((requestId) => { .forEach((requestId) => {
if (pendingRequests[requestId] + timeout < now) { if (pendingRequests[requestId] + timeout < now) {
delete pendingRequests[requestId]; delete pendingRequests[requestId];
resourcesStats.forgotten++; resourcesStats.forgotten++;
} }
}); });
logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`); logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`);

11
src/main.js
View File

@ -30,7 +30,8 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => {
}; };
Apify.main(async () => { Apify.main(async () => {
const { searchString, lat, lng } = await Apify.getValue('INPUT'); const input = await Apify.getValue('INPUT');
const { searchString, lat, lng } = input;
if (!searchString) throw new Error('Attribute searchString missing in input.'); if (!searchString) throw new Error('Attribute searchString missing in input.');
@ -73,10 +74,10 @@ Apify.main(async () => {
await sleep(5000); await sleep(5000);
await page.click('#searchbox-searchbutton'); await page.click('#searchbox-searchbutton');
await sleep(5000); await sleep(5000);
while(true) { while (true) {
await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT }); await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT });
const paginationText = await page.$eval('.section-pagination-right', el => el.innerText); const paginationText = await page.$eval('.section-pagination-right', (el) => el.innerText);
const [fromString , toString] = paginationText.match(/\d+/g); const [fromString, toString] = paginationText.match(/\d+/g);
const from = parseInt(fromString); const from = parseInt(fromString);
const to = parseInt(toString); const to = parseInt(toString);
if (listingPagination.to && to <= listingPagination.to) { if (listingPagination.to && to <= listingPagination.to) {
@ -113,7 +114,7 @@ Apify.main(async () => {
placeDetail.reviews = []; placeDetail.reviews = [];
if (placeDetail.totalScore) { if (placeDetail.totalScore) {
placeDetail.reviewsCount = await page.evaluate(() => { placeDetail.reviewsCount = await page.evaluate(() => {
const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim() const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim();
return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null; return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null;
}); });
// Get all reviews // Get all reviews