diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..c7a5a28 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,40 @@ +{ + "extends": ["airbnb-base"], + "plugins": [ + "import", + "promise" + ], + "parserOptions": { + "ecmaVersion": 8, + "ecmaFeatures": { + "jsx": true + } + }, + "globals": { + "Npm": true + }, + "rules": { + "indent": ["error", 4, { "SwitchCase": 1 }], + "no-underscore-dangle": [2, { + "allow": ["_id", "_outputSeqNo", "_queueOrderNo", "_skipOutput", "_retryCount", "_crashesCount", "_pageFailed", "_pageCrashed"], + "allowAfterThis": true + }], + "no-param-reassign": 0, + "consistent-return": 0, + "array-callback-return": 0, + "no-plusplus": 0, + "max-len": ["error", 150], + "func-names": 0, + "import/no-extraneous-dependencies": ["error", { "devDependencies": true }], + "import/prefer-default-export": 0, + "import/no-unresolved": 0, + "class-methods-use-this": 0, + "strict": 0, + "object-curly-newline": 0, + "no-await-in-loop": 0, + "arrow-body-style": 0, + "arrow-parens": ["error", "always"], + "no-restricted-syntax": 0, + "no-console": 0 + } +} diff --git a/src/infinite_scroll.js b/src/infinite_scroll.js index 3250b84..2213cd6 100644 --- a/src/infinite_scroll.js +++ b/src/infinite_scroll.js @@ -1,6 +1,6 @@ const Apify = require('apify'); -const sleepPromised = ms => new Promise(resolve => setTimeout(resolve, ms)); +const sleepPromised = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); const logError = (msg, e) => { console.log(`ERROR: ${msg}`); @@ -65,7 +65,6 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => { delete pendingRequests[msg._requestId]; resourcesStats.failed++; } - } }); page.on('requestfinished', (msg) => { @@ -89,12 +88,12 @@ module.exports = async (page, maxHeight, elementToScroll = 'body') => { const now = Date.now(); const timeout = 30000; // TODO: use resourceTimeout Object.keys(pendingRequests) - .forEach((requestId) => { - if (pendingRequests[requestId] + timeout < now) { - delete pendingRequests[requestId]; - resourcesStats.forgotten++; - } - }); + .forEach((requestId) => { + if (pendingRequests[requestId] + timeout < now) { + delete pendingRequests[requestId]; + resourcesStats.forgotten++; + } + }); logDebug(`Infinite scroll stats (${stringifyScrollInfo(scrollInfo)} resourcesStats=${JSON.stringify(resourcesStats)}).`); diff --git a/src/main.js b/src/main.js index f5d79bc..9092729 100644 --- a/src/main.js +++ b/src/main.js @@ -30,7 +30,8 @@ const enqueueAllUrlsFromPagination = async (page, requestQueue) => { }; Apify.main(async () => { - const { searchString, lat, lng } = await Apify.getValue('INPUT'); + const input = await Apify.getValue('INPUT'); + const { searchString, lat, lng } = input; if (!searchString) throw new Error('Attribute searchString missing in input.'); @@ -73,10 +74,10 @@ Apify.main(async () => { await sleep(5000); await page.click('#searchbox-searchbutton'); await sleep(5000); - while(true) { + while (true) { await page.waitForSelector('#section-pagination-button-next', { timeout: DEFAULT_TIMEOUT }); - const paginationText = await page.$eval('.section-pagination-right', el => el.innerText); - const [fromString , toString] = paginationText.match(/\d+/g); + const paginationText = await page.$eval('.section-pagination-right', (el) => el.innerText); + const [fromString, toString] = paginationText.match(/\d+/g); const from = parseInt(fromString); const to = parseInt(toString); if (listingPagination.to && to <= listingPagination.to) { @@ -113,7 +114,7 @@ Apify.main(async () => { placeDetail.reviews = []; if (placeDetail.totalScore) { placeDetail.reviewsCount = await page.evaluate(() => { - const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim() + const numberReviewsText = $('button.section-reviewchart-numreviews').text().trim(); return (numberReviewsText) ? numberReviewsText.match(/\d+/)[0] : null; }); // Get all reviews