From 4f7842583b3257b28a42462a75194710206597b1 Mon Sep 17 00:00:00 2001 From: Rich Trott Date: Wed, 1 Feb 2023 15:11:27 -0800 Subject: [PATCH] chore: enforce linting An ESLint rule plugin is installed but is not enforced. This adds linting to the test job and fixes many lint errors in the existing code base. --- package.json | 5 +++-- src/index.js | 2 +- src/parser.js | 7 +++---- src/robots.js | 10 +++++----- test/.eslintrc | 8 ++++++++ test/parser/can-parse-allow.js | 4 ++-- test/parser/can-parse-crawl-delay.js | 4 ++-- test/parser/can-parse-disallow.js | 4 ++-- test/parser/can-parse-sitemaps.js | 2 +- test/parser/can-parse-test-files.js | 12 ++++++------ test/parser/can-parse-user-agents.js | 4 ++-- test/parser/correct-parse-formatting.js | 4 ++-- test/parser/ignores-malformed-values.js | 4 ++-- test/robots/can-crawl.js | 10 +++++----- test/robots/fetch.js | 2 +- test/robots/get-crawl-delay.js | 6 +++--- test/robots/get-crawlable-links.js | 4 ++-- test/robots/get-host.js | 4 ++-- test/robots/get-sitemaps.js | 6 +++--- test/robots/is-cached.js | 4 ++-- test/robots/use-robots-for.js | 2 +- test/util/format-link.js | 2 +- test/util/url-interactions.js | 4 ++-- 23 files changed, 61 insertions(+), 53 deletions(-) create mode 100644 test/.eslintrc diff --git a/package.json b/package.json index 5be0b1a..193007e 100644 --- a/package.json +++ b/package.json @@ -18,12 +18,13 @@ "test": "test" }, "scripts": { - "test": "mocha test/**/*.js", + "test": "npm run lint && mocha test/**/*.js", "test-robots": "mocha test/robots/*.js", "test-parser": "mocha test/parser/*.js", "test-util": "mocha test/util/*.js", "coverage": "nyc --reporter=lcov mocha test/**/*.js --timeout 10s", - "local-coverage": "nyc mocha test/**/*.js" + "local-coverage": "nyc mocha test/**/*.js", + "lint": "eslint src test" }, "repository": { "type": "git", diff --git a/src/index.js b/src/index.js index 5c0eee7..3f3e70a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,3 +1,3 @@ -const Robots = require('./robots.js'); +const Robots = require('./robots'); module.exports = (opts) => new Robots(opts); diff --git a/src/parser.js b/src/parser.js index a7c8f37..efe6d63 100644 --- a/src/parser.js +++ b/src/parser.js @@ -19,9 +19,7 @@ const cleanSpaces = (rawString) => rawString.replace(whitespace, '').trim(); const splitOnLines = (string) => string.split(lineEndings); -const robustSplit = (string) => { - return !string.includes('') ? [...string.match(recordSlices)].map(cleanSpaces) : []; -}; +const robustSplit = (string) => (!string.includes('') ? [...string.match(recordSlices)].map(cleanSpaces) : []); const parseRecord = (line) => { // Find first colon and assume is the field delimiter. @@ -70,7 +68,7 @@ const parser = (rawString) => { lines.forEach((line) => { const record = parseRecord(line); switch (record.field) { - case USER_AGENT: + case USER_AGENT: { const recordValue = record.value.toLowerCase(); if (recordValue !== agent && recordValue.length > 0) { // Bot names are non-case sensitive. @@ -84,6 +82,7 @@ const parser = (rawString) => { agent = ''; } break; + } // https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#order-of-precedence-for-group-member-records case ALLOW: if (agent.length > 0 && record.value.length > 0) { diff --git a/src/robots.js b/src/robots.js index c7f6ddc..d94cdbb 100644 --- a/src/robots.js +++ b/src/robots.js @@ -1,6 +1,6 @@ -const get = require('./get.js'); -const parser = require('./parser.js'); -const util = require('./util.js'); +const get = require('./get'); +const parser = require('./parser'); +const util = require('./util'); const DFLT_OPTS = { userAgent: '*', @@ -25,7 +25,7 @@ class Robots { const otherBots = '*' in domainBots; if (ourBotInBots) { return domainBots[this.opts.userAgent]; - } else if (otherBots) { + } if (otherBots) { return domainBots['*']; } return false; @@ -44,7 +44,7 @@ class Robots { if (noDisallows || (allow.maxSpecificity > disallow.maxSpecificity)) { return true; - } else if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) { + } if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) { return false; } return this.opts.allowOnNeutral; diff --git a/test/.eslintrc b/test/.eslintrc new file mode 100644 index 0000000..830273e --- /dev/null +++ b/test/.eslintrc @@ -0,0 +1,8 @@ +{ + "env": { + "mocha": true + }, + "rules": { + "no-unused-expressions": 0 + } +} \ No newline at end of file diff --git a/test/parser/can-parse-allow.js b/test/parser/can-parse-allow.js index d778f13..11e3f46 100644 --- a/test/parser/can-parse-allow.js +++ b/test/parser/can-parse-allow.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const allowRobots = require('../test-data/example-robots-txt-allow.js'); -const parse = require('../../src/parser.js'); +const allowRobots = require('../test-data/example-robots-txt-allow'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/can-parse-crawl-delay.js b/test/parser/can-parse-crawl-delay.js index 58e1d20..6ab2322 100644 --- a/test/parser/can-parse-crawl-delay.js +++ b/test/parser/can-parse-crawl-delay.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); -const parse = require('../../src/parser.js'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/can-parse-disallow.js b/test/parser/can-parse-disallow.js index 6c39ad9..23801a6 100644 --- a/test/parser/can-parse-disallow.js +++ b/test/parser/can-parse-disallow.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const disallowRobots = require('../test-data/example-robots-txt-disallow.js'); -const parse = require('../../src/parser.js'); +const disallowRobots = require('../test-data/example-robots-txt-disallow'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/can-parse-sitemaps.js b/test/parser/can-parse-sitemaps.js index 008fe34..78f00d7 100644 --- a/test/parser/can-parse-sitemaps.js +++ b/test/parser/can-parse-sitemaps.js @@ -1,7 +1,7 @@ const chai = require('chai'); const uniq = require('lodash/uniq'); const sitemaps = require('../test-data/example-robots-txt-sitemaps'); -const parse = require('../../src/parser.js'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/can-parse-test-files.js b/test/parser/can-parse-test-files.js index ba37c89..6e86552 100644 --- a/test/parser/can-parse-test-files.js +++ b/test/parser/can-parse-test-files.js @@ -1,10 +1,10 @@ const chai = require('chai'); -const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc.js'); -const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc.js'); -const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); -const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando.js'); -const parse = require('../../src/parser.js'); +const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc'); +const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc'); +const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); +const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/can-parse-user-agents.js b/test/parser/can-parse-user-agents.js index 7142d36..cf23be7 100644 --- a/test/parser/can-parse-user-agents.js +++ b/test/parser/can-parse-user-agents.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); -const parse = require('../../src/parser.js'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/correct-parse-formatting.js b/test/parser/correct-parse-formatting.js index 07be28a..b98a49e 100644 --- a/test/parser/correct-parse-formatting.js +++ b/test/parser/correct-parse-formatting.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const exampleShort = require('../test-data/example-robots-txt-short.js'); -const parse = require('../../src/parser.js'); +const exampleShort = require('../test-data/example-robots-txt-short'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/parser/ignores-malformed-values.js b/test/parser/ignores-malformed-values.js index 682beb0..b3ab69f 100644 --- a/test/parser/ignores-malformed-values.js +++ b/test/parser/ignores-malformed-values.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short.js'); -const parse = require('../../src/parser.js'); +const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short'); +const parse = require('../../src/parser'); const { expect } = chai; diff --git a/test/robots/can-crawl.js b/test/robots/can-crawl.js index f64f723..489183a 100644 --- a/test/robots/can-crawl.js +++ b/test/robots/can-crawl.js @@ -1,9 +1,9 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const parser = require('../../src/parser.js'); -const util = require('../../src/util.js'); -const testData = require('../test-data/can-crawl-test-data.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const parser = require('../../src/parser'); +const util = require('../../src/util'); +const testData = require('../test-data/can-crawl-test-data'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/fetch.js b/test/robots/fetch.js index a41c84c..a4603f8 100644 --- a/test/robots/fetch.js +++ b/test/robots/fetch.js @@ -1,5 +1,5 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); +const robots = require('../../src/index'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/get-crawl-delay.js b/test/robots/get-crawl-delay.js index cf74827..f4364a2 100644 --- a/test/robots/get-crawl-delay.js +++ b/test/robots/get-crawl-delay.js @@ -1,7 +1,7 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const parser = require('../../src/parser.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const parser = require('../../src/parser'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/get-crawlable-links.js b/test/robots/get-crawlable-links.js index 73b7ca4..4385f0a 100644 --- a/test/robots/get-crawlable-links.js +++ b/test/robots/get-crawlable-links.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/get-host.js b/test/robots/get-host.js index 704bd31..c25aa46 100644 --- a/test/robots/get-host.js +++ b/test/robots/get-host.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/get-sitemaps.js b/test/robots/get-sitemaps.js index c22c600..73a060b 100644 --- a/test/robots/get-sitemaps.js +++ b/test/robots/get-sitemaps.js @@ -1,7 +1,7 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const parser = require('../../src/parser.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const parser = require('../../src/parser'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/is-cached.js b/test/robots/is-cached.js index c3d8e2e..d517baf 100644 --- a/test/robots/is-cached.js +++ b/test/robots/is-cached.js @@ -1,6 +1,6 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); -const exampleRobotsShort = require('../test-data/example-robots-txt-short.js'); +const robots = require('../../src/index'); +const exampleRobotsShort = require('../test-data/example-robots-txt-short'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/robots/use-robots-for.js b/test/robots/use-robots-for.js index d0e3b20..e959b9f 100644 --- a/test/robots/use-robots-for.js +++ b/test/robots/use-robots-for.js @@ -1,5 +1,5 @@ const chai = require('chai'); -const robots = require('../../src/index.js'); +const robots = require('../../src/index'); const { expect } = chai; const robotsParser = robots(); diff --git a/test/util/format-link.js b/test/util/format-link.js index 898ea4a..b56de0f 100644 --- a/test/util/format-link.js +++ b/test/util/format-link.js @@ -1,5 +1,5 @@ const chai = require('chai'); -const util = require('../../src/util.js'); +const util = require('../../src/util'); const { expect } = chai; diff --git a/test/util/url-interactions.js b/test/util/url-interactions.js index 46f8ff7..d6d50bc 100644 --- a/test/util/url-interactions.js +++ b/test/util/url-interactions.js @@ -1,6 +1,6 @@ const chai = require('chai'); const url = require('fast-url-parser'); -const util = require('../../src/util.js'); +const util = require('../../src/util'); const noProtocol = ['bbc.co.uk', 'google.com/robots.txt', 'chaijs.com/api/bdd/', 'www.reddit.com/r/news', 'example.com/example/example.html']; @@ -26,7 +26,7 @@ describe('url-interaction', () => { describe('has-http-protocol', () => { it('Expect all links to have a http protocol.', () => { - hasHttpsProtocol.forEach((link) => { + hasHttpProtocol.forEach((link) => { const { protocol } = url.parse(link); expect(util.hasHttpProtocol(protocol)).to.be.true; });