Skip to content

chore: enforce linting #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

chore: enforce linting #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@
"test": "test"
},
"scripts": {
"test": "mocha test/**/*.js",
"test": "npm run lint && mocha test/**/*.js",
"test-robots": "mocha test/robots/*.js",
"test-parser": "mocha test/parser/*.js",
"test-util": "mocha test/util/*.js",
"coverage": "nyc --reporter=lcov mocha test/**/*.js --timeout 10s",
"local-coverage": "nyc mocha test/**/*.js"
"local-coverage": "nyc mocha test/**/*.js",
"lint": "eslint src test"
},
"repository": {
"type": "git",
Expand Down
2 changes: 1 addition & 1 deletion src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
const Robots = require('./robots.js');
const Robots = require('./robots');

module.exports = (opts) => new Robots(opts);
7 changes: 3 additions & 4 deletions src/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ const cleanSpaces = (rawString) => rawString.replace(whitespace, '').trim();

const splitOnLines = (string) => string.split(lineEndings);

const robustSplit = (string) => {
return !string.includes('<html>') ? [...string.match(recordSlices)].map(cleanSpaces) : [];
};
const robustSplit = (string) => (!string.includes('<html>') ? [...string.match(recordSlices)].map(cleanSpaces) : []);

const parseRecord = (line) => {
// Find first colon and assume is the field delimiter.
Expand Down Expand Up @@ -70,7 +68,7 @@ const parser = (rawString) => {
lines.forEach((line) => {
const record = parseRecord(line);
switch (record.field) {
case USER_AGENT:
case USER_AGENT: {
const recordValue = record.value.toLowerCase();
if (recordValue !== agent && recordValue.length > 0) {
// Bot names are non-case sensitive.
Expand All @@ -84,6 +82,7 @@ const parser = (rawString) => {
agent = '';
}
break;
}
// https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt#order-of-precedence-for-group-member-records
case ALLOW:
if (agent.length > 0 && record.value.length > 0) {
Expand Down
10 changes: 5 additions & 5 deletions src/robots.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const get = require('./get.js');
const parser = require('./parser.js');
const util = require('./util.js');
const get = require('./get');
const parser = require('./parser');
const util = require('./util');

const DFLT_OPTS = {
userAgent: '*',
Expand All @@ -25,7 +25,7 @@ class Robots {
const otherBots = '*' in domainBots;
if (ourBotInBots) {
return domainBots[this.opts.userAgent];
} else if (otherBots) {
} if (otherBots) {
return domainBots['*'];
}
return false;
Expand All @@ -44,7 +44,7 @@ class Robots {

if (noDisallows || (allow.maxSpecificity > disallow.maxSpecificity)) {
return true;
} else if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) {
} if (noAllows || (allow.maxSpecificity < disallow.maxSpecificity)) {
return false;
}
return this.opts.allowOnNeutral;
Expand Down
8 changes: 8 additions & 0 deletions test/.eslintrc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"env": {
"mocha": true
},
"rules": {
"no-unused-expressions": 0
}
}
4 changes: 2 additions & 2 deletions test/parser/can-parse-allow.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const allowRobots = require('../test-data/example-robots-txt-allow.js');
const parse = require('../../src/parser.js');
const allowRobots = require('../test-data/example-robots-txt-allow');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/parser/can-parse-crawl-delay.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const parse = require('../../src/parser.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/parser/can-parse-disallow.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const disallowRobots = require('../test-data/example-robots-txt-disallow.js');
const parse = require('../../src/parser.js');
const disallowRobots = require('../test-data/example-robots-txt-disallow');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
2 changes: 1 addition & 1 deletion test/parser/can-parse-sitemaps.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const chai = require('chai');
const uniq = require('lodash/uniq');
const sitemaps = require('../test-data/example-robots-txt-sitemaps');
const parse = require('../../src/parser.js');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
12 changes: 6 additions & 6 deletions test/parser/can-parse-test-files.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
const chai = require('chai');
const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc.js');
const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc.js');
const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando.js');
const parse = require('../../src/parser.js');
const exampleRobotsBBC = require('../test-data/example-robots-txt-bbc');
const exampleRobotsBcc = require('../test-data/example-robots-txt-bcc');
const exampleRobotsKarwei = require('../test-data/example-robots-txt-karwei');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');
const exampleRobotsZalando = require('../test-data/example-robots-txt-zalando');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/parser/can-parse-user-agents.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const parse = require('../../src/parser.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/parser/correct-parse-formatting.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const exampleShort = require('../test-data/example-robots-txt-short.js');
const parse = require('../../src/parser.js');
const exampleShort = require('../test-data/example-robots-txt-short');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/parser/ignores-malformed-values.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short.js');
const parse = require('../../src/parser.js');
const exampleRobotsMalformed = require('../test-data/example-robots-malformed-short');
const parse = require('../../src/parser');

const { expect } = chai;

Expand Down
10 changes: 5 additions & 5 deletions test/robots/can-crawl.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const parser = require('../../src/parser.js');
const util = require('../../src/util.js');
const testData = require('../test-data/can-crawl-test-data.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const parser = require('../../src/parser');
const util = require('../../src/util');
const testData = require('../test-data/can-crawl-test-data');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
2 changes: 1 addition & 1 deletion test/robots/fetch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const robots = require('../../src/index');

const { expect } = chai;
const robotsParser = robots();
Expand Down
6 changes: 3 additions & 3 deletions test/robots/get-crawl-delay.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const parser = require('../../src/parser.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const parser = require('../../src/parser');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
4 changes: 2 additions & 2 deletions test/robots/get-crawlable-links.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
4 changes: 2 additions & 2 deletions test/robots/get-host.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
6 changes: 3 additions & 3 deletions test/robots/get-sitemaps.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const parser = require('../../src/parser.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const parser = require('../../src/parser');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
4 changes: 2 additions & 2 deletions test/robots/is-cached.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const exampleRobotsShort = require('../test-data/example-robots-txt-short.js');
const robots = require('../../src/index');
const exampleRobotsShort = require('../test-data/example-robots-txt-short');

const { expect } = chai;
const robotsParser = robots();
Expand Down
2 changes: 1 addition & 1 deletion test/robots/use-robots-for.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const chai = require('chai');
const robots = require('../../src/index.js');
const robots = require('../../src/index');

const { expect } = chai;
const robotsParser = robots();
Expand Down
2 changes: 1 addition & 1 deletion test/util/format-link.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const chai = require('chai');
const util = require('../../src/util.js');
const util = require('../../src/util');

const { expect } = chai;

Expand Down
4 changes: 2 additions & 2 deletions test/util/url-interactions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const chai = require('chai');
const url = require('fast-url-parser');
const util = require('../../src/util.js');
const util = require('../../src/util');

const noProtocol = ['bbc.co.uk', 'google.com/robots.txt', 'chaijs.com/api/bdd/', 'www.reddit.com/r/news',
'example.com/example/example.html'];
Expand All @@ -26,7 +26,7 @@ describe('url-interaction', () => {

describe('has-http-protocol', () => {
it('Expect all links to have a http protocol.', () => {
hasHttpsProtocol.forEach((link) => {
hasHttpProtocol.forEach((link) => {
const { protocol } = url.parse(link);
expect(util.hasHttpProtocol(protocol)).to.be.true;
});
Expand Down