robotstxt
robotstxt copied to clipboard
comments cause an exception
It looks like comments in a robots.txt cause an error to be thrown:
robots = require('robotstxt');
r = robots("http://findingaids.loc.gov/robots.txt", "test");
r.on("ready", function(gk) {
console.log(gk.isAllowed("http://findingaids.loc.gov/"));
});
which results in:
/home/ed/Projects/robots/node_modules/robotstxt/index.js:364
return myGateKeeper.comments.push({
^
TypeError: Cannot read property 'comments' of undefined
at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:364:30)
at /home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63
at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:373:9)
at RobotsTxt.parse (/home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63)
at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:235:25)
at IncomingMessage.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63)
at IncomingMessage.emit (events.js:88:20)
at HTTPParser.onMessageComplete (http.js:137:23)
at Socket.ondata (http.js:1147:24)
at TCP.onread (net.js:354:27)