robotstxt icon indicating copy to clipboard operation
robotstxt copied to clipboard

comments cause an exception

Open edsu opened this issue 13 years ago • 0 comments

It looks like comments in a robots.txt cause an error to be thrown:

robots = require('robotstxt');

r = robots("http://findingaids.loc.gov/robots.txt", "test");
r.on("ready", function(gk) {
  console.log(gk.isAllowed("http://findingaids.loc.gov/"));
});

which results in:

/home/ed/Projects/robots/node_modules/robotstxt/index.js:364
          return myGateKeeper.comments.push({
                             ^
TypeError: Cannot read property 'comments' of undefined
    at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:364:30)
    at /home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63
    at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:373:9)
    at RobotsTxt.parse (/home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63)
    at RobotsTxt.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:235:25)
    at IncomingMessage.<anonymous> (/home/ed/Projects/robots/node_modules/robotstxt/index.js:3:63)
    at IncomingMessage.emit (events.js:88:20)
    at HTTPParser.onMessageComplete (http.js:137:23)
    at Socket.ondata (http.js:1147:24)
    at TCP.onread (net.js:354:27)

edsu avatar Nov 18 '12 00:11 edsu