Assertion failure in the hook is not treated as a failure

[tu1ly]

It is treated as error.

[smizell]

@tu1ly is this what you are seeing (see output below)?

API Blueprint

FORMAT: 1A

# Machines API

# Group Machines

# Machines collection [/machines]

## Get Machines [GET]

- Response 200 (application/json)

        [{"type": "bulldozer", "name": "willy"}]

Hook File (test_hooks.js)

var hooks = require('hooks');
var assert = require('assert');

hooks.before("Machines > Machines collection > Get Machines", function (transaction) {
  assert(false);
});

Server

var http = require('http');

http.createServer(function(request, response) {
  var body = [];
  request.on('data', function(chunk) {
    body.push(chunk);
  }).on('end', function() {
    console.log(request.url);
    response.statusCode = 200;
    response.setHeader('Content-Type', 'application/json');
    response.end('[{"type": "bulldozer", "name": "willy"}]');
  });
}).listen(3000);

Dredd Command and Output

$ dredd ./apiary.apib http://localhost:3000 --hookfiles=*_hooks.js
info: Beginning Dredd testing...
info: Found Hookfiles: test_hooks.js
error: GET /machines duration: NaNms
error: AssertionError: false == true
  at /Users/smizell/Sandbox/dredd-testing/test_hooks.js:5:3
  at TransactionRunner.runHook (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:314:9)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:151:28
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:246:17
  at iterate (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:146:13)
  at async.eachSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:162:9)
  at _asyncMap (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:245:13)
  at Object.mapSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:228:23)
  at Object.async.timesSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:1048:22)
  at TransactionRunner.runHooksForData (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:206:20)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:92:26
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:207:16
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:251:17
  at async.eachSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:142:20)
  at _asyncMap (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:245:13)
  at Object.mapSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:228:23)
  at Object.async.timesSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:1048:22)
  at TransactionRunner.runHooksForData (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:206:20)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:88:24
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:246:17
  at iterate (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:146:13)
  at async.eachSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:162:9)
  at _asyncMap (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:245:13)
  at Object.mapSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:228:23)
  at Object.async.timesSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:1048:22)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:86:22
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:207:16
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:251:17
  at async.eachSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:142:20)
  at _asyncMap (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:245:13)
  at Object.mapSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:228:23)
  at Object.async.timesSeries (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:1048:22)
  at TransactionRunner.runHooksForData (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:206:20)
  at TransactionRunner.executeAllTransactions (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:81:17)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:64:22
  at addHooks (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/add-hooks.js:142:16)
  at TransactionRunner.run (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/transaction-runner.js:59:12)
  at Dredd.startRunner (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:274:24)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:115:28
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:266:18
  at EventEmitter.<anonymous> (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/reporters/cli-reporter.js:24:14)
  at emitTwo (events.js:92:20)
  at EventEmitter.emit (events.js:172:7)
  at Dredd.emitStart (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:258:39)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:111:26
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:249:16
  at done (/Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:132:19)
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/async/lib/async.js:32:16
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/lib/dredd.js:241:18
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/dredd-transactions/lib/dredd-transactions.js:31:14
  at /Users/smizell/.nvm/versions/node/v5.0.0/lib/node_modules/dredd/node_modules/dredd-transactions/lib/parse.js:17:12

pass: GET /machines duration: 45ms
info: Displaying failed tests...
fail: GET /machines duration: NaNms
fail: Machines > Machines collection > Get Machines
request:
method: GET
uri: /machines
headers:
    User-Agent: Dredd/1.1.0-pre.0 (Darwin 15.3.0; x64)

body:



complete: 1 passing, 0 failing, 1 errors, 0 skipped, 1 total
complete: Tests took 59ms

[honzajavorek]

Related to https://github.com/apiaryio/dredd/issues/692

[artem-zakharchenko]

@honzajavorek could you please share more technical background on this?

I used to think error originates from a test assertion, and failure indicates any kind of test-unrelated exception happening during the test run.

[honzajavorek]

I used to think error originates from a test assertion, and failure indicates any kind of test-unrelated exception happening during the test run.

There are several ways the test can be failed, some work and some are buggy at the moment. I think https://github.com/apiaryio/dredd/issues/692#issuecomment-285015678 sums it well. I'm unsure what this particular issue was exactly about, @tu1ly's description is very brief and he didn't reply to @smizell's attempt to reproduce.

In my opinion, this issue can be closed if we focus on testing and fixing all the cases mentioned in https://github.com/apiaryio/dredd/issues/692#issuecomment-285015678. Collaterally I think that would fix any problem this issue might have been about.