mocha-steps
mocha-steps copied to clipboard
Published
20 hours ago •
rprieto
rprieto
Failed step causes previous steps to be marked "pending" as well as "passed"
When running the following test suite, the failed middle step causes all tests, including previously passed tests, to be marked "pending". This behavior seems unintentional, or at least ambiguous. Am I using this library correctly? Also, I can fix the JSON output programmatically by checking for test.pass && test.pending then setting test.pending = false, but fixing the mochawesome HTML report in the same way is annoying because the test results are embedded and url-encoded.
describe("Steps Sample", function () {
// This step succeeds
step('1 should equal 1', function () {
expect(1).to.equal(1);
});
// This step should fail
step('2 should equal 2', function () {
expect(2).to.equal(0);
});
// This step is skipped due to an earlier failure
step('3 should equal 3', function () {
expect(3).to.equal(3);
});
});
Here is the JSON result:
{
"stats": {
"suites": 1,
"tests": 3,
"passes": 1,
"pending": 1,
"failures": 1,
"start": "2021-04-16T00:57:36.700Z",
"end": "2021-04-16T00:57:36.714Z",
"duration": 14,
"testsRegistered": 3,
"passPercent": 50,
"pendingPercent": 33.33333333333333,
"other": 0,
"hasOther": false,
"skipped": 0,
"hasSkipped": false
},
"results": [
{
"uuid": "e0be6bcc-2868-429e-b26c-7395209fee91",
"title": "",
"fullFile": "",
"file": "",
"beforeHooks": [],
"afterHooks": [],
"tests": [],
"suites": [
{
"uuid": "55ff9b71-7e98-4549-9298-a82a56f26087",
"title": "Steps Sample",
"fullFile": "C:\\projects\\test\\index.ts",
"file": "\\test\\index.ts",
"beforeHooks": [],
"afterHooks": [],
"tests": [
{
"title": "1 should equal 1",
"fullTitle": "Steps Sample 1 should equal 1",
"timedOut": false,
"duration": 1,
"state": "passed",
"speed": "fast",
"pass": true, // expected
"fail": false,
"pending": true, // not expected
"context": null,
"code": "",
"err": {},
"uuid": "0b90cbec-2050-4932-b689-40e7544d4f7e",
"parentUUID": "55ff9b71-7e98-4549-9298-a82a56f26087",
"isHook": false,
"skipped": false
},
{
"title": "2 should equal 2",
"fullTitle": "Steps Sample 2 should equal 2",
"timedOut": false,
"duration": 0,
"state": "failed",
"speed": null,
"pass": false,
"fail": true,
"pending": false,
"context": null,
"code": "",
"err": {
"message": "AssertionError: expected 2 to equal 0",
"estack": "AssertionError: expected 2 to equal 0\n at Context.<anonymous> (test\\index.ts:22:22)\n at Context.sync (node_modules\\mocha-steps\\lib\\step.js:29:24)\n at processImmediate (internal/timers.js:456:21)\n at process.topLevelDomainCallback (domain.js:137:15)",
"diff": "- 2\n+ 0\n"
},
"uuid": "bf09e5ec-ce3c-48df-b88e-b75be50131b3",
"parentUUID": "55ff9b71-7e98-4549-9298-a82a56f26087",
"isHook": false,
"skipped": false
},
{
"title": "3 should equal 3",
"fullTitle": "Steps Sample 3 should equal 3",
"timedOut": false,
"duration": 0,
"state": "pending",
"speed": null,
"pass": false,
"fail": false,
"pending": true,
"context": null,
"code": "",
"err": {},
"uuid": "b598469d-daa7-40d5-a3dd-b65542318b8c",
"parentUUID": "55ff9b71-7e98-4549-9298-a82a56f26087",
"isHook": false,
"skipped": false
}
],
"suites": [],
"passes": [
"0b90cbec-2050-4932-b689-40e7544d4f7e"
],
"failures": [
"bf09e5ec-ce3c-48df-b88e-b75be50131b3"
],
"pending": [
"0b90cbec-2050-4932-b689-40e7544d4f7e", // not expected
"b598469d-daa7-40d5-a3dd-b65542318b8c"
],
"skipped": [],
"duration": 1,
"root": false,
"rootEmpty": false,
"_timeout": 2000
}
],
"passes": [],
"failures": [],
"pending": [],
"skipped": [],
"duration": 0,
"root": true,
"rootEmpty": true,
"_timeout": 2000
}
],
"meta": {
"mocha": {
"version": "8.3.2"
},
"mochawesome": {
"options": {
"quiet": false,
"reportFilename": "test-results",
"saveHtml": true,
"saveJson": true,
"consoleReporter": "spec",
"useInlineDiffs": false,
"code": false
},
"version": "6.2.2"
},
"marge": {
"options": {
"reportDir": "artifacts",
"reportFilename": "test-results",
"code": "false",
"charts": "true",
"reportTitle": "Tests",
"reportPageTitle": "Tests",
"saveJson": "true",
"html": "true"
},
"version": "5.2.0"
}
}
}
And here is my list of package versions in the project:
"dependencies": {
"chai": "^4.3.4",
"dotenv": "^8.2.0",
"mocha": "^8.3.2",
"mocha-steps": "^1.3.0",
"mochawesome": "^6.2.2",
"replace": "^1.2.1",
"supertest": "^6.1.3",
"ts-node": "^9.1.1",
"typescript": "^4.2.4"
},
"devDependencies": {
"@types/chai": "^4.2.16",
"@types/mocha": "^8.2.2",
"@types/node": "^14.14.37",
"@types/supertest": "^2.0.11"
}
It appears the issue is caused by the following lines in step.js:
console.log(tests.indexOf(currentTest); // prints '-1'
for (var index = tests.indexOf(currentTest) + 1; index < tests.length; index++) {
var test = tests[index];
test.pending = true;
}
In my case, I would expect tests.indexOf(currentTest) to equal 1 (since it was the second test out of three). Instead, the console log prints -1.
I added a console log to step.js which checks the index of the current test, and it seems like the failed test is triggering the step handler twice. Here is the code including my console log:
function sync() {
console.log(`~ Running test step: [${this.test.title}] checking for indexOf(this.test): [${this.test.parent.tests.indexOf(this.test)}]`);
var context = this;
try {
var promise = fn.call(context);
if (promise != null && promise.then != null && promise.catch != null) {
return promise.catch(function(err) {
markRemainingTestsAndSubSuitesAsPending(context.test);
throw err;
});
} else {
return promise;
}
} catch (ex) {
markRemainingTestsAndSubSuitesAsPending(context.test);
throw ex;
}
}
And here is the console output:
> mocha
Steps Sample
~ Running test step: [1 should equal 1] checking for indexOf(this.test): [0]
√ 1 should equal 1
~ Running test step: [2 should equal 2] checking for indexOf(this.test): [1]
~ Running test step: [2 should equal 2] checking for indexOf(this.test): [-1]
1) 2 should equal 2
- 3 should equal 3
1 passing (14ms)
1 pending
1 failing
I was able to solve my problem by explicitly setting this.retries(0); from within my describe block. I'm not sure why the second attempt at the test did not seem to get the right value from indexOf as shown above, and I was unaware in the first place that mocha was even supposed to be retrying my test cases by default.
Since I do not want anything retried this workaround is fine for me...but I suspect others may want to have retries, so hopefully someone sees this issue and has better luck determining root cause. I am also still unsure whether the issue lies within mocha-steps or if the root cause lies in some part of mocha itself.
For posterity, here is my working test:
describe("Steps Sample", function () {
this.retries(0);
step('1 should equal 1', function () {
expect(1).to.equal(1);
});
step('2 should equal 2', function () {
expect(2).to.equal(7);
});
step('3 should equal 3', function () {
expect(3).to.equal(3);
});
});
And the resulting JSON:
{
"stats": {
"suites": 1,
"tests": 3,
"passes": 1,
"pending": 1,
"failures": 1,
"start": "2021-04-16T02:25:20.153Z",
"end": "2021-04-16T02:25:20.165Z",
"duration": 12,
"testsRegistered": 3,
"passPercent": 50,
"pendingPercent": 33.33333333333333,
"other": 0,
"hasOther": false,
"skipped": 0,
"hasSkipped": false
},
"results": [
{
"uuid": "98af79fd-a919-4f36-92a9-6f7f6f1cb9a3",
"title": "",
"fullFile": "",
"file": "",
"beforeHooks": [],
"afterHooks": [],
"tests": [],
"suites": [
{
"uuid": "ee530de9-52ef-40fc-b6c5-99e5b6fa9827",
"title": "Steps Sample",
"fullFile": "C:\\projects\\test\\index.ts",
"file": "\\test\\index.ts",
"beforeHooks": [],
"afterHooks": [],
"tests": [
{
"title": "1 should equal 1",
"fullTitle": "Steps Sample 1 should equal 1",
"timedOut": false,
"duration": 1,
"state": "passed",
"speed": "fast",
"pass": true, // expected
"fail": false,
"pending": false, // much better!
"context": null,
"code": "",
"err": {},
"uuid": "b19b5e4f-1645-49ea-9e25-ad4f49dc29c0",
"parentUUID": "ee530de9-52ef-40fc-b6c5-99e5b6fa9827",
"isHook": false,
"skipped": false
},
{
"title": "2 should equal 2",
"fullTitle": "Steps Sample 2 should equal 2",
"timedOut": false,
"duration": 1,
"state": "failed",
"speed": null,
"pass": false,
"fail": true, // expected
"pending": false,
"context": null,
"code": "",
"err": {
"message": "AssertionError: expected 2 to equal 7",
"estack": "AssertionError: expected 2 to equal 7\n at Context.<anonymous> (test\\index.ts:22:22)\n at Context.sync (node_modules\\mocha-steps\\lib\\step.js:36:24)\n at processImmediate (internal/timers.js:456:21)\n at process.topLevelDomainCallback (domain.js:137:15)",
"diff": "- 2\n+ 7\n"
},
"uuid": "34ea4dce-9860-4040-a7d0-3d294e8fe03d",
"parentUUID": "ee530de9-52ef-40fc-b6c5-99e5b6fa9827",
"isHook": false,
"skipped": false
},
{
"title": "3 should equal 3",
"fullTitle": "Steps Sample 3 should equal 3",
"timedOut": false,
"duration": 0,
"state": "pending",
"speed": null,
"pass": false,
"fail": false,
"pending": true, // expected
"context": null,
"code": "",
"err": {},
"uuid": "a8b10a7e-c731-4ec4-b282-ff17ebb87a72",
"parentUUID": "ee530de9-52ef-40fc-b6c5-99e5b6fa9827",
"isHook": false,
"skipped": false
}
],
"suites": [],
"passes": [
"b19b5e4f-1645-49ea-9e25-ad4f49dc29c0"
],
"failures": [
"34ea4dce-9860-4040-a7d0-3d294e8fe03d"
],
"pending": [
"a8b10a7e-c731-4ec4-b282-ff17ebb87a72"
],
"skipped": [],
"duration": 2,
"root": false,
"rootEmpty": false,
"_timeout": 2000
}
],
"passes": [],
"failures": [],
"pending": [],
"skipped": [],
"duration": 0,
"root": true,
"rootEmpty": true,
"_timeout": 2000
}
],
"meta": {
"mocha": {
"version": "8.3.2"
},
"mochawesome": {
"options": {
"quiet": false,
"reportFilename": "test-results",
"saveHtml": true,
"saveJson": true,
"consoleReporter": "spec",
"useInlineDiffs": false,
"code": false
},
"version": "6.2.2"
},
"marge": {
"options": {
"reportDir": "artifacts",
"reportFilename": "test-results",
"code": "false",
"charts": "true",
"reportTitle": "Tests",
"reportPageTitle": "Tests",
"saveJson": "true",
"html": "true"
},
"version": "5.2.0"
}
}
}