alcotest
alcotest copied to clipboard
Should CLI filters override `filter` from `with_options`?
I'm disabling a set of tests that should run e.g., on Windows, using the filter parameter from 'a with_options. I've discovered that this filter parameter is overridden by the CLI arguments, and I think that's surprising. If the test suite is itself disabling some of its test cases, maybe they shouldn't be run even at all. For instance:
(* A module with functions to test *)
module To_test = struct
let lowercase = String.lowercase_ascii
let capitalize = String.capitalize_ascii
end
(* The tests *)
let test_lowercase () =
Alcotest.(check string) "same string" "hello!" (To_test.lowercase "hELLO!")
let test_capitalize () =
Alcotest.(check string) "same string" "World." (To_test.capitalize "world.")
(* Run it *)
let () =
let open Alcotest in
let filter ~name ~index =
if name = "string-case" && index = 0 then `Skip
else `Run
in
run ~filter "Utils" [
"string-case", [
test_case "Lower case" `Quick test_lowercase;
test_case "Capitalization" `Quick test_capitalize;
];
]
$ dune runtest
$ (cd _build/default && ./alcotest_test.exe test string-case)
Testing `Utils'.
This run has ID `8YC0XDXU'.
[OK] string-case 0 Lower case.
[OK] string-case 1 Capitalization.
Full test results in `~\Tarides\alcotest_test\_build\default\_build\_tests\8YC0XDXU.
Test Successful in 0.001s. 2 tests run.
IMO Lower case shouldn't be run. Arguably the user knows better and the cli overrides the code, but as a dev if I've disabled tests in the code, if I want to change that I'm going to change the code directly.