Is parameterised testing possible in Ceedling?

[TAGC]

I've seen that Unity has support for parameterised testing but there's very little documentation available.

I'm trying to create a parametised test like this:

#define TEST_CASE(...)
// ...
TEST_CASE(0)
TEST_CASE(10)
void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _)
{
  // ...
}

However, when I run ceedling, I get this error:

Test 'test_basicPwmControl.c'
-----------------------------
Creating mock for pwmHandler...
Creating mock for pwmHardware...
Creating mock for commsHandler...
Creating mock for versionHandler...
Generating runner for test_basicPwmControl.c...
Compiling test_basicPwmControl_runner.c...
build/test/runners/test_basicPwmControl_runner.c: In function ‘main’:
build/test/runners/test_basicPwmControl_runner.c:87:12: error: too few arguments to function ‘test_Device_Controller_Enables_PWM_When_Requested_By_PC’
   RUN_TEST(test_Device_Controller_Enables_PWM_When_Requested_By_PC, 44);
            ^
build/test/runners/test_basicPwmControl_runner.c:14:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(); \
       ^~~~~~~~
build/test/runners/test_basicPwmControl_runner.c:42:13: note: declared here
 extern void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _);
             ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ERROR: Shell command failed.
> Shell executed command:
'gcc.exe -std=c99 -I"test" -I"test/support" -I"src" -I"src/c" -I"src/include" -I"lib" -I"lib/c" -I"lib/include" -I"C:/Users/<me>/Documents/IAR Embedded Workspace/testcube-firmware-v2/vendor/ceedling/vendor/unity/src" -I"C:/Users/<me>/Documents/IAR Embedded Workspace/testcube-firmware-v2/vendor/ceedling/vendor/cmock/src" -I"build/test/mocks" -DTEST -DUNITY_SUPPORT_TEST_CASES -DGNU_COMPILER -g -c "build/test/runners/test_basicPwmControl_runner.c" -o "build/test/out/test_basicPwmControl_runner.o"'
> And exited with status: [1].

This is the output from ceedling version:

Ceedling:: 0.28.2
CException:: 1.3.1.18
     CMock:: 2.4.4.215
     Unity:: 2.4.1.120

This is my project.yml file:

:project:
  :use_exceptions: FALSE
  :use_test_preprocessor: TRUE
  :use_auxiliary_dependencies: TRUE
  :build_root: build
  :release_build: FALSE
  :test_file_prefix: test_

:extension:
  :executable: .out

:flags:
  :test:
    :compile:
      :*:
        - -std=c99
        
:paths:
  :test:
    - +:test/**
    - -:test/support
  :source:
    - src/**
    - lib/**
  :support:
    - test/support

:defines:
  :commmon: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

:cmock:
  :mock_prefix: "mock_"
  :when_no_prototypes: :warn
  :enforce_strict_ordering: TRUE
  :plugins:
    - :ignore
    - :callback
    - :expect_any_args
    - :array
    - :return_thru_ptr
  :treat_as:
    uint8:    HEX8
    uint16:   HEX16
    uint32:   UINT32
    int8:     INT8
    bool:     UINT8

# LIBRARIES
# These libraries are automatically injected into the build process. Those specified as
# common will be used in all types of builds. Otherwise, libraries can be injected in just
# tests or releases. These options are MERGED with the options in supplemental yaml files.
:libraries:
  :placement: :end
  :flag: "${1}"  # or "-L ${1}" for example
  :common: &common_libraries []
  :test:
    - *common_libraries
  :release:
    - *common_libraries

:plugins:
  :load_paths:
    - vendor/ceedling/plugins
  :enabled:
    - stdout_pretty_tests_report
    - module_generator

[TAGC]

I've tried adding the following to my project.yml

:unity:
  :use_param_tests: true

Now the error is a little different:

Compiling test_basicPwmControl_runner.c...
build/test/runners/test_basicPwmControl_runner.c: In function ‘main’:
build/test/runners/test_basicPwmControl_runner.c:88:12: error: too few arguments to function ‘test_Device_Controller_Enables_PWM_When_Requested_By_PC’
   RUN_TEST(test_Device_Controller_Enables_PWM_When_Requested_By_PC, 44, RUN_TEST_NO_ARGS);
            ^
build/test/runners/test_basicPwmControl_runner.c:15:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(__VA_ARGS__); \
       ^~~~~~~~
build/test/runners/test_basicPwmControl_runner.c:43:13: note: declared here
 extern void test_Device_Controller_Enables_PWM_When_Requested_By_PC(uint32_t _);

[mvandervoord]

Try adding that same line to the :test_runner: section as well. :)

[TAGC]

You mean like this?

...
:unity:
  :use_param_tests: true
  
:test_runner:
  :use_param_tests: true
...

I still get the same error.

[mvandervoord]

Hm. I'm surprised.

(1) I'd force everything to rebuild by issuing a rake clobber. (2) If there is still a problem, can you post what the RUN_TEST macro looks like in your build/test/runners/test_basicPwmControl_runner.c file?

[TAGC]

ceedling clobber / rake clobber doesn't have any effect.

Given this project.yml:

:unity:
  :use_param_tests: true
  
:test_runner:
  :use_param_tests: true

...

:defines:
  :commmon: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

And this test file:

#include "unity.h"

#define TEST_CASE(...)

void setUp(void)
{
}

void tearDown(void)
{
}

TEST_CASE(1)
TEST_CASE(2)
void test_Foo(uint32_t _)
{
}

Ceedling generates this runner:

/* AUTOGENERATED FILE. DO NOT EDIT. */

/*=======Test Runner Used To Run Each Test Below=====*/
#define RUN_TEST_NO_ARGS
#define RUN_TEST(TestFunc, TestLineNum, ...) \
{ \
  Unity.CurrentTestName = #TestFunc "(" #__VA_ARGS__ ")"; \
  Unity.CurrentTestLineNumber = TestLineNum; \
  Unity.NumberOfTests++; \
  if (TEST_PROTECT()) \
  { \
      setUp(); \
      TestFunc(__VA_ARGS__); \
  } \
  if (TEST_PROTECT()) \
  { \
    tearDown(); \
  } \
  UnityConcludeTest(); \
}

/*=======Automagically Detected Files To Include=====*/
#include "unity.h"
#include <setjmp.h>
#include <stdio.h>

int GlobalExpectCount;
int GlobalVerifyOrder;
char* GlobalOrderError;

/*=======External Functions This Runner Calls=====*/
extern void setUp(void);
extern void tearDown(void);
extern void test_Foo(uint32_t _);


/*=======Test Reset Option=====*/
void resetTest(void);
void resetTest(void)
{
  tearDown();
  setUp();
}


/*=======MAIN=====*/
int main(void)
{
  UnityBegin("test_temp.c");
  RUN_TEST(test_Foo, 15, RUN_TEST_NO_ARGS);

  return (UnityEnd());
}

And outputs this error:

Compiling test_temp_runner.c...
build/test/runners/test_temp_runner.c: In function ‘main’:
build/test/runners/test_temp_runner.c:50:12: error: too few arguments to function ‘test_Foo’
   RUN_TEST(test_Foo, 15, RUN_TEST_NO_ARGS);
            ^
build/test/runners/test_temp_runner.c:13:7: note: in definition of macro ‘RUN_TEST’
       TestFunc(__VA_ARGS__); \
       ^~~~~~~~
build/test/runners/test_temp_runner.c:34:13: note: declared here
 extern void test_Foo(uint32_t _);

[mvandervoord]

This definitely seems like a bug in the runner generator. Thanks for building a minimal example to reproduce it.

[austinglaser]

I believe I've run into this issue before.

A workaround

The problem I encountered is when setting

:project:
  # ...
  :use_test_preprocessor: TRUE

In this case, the preprocessor expands the TEST_CASE() macro away before the runner generator gets ahold of it. I made the following definition in a test-support header file:

#if defined(TEST_PP)
#   define TEST_VALUE(...) TEST_CASE(__VA_ARGS__)
#else
#   define TEST_VALUE(...)
#endif

and then added to my project.yml:

:defines:
  # ...
  :test_preprocess:
    # ...
    - TEST_PP

The reason to do things this way (rather than just disabling preprocessing) is that

• There may be other good reasons to do preprocessing
• Without preprocessing, parametrized tests like

#define A_LOCAL_CONSTANT 5

TEST_CASE(A_LOCAL_CONSTANT)
void test_works(uint32_t v)
{
    /* ... */
}

will fail due to A_LOCAL_CONSTANT being undefined in the runner.

An opinionated rant

With all that said, I still feel that Unity's parametrized testing story is extremely weak. Consider, for instance, this case (assuming the availability of the definition above):

#include <stdint.h>

TEST_VALUE(UINT32_MAX)
void test_works(uint32_t v)
{
    /* ... */
}

This won't match, because the TEST_VALUE definition expands to something like

TEST_CASE(
(4294967295U)
)

which the runner generator won't match. This can cause serious issues if, for instance, you define a test like

#include <stdint.h>

TEST_VALUE(2)
TEST_VALUE(3)
TEST_VALUE(UINT32_MAX)
TEST_VALUE(0)
void test_works(uint32_t v)
{
    /* ... */
}

where the preprocessor will silently ignore everything before the TEST_VALUE(UINT32_MAX), and only run the test with the value 0.

Of course, the current approach has ergonomic problems as well:

• It's impossible to re-use a list of parameters for multiple tests, which can present a maintainability problem
• It's extremely difficult to pass struct values, since C has little concept of a struct "literal"
• Editors get confused by the non-semicolon-terminated lines

What to do about it

I'm interested in helping to build better parametrized testing into Unity and Ceedling, but it probably makes sense to discuss first what the best approach to take would be, and if it has a chance to be merged in (given considerations like backwards compatibility, etc).

My first idea is to take a fixture-like approach, which defines an array to be passed in element-by-element. It might look something like

/*
 * TEST_FIXTURE causes the runner to put an extern definition in the runner.
 * Alternatively, to match the current convention for tests, this could just be
 * a naming convention like uint32_t fixture_invalid_values[] = { ... };
 */
TEST_FIXTURE(uint32_t invalid_values[] = {
    0,
    UINT32_MAX,
    5,
    100,
});

/*
 * WITH_FIXTURE works similarly to the current TEST_CASE, but:
 *
 * -    Sidesteps the preprocessing question, since it takes an actual language
 *      symbol rather than a constant
 * -    Brings along a whole set of test-case values, improving re-usability
 *  -   Would allow a trailing semicolon
 */
WITH_FIXTURE(invalid_values);
void test_catches_bad_values(uint32_t val)
{
}

/*
 * WITH_FIXTURE would allow multiple fixtures, and give all possible
 * combinations (the cross product) of all values.
 */
WITH_FIXTURE(invalid_vals_1, invalid_names);
void test_handles_all_bad_things(uint32_t val, char const * name)
{
}

I can open an issue for further discussion if you're at all open to something like this, Mark

[Miv13]

I get the same error as @TAGC even with @austinglaser 's workaround. Any news on this issue?

[joel-felcana]

Hi,

I'm trying to use this in my tests, but I can't make it work, and I can't find any documentation about it.

I created a new test with the latest ceedling:

$ ceedling version
   Ceedling:: 0.29.1
      Unity:: 2.5.0
      CMock:: 2.5.1
 CException:: 1.3.2

My test is:

#include <unity.h>

#include <stdio.h>

#define TEST_CASE(...)

/*
 */

void setUp(void) {
}

void tearDown(void) {
}

TEST_CASE(1)
TEST_CASE(100)
void test_parameterised(int num) {
    printf("Num is %d\n", num);
    TEST_ASSERT_EQUAL(1, num);
}

What I expect to get is:

• 1 test passed and one failed
• Two printfs, one with Num is 1, and one with Num is 100.

What I get? One of the two options:

1. Either only one test is run, and num contains dirty memory:

Test 'test_parameterised.c'
---------------------------
Running test_parameterised.out...

-----------
TEST OUTPUT
-----------
[test_parameterised.c]
  - "Num is 4265064"

-------------------
FAILED TEST SUMMARY
-------------------
[test_parameterised.c]
  Test: test_parameterised
  At line (20): "Expected 1 Was 4265064"

--------------------
OVERALL TEST SUMMARY
--------------------
TESTED:  1
PASSED:  0
FAILED:  1
IGNORED: 0

---------------------
BUILD FAILURE SUMMARY
---------------------
Unit test failures.

2. Ceedling complainign that my test function is not a void func(void) type of function

$ ceedling test


Test 'test_parameterised.c'
---------------------------
Generating runner for test_parameterised.c...
Compiling test_parameterised_runner.c...
build/test/runners/test_parameterised_runner.c: In function 'main':
build/test/runners/test_parameterised_runner.c:82:12: warning: passing argument 1 of 'run_test' from incompatible pointer type [-Wincompatible-pointer-types]
   82 |   run_test(test_parameterised, "test_parameterised", 18);
      |            ^~~~~~~~~~~~~~~~~~
      |            |
      |            void (*)(int)
build/test/runners/test_parameterised_runner.c:46:40: note: expected 'UnityTestFunction' {aka 'void (*)(void)'} but argument is of type 'void (*)(int)'
   46 | static void run_test(UnityTestFunction func, const char* name, int line_num)
      |                      ~~~~~~~~~~~~~~~~~~^~~~
Compiling test_parameterised.c...
Compiling unity.c...
Compiling cmock.c...
Linking test_parameterised.out...
Running test_parameterised.out...

-----------
TEST OUTPUT
-----------
[test_parameterised.c]
  - "Num is 4265064"

-------------------
FAILED TEST SUMMARY
-------------------
[test_parameterised.c]
  Test: test_parameterised
  At line (20): "Expected 1 Was 4265064"

--------------------
OVERALL TEST SUMMARY
--------------------
TESTED:  1
PASSED:  0
FAILED:  1
IGNORED: 0

---------------------
BUILD FAILURE SUMMARY
---------------------
Unit test failures.

My project file is the standard, as generated by ceedling. The funny thing is that the result seems to be totally random. Sometimes I get the complaint about the func type, sometimes it runs. I tried adding

:unity:
  :use_param_tests: true
:cmock:
  :use_param_tests: true
:test_runner:
  :use_param_tests: true

And

:defines:
  # in order to add common defines:
  #  1) remove the trailing [] from the :common: section
  #  2) add entries to the :common: section (e.g. :test: has TEST defined)
  :common: &common_defines []
  :test:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES
  :test_preprocess:
    - *common_defines
    - TEST
    - UNITY_SUPPORT_TEST_CASES

But nothing makes the parameterised test run. Sometimes adding something makes it run, sometimes breaks it... it's not like a config works 100% all the time, and changing the project makes it work or not.... For example, the first time I run it with the default config it failed, then adding the unity option made it work, then adding something else broke it, and then deleting everything and coming back to the default, which didn't work in the first place, then worked. I run clean after each config change.

[fj-sanchez]

I'm not sure if this is still relevant after https://github.com/ThrowTheSwitch/Ceedling/pull/497, I have managed to get it working by setting this in my project.yml:

:project:
  :use_test_preprocessor: TRUE
  :use_preprocessor_directives: TRUE

:unity:
  :use_param_tests: true

And then, in my test files I had to add this:

#define TEST_CASE(...)
#define TEST_RANGE(...)

[pez3]

hi there... i'm trying to write some parameterized unit tests with unity. is there the possiblility to hand over arrays via TEST_CASE ?

thank you for your help greets peter

[lapo4719]

I got TEST_CASE() TEST_RANGE() and TEST_MATRIX() working with this setup!

1. I installed Ceedling 0.31.0 (latest as of today) and did a local install 2. ceedling new your_project --local
2. I did local install so I could swap the Unity vendor/ folder (both the auto/ and src/ code subfolders) with the latest Unity source v1.6.0 code which has support for these macros. You'll need the new Unity ruby scripts and the new Unity C code from that git repo.

• Then to my project.yml I added many of the things in these posts above and flag/#defines I found in the latest Unity documentation:

:project:
  :use_exceptions: FALSE
  :use_test_preprocessor: TRUE
  :use_preprocessor_directives: TRUE # ADD THIS
  :use_auxiliary_dependencies: TRUE # ADD THIS
  :build_root: build
#  :release_build: TRUE
  :test_file_prefix: test_
  :which_ceedling: vendor/ceedling
  :ceedling_version: 0.31.1
  :default_tasks:
    - test:all

:unity: # ADD THIS
  :use_param_tests: true
  :includes:
    - parametric_macros.h # THIS HAS MY #defines for the TEST_xxx() macros, I have added the file below
  
:test_runner: # ADD THIS
  :use_param_tests: true

#:test_build:
#  :use_assembly: TRUE

#:release_build:
#  :output: MyApp.out
#  :use_assembly: FALSE

:environment:

:extension:
  :executable: .out

:paths:
  :test:
    - +:test/unit_tests/**
    - -:test/support
  :source:
    - src/**
  :support:
    - test/support
  :libraries: []

:defines:
  # in order to add common defines:
  #  1) remove the trailing [] from the :common: section
  #  2) add entries to the :common: section (e.g. :test: has TEST defined)
  :common: &common_defines []
  :test:
    - *common_defines
    - CEEDLING_UNIT_TEST # I use this instead of normal TEST. Doesn't matter
    - UNITY_SUPPORT_TEST_CASES # ADD THIS!
  :test_preprocess:
    - *common_defines
    - CEEDLING_UNIT_TEST
    - UNITY_SUPPORT_TEST_CASES # # ADD THIS!

:cmock:
  :mock_prefix: mock_
  :when_no_prototypes: :warn
  :enforce_strict_ordering: TRUE
  :plugins:
    - :ignore
    - :callback
  :treat_as:
    uint8:    HEX8
    uint16:   HEX16
    uint32:   UINT32
    int8:     INT8
    bool:     UINT8

# Add -gcov to the plugins list to make sure of the gcov plugin
# You will need to have gcov and gcovr both installed to make it work.
# For more information on these options, see docs in plugins/gcov
:gcov:
  :reports:
    - HtmlDetailed
  :gcovr:
    :html_medium_threshold: 75
    :html_high_threshold: 90

#:tools:
# Ceedling defaults to using gcc for compiling, linking, etc.
# As [:tools] is blank, gcc will be used (so long as it's in your system path)
# See documentation to configure a given toolchain for use

# LIBRARIES
# These libraries are automatically injected into the build process. Those specified as
# common will be used in all types of builds. Otherwise, libraries can be injected in just
# tests or releases. These options are MERGED with the options in supplemental yaml files.
:libraries:
  :placement: :end
  :flag: "-l${1}"
  :path_flag: "-L ${1}"
  :system: []    # for example, you might list 'm' to grab the math library
  :test: []
  :release: []

:plugins:
  :load_paths:
    - vendor/ceedling/plugins
  :enabled:
    - stdout_pretty_tests_report
    - module_generator
    - raw_output_report
    - xml_tests_report

Maybe there are too many flags used here, but together, and with this parametrics_macro.h file, it lets me do use parametric features:

// parametric_macros.h
#ifndef PARAMETRIC_MACROS_H
#define PARAMETRIC_MACROS_H
// import this to use TEST_CASE() AND TEST_RANGE() -- these are unity features that ceedling supports
// unity documentation for more details: https://github.com/ThrowTheSwitch/Unity/blob/master/docs/UnityHelperScriptsGuide.md

#ifndef TEST_CASE
#define TEST_CASE(...)
#endif  // TEST_CASE


// THESE ONLY WORK W/ UNITY > V1.6.0
#ifndef TEST_RANGE
#define TEST_RANGE(...)
#endif  // TEST_RANGE

#ifndef TEST_MATRIX
#define TEST_MATRIX(...)
#endif  // TEST_MATRIX

#endif // PARAMETRIC_MACROS_H

You Must have the setUp() and tearDown() function definitions, for some reason they were getting screwed up in the generated runner if I didn't add them myself. But no big deal just leave them empty.

// /****************************************
// 	File Name: Tests
// *************************************** */
#include <stdlib.h>
#include "unity.h"
#include "parametric_macros.h"
#include "stdbool.h"


void setUp(void) {

}

void tearDown(void) {

}


void test_hello_world(void)
{
    bool myvar = true;
    TEST_ASSERT_EQUAL(myvar, true);
}

TEST_CASE(1)
TEST_CASE(3)
void test_parametric_arguments(uint8_t my_param)
{
    printf("testing parametric arguments: %d", my_param);
    TEST_ASSERT_TRUE_MESSAGE(my_param, "I print if assert is false");
}

TEST_RANGE([0, 0, 1])
void test_parametric_arguments_inclusive_range(uint8_t my_param)
{
    printf("testing parametric arguments: %d", my_param);

	// make a little debug message:
	char *debug_msg = (char*)malloc(128 * sizeof(char));
	sprintf(debug_msg, "my_param: %d", my_param);

	// assert:
    TEST_ASSERT_TRUE_MESSAGE(my_param == 0, debug_msg);
}
TEST_RANGE(<0,1,1>)
void test_parametric_arguments_exclusive_range(uint8_t my_param)
{
	// make a little debug message:
	char *debug_msg = (char*)malloc(128 * sizeof(char));
	sprintf(debug_msg, "my_param: %d", my_param);

    TEST_ASSERT_TRUE_MESSAGE(my_param < 1, debug_msg);
}

TEST_MATRIX([3, 4, 7])
void test_parametric_arguments_with_test_matrix(uint8_t my_param)
{
	// make a little debug message:
	char *debug_msg = (char*)malloc(128 * sizeof(char));
	sprintf(debug_msg, "my_param: %d", my_param);

	printf("I AM HERE! Running parametric MATRIX tests");

    TEST_ASSERT_TRUE_MESSAGE(my_param < 8, debug_msg);
}

Output on command line:

>> ceedling verbosity[3] test:all

Test 'test_hello_world.c'
-------------------------
Generating runner for test_hello_world.c...
Compiling test_hello_world_runner.c...
Compiling test_hello_world.c...
Linking test_hello_world.out...
Running test_hello_world.out...


Test 'test_hello_world_cmock.c'
-------------------------------
Generating include list for hello_world.h...
Creating mock for hello_world...
Generating runner for test_hello_world_cmock.c...
Compiling test_hello_world_cmock_runner.c...
Compiling test_hello_world_cmock.c...
Compiling mock_hello_world.c...
Linking test_hello_world_cmock.out...
Running test_hello_world_cmock.out...


Test 'test_hello_world_parametric.c'
------------------------------------
Generating runner for test_hello_world_parametric.c...
Compiling test_hello_world_parametric_runner.c...
Compiling test_hello_world_parametric.c...
Linking test_hello_world_parametric.out...
Running test_hello_world_parametric.out...

-----------
TEST OUTPUT
-----------
[test_hello_world_parametric.c]
  - "testing parametric arguments: 1"
  - "testing parametric arguments: 3"
  - "testing parametric arguments: 0"
  - "I AM HERE AND RUNNING"
  - "I AM HERE AND RUNNING"
  - "I AM HERE AND RUNNING"

--------------------
OVERALL TEST SUMMARY
--------------------
TESTED:  11
PASSED:  10
FAILED:   0
IGNORED:  1