Table/data driven tests

[raniejade]

The current recommended approach is by using listOf and nested listOfs for multiple rows.

listOf(
  listOf(1, 2, 3),
  listOf(1, 2, 3)
).forEach { data -> {
  ...
}

It has some major caveats:

• api is clunky
• uses list as a container for rows, so it's possible to have non-uniform row sizes.
• row values has no meaningful names, you access them via data[i] (although you can assign them to a local variable to give it a meaningful name).

Spek 1 has a data driven extension but it is tightly coupled into the DSL and is very opinionated to particular usage of Spek (on).

I'd like to bring some form of the Spek 1 plugin into 2.x:

withData(
  r(1, 2, 3),
  r(2, -1, 1)
) { (a, b, c) ->
  test("$a + $b == $c") { ... }
}

The whole implementation is the following:

interface RowN

data class Row1<A>(val a: A) : RowN
data class Row2<A, B>(val a: A, val b: B) : RowN
data class Row3<A, B, C>(val a: A, val b: B, val c: C) : RowN
data class Row4<A, B, C, D>(val a: A, val b: B, val c: C, val d: D) : RowN
data class Row5<A, B, C, D, E>(val a: A, val b: B, val c: C, val d: D, val e: E) : RowN
data class Row6<A, B, C, D, E, F>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F) : RowN
data class Row7<A, B, C, D, E, F, G>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G) : RowN
data class Row8<A, B, C, D, E, F, G, H>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H) : RowN
data class Row9<A, B, C, D, E, F, G, H, I>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I) : RowN
data class Row10<A, B, C, D, E, F, G, H, I, J>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J) : RowN
data class Row11<A, B, C, D, E, F, G, H, I, J, K>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K) : RowN
data class Row12<A, B, C, D, E, F, G, H, I, J, K, L>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L) : RowN
data class Row13<A, B, C, D, E, F, G, H, I, J, K, L, M>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M) : RowN
data class Row14<A, B, C, D, E, F, G, H, I, J, K, L, M, N>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N) : RowN
data class Row15<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O) : RowN
data class Row16<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P) : RowN
data class Row17<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q) : RowN
data class Row18<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R) : RowN
data class Row19<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S) : RowN
data class Row20<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T) : RowN
data class Row21<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U) : RowN
data class Row22<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V) : RowN
data class Row23<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W) : RowN
data class Row24<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X) : RowN
data class Row25<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X, val y: Y) : RowN
data class Row26<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z>(val a: A, val b: B, val c: C, val d: D, val e: E, val f: F, val g: G, val h: H, val i: I, val j: J, val k: K, val l: L, val m: M, val n: N, val o: O, val p: P, val q: Q, val r: R, val s: S, val t: T, val u: U, val v: V, val w: W, val x: X, val y: Y, val z: Z) : RowN


fun<A> r(a: A) = Row1(a)
fun<A, B> r(a: A, b: B) = Row2(a, b)
fun<A, B, C> r(a: A, b: B, c: C) = Row3(a, b, c)
fun<A, B, C, D> r(a: A, b: B, c: C, d: D) = Row4(a, b, c, d)
fun<A, B, C, D, E> r(a: A, b: B, c: C, d: D, e: E) = Row5(a, b, c, d, e)
fun<A, B, C, D, E, F> r(a: A, b: B, c: C, d: D, e: E, f: F) = Row6(a, b, c, d, e, f)
fun<A, B, C, D, E, F, G> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G) = Row7(a, b, c, d, e, f, g)
fun<A, B, C, D, E, F, G, H> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H) = Row8(a, b, c, d, e, f, g, h)
fun<A, B, C, D, E, F, G, H, I> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I) = Row9(a, b, c, d, e, f, g, h, i)
fun<A, B, C, D, E, F, G, H, I, J> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J) = Row10(a, b, c, d, e, f, g, h, i, j)
fun<A, B, C, D, E, F, G, H, I, J, K> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K) = Row11(a, b, c, d, e, f, g, h, i, j, k)
fun<A, B, C, D, E, F, G, H, I, J, K, L> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L) = Row12(a, b, c, d, e, f, g, h, i, j, k, l)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M) = Row13(a, b, c, d, e, f, g, h, i, j, k, l, m)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N) = Row14(a, b, c, d, e, f, g, h, i, j, k, l, m, n)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O) = Row15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P) = Row16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q) = Row17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R) = Row18(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S) = Row19(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T) = Row20(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U) = Row21(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V) = Row22(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W) = Row23(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X) = Row24(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X, y: Y) = Row25(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y)
fun<A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z> r(a: A, b: B, c: C, d: D, e: E, f: F, g: G, h: H, i: I, j: J, k: K, l: L, m: M, n: N, o: O, p: P, q: Q, r: R, s: S, t: T, u: U, v: V, w: W, x: X, y: Y, z: Z) = Row26(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q, r, s, t, u, v, w, x, y, z)

fun<T: RowN> withData(vararg rows: T, block: (T) -> Unit) {
    rows.forEach(block)
}

[raniejade]

Another option which is somewhat dynamic:

val x by memoized<Int>()
val y by memoized<Int>()
val expected by memoized<Int>()

it("$x + $y = $expected") {
    assertEqual(expected, x + y)
}

where(
    c("x",  "y", "expected"),
    r( 1 ,   2 ,         3 ),
    r( 2 ,   2 ,         4 ),
    r( 1 ,  -1 ,         0 ),
)

[raniejade]

It might be also possible to do:

with { (a, b, c) ->

} where (
    r(1, 2, 3),
    r(1, 2, 3)
)

// or
with { (a, b, c) ->

} where {
    r(1, 2, 3)
    r(1, 2, 3)
}

[schleifenkauz]

Which options do you consider the best at the moment? I would really like to contribute by implementing it.

[raniejade]

@NKb03 I have a local branch that I'm experimenting on. The approach I'm leaning on is the same as what we have in Spek 1.

withData(
  r(1, 2, 3),
  r(2, -1, 1)
) { (a, b, c) ->
  test("$a + $b == $c") { ... }
}

[schleifenkauz]

I actually like the approach, which has the lambda expression after the where the most, because with this approach the data would not have to be actually stored. The where function could be inline too, eliminating any performance overhead. The memory and performance improvement may be neglectable although.

[raniejade]

@NKb03 the where approach won't work unless you switch the ordering (do where first - then the with) as the compiler won't have enough information about what the actual parameter types are.

[ravenblackdusk]

I've been implementing data driven tests with spek for my project and I'd like to share my experience, so that the problems that I faced could perhaps be better handled in your design.

class FooIT : Spek({
    setup()
    Feature("f") {
        Scenario("s") {
            val setup by memoized<Setup>()
            equalize(
                listOf(1, 2, 3, 4, 5),
                listOf("a", "b", "c"),
                listOf(true, false),
                Iterable { iterator { yieldAll(setup.users) } }, // setup cannot be accessed here so it must be evaluated lazily
            ).forEach {
                lateinit var first: Serializable // lots of extra lateinit lines
                lateinit var second: Serializable
                lateinit var third: Serializable
                lateinit var fourth: Serializable
                Given("g") {
                    val iterator = it.iterator()
                    first = iterator.next()
                    second = iterator.next()
                    third = iterator.next()
                    fourth = iterator.next()
                    // setup cannot be accessed in the description so you have to do something like this, which is not so good
                    println("given first $first second $second third $third fourth $fourth")
                }
                When("w") {
                }
                Then("t") {
                }
            }
        }
    }
})

in short not being able to access memoized in scenario scope causes a lot of problems. this is equalize in case you are intrested:

fun <T> equalize(vararg iterables: Iterable<T>): Iterable<Sequence<T>> {
    val max = iterables.filterIsInstance<Collection<T>>().maxOf { it.size }
    return Iterable {
        object : Iterator<Sequence<T>> {
            var index = 0
            val iterators = iterables.map { it.iterator() }.toMutableList()
            override fun hasNext() = index < max
            override fun next() = sequence {
                val listIterator = iterators.listIterator()
                while (listIterator.hasNext()) {
                    val iterable = iterables[listIterator.nextIndex()]
                    val iterator = listIterator.next()
                    yield(
                        when {
                            iterable is List<T> -> iterable[index % iterable.size]
                            iterator.hasNext() -> iterator.next()
                            else -> {
                                val nextIterator = iterable.iterator()
                                listIterator.set(nextIterator)
                                nextIterator.next()
                            }
                        }
                    )
                }
            }.also { index++ }
        }
    }
}