Memory issues

[epolack]

When using TestItemRunner interactively, there are memory issues that I cannot solve.

@testitem "tt" begin
    a = ones(1000, 1000, 1000)
end

If I run @run_package_tests multiple times, then the memory fills up to crashing the computer. To keep it low, I need to manually set the variable to nothing:

@testitem "tt" begin
    a = ones(1000, 1000, 1000)
    a = nothing
end

This does not happen for example with

using Test
@testset "tt" begin
    a = ones(1000, 1000, 1000)
end

[davidanthoff]

We create a new module every time you run a test, and I think they stick around... I think (but not sure) there is no way in Julia to unload a module, but I might be wrong. Maybe we could do something to iterate over all global names in a module that is no longer needed and at least set all mutable global variables to something like nothing so that some memory is freed? Not sure...

[epolack]

This seems to work:

using TestItemRunner

function TestItemRunner.run_testitem(filepath, use_default_usings, setups, package_name, original_code, line, column, test_setup_module_set)
    # with fixed module name (no gensym)
    mod = Core.eval(Main, :(module AA end))

    if use_default_usings
        Core.eval(mod, :(using Test))

        if package_name!=""
            Core.eval(mod, :(using $(Symbol(package_name))))
        end
    end

    for m in setups
        Core.eval(mod, Expr(:using, Expr(:., :., :., nameof(test_setup_module_set.setupmodule), m)))
    end

    code = string('\n'^line, ' '^column, original_code)

    TestItemRunner.withpath(filepath) do
        Base.invokelatest(include_string, mod, code, filepath)
    end

    for name in names(Main.AA; all=true)
        contains("$(name)", "#") && continue
        eval(:(Main.AA.$name isa Function ? Base.delete_method.(methods(Main.AA.$name)) : Main.AA.$name isa Module || (Main.AA.$name = nothing)))
    end
end

@testitem "tt" begin
    a = ones(1000, 1000, 100)
end

@run_package_tests

I do not know how to use the gensymed name. So far I got something like this:

using TestItemRunner

function TestItemRunner.run_testitem(filepath, use_default_usings, setups, package_name, original_code, line, column, test_setup_module_set)
    mod = Core.eval(Main, :(module $(gensym()) end))

    if use_default_usings
        Core.eval(mod, :(using Test))

        if package_name!=""
            Core.eval(mod, :(using $(Symbol(package_name))))
        end
    end

    for m in setups
        Core.eval(mod, Expr(:using, Expr(:., :., :., nameof(test_setup_module_set.setupmodule), m)))
    end

    code = string('\n'^line, ' '^column, original_code)

    TestItemRunner.withpath(filepath) do
        Base.invokelatest(include_string, mod, code, filepath)
    end

    for name in names(mod; all=true)
        contains("$(name)", "#") && continue
        "$(name)" == "eval" && continue
        "$(name)" == "include" && continue
        setproperty!(mod, name, nothing)
    end
end

@testitem "tt" begin
    a = ones(1000, 1000, 100)
end

@run_package_tests