Dagger.jl `Threads.@threads` doesn't compose with `ThreadProc`

I may be missing something, but it doesn't seem like ThreadProc is doing anything..?

using Dagger
import Dagger: @par

function count_threads()
    n = Threads.nthreads()
    a = zeros(n)
    Threads.@threads for i = 1:n
           a[i] = Threads.threadid()
    end
    a
end
procutil=Dict(Dagger.ThreadProc => 36.0)
x = @par procutil=procutil count_threads()
length(unique(collect(x))) # returns 1.0
length(unique(count_threads())) # return 36.0

Aug 05 '21 02:08 tbenst

Threads.@threads doesn't appear to use multithreading when run on a thread other than 1 (which is what ThreadProc does):

10> Threads.@spawn begin
    @show length(unique(count_threads()))
    end
Task (runnable) @0x00007fe09e8a8be0

length(unique(count_threads())) = 1

11> length(unique(count_threads()))
6

14> t = Task() do
    length(unique(count_threads()))
    end
Task (runnable) @0x00007fe0a1adc010

16> ccall(:jl_set_task_tid, Cint, (Any, Cint), t, 0)
1

17> schedule(t)
Task (done) @0x00007fe0a1adc010

18> fetch(t)
6

14/19> t = Task() do
       length(unique(count_threads()))
       end
Task (runnable) @0x00007fe0a1b8c160

16/21> ccall(:jl_set_task_tid, Cint, (Any, Cint), t, 1)
1

22> schedule(t)
Task (runnable) @0x00007fe0a1b8c160

18/23> fetch(t)
1

This isn't a Dagger bug per-se, but really a lack of composability of Threads.@threads. We could probably work around this by using @async instead of Threads.@spawn when procutil is greater than 1, but I'll need to think for a bit on the potential consequences of that...

Aug 05 '21 11:08 jpsamaroo

Thanks for the great detective work! Seems like a surprising limitation of @threads to me, and this may be unintended behavior on the Julia side: https://discourse.julialang.org/t/understanding-nested-threads-threads-scheduling/40400/3 and https://github.com/JuliaLang/julia/pull/35646#issuecomment-622012366

P.S. doh sorry for my bad DRY function... improved ;)

function count_threads()
    n = Threads.nthreads()
    a = zeros(n)
    Threads.@threads for i = 1:n
           a[i] = Threads.threadid()
    end
    length(unique(a))
end

Aug 05 '21 15:08 tbenst

I did some benchmarking, and seems better to use @spawn than @threads:

function count_threads()
    n = Threads.nthreads()
    a = zeros(n)
    Threads.@threads for i = 1:n
           a[i] = Threads.threadid()
    end
    length(unique(a))
end

function count_threads_floop(n=-1)
    if n == -1
        n = Threads.nthreads()
    end
    a = zeros(n)
    @floop for i = 1:n
           a[i] = Threads.threadid()
    end
    length(unique(a))
end

function count_threads_spawn(n=-1)
    if n == -1
        n = Threads.nthreads()
    end
    a = zeros(n)
    for i = 1:n
           Threads.@spawn a[i] = Threads.threadid()
    end
    length(unique(a))
end

@show count_threads() # 36
Threads.@spawn @show count_threads() # 1
Threads.@spawn @show count_threads_floop() # 1
Threads.@spawn @show count_threads_spawn() # 1
Threads.@spawn @show count_threads(10000) # 1
Threads.@spawn @show count_threads_floop(10000) # 1
Threads.@spawn @show count_threads_spawn(10000) # 35
Threads.@spawn @show count_threads_spawn(80) # 10

Aug 05 '21 15:08 tbenst

Figured I might ask a more direct question for my usecase. I have a function that reads ~100k tiffs in parallel from a directory. For example, using 36 threads this takes 48 seconds to load 60GB of data from disk. The task is partially CPU bound, as the RAID0 NVME array can sustain >7GB/s, and the function call saturates all threads at 100%.

Any thoughts as to how I should go about this with Dagger? I tried changing the Threads.@threads call to Threads.@sync Threads.@spawn, and this seemed a bit faster initially when called directly (38 seconds), but it's still single threaded when called via Dagger @par / collect.

"Read a 4D (HWZT) tiff stack in parallel from a folder."
function loadTseries(tifdir, containsStr::String="Ch3")
    H, W, Z, T, framePlane2tiffPath = tseriesTiffDirMetadata(tifdir, containsStr)
    tseries = Array{UInt16}(undef, H, W, Z, T)
    memory_size_bytes = prod(size(tseries)) * 2
    memory_size_gb = round(memory_size_bytes / 1024^3, digits=1)
    println("estimated memory usage: $memory_size_gb")
    p = Progress(T, 1, "Load Tseries: ")

    @threads for t in 1:T
        for z in 1:Z
            tp = framePlane2tiffPath[t,z]
            tseries[:,:,z,t] .= reinterpret(UInt16, ImageMagick.load(tp))
        end
        next!(p)
    end
    tseries
end

Aug 06 '21 06:08 tbenst

Instead of using Threads.@threads to parallelize this, you should use the Threads.@spawn approach but replace that with Dagger.@spawn. You almost always want to write the function you pass to Dagger as a serial program, and let Dagger schedule all of those serial programs onto whatever parallel resources (threads, processes, etc.) that you have available. Since this is an embarrassingly parallel task, Dagger should fully exploit all threads on all processes (and if it doesn't, let me know).

Aug 06 '21 13:08 jpsamaroo

I get the following error on this line Threads.@sync Dagger.@spawn for t in 1:T:

ERROR: LoadError: MethodError: no method matching iterate(::Dagger.EagerThunk)
Closest candidates are:
  iterate(::Union{LinRange, StepRangeLen}) at range.jl:664
  iterate(::Union{LinRange, StepRangeLen}, ::Int64) at range.jl:664
  iterate(::T) where T<:Union{Base.KeySet{var"#s79", var"#s78"} where {var"#s79", var"#s78"<:Dict}, Base.ValueIterator{var"#s77"} where var"#s77"<:Dict} at dict.jl:693

So I refactored a little:

using Dagger, ProgressMeter, Base.Threads
import Dagger: @par

function _loadrand!(tseries, t, p, Z, H, W)
    for z in 1:Z
        tseries[:,:,z,t] .= rand(UInt16,H,W)
    end
    next!(p)
end

"Mock read a 4D (HWZT) tiff stack in parallel from a folder."
function loadrand(H,W,Z,T)
    tseries = Array{UInt16}(undef, H, W, Z, T)
    memory_size_bytes = prod(size(tseries)) * sizeof(UInt16)
    memory_size_gb = round(memory_size_bytes / 1024^3, digits=1)
    println("estimated memory usage: $memory_size_gb")
    p = Progress(T, 1, "Load Tseries: ")
    # Threads.@sync for t in 1:T
    for t in 1:T
        # Threads.@spawn _loadrand!(tseries, t, p, Z, H, W)
        Dagger.@spawn _loadrand!(tseries, t, p, Z, H, W)
    end
    tseries
end

@par tseries_lazy = loadrand(100,100,10,10);
tseries = collect(tseries_lazy);

And this seems to run! Although how can I tell if all threads have finished? I tried wrapping in Threads.@sync but got an error. Is there an equivalent operation for Dagger?

Error in eager scheduler in wrapped in Threads.@sync

Error in eager scheduler:
KeyError: key 49 not found
Stacktrace:
 [1] getindex(h::Dict{Int64, Any}, key::Int64)
   @ Base ./dict.jl:482
 [2] compute_dag(ctx::Dagger.Context, d::Dagger.Thunk; options::Dagger.Sch.SchedulerOptions)
   @ Dagger.Sch ~/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:417
 [3] compute(ctx::Dagger.Context, d::Dagger.Thunk; options::Dagger.Sch.SchedulerOptions)
   @ Dagger ~/.julia/packages/Dagger/3FRGD/src/compute.jl:31
 [4] (::Dagger.Sch.var"#43#44"{Dagger.Context})()
   @ Dagger.Sch ./task.jl:411
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./lock.jl:185
lock at ./weakkeydict.jl:87 [inlined]
delete! at ./weakkeydict.jl:166 [inlined]
finalize_ref at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Distributed/src/remotecall.jl:92
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./lock.jl:185
lock at ./weakkeydict.jl:87 [inlined]
delete! at ./weakkeydict.jl:166 [inlined]
finalize_ref at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Distributed/src/remotecall.jl:92
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))
error in running finalizer: ErrorException("task switch not allowed from inside gc finalizer")
jl_error at /buildworker/worker/package_linux64/build/src/rtutils.c:41
jl_switch at /buildworker/worker/package_linux64/build/src/task.c:501
try_yieldto at ./task.jl:700
wait at ./task.jl:769 [inlined]
wait at ./condition.jl:106
lock at ./lock.jl:100
lock at ./condition.jl:73 [inlined]
lock at ./channels.jl:424 [inlined]
put_buffered at ./channels.jl:320
put! at ./channels.jl:316
unknown function (ip: 0x7fafce849b3d)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
_enqueue_work at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:111
#25 at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:152
macro expansion at /home/tyler/.julia/packages/MemPool/RPu2r/src/lock.jl:62 [inlined]
with_datastore_lock at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:204
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:144
poolunref at /home/tyler/.julia/packages/MemPool/RPu2r/src/datastore.jl:143
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
run_finalizer at /buildworker/worker/package_linux64/build/src/gc.c:278
jl_gc_run_finalizers_in_list at /buildworker/worker/package_linux64/build/src/gc.c:365
run_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:394
jl_gc_run_pending_finalizers at /buildworker/worker/package_linux64/build/src/gc.c:404
enable_finalizers at ./gcutils.jl:123 [inlined]
unlock at ./locks-mt.jl:92
unlock at ./condition.jl:74 [inlined]
unlock at ./lock.jl:139
lock at ./lock.jl:189
macro expansion at /home/tyler/.julia/packages/Dagger/3FRGD/src/sch/Sch.jl:383 [inlined]
#75 at ./task.jl:411
unknown function (ip: 0x7fafce7ee52c)
_jl_invoke at /buildworker/worker/package_linux64/build/src/gf.c:2237 [inlined]
jl_apply_generic at /buildworker/worker/package_linux64/build/src/gf.c:2419
jl_apply at /buildworker/worker/package_linux64/build/src/julia.h:1703 [inlined]
start_task at /buildworker/worker/package_linux64/build/src/task.c:839
unknown function (ip: (nil))

Aug 06 '21 15:08 tbenst