Elemental.jl icon indicating copy to clipboard operation
Elemental.jl copied to clipboard

Can not copy DistMatrix

Open dh-ilight opened this issue 3 years ago • 0 comments

# mpiexecjl -n 2 julia dcopy.jl

# This code trys to copy in parallel a El.DistMatrix to another El.DistMatrix
# The intention is for each rank to deal with just its portion of the data.
# My actual goal is to create a El.DistMatrix with 1 less column

# Please show me how do a copy with loops not just an assignment.
# The code works when a single rank is used.

using MPIClusterManagers
manager = MPIClusterManagers.start_main_loop(MPI_TRANSPORT_ALL)
@mpi_do manager begin
        using Elemental
        const El = Elemental
end

@mpi_do manager begin
   using MPI
   comm = MPI.COMM_WORLD
   myrank = MPI.Comm_rank(comm)
   num_ranks = MPI.Comm_size(comm)
   MPI.Barrier(comm)
end

@mpi_do manager begin
    height = 4
    width = 6
    dfrom = El.DistMatrix(Float32, El.STAR, El.VC)
    El.zeros!(dfrom, height, width)
    mul = 1.0/num_ranks
    
    # dfrom is a DistMatrix where each value is its coordinate in the array
    for i = 1:width
        for j = 1:height
            El.queueUpdate(dfrom, j, i, Float32(mul * (i*1000+j)))
        end
    end
    El.processQueues(dfrom)
    MPI.Barrier(comm)
    
    dto = El.DistMatrix(Float32, El.STAR, El.VC)
    
    # dto is the copy
    El.zeros!(dto, height, width) 
    lh = El.localHeight(dfrom)
    if lh != height
        println("$myrank lh != height  ")
        return
    end    
    lw = El.localWidth(dto)
    # All members of a column are in a single rank.
    for i = 1:lw 
        colto = El.globalCol(dto, i)
        #println("$myrank $colto  ")
        for j = 1:height
            #println("$myrank j= $j, colto= $colto\n")
            El.queueUpdate(dto, j, colto, dfrom[j, colto])
        end # j
    end # i

    El.processQueues(dto)
    MPI.Barrier(comm)
    # print dto
    d = collect(dto)
    dfromc = collect(dfrom)
        #println("$myrank dfrom= $dfrom")
        #println("$myrank dto= $dto")
    if myrank == 0

        println("------\n  ")
        println("* Lines with expected value.")
        for i = 1:height
            s = " "
            sf = "*"
            for j = 1:width
                sf = sf * "$(dfromc[i, j]),  "
                s = s * "$(d[i, j]),  "
            end
            println(sf, " ")
            println(s,"  ")
        end
    end
end

@mpi_do manager Elemental.Finalize()
 
MPIClusterManagers.stop_main_loop(manager)
println("main loop stopped")

dh-ilight avatar Mar 10 '22 19:03 dh-ilight