FilePathsBase.jl icon indicating copy to clipboard operation
FilePathsBase.jl copied to clipboard
verified publisher icon rofinn

Too many requests while running test suite with `S3Path`

Open omus opened this issue 4 years ago • 2 comments

May want to revise the test_download test to avoid failing due to rate limiting:

download: Error During Test at /Users/runner/.julia/packages/FilePathsBase/kMipS/src/test.jl:893
  Got exception outside of a @test
  IOError: unlink("/var/folders/24/8k48jl6d249_n_qfxwsl6xvm0000gn/T/jl_2xOPsw"): no such file or directory (ENOENT)
  Stacktrace:
    [1] uv_error
      @ ./libuv.jl:97 [inlined]
    [2] unlink(p::String)
      @ Base.Filesystem ./file.jl:934
    [3] rm(path::String; force::Bool, recursive::Bool)
      @ Base.Filesystem ./file.jl:272
    [4] rm
      @ ./file.jl:263 [inlined]
    [5] #rm#34
      @ ~/.julia/packages/FilePathsBase/kMipS/src/system.jl:232 [inlined]
    [6] rm
      @ ~/.julia/packages/FilePathsBase/kMipS/src/system.jl:232 [inlined]
    [7] mktemp(fn::FilePathsBase.var"#22#23"{String, S3Path{Nothing}}, parent::PosixPath)
      @ FilePathsBase ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:802
    [8] mktemp(fn::Function, P::Type{S3Path{Nothing}})
      @ FilePathsBase ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:772
    [9] download
      @ ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:684 [inlined]
   [10] macro expansion
      @ ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:895 [inlined]
   [11] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [12] test_download(ps::PathSet{S3Path{Nothing}})
      @ FilePathsBase.TestPaths ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:894
   [13] test(ps::PathSet{S3Path{Nothing}}, test_sets::Vector{Function})
      @ FilePathsBase.TestPaths ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:971
   [14] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/s3path.jl:369 [inlined]
   [15] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [16] s3path_tests(config::AWSConfig)
      @ Main ~/work/AWSS3.jl/AWSS3.jl/test/s3path.jl:307
   [17] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:55 [inlined]
   [18] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [19] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:54 [inlined]
   [20] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [21] top-level scope
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:22
   [22] include(fname::String)
      @ Base.MainInclude ./client.jl:444
   [23] top-level scope
      @ none:6
   [24] eval
      @ ./boot.jl:360 [inlined]
   [25] exec_options(opts::Base.JLOptions)
      @ Base ./client.jl:261
   [26] _start()
      @ Base ./client.jl:485
  
  caused by: HTTP/1.1 429 too many requests while requesting https://github.com/rofinn/FilePathsBase.jl/blob/master/README.md
  Stacktrace:
    [1] (::Downloads.var"#3#4"{Nothing, Vector{Pair{String, String}}, Float64, Nothing, Bool, Nothing, String})(output::IOStream)
      @ Downloads /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Downloads/src/Downloads.jl:233
    [2] open(f::Downloads.var"#3#4"{Nothing, Vector{Pair{String, String}}, Float64, Nothing, Bool, Nothing, String}, args::String; kwargs::Base.Iterators.Pairs{Symbol, Bool, Tuple{Symbol}, NamedTuple{(:write,), Tuple{Bool}}})
      @ Base ./io.jl:330
    [3] arg_write(f::Function, arg::String)
      @ ArgTools /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/ArgTools/src/ArgTools.jl:86
    [4] #download#2
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Downloads/src/Downloads.jl:221 [inlined]
    [5] download(url::String, output::String)
      @ Downloads /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Downloads/src/Downloads.jl:221
    [6] #invokelatest#2
      @ ./essentials.jl:708 [inlined]
    [7] invokelatest
      @ ./essentials.jl:706 [inlined]
    [8] do_download
      @ ./download.jl:33 [inlined]
    [9] download
      @ ./download.jl:28 [inlined]
   [10] download(url::String, dest::PosixPath)
      @ FilePathsBase ~/.julia/packages/FilePathsBase/kMipS/src/system.jl:389
   [11] #22
      @ ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:685 [inlined]
   [12] mktemp(fn::FilePathsBase.var"#22#23"{String, S3Path{Nothing}}, parent::PosixPath)
      @ FilePathsBase ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:799
   [13] mktemp(fn::Function, P::Type{S3Path{Nothing}})
      @ FilePathsBase ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:772
   [14] download
      @ ~/.julia/packages/FilePathsBase/kMipS/src/path.jl:684 [inlined]
   [15] macro expansion
      @ ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:895 [inlined]
   [16] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [17] test_download(ps::PathSet{S3Path{Nothing}})
      @ FilePathsBase.TestPaths ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:894
   [18] test(ps::PathSet{S3Path{Nothing}}, test_sets::Vector{Function})
      @ FilePathsBase.TestPaths ~/.julia/packages/FilePathsBase/kMipS/src/test.jl:971
   [19] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/s3path.jl:369 [inlined]
   [20] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [21] s3path_tests(config::AWSConfig)
      @ Main ~/work/AWSS3.jl/AWSS3.jl/test/s3path.jl:307
   [22] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:55 [inlined]
   [23] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [24] macro expansion
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:54 [inlined]
   [25] macro expansion
      @ /Users/julia/buildbot/worker/package_macos64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [26] top-level scope
      @ ~/work/AWSS3.jl/AWSS3.jl/test/runtests.jl:22
   [27] include(fname::String)
      @ Base.MainInclude ./client.jl:444
   [28] top-level scope
      @ none:6
   [29] eval
      @ ./boot.jl:360 [inlined]
   [30] exec_options(opts::Base.JLOptions)
      @ Base ./client.jl:261
   [31] _start()
      @ Base ./client.jl:485

– https://github.com/JuliaCloud/AWSS3.jl/runs/4025171843?check_suite_focus=true

omus avatar Oct 27 '21 17:10 omus

I'd recommend fixing this in AWSS3 for now by just writing your own test_download there. We already do that for several other test suites if I recall. I've yet to hit this during my builds here which make me think that AWSS3 is running more concurrent pipelines than I am. Depending on your proposed solution there I may choose to incorporate it here if there are no tradeoffs (e.g., mocking, complicated tests with authentication).

rofinn avatar Oct 27 '21 20:10 rofinn

Thanks @rofinn. I probably won't get around to fixing this myself but I wanted to make sure I recorded the issue.

omus avatar Oct 27 '21 20:10 omus