astroquery
astroquery copied to clipboard
astropy
Unit tests failed during parallel invocation: FileNotFoundError: [Errno 2] No such file or directory
Hi,
During scheduled update of astroquery in Guix I tried to speed up unit tests
with pytest-xdist. It's minimized the wait time especially on multi threaded
CI farm. Unfortunately by enabling it I've got 2 tests failed:
python pytest \
--color=yes \
--doctest-continue-on-failure \
--doctest-rst \
--pyargs astroquery \
-m not remote_data \
-n auto \
-v
...
____________________ TestXMMNewton.test_get_epic_lightcurve ____________________
[gw7] linux -- Python 3.10.7 /gnu/store/jh59fh48mcffyz5wmsjj0p96xkkflbz0-python-wrapper-3.10.7/bin/python
self = <astroquery.esa.xmm_newton.tests.test_xmm_newton.TestXMMNewton object at 0x7fff62909120>
tmp_path = PosixPath('/tmp/guix-build-python-astroquery-0.4.7.drv-0/pytest-of-nixbld/pytest-0/popen-gw7/test_get_epic_lightcurve0')
def test_get_epic_lightcurve(self, tmp_path):
path = Path(tmp_path, "tarfile.tar")
> self._create_tar(path, self._files)
/gnu/store/8xaif1f5787v7kmvw3xvpng36z2k36y9-python-astroquery-0.4.7/lib/python3.10/site-packages/astroquery/esa/xmm_newton/tests/test_xmm_newton.py:473:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/gnu/store/8xaif1f5787v7kmvw3xvpng36z2k36y9-python-astroquery-0.4.7/lib/python3.10/site-packages/astroquery/esa/xmm_newton/tests/test_xmm_newton.py:265: in _create_tar
shutil.rmtree(os.path.join(ob_name, ftype))
/gnu/store/4r7k7ipiaqkdf4lmnxwmbz0wx2yzygzc-python-3.10.7/lib/python3.10/shutil.py:724: in rmtree
_rmtree_safe_fd(fd, path, onerror)
/gnu/store/4r7k7ipiaqkdf4lmnxwmbz0wx2yzygzc-python-3.10.7/lib/python3.10/shutil.py:680: in _rmtree_safe_fd
onerror(os.unlink, fullname, sys.exc_info())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
topfd = 14, path = '0405320501/pps'
onerror = <function rmtree.<locals>.onerror at 0x7fff646c36d0>
def _rmtree_safe_fd(topfd, path, onerror):
try:
with os.scandir(topfd) as scandir_it:
entries = list(scandir_it)
except OSError as err:
err.filename = path
onerror(os.scandir, path, sys.exc_info())
return
for entry in entries:
fullname = os.path.join(path, entry.name)
try:
is_dir = entry.is_dir(follow_symlinks=False)
except OSError:
is_dir = False
else:
if is_dir:
try:
orig_st = entry.stat(follow_symlinks=False)
is_dir = stat.S_ISDIR(orig_st.st_mode)
except OSError:
onerror(os.lstat, fullname, sys.exc_info())
continue
if is_dir:
try:
dirfd = os.open(entry.name, os.O_RDONLY, dir_fd=topfd)
dirfd_closed = False
except OSError:
onerror(os.open, fullname, sys.exc_info())
else:
try:
if os.path.samestat(orig_st, os.fstat(dirfd)):
_rmtree_safe_fd(dirfd, fullname, onerror)
try:
os.close(dirfd)
dirfd_closed = True
os.rmdir(entry.name, dir_fd=topfd)
except OSError:
onerror(os.rmdir, fullname, sys.exc_info())
else:
try:
# This can only happen if someone replaces
# a directory with a symlink after the call to
# os.scandir or stat.S_ISDIR above.
raise OSError("Cannot call rmtree on a symbolic "
"link")
except OSError:
onerror(os.path.islink, fullname, sys.exc_info())
finally:
if not dirfd_closed:
os.close(dirfd)
else:
try:
> os.unlink(entry.name, dir_fd=topfd)
E FileNotFoundError: [Errno 2] No such file or directory: 'P0405320501PNS001EXPMAP4000.FTZ'
/gnu/store/4r7k7ipiaqkdf4lmnxwmbz0wx2yzygzc-python-3.10.7/lib/python3.10/shutil.py:678: FileNotFoundError
_______________ TestXMMNewton.test_get_epic_images_invalid_band ________________
[gw5] linux -- Python 3.10.7 /gnu/store/jh59fh48mcffyz5wmsjj0p96xkkflbz0-python-wrapper-3.10.7/bin/python
self = <astroquery.esa.xmm_newton.tests.test_xmm_newton.TestXMMNewton object at 0x7fff6282e4d0>
tmp_path = PosixPath('/tmp/guix-build-python-astroquery-0.4.7.drv-0/pytest-of-nixbld/pytest-0/popen-gw5/test_get_epic_images_invalid_b0')
capsys = <_pytest.capture.CaptureFixture object at 0x7fff646cf3d0>
def test_get_epic_images_invalid_band(self, tmp_path, capsys):
path = Path(tmp_path, "tarfile.tar")
_invalid_band = 10
> self._create_tar(path, self._files)
/gnu/store/8xaif1f5787v7kmvw3xvpng36z2k36y9-python-astroquery-0.4.7/lib/python3.10/site-packages/astroquery/esa/xmm_newton/tests/test_xmm_newton.py:375:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/gnu/store/8xaif1f5787v7kmvw3xvpng36z2k36y9-python-astroquery-0.4.7/lib/python3.10/site-packages/astroquery/esa/xmm_newton/tests/test_xmm_newton.py:263: in _create_tar
tar.add(os.path.join(ob_name, ftype, f))
/gnu/store/4r7k7ipiaqkdf4lmnxwmbz0wx2yzygzc-python-3.10.7/lib/python3.10/tarfile.py:1986: in add
tarinfo = self.gettarinfo(name, arcname)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <tarfile.TarFile object at 0x7fff646cd630>
name = '0405320501/pps/P0405320501M2S003EXPMAP2000.FTZ'
arcname = '0405320501/pps/P0405320501M2S003EXPMAP2000.FTZ', fileobj = None
def gettarinfo(self, name=None, arcname=None, fileobj=None):
"""Create a TarInfo object from the result of os.stat or equivalent
on an existing file. The file is either named by `name', or
specified as a file object `fileobj' with a file descriptor. If
given, `arcname' specifies an alternative name for the file in the
archive, otherwise, the name is taken from the 'name' attribute of
'fileobj', or the 'name' argument. The name should be a text
string.
"""
self._check("awx")
# When fileobj is given, replace name by
# fileobj's real name.
if fileobj is not None:
name = fileobj.name
# Building the name of the member in the archive.
# Backward slashes are converted to forward slashes,
# Absolute paths are turned to relative paths.
if arcname is None:
arcname = name
drv, arcname = os.path.splitdrive(arcname)
arcname = arcname.replace(os.sep, "/")
arcname = arcname.lstrip("/")
# Now, fill the TarInfo object with
# information specific for the file.
tarinfo = self.tarinfo()
tarinfo.tarfile = self # Not needed
# Use os.stat or os.lstat, depending on if symlinks shall be resolved.
if fileobj is None:
if not self.dereference:
> statres = os.lstat(name)
E FileNotFoundError: [Errno 2] No such file or directory: '0405320501/pps/P0405320501M2S003EXPMAP2000.FTZ'
...
/gnu/store/4r7k7ipiaqkdf4lmnxwmbz0wx2yzygzc-python-3.10.7/lib/python3.10/tarfile.py:1865: FileNotFoundError
FAILED esa/xmm_newton/tests/test_xmm_newton.py::TestXMMNewton::test_get_epic_lightcurve
FAILED esa/xmm_newton/tests/test_xmm_newton.py::TestXMMNewton::test_get_epic_images_invalid_band
...
===== 2 failed, 1456 passed, 8 skipped, 2 xfailed, 114 warnings in 13.25s ======
...
``
I can't reproduce this with a standard test run. Is it possible this was a temporary failure driven by the parallel run?
$ pytest astroquery/esa/xmm_newton --remote-data
============================================================================================================= test session starts =============================================================================================================
platform darwin -- Python 3.10.12, pytest-7.4.2, pluggy-1.3.0
Running tests with astroquery version 0.4.7.dev9202_testrun_testrun.
Running tests with astropy_helpers version 2.0.9.
Running tests in astroquery/esa/xmm_newton.
Date: 2024-03-18T13:39:05
Platform: macOS-13.6-arm64-i386-64bit
Executable: /Users/adam/mambaforge/envs/py310forge/bin/python3.10
Full Python Version:
3.10.12 | packaged by conda-forge | (main, Jun 23 2023, 22:41:52) [Clang 15.0.7 ]
encodings: sys: utf-8, locale: UTF-8, filesystem: utf-8
byteorder: little
float info: dig: 15, mant_dig: 15
Package versions:
Numpy: 1.26.2
Matplotlib: 3.7.2
Astropy: 6.1.dev589+g2390bf8f8f
regions: 0.8
pyVO: 1.5.dev228+ga035576
mocpy: not available
astropy-healpix: 1.0.0
vamdclib: not available
Using Astropy options: remote_data: any.
rootdir: /Users/adam/repos/astroquery
configfile: setup.cfg
plugins: anyio-4.0.0, remotedata-0.4.1, asdf-3.0.0, doctestplus-1.2.1, astropy-header-0.2.2
collected 34 items
astroquery/esa/xmm_newton/tests/test_xmm_newton.py .......................... [ 76%]
astroquery/esa/xmm_newton/tests/test_xmm_newton_remote.py ........ [100%]
======================================================================================================== 34 passed in 84.28s (0:01:24) ========================================================================================================
It's totally possible to have bugs in these tests e.g. one assumes a file is around that a previous test was creating and that hasn't yet been cleaned up. Any further investigation and PRs to fix it are more than welcome!
@keflavich as @bsipocz mentioned it could be a race condition during parallel run of the tests which I tried to enable. It passes just fine in single thread invocation.
(fwiw, I do see these failures locally, in fact I see a couple more of the same kind for the esa.xmm_newton module (6 instead of 2))