pytest-snapshot
pytest-snapshot copied to clipboard
Published
20 hours ago •
joseph-roitman
joseph-roitman
Test failures with pytest 8.0.2
Hi!
After updating pytest to 8.x we are seeing test failures with pytest-snapshot 0.9.0 over in nixpkgs.
============================= test session starts ==============================
platform linux -- Python 3.11.8, pytest-8.0.2, pluggy-1.4.0
rootdir: /build/source
plugins: snapshot-0.9.0
collected 108 items
tests/test_assert_match.py ....F.............................x.... [ 36%]
tests/test_assert_match_dir.py ............................ [ 62%]
tests/test_misc.py ......................................... [100%]
=================================== FAILURES ===================================
_______________________ test_assert_match_failure_bytes ________________________
request = <FixtureRequest for <Function test_assert_match_failure_bytes>>
testdir = <Testdir local('/build/pytest-of-nixbld/pytest-0/test_assert_match_failure_bytes0')>
basic_case_dir = local('/build/pytest-of-nixbld/pytest-0/test_assert_match_failure_bytes0/case_dir')
def test_assert_match_failure_bytes(request, testdir, basic_case_dir):
testdir.makepyfile(r"""
import os
def test_sth(snapshot):
snapshot.snapshot_dir = 'case_dir'
snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')
""")
result = runpytest_with_assert_mode(testdir, request, '-v', '--assert=rewrite')
> result.stdout.fnmatch_lines([
r'*::test_sth FAILED*',
r">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')",
r'E* AssertionError: value does not match the expected value in snapshot case_dir?snapshot1.txt',
r'E* (run pytest with --snapshot-update to update snapshots)',
r"E* assert * == *",
r"E* At index 4 diff: * != *",
r"E* Full diff:",
r"E* - b'the valu\xc3\x89 of snapshot1.txt{}'".format(repr(os.linesep)[1:-1]),
r"E* + b'the INCORRECT value of snapshot1.txt{}'".format(repr(os.linesep)[1:-1]),
])
E Failed: nomatch: '*::test_sth FAILED*'
E and: '============================= test session starts =============================='
E and: 'platform linux -- Python 3.11.8, pytest-8.0.2, pluggy-1.4.0 -- /nix/store/vhlk4r8ralh744khcy3vy47ijw5rvpw5-python3-3.1
E and: 'cachedir: .pytest_cache'
E and: 'rootdir: /build/pytest-of-nixbld/pytest-0/test_assert_match_failure_bytes0'
E and: 'plugins: snapshot-0.9.0'
E and: 'collecting ... collected 1 item'
E and: ''
E fnmatch: '*::test_sth FAILED*'
E with: 'test_assert_match_failure_bytes.py::test_sth FAILED [100%]'
E nomatch: ">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E and: ''
E and: '=================================== FAILURES ==================================='
E and: '___________________________________ test_sth ___________________________________'
E and: ''
E and: 'snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7ffff5e1c2d0>'
E and: ''
E and: ' def test_sth(snapshot):'
E and: " snapshot.snapshot_dir = 'case_dir'"
E fnmatch: ">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E with: "> snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E fnmatch: 'E* AssertionError: value does not match the expected value in snapshot case_dir?snapshot1.txt'
E with: 'E AssertionError: value does not match the expected value in snapshot case_dir/snapshot1.txt'
E fnmatch: 'E* (run pytest with --snapshot-update to update snapshots)'
E with: 'E (run pytest with --snapshot-update to update snapshots)'
E fnmatch: 'E* assert * == *'
E with: "E assert b'the INCORRE...apshot1.txt\\n' == b'the valu\\xc...apshot1.txt\\n'"
E nomatch: 'E* At index 4 diff: * != *'
E and: 'E '
E fnmatch: 'E* At index 4 diff: * != *'
E with: "E At index 4 diff: b'I' != b'v'"
E nomatch: 'E* Full diff:'
E and: 'E '
E fnmatch: 'E* Full diff:'
E with: 'E Full diff:'
E nomatch: "E* - b'the valu\\xc3\\x89 of snapshot1.txt\\n'"
E and: "E - (b'the valu\\xc3\\x89 of snapshot1.txt\\n')"
E and: 'E ? ^^^^^^^^'
E and: "E + (b'the INCORRECT value of snapshot1.txt\\n')"
E and: 'E ? ++++++++++ ^'
E and: ''
E and: 'test_assert_match_failure_bytes.py:4: AssertionError'
E and: '=========================== short test summary info ============================'
E and: 'FAILED test_assert_match_failure_bytes.py::test_sth - AssertionError: value d...'
E and: '============================== 1 failed in 0.01s ==============================='
E remains unmatched: "E* - b'the valu\\xc3\\x89 of snapshot1.txt\\n'"
/build/source/tests/test_assert_match.py:82: Failed
----------------------------- Captured stdout call -----------------------------
============================= test session starts ==============================
platform linux -- Python 3.11.8, pytest-8.0.2, pluggy-1.4.0 -- /nix/store/vhlk4r8ralh744khcy3vy47ijw5rvpw5-python3-3.11.8/bin/python3.11
cachedir: .pytest_cache
rootdir: /build/pytest-of-nixbld/pytest-0/test_assert_match_failure_bytes0
plugins: snapshot-0.9.0
collecting ... collected 1 item
test_assert_match_failure_bytes.py::test_sth FAILED [100%]
=================================== FAILURES ===================================
___________________________________ test_sth ___________________________________
snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7ffff5e1c2d0>
def test_sth(snapshot):
snapshot.snapshot_dir = 'case_dir'
> snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')
E AssertionError: value does not match the expected value in snapshot case_dir/snapshot1.txt
E (run pytest with --snapshot-update to update snapshots)
E assert b'the INCORRE...apshot1.txt\n' == b'the valu\xc...apshot1.txt\n'
E
E At index 4 diff: b'I' != b'v'
E
E Full diff:
E - (b'the valu\xc3\x89 of snapshot1.txt\n')
E ? ^^^^^^^^
E + (b'the INCORRECT value of snapshot1.txt\n')
E ? ++++++++++ ^
test_assert_match_failure_bytes.py:4: AssertionError
=========================== short test summary info ============================
FAILED test_assert_match_failure_bytes.py::test_sth - AssertionError: value d...
============================== 1 failed in 0.01s ===============================
=========================== short test summary info ============================
FAILED tests/test_assert_match.py::test_assert_match_failure_bytes - Failed: nomatch: '*::test_sth FAILED*'
=================== 1 failed, 106 passed, 1 xfailed in 4.55s ===================
Similar with pytest 8.1.1
Here is pytest output:
+ PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib64/python3.9/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib/python3.9/site-packages
+ /usr/bin/pytest -ra -m 'not network'
==================================================================================== test session starts ====================================================================================
platform linux -- Python 3.9.18, pytest-8.1.1, pluggy-1.4.0
rootdir: /home/tkloczko/rpmbuild/BUILD/pytest-snapshot-0.9.0
configfile: pyproject.toml
plugins: snapshot-0.9.0
collected 108 items
tests/test_assert_match.py ....F.............................x.... [ 36%]
tests/test_assert_match_dir.py ............................ [ 62%]
tests/test_misc.py ......................................... [100%]
========================================================================================= FAILURES ==========================================================================================
______________________________________________________________________________ test_assert_match_failure_bytes ______________________________________________________________________________
request = <FixtureRequest for <Function test_assert_match_failure_bytes>>, testdir = <Testdir local('/tmp/pytest-of-tkloczko/pytest-30/test_assert_match_failure_bytes0')>
basic_case_dir = local('/tmp/pytest-of-tkloczko/pytest-30/test_assert_match_failure_bytes0/case_dir')
def test_assert_match_failure_bytes(request, testdir, basic_case_dir):
testdir.makepyfile(r"""
import os
def test_sth(snapshot):
snapshot.snapshot_dir = 'case_dir'
snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')
""")
result = runpytest_with_assert_mode(testdir, request, '-v', '--assert=rewrite')
> result.stdout.fnmatch_lines([
r'*::test_sth FAILED*',
r">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')",
r'E* AssertionError: value does not match the expected value in snapshot case_dir?snapshot1.txt',
r'E* (run pytest with --snapshot-update to update snapshots)',
r"E* assert * == *",
r"E* At index 4 diff: * != *",
r"E* Full diff:",
fr"E* - b'the valu\xc3\x89 of snapshot1.txt{repr(os.linesep)[1:-1]}'",
fr"E* + b'the INCORRECT value of snapshot1.txt{repr(os.linesep)[1:-1]}'",
])
E Failed: nomatch: '*::test_sth FAILED*'
E and: '============================= test session starts =============================='
E and: 'platform linux -- Python 3.9.18, pytest-8.1.1, pluggy-1.4.0 -- /usr/bin/python3'
E and: 'cachedir: .pytest_cache'
E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-30/test_assert_match_failure_bytes0'
E and: 'plugins: snapshot-0.9.0'
E and: 'collecting ... collected 1 item'
E and: ''
E fnmatch: '*::test_sth FAILED*'
E with: 'test_assert_match_failure_bytes.py::test_sth FAILED [100%]'
E nomatch: ">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E and: ''
E and: '=================================== FAILURES ==================================='
E and: '___________________________________ test_sth ___________________________________'
E and: ''
E and: 'snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7efe791e08e0>'
E and: ''
E and: ' def test_sth(snapshot):'
E and: " snapshot.snapshot_dir = 'case_dir'"
E fnmatch: ">* snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E with: "> snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')"
E fnmatch: 'E* AssertionError: value does not match the expected value in snapshot case_dir?snapshot1.txt'
E with: 'E AssertionError: value does not match the expected value in snapshot case_dir/snapshot1.txt'
E fnmatch: 'E* (run pytest with --snapshot-update to update snapshots)'
E with: 'E (run pytest with --snapshot-update to update snapshots)'
E fnmatch: 'E* assert * == *'
E with: "E assert b'the INCORRE...apshot1.txt\\n' == b'the valu\\xc...apshot1.txt\\n'"
E nomatch: 'E* At index 4 diff: * != *'
E and: 'E '
E fnmatch: 'E* At index 4 diff: * != *'
E with: "E At index 4 diff: b'I' != b'v'"
E nomatch: 'E* Full diff:'
E and: 'E '
E fnmatch: 'E* Full diff:'
E with: 'E Full diff:'
E nomatch: "E* - b'the valu\\xc3\\x89 of snapshot1.txt\\n'"
E and: "E - (b'the valu\\xc3\\x89 of snapshot1.txt\\n')"
E and: 'E ? ^^^^^^^^'
E and: "E + (b'the INCORRECT value of snapshot1.txt\\n')"
E and: 'E ? ++++++++++ ^'
E and: ''
E and: 'test_assert_match_failure_bytes.py:4: AssertionError'
E and: '=========================== short test summary info ============================'
E and: 'FAILED test_assert_match_failure_bytes.py::test_sth - AssertionError: value d...'
E and: '============================== 1 failed in 0.01s ==============================='
E remains unmatched: "E* - b'the valu\\xc3\\x89 of snapshot1.txt\\n'"
/home/tkloczko/rpmbuild/BUILD/pytest-snapshot-0.9.0/tests/test_assert_match.py:82: Failed
----------------------------------------------------------------------------------- Captured stdout call ------------------------------------------------------------------------------------
============================= test session starts ==============================
platform linux -- Python 3.9.18, pytest-8.1.1, pluggy-1.4.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /tmp/pytest-of-tkloczko/pytest-30/test_assert_match_failure_bytes0
plugins: snapshot-0.9.0
collecting ... collected 1 item
test_assert_match_failure_bytes.py::test_sth FAILED [100%]
=================================== FAILURES ===================================
___________________________________ test_sth ___________________________________
snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7efe791e08e0>
def test_sth(snapshot):
snapshot.snapshot_dir = 'case_dir'
> snapshot.assert_match(b'the INCORRECT value of snapshot1.txt' + os.linesep.encode(), 'snapshot1.txt')
E AssertionError: value does not match the expected value in snapshot case_dir/snapshot1.txt
E (run pytest with --snapshot-update to update snapshots)
E assert b'the INCORRE...apshot1.txt\n' == b'the valu\xc...apshot1.txt\n'
E
E At index 4 diff: b'I' != b'v'
E
E Full diff:
E - (b'the valu\xc3\x89 of snapshot1.txt\n')
E ? ^^^^^^^^
E + (b'the INCORRECT value of snapshot1.txt\n')
E ? ++++++++++ ^
test_assert_match_failure_bytes.py:4: AssertionError
=========================== short test summary info ============================
FAILED test_assert_match_failure_bytes.py::test_sth - AssertionError: value d...
============================== 1 failed in 0.01s ===============================
========================================================================================= XFAILURES =========================================================================================
___________________________________________________________________________ test_assert_match_edge_cases[slash-r] ___________________________________________________________________________
testdir = <Testdir local('/tmp/pytest-of-tkloczko/pytest-30/test_assert_match_edge_cases5')>
basic_case_dir = local('/tmp/pytest-of-tkloczko/pytest-30/test_assert_match_edge_cases5/case_dir'), tested_value = '\r'
@pytest.mark.parametrize('tested_value', [
b'',
'',
bytes(bytearray(range(256))),
''.join(chr(i) for i in range(0, 10000)).replace('\r', ''),
' \n \t \n Whitespace! \n\t Whitespace! \n \t \n ',
# We don't support \r due to cross-compatibility and git by default modifying snapshot files...
pytest.param('\r', marks=pytest.mark.xfail(strict=True)),
], ids=[
'empty-bytes',
'empty-string',
'all-bytes',
'unicode',
'whitespace',
'slash-r',
])
def test_assert_match_edge_cases(testdir, basic_case_dir, tested_value):
"""
This test tests many possible values to snapshot test.
This test will fail if we change the snapshot file format in any way.
This test also checks that assert_match will pass after a snapshot update.
"""
testdir.makepyfile(r"""
def test_sth(snapshot):
tested_value = {tested_value!r}
snapshot.snapshot_dir = 'case_dir'
snapshot.assert_match(tested_value, 'tested_value_snapshot')
""".format(tested_value=tested_value))
result = testdir.runpytest('-v', '--snapshot-update')
> result.stdout.fnmatch_lines([
'*::test_sth PASSED*',
'*::test_sth ERROR*',
])
E Failed: nomatch: '*::test_sth PASSED*'
E and: '============================= test session starts =============================='
E and: 'platform linux -- Python 3.9.18, pytest-8.1.1, pluggy-1.4.0 -- /usr/bin/python3'
E and: 'cachedir: .pytest_cache'
E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-30/test_assert_match_edge_cases5'
E and: 'plugins: snapshot-0.9.0'
E and: 'collecting ... collected 1 item'
E and: ''
E and: 'test_assert_match_edge_cases.py::test_sth FAILED [100%]'
E and: ''
E and: '=================================== FAILURES ==================================='
E and: '___________________________________ test_sth ___________________________________'
E and: ''
E and: 'snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7efe78f5ed00>'
E and: ''
E and: ' def test_sth(snapshot):'
E and: " tested_value = '\\r'"
E and: " snapshot.snapshot_dir = 'case_dir'"
E and: "> snapshot.assert_match(tested_value, 'tested_value_snapshot')"
E and: ''
E and: 'test_assert_match_edge_cases.py:4: '
E and: '_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ '
E and: '/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib/python3.9/site-packages/pytest_snapshot/plugin.py:173: in assert_match'
E and: ' encoded_value = encode(value)'
E and: '_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ '
E and: ''
E and: "string = '\\r'"
E and: ''
E and: ' def _file_encode(string: str) -> bytes:'
E and: ' """'
E and: ' Returns the bytes that would be in a file created using ``path.write_text(string)``.'
E and: ' See universal newlines documentation.'
E and: ' """'
E and: " if '\\r' in string:"
E and: "> raise ValueError('''\\"
E and: ' Snapshot testing strings containing "\\\\r" is not supported.'
E and: ' To snapshot test non-standard newlines you should convert the tested value to bytes.'
E and: ' Warning: git may decide to modify the newlines in the snapshot file.'
E and: ' To avoid this read \\'
E and: " https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings''')"
E and: 'E ValueError: Snapshot testing strings containing "\\r" is not supported.'
E and: 'E To snapshot test non-standard newlines you should convert the tested value to bytes.'
E and: 'E Warning: git may decide to modify the newlines in the snapshot file.'
E and: 'E To avoid this read https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings'
E and: ''
E and: '/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib/python3.9/site-packages/pytest_snapshot/plugin.py:52: ValueError'
E and: '=========================== short test summary info ============================'
E and: 'FAILED test_assert_match_edge_cases.py::test_sth - ValueError: Snapshot testi...'
E and: '============================== 1 failed in 0.01s ==============================='
E remains unmatched: '*::test_sth PASSED*'
/home/tkloczko/rpmbuild/BUILD/pytest-snapshot-0.9.0/tests/test_assert_match.py:331: Failed
----------------------------------------------------------------------------------- Captured stdout call ------------------------------------------------------------------------------------
============================= test session starts ==============================
platform linux -- Python 3.9.18, pytest-8.1.1, pluggy-1.4.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /tmp/pytest-of-tkloczko/pytest-30/test_assert_match_edge_cases5
plugins: snapshot-0.9.0
collecting ... collected 1 item
test_assert_match_edge_cases.py::test_sth FAILED [100%]
=================================== FAILURES ===================================
___________________________________ test_sth ___________________________________
snapshot = <pytest_snapshot.plugin.Snapshot object at 0x7efe78f5ed00>
def test_sth(snapshot):
tested_value = '\r'
snapshot.snapshot_dir = 'case_dir'
> snapshot.assert_match(tested_value, 'tested_value_snapshot')
test_assert_match_edge_cases.py:4:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib/python3.9/site-packages/pytest_snapshot/plugin.py:173: in assert_match
encoded_value = encode(value)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
string = '\r'
def _file_encode(string: str) -> bytes:
"""
Returns the bytes that would be in a file created using ``path.write_text(string)``.
See universal newlines documentation.
"""
if '\r' in string:
> raise ValueError('''\
Snapshot testing strings containing "\\r" is not supported.
To snapshot test non-standard newlines you should convert the tested value to bytes.
Warning: git may decide to modify the newlines in the snapshot file.
To avoid this read \
https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings''')
E ValueError: Snapshot testing strings containing "\r" is not supported.
E To snapshot test non-standard newlines you should convert the tested value to bytes.
E Warning: git may decide to modify the newlines in the snapshot file.
E To avoid this read https://docs.github.com/en/get-started/getting-started-with-git/configuring-git-to-handle-line-endings
/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-snapshot-0.9.0-2.fc36.x86_64/usr/lib/python3.9/site-packages/pytest_snapshot/plugin.py:52: ValueError
=========================== short test summary info ============================
FAILED test_assert_match_edge_cases.py::test_sth - ValueError: Snapshot testi...
============================== 1 failed in 0.01s ===============================
================================================================================== short test summary info ==================================================================================
XFAIL tests/test_assert_match.py::test_assert_match_edge_cases[slash-r]
FAILED tests/test_assert_match.py::test_assert_match_failure_bytes - Failed: nomatch: '*::test_sth FAILED*'
========================================================================= 1 failed, 106 passed, 1 xfailed in 7.51s ==========================================================================
Seems like PyTest added '(' to its message
E nomatch: "E* - b'the valu\\xc3\\x89 of snapshot1.txt\\r\\n'"
E and: "E - (b'the valu\\xc3\\x89 of snapshot1.txt\\r\\n')"