Investigate podman checkpoint test failures on centos-stream-10-x86_64

[lsm5]

trafficstars

podman checkpoint tests fail on centos-stream-10-x86_64 env like so:

not ok 1 [520] podman checkpoint - basic test
# tags: ci:parallel
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 45)
#   `run_podman container checkpoint $cid' failed
#
# [12:32:37.289378046] # podman run -d quay.io/libpod/testimage:20241011 sh -c while :;do cat /proc/uptime; sleep 0.1;done
# [12:32:37.460386134] aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895
#
# [12:32:37.476266893] # podman logs aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895
# [12:32:37.510535848] 397.05 455.99
#
# [12:32:37.518142895] # podman container checkpoint aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895
# [12:32:37.602616185] CRIU checkpointing failed -52.  Please check CRIU logfile /var/lib/containers/storage/overlay-containers/aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /var/lib/containers/storage/overlay-containers/aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895/userdata/checkpoint --work-path /var/lib/containers/storage/overlay-containers/aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895/userdata aa754504d33332cccc1798ff687d20164cce1219f9379886a75d4a7855dad895` failed: exit status 1
# [12:32:37.605653965] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# # [teardown]
not ok 2 [520] podman checkpoint/restore print IDs or raw input
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 117)
#   `run_podman container checkpoint -a' failed
#
# [12:32:38.107536812] # podman run -d quay.io/libpod/testimage:20241011 top
# [12:32:38.283187758] f380fdc16b2c0ba8aa1f3953a3b293f30337b1bde3cdbad85417bdd821331c2a
#
# [12:32:38.288706676] # podman container checkpoint -a
# [12:32:38.353895738] CRIU checkpointing failed -52.  Please check CRIU logfile /var/lib/containers/storage/overlay-containers/f380fdc16b2c0ba8aa1f3953a3b293f30337b1bde3cdbad85417bdd821331c2a/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /var/lib/containers/storage/overlay-containers/f380fdc16b2c0ba8aa1f3953a3b293f30337b1bde3cdbad85417bdd821331c2a/userdata/checkpoint --work-path /var/lib/containers/storage/overlay-containers/f380fdc16b2c0ba8aa1f3953a3b293f30337b1bde3cdbad85417bdd821331c2a/userdata f380fdc16b2c0ba8aa1f3953a3b293f30337b1bde3cdbad85417bdd821331c2a` failed: exit status 1
# [12:32:38.356880607] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# # [teardown]
not ok 3 [520] podman checkpoint --export, with volumes
# tags: ci:parallel
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 163)
#   `run_podman $p_opts container checkpoint \' failed
#
# [12:32:38.859512031] # podman save -o /tmp/podman_bats.QusMCm/image.tar quay.io/libpod/testimage:20241011
# [12:32:38.981361509] Copying blob sha256:b66a884aaf08f1c410c61682a7072d68a0d837ba8dc16ff13b9509bdceb32fd2
# Copying blob sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef
# Copying config sha256:b82e560ed57b77a897379e160371adcf1b000ca885e69c62cbec674777a83850
# Writing manifest to image destination
#
# [12:32:38.986872767] # podman --root /tmp/podman_bats.QusMCm/root --runroot /tmp/podman_bats.QusMCm/runroot --tmpdir /tmp/podman_bats.QusMCm/tmpdir --events-backend file load -i /tmp/podman_bats.QusMCm/image.tar
# [12:32:39.273103249] Getting image source signatures
# Copying blob sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef
# Copying blob sha256:b66a884aaf08f1c410c61682a7072d68a0d837ba8dc16ff13b9509bdceb32fd2
# Copying config sha256:b82e560ed57b77a897379e160371adcf1b000ca885e69c62cbec674777a83850
# Writing manifest to image destination
# Loaded image: quay.io/libpod/testimage:20241011
#
# [12:32:39.287905702] # podman --root /tmp/podman_bats.QusMCm/root --runroot /tmp/podman_bats.QusMCm/runroot --tmpdir /tmp/podman_bats.QusMCm/tmpdir --events-backend file volume create v-t3-rvymbgki
# [12:32:39.318728587] v-t3-rvymbgki
#
# [12:32:39.356315820] # podman --root /tmp/podman_bats.QusMCm/root --runroot /tmp/podman_bats.QusMCm/runroot --tmpdir /tmp/podman_bats.QusMCm/tmpdir --events-backend file run -d --name c-t3-rvymbgki --volume v-t3-rvymbgki:/myvol -p 5521:80 -w /myvol quay.io/libpod/testimage:20241011 sh -c /bin/busybox-extras httpd -p 80;echo c-t3-rvymbgki >cname;echo READY;while :;do cat /proc/uptime >mydate.tmp;mv -f mydate.tmp mydate;sleep 0.1;done
# [12:32:39.515414272] b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31
#
# [12:32:39.532909998] # podman logs b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31
# [12:32:39.567719236] READY
#
# [12:32:39.593373471] # podman --root /tmp/podman_bats.QusMCm/root --runroot /tmp/podman_bats.QusMCm/runroot --tmpdir /tmp/podman_bats.QusMCm/tmpdir --events-backend file container checkpoint --ignore-rootfs --export=/tmp/podman_bats.QusMCm/c-t3-rvymbgki.tar.gz c-t3-rvymbgki
# [12:32:39.661089902] CRIU checkpointing failed -52.  Please check CRIU logfile /tmp/podman_bats.QusMCm/root/overlay-containers/b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /tmp/podman_bats.QusMCm/root/overlay-containers/b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31/userdata/checkpoint --work-path /tmp/podman_bats.QusMCm/root/overlay-containers/b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31/userdata b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31` failed: exit status 1
# [12:32:39.664092862] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# # [teardown]
# rm: cannot remove '/tmp/podman_bats.QusMCm/root/overlay/e293d086818e740c8691f406e6eaf3deb0ba7fb86f2413088f0f2543280fe778/merged': Device or resource busy
# rm: cannot remove '/tmp/podman_bats.QusMCm/root/overlay-containers/b1348e79e724bf35495ecaaefdc6e18964edd1bb4de9700d14c24ac859d47a31/userdata/shm': Device or resource busy
not ok 4 [520] podman checkpoint --file-locks
# tags: ci:parallel
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 208)
#   `run_podman container checkpoint --file-locks $cid' failed
#
# [12:32:40.063568252] # podman run -d quay.io/libpod/testimage:20241011 sh -c touch /wait; touch test.lock; echo READY; flock test.lock sh -c "while [ -e /wait ];do sleep 0.5;done;for i in 1 2 3;do echo \$i;sleep 0.5;done" & flock test.lock sh -c "while [ -e /wait ];do sleep 0.5;done;for i in 1 2 3;do echo \$i;sleep 0.5;done" & wait
# [12:32:40.207385819] 67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7
#
# [12:32:40.224641300] # podman logs 67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7
# [12:32:40.260058042] READY
#
# [12:32:40.267968915] # podman container checkpoint --file-locks 67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7
# [12:32:40.338913292] CRIU checkpointing failed -52.  Please check CRIU logfile /var/lib/containers/storage/overlay-containers/67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /var/lib/containers/storage/overlay-containers/67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7/userdata/checkpoint --work-path /var/lib/containers/storage/overlay-containers/67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7/userdata --file-locks 67881fd268c64a4c616ac0f84acb109642640c4e6ecfd1f0e1525b97479921d7` failed: exit status 1
# [12:32:40.342021564] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# # [teardown]
not ok 5 [520] podman checkpoint/restore ip and mac handling
# tags: ci:parallel
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 253)
#   `run_podman container checkpoint $cid' failed
#
# [12:32:40.817375344] # podman network create --subnet 172.11.73.0/24 net-t5-euyssq6n
# [12:32:40.846512560] net-t5-euyssq6n
#
# [12:32:40.851935136] # podman run -d --network net-t5-euyssq6n quay.io/libpod/testimage:20241011 top
# [12:32:41.053557635] ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42
#
# [12:32:41.059352370] # podman inspect ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42 --format {{(index .NetworkSettings.Networks "net-t5-euyssq6n").IPAddress}}
# [12:32:41.094562587] 172.11.73.2
#
# [12:32:41.100312280] # podman inspect ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42 --format {{(index .NetworkSettings.Networks "net-t5-euyssq6n").MacAddress}}
# [12:32:41.133997952] be:7a:a0:c5:08:60
#
# [12:32:41.139669793] # podman exec ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42 cat /etc/hosts /etc/resolv.conf
# [12:32:41.278332181] 127.0.0.1	localhost localhost.localdomain localhost4 localhost4.localdomain4
# ::1	localhost localhost.localdomain localhost6 localhost6.localdomain6
# 172.11.73.1	host.containers.internal host.docker.internal
# 172.11.73.2	ec90e9097fc6 stoic_cannon
# search us-east-2.compute.internal
# nameserver 172.11.73.1
#
# [12:32:41.284015513] # podman container checkpoint ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42
# [12:32:41.357133430] CRIU checkpointing failed -52.  Please check CRIU logfile /var/lib/containers/storage/overlay-containers/ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /var/lib/containers/storage/overlay-containers/ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42/userdata/checkpoint --work-path /var/lib/containers/storage/overlay-containers/ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42/userdata ec90e9097fc6a98a560971fce4f97e75cc6e90afeeb9c4793c5a13ad304d9c42` failed: exit status 1
# [12:32:41.360221952] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# # [teardown]
not ok 6 [520] podman checkpoint/restore the latest container
# (from function `bail-now' in file /usr/share/podman/test/system/helpers.bash, line 187,
#  from function `die' in file /usr/share/podman/test/system/helpers.bash, line 970,
#  from function `run_podman' in file /usr/share/podman/test/system/helpers.bash, line 572,
#  in test file /usr/share/podman/test/system/520-checkpoint.bats, line 432)
#   `run_podman container checkpoint --latest' failed
#
# [12:32:41.873544793] # podman run -d quay.io/libpod/testimage:20241011 top
# [12:32:42.042898743] 9b4501d035183dbd21eaaf20d12404d7d8124c851636441008f040fed1c4a170
#
# [12:32:42.048453522] # podman container checkpoint --latest
# [12:32:42.111486045] CRIU checkpointing failed -52.  Please check CRIU logfile /var/lib/containers/storage/overlay-containers/9b4501d035183dbd21eaaf20d12404d7d8124c851636441008f040fed1c4a170/userdata/dump.log: Invalid exchange
# Error: `/usr/bin/crun checkpoint --image-path /var/lib/containers/storage/overlay-containers/9b4501d035183dbd21eaaf20d12404d7d8124c851636441008f040fed1c4a170/userdata/checkpoint --work-path /var/lib/containers/storage/overlay-containers/9b4501d035183dbd21eaaf20d12404d7d8124c851636441008f040fed1c4a170/userdata 9b4501d035183dbd21eaaf20d12404d7d8124c851636441008f040fed1c4a170` failed: exit status 1
# [12:32:42.114521185] [ rc=125 (** EXPECTED 0 **) ]
# #/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
# #| FAIL: exit code is 125; expected 0
# #\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^