ruamel/yaml errors in Jenkins PR with node _update_config

[cezarmoise]

Does not happen locally, tested with up to date branch

Error 1

ValueError: I/O operation on closed file.

Stacktrace

self = <secondary_indexes_test.TestSecondaryIndexes object at 0x7fd9fc5f20f0>

    @pytest.mark.scylla_mode("!debug")
    def test_stop_node_during_index_build(self):
        """
        Stop one node during index building and read data by index
        """
>       self._node_action_during_index_build(node_action="stop", nodes=4, rf=3, num_rows=500_000)

secondary_indexes_test.py:1362: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
secondary_indexes_test.py:1391: in _node_action_during_index_build
    session = self.prepare(nodes=nodes, rf=rf, keyspace_name=keyspace_name, session_node=3)
secondary_indexes_test.py:138: in prepare
    cluster.populate(populate, use_vnodes=True)
../scylla/.local/lib/python3.12/site-packages/ccmlib/cluster.py:329: in populate
    self.new_node(i, debug=debug, initial_token=tk, data_center=dc, rack=rack)
../scylla/.local/lib/python3.12/site-packages/ccmlib/cluster.py:336: in new_node
    node = self.create_node(name=f'node{i}',
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_cluster.py:95: in create_node
    return ScyllaNode(name, self, auto_bootstrap, None,
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:54: in __init__
    super().__init__(name, cluster, auto_bootstrap,
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:125: in __init__
    self.import_config_files()
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:980: in import_config_files
    self._update_config()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1589: in _update_config
    yaml.dump(values, f)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:590: in dump
    self._context_manager.dump(data)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:953: in dump
    self._yaml.representer.represent(data)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/representer.py:84: in represent
    self.serializer.serialize(node)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/serializer.py:105: in serialize
    self.emitter.emit(
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:267: in emit
    self.state()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:370: in expect_document_start
    self.write_indicator('---', True)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ruamel.yaml.emitter.RoundTripEmitter object at 0x7fd9fae4ac60>
indicator = '---', need_whitespace = True, whitespace = False, indention = False

    def write_indicator(
        self,
        indicator: Any,
        need_whitespace: Any,
        whitespace: bool = False,
        indention: bool = False,
    ) -> None:
        if self.whitespace or not need_whitespace:
            data = indicator
        else:
            data = ' ' + indicator
        self.whitespace = whitespace
        self.indention = self.indention and indention
        self.column += len(data)
        self.open_ended = False
        if bool(self.encoding):
            data = data.encode(self.encoding)
>       self.stream.write(data)
E       ValueError: I/O operation on closed file.

../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:1249: ValueError

Error 2

ruamel.yaml.emitter.EmitterError: expected DocumentStartEvent, but got MappingStartEvent(tag='tag:yaml.org,2002:map', implicit=True, flow_style=False)

Stacktrace

self = <materialized_views_test.TestInterruptBuildProcess object at 0x7fd9fc8b1d60>

    @pytest.mark.scylla_mode("!debug")
    @pytest.mark.skip_if(with_feature("tablets") & issue_open("#16739"))
    def test_interrupt_build_process_and_resharding_max_to_half_test(self):
        """Test that an interrupted MV build process is resumed after interrupted resharding,
        with resharding cpu_count() -> cpu_count() // 2"""
        # For some reason, Scylla's hwloc only sees cpu_count() - 1 cpus
>       self._do_resharding_test(self._max_shards(), self._half_shards(), interrupt_resharding=True)

materialized_views_test.py:4311: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
materialized_views_test.py:4333: in _do_resharding_test
    self.__do_resharding_test(smp_before, smp_after, compression, interrupt_resharding)
materialized_views_test.py:4405: in __do_resharding_test
    [logger.debug(future.result()) for future in futures]
/usr/local/lib/python3.12/concurrent/futures/_base.py:456: in result
    return self.__get_result()
/usr/local/lib/python3.12/concurrent/futures/_base.py:401: in __get_result
    raise self._exception
/usr/local/lib/python3.12/concurrent/futures/thread.py:58: in run
    result = self.fn(*self.args, **self.kwargs)
materialized_views_test.py:4340: in __do_start_node_with_interrupt_resharding
    node.start(jvm_args=jvm_args, no_wait=True)
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:711: in start
    scylla_process = self._start_scylla(args=args, marks=marks, update_pid=update_pid,
../scylla/.local/lib/python3.12/site-packages/ccmlib/scylla_node.py:340: in _start_scylla
    self._update_pid(self._process_scylla)
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1900: in _update_pid
    self.__update_status()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1777: in __update_status
    self._update_config()
../scylla/.local/lib/python3.12/site-packages/ccmlib/node.py:1589: in _update_config
    yaml.dump(values, f)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:594: in dump
    return self.dump_all([data], stream, transform=transform)
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:605: in dump_all
    self._context_manager.teardown_output()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/main.py:914: in teardown_output
    self._yaml.serializer.close()
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/serializer.py:91: in close
    self.emitter.emit(StreamEndEvent())
../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:267: in emit
    self.state()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <ruamel.yaml.emitter.RoundTripEmitter object at 0x7fd9fae4ac60>
first = False

    def expect_document_start(self, first: bool = False) -> None:
        if isinstance(self.event, DocumentStartEvent):
            if (self.event.version or self.event.tags) and self.open_ended:
                self.write_indicator('...', True)
                self.write_indent()
            if self.event.version:
                version_text = self.prepare_version(self.event.version)
                self.write_version_directive(version_text)
            self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy()
            if self.event.tags:
                handles = sorted(self.event.tags.keys())
                for handle in handles:
                    prefix = self.event.tags[handle]
                    self.tag_prefixes[prefix] = handle
                    handle_text = self.prepare_tag_handle(handle)
                    prefix_text = self.prepare_tag_prefix(prefix)
                    self.write_tag_directive(handle_text, prefix_text)
            implicit = (
                first
                and not self.event.explicit
                and not self.canonical
                and not self.event.version
                and not self.event.tags
                and not self.check_empty_document()
            )
            if not implicit:
                self.write_indent()
                self.write_indicator('---', True)
                if self.canonical:
                    self.write_indent()
            self.state = self.expect_document_root
        elif isinstance(self.event, StreamEndEvent):
            if self.open_ended:
                self.write_indicator('...', True)
                self.write_indent()
            self.write_stream_end()
            self.state = self.expect_nothing
        else:
>           raise EmitterError(f'expected DocumentStartEvent, but got {self.event!s}')
E           ruamel.yaml.emitter.EmitterError: expected DocumentStartEvent, but got MappingStartEvent(tag='tag:yaml.org,2002:map', implicit=True, flow_style=False)

../scylla/.local/lib/python3.12/site-packages/ruamel/yaml/emitter.py:381: EmitterError

[fruch]

is that consistent ? i.e. if you trigger again that CI run ? it happens every single time ?

[cezarmoise]

The first commit where this happens is 27475153553311c9c9dab1aaf9bd6b5f4f0e4925, the commit that merges the yaml changes. However I see that this run has errors while this one you rebuit does not, even though they are on the same ccm commit (9370f298cec8f6ce06468aa12a5aa33d38b0a7bf). I also did a rebuild and it failed again

[cezarmoise]

Closing as it not longer reproduces