Flaky sqlalchemy test with pypy

[xrmx]

Describe your environment

Steps to reproduce

tox -e pypy3-test-instrumentation-sqlalchemy-1

What is the expected behavior?

Tests pass

What is the actual behavior?

=================================== FAILURES ===================================
______ TestSqlalchemyMetricsInstrumentation.test_metrics_two_connections _______

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_two_connections>

    def tearDown(self):
        super().tearDown()
>       SQLAlchemyInstrumentor().uninstrument()

instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
opentelemetry-instrumentation/src/opentelemetry/instrumentation/instrumentor.py:128: in uninstrument
    result = self._uninstrument(**kwargs)
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/__init__.py:229: in _uninstrument
    EngineTracer.remove_all_event_listeners()
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/engine.py:187: in remove_all_event_listeners
    remove(weak_ref_target(), identifier, func)
.tox/pypy3-test-instrumentation-sqlalchemy-1/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:213: in remove
    _event_key(target, identifier, fn).remove()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

target = None, identifier = 'handle_error'
fn = <function _handle_error at 0x00007fb2bec93100>

    def _event_key(target, identifier, fn):
        for evt_cls in _registrars[identifier]:
            tgt = evt_cls._accept_with(target)
            if tgt is not None:
                return _EventKey(target, identifier, fn, tgt)
        else:
>           raise exc.InvalidRequestError(
                "No such event '%s' for target '%s'" % (identifier, target)
            )
E           sqlalchemy.exc.InvalidRequestError: No such event 'handle_error' for target 'None'

.tox/pypy3-test-instrumentation-sqlalchemy-1/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:29: InvalidRequestError
_____ TestSqlalchemyMetricsInstrumentation.test_metrics_without_pool_name ______

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_without_pool_name>

    def test_metrics_without_pool_name(self):
        pool_name = "pool_test_name"
        engine = sqlalchemy.create_engine(
            "sqlite:///:memory:",
            pool_size=5,
            poolclass=QueuePool,
            pool_logging_name=pool_name,
        )
    
        self.assertIsNone(self.memory_metrics_reader.get_metrics_data())
    
        with engine.connect():
>           self.assert_pool_idle_used_expected(
                pool_name=pool_name, idle=0, used=1
            )

instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:83: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:34: in assert_pool_idle_used_expected
    metrics = self.get_sorted_metrics()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_without_pool_name>

    def get_sorted_metrics(self):
        resource_metrics = (
>           self.memory_metrics_reader.get_metrics_data().resource_metrics
        )
E       AttributeError: 'NoneType' object has no attribute 'resource_metrics'

.tox/pypy3-test-instrumentation-sqlalchemy-1/lib/pypy3.8/site-packages/opentelemetry/test/test_base.py:148: AttributeError
------------------------------ Captured log call -------------------------------
WARNING  opentelemetry.instrumentation.instrumentor:instrumentor.py:100 Attempting to instrument while already instrumented

Additional context Add any other context about the problem here.

[xrmx]

Another similar:

_____ TestSqlalchemyMetricsInstrumentation.test_metrics_without_pool_name ______

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_without_pool_name>

    def tearDown(self):
        super().tearDown()
>       SQLAlchemyInstrumentor().uninstrument()

instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
opentelemetry-instrumentation/src/opentelemetry/instrumentation/instrumentor.py:128: in uninstrument
    result = self._uninstrument(**kwargs)
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/__init__.py:229: in _uninstrument
    EngineTracer.remove_all_event_listeners()
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/engine.py:187: in remove_all_event_listeners
    remove(weak_ref_target(), identifier, func)
.tox/pypy3-test-instrumentation-sqlalchemy-0/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:178: in remove
    _event_key(target, identifier, fn).remove()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

target = None, identifier = 'before_cursor_execute'
fn = <bound method EngineTracer._before_cur_exec of <opentelemetry.instrumentation.sqlalchemy.engine.EngineTracer object at 0x00000000037d5590>>

    def _event_key(target, identifier, fn):
        for evt_cls in _registrars[identifier]:
            tgt = evt_cls._accept_with(target)
            if tgt is not None:
                return _EventKey(target, identifier, fn, tgt)
        else:
>           raise exc.InvalidRequestError("No such event '%s' for target '%s'" %
                                          (identifier, target))
E           sqlalchemy.exc.InvalidRequestError: No such event 'before_cursor_execute' for target 'None'

.tox/pypy3-test-instrumentation-sqlalchemy-0/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:27: InvalidRequestError

[xyq175com]

Is this still a issue, I run test in my local got all passed

[xrmx]

Flaky does not mean it fails everytime but that it fails sometime and often it's not easy to reproduce.

[tammy-baylis-swi]

This could relate to sqlalchemy instrumentor engine.py's use of weakref, and weak references staying alive longer than expected in pypy compared to cpython. The dep was first added in https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1771 to fix a memory leak. Another part of the fix is in https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3053 which would be good to review in case it helps with the flaky tests.

[emdneto]

Thanks for the analysis @tammy-baylis-swi . Another related: https://github.com/open-telemetry/opentelemetry-python-contrib/actions/runs/12129370868/job/33817773387?pr=3060

[xrmx]

Another one

=================================== FAILURES ===================================
______ TestSqlalchemyMetricsInstrumentation.test_metrics_two_connections _______

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_two_connections>

    def tearDown(self):
        super().tearDown()
>       SQLAlchemyInstrumentor().uninstrument()

instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:31: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
opentelemetry-instrumentation/src/opentelemetry/instrumentation/instrumentor.py:130: in uninstrument
    result = self._uninstrument(**kwargs)
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/__init__.py:244: in _uninstrument
    EngineTracer.remove_all_event_listeners()
instrumentation/opentelemetry-instrumentation-sqlalchemy/src/opentelemetry/instrumentation/sqlalchemy/engine.py:200: in remove_all_event_listeners
    remove(weak_ref_target(), identifier, func)
.tox/pypy3-test-instrumentation-sqlalchemy-1/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:213: in remove
    _event_key(target, identifier, fn).remove()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

target = None, identifier = 'before_cursor_execute'
fn = <bound method EngineTracer._before_cur_exec of <opentelemetry.instrumentation.sqlalchemy.engine.EngineTracer object at 0x0000000004e1ba28>>

    def _event_key(target, identifier, fn):
        for evt_cls in _registrars[identifier]:
            tgt = evt_cls._accept_with(target)
            if tgt is not None:
                return _EventKey(target, identifier, fn, tgt)
        else:
>           raise exc.InvalidRequestError(
                "No such event '%s' for target '%s'" % (identifier, target)
            )
E           sqlalchemy.exc.InvalidRequestError: No such event 'before_cursor_execute' for target 'None'

.tox/pypy3-test-instrumentation-sqlalchemy-1/lib/pypy3.8/site-packages/sqlalchemy/event/api.py:29: InvalidRequestError
_____ TestSqlalchemyMetricsInstrumentation.test_metrics_without_pool_name ______

self = <tests.test_sqlalchemy_metrics.TestSqlalchemyMetricsInstrumentation testMethod=test_metrics_without_pool_name>

    def test_metrics_without_pool_name(self):
        pool_name = "pool_test_name"
        engine = sqlalchemy.create_engine(
            "sqlite:///:memory:",
            pool_size=5,
            poolclass=QueuePool,
            pool_logging_name=pool_name,
        )
    
        self.assertIsNone(self.memory_metrics_reader.get_metrics_data())
    
        with engine.connect():
>           self.assert_pool_idle_used_expected(
                pool_name=pool_name, idle=0, used=1
            )

instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:83: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
instrumentation/opentelemetry-instrumentation-sqlalchemy/tests/test_sqlalchemy_metrics.py:35: in assert_pool_idle_used_expected
    self.assertEqual(len(metrics), 1)
E   AssertionError: 0 != 1