cannot switch to state 15; another operation (2) is in progress

[ale-dd]

• asyncpg version: 0.23.0
• PostgreSQL version: Aurora PostgreSQL 10.14
• Do you use a PostgreSQL SaaS? If so, which? Can you reproduce the issue with a local PostgreSQL install?: RDS Aurora (and hard to repro)
• Python version: 3.9
• Platform: Linux
• Do you use pgbouncer?: no
• Did you install asyncpg with pip?: pip3.9 install --user
• If you built asyncpg locally, which version of Cython did you use?:
• Can the issue be reproduced under both asyncio and uvloop?: repro'd with uvloop

  File "/root/.local/lib/python3.9/site-packages/asyncpg/pool.py", line 792, in __aexit__
    await self.pool.release(con)
  File "/root/.local/lib/python3.9/site-packages/asyncpg/pool.py", line 666, in release
    return await asyncio.shield(ch.release(timeout))
  File "/usr/lib/python3.9/asyncio/futures.py", line 284, in __await__
    yield self  # This tells Task to wait for completion.
  File "/usr/lib/python3.9/asyncio/tasks.py", line 328, in __wakeup
    future.result()
  File "/usr/lib/python3.9/asyncio/futures.py", line 201, in result
    raise self._exception
  File "/usr/lib/python3.9/asyncio/tasks.py", line 256, in __step
    result = coro.send(None)
  File "/root/.local/lib/python3.9/site-packages/asyncpg/pool.py", line 218, in release
    raise ex
  File "/root/.local/lib/python3.9/site-packages/asyncpg/pool.py", line 208, in release
    await self._con.reset(timeout=budget)
  File "/root/.local/lib/python3.9/site-packages/asyncpg/connection.py", line 1311, in reset
    await self.execute(reset_query, timeout=timeout)
  File "lib/common.py", line 164, in execute
    return await self._with_logger(super(PSQLLoggerMixin, self).execute)(*args, **kwargs)
  File "lib/common.py", line 119, in wrapper
    ret_val = await func(*args, **kwargs)
  File "/root/.local/lib/python3.9/site-packages/asyncpg/connection.py", line 297, in execute
    return await self._protocol.query(query, timeout)
  File "asyncpg/protocol/protocol.pyx", line 338, in query
  File "/usr/lib/python3.9/asyncio/futures.py", line 287, in __await__
    return self.result()  # May raise too.
  File "/usr/lib/python3.9/asyncio/futures.py", line 201, in result
    raise self._exception
  File "asyncpg/protocol/protocol.pyx", line 331, in asyncpg.protocol.protocol.BaseProtocol.query
  File "asyncpg/protocol/coreproto.pyx", line 1061, in asyncpg.protocol.protocol.CoreProtocol._simple_query
  File "asyncpg/protocol/coreproto.pyx", line 768, in asyncpg.protocol.protocol.CoreProtocol._set_state
asyncpg.exceptions._base.InternalClientError: cannot switch to state 15; another operation (2) is in progress

[elprans]

Please provide a way to reproduce, the traceback isn't much help here, I'm afraid.

[villasv]

I've also observed this but couldn't built a MWE either, it happens intermittently only in production.

This is my stack trace:

  File "___/query_runner.py", line 106, in __run
    query_results = await asyncio.gather(*map(query_to_pd, prepared_queries.values()))
  File "___/connection/pool.py", line 109, in query_to_pd
    return pd.DataFrame(result.all(), columns=result.keys())
  File "sqlalchemy/ext/asyncio/engine.py", line 472, in __aexit__
    await self.close()
  File "sqlalchemy/ext/asyncio/engine.py", line 309, in close
    await greenlet_spawn(conn.close)
  File "sqlalchemy/util/_concurrency_py3k.py", line 127, in greenlet_spawn
    result = context.throw(*sys.exc_info())
  File "sqlalchemy/future/engine.py", line 246, in close
    super(Connection, self).close()
  File "sqlalchemy/engine/base.py", line 1134, in close
    self._transaction.close()
  File "sqlalchemy/engine/base.py", line 2286, in close
    self._do_close()
  File "sqlalchemy/engine/base.py", line 2502, in _do_close
    self._close_impl()
  File "sqlalchemy/engine/base.py", line 2488, in _close_impl
    self._connection_rollback_impl()
  File "sqlalchemy/engine/base.py", line 2480, in _connection_rollback_impl
    self.connection._rollback_impl()
  File "sqlalchemy/engine/base.py", line 960, in _rollback_impl
    self._handle_dbapi_exception(e, None, None, None, None)
  File "sqlalchemy/engine/base.py", line 1998, in _handle_dbapi_exception
    util.raise_(exc_info[1], with_traceback=exc_info[2])
  File "sqlalchemy/util/compat.py", line 207, in raise_
    raise exception
  File "sqlalchemy/engine/base.py", line 958, in _rollback_impl
    self.engine.dialect.do_rollback(self.connection)
  File "sqlalchemy/engine/default.py", line 667, in do_rollback
    dbapi_connection.rollback()
  File "sqlalchemy/dialects/postgresql/asyncpg.py", line 701, in rollback
    self._handle_exception(error)
  File "sqlalchemy/dialects/postgresql/asyncpg.py", line 654, in _handle_exception
    raise error
  File "sqlalchemy/dialects/postgresql/asyncpg.py", line 699, in rollback
    self.await_(self._transaction.rollback())
  File "sqlalchemy/util/_concurrency_py3k.py", line 69, in await_only
    return current.driver.switch(awaitable)
  File "sqlalchemy/util/_concurrency_py3k.py", line 122, in greenlet_spawn
    value = await result
  File "asyncpg/transaction.py", line 219, in rollback
    await self.__rollback()
  File "asyncpg/transaction.py", line 198, in __rollback
    await self._connection.execute(query)
  File "asyncpg/connection.py", line 295, in execute
    return await self._protocol.query(query, timeout)
  File "asyncpg/protocol/protocol.pyx", line 316, in query
  File "asyncpg/protocol/protocol.pyx", line 309, in asyncpg.protocol.protocol.BaseProtocol.query
  File "asyncpg/protocol/coreproto.pyx", line 976, in asyncpg.protocol.protocol.CoreProtocol._simple_query
    self._set_state(PROTOCOL_SIMPLE_QUERY)
  File "asyncpg/protocol/coreproto.pyx", line 774, in asyncpg.protocol.protocol.CoreProtocol._set_state
    raise apg_exc.InternalClientError(

This part of the stack trace:

  File "___/connection/pool.py", line 109, in query_to_pd
    return pd.DataFrame(result.all(), columns=result.keys())
  File "sqlalchemy/ext/asyncio/engine.py", line 472, in __aexit__
    await self.close()

Corresponds to this bit of code using SQLAlchemy to return a Pandas DataFrame

async def query_to_pd(query):
    async with db_engine.connect() as conn:
        result = await conn.execute(text(query))
        return pd.DataFrame(result.all(), columns=result.keys())