salt icon indicating copy to clipboard operation
salt copied to clipboard

Published 20 hours ago • verified publisher icon saltstack

[BUG] TypeError in transport/tcp.py

Open bgdnlp opened this issue 1 year ago • 13 comments

Description Issue in SaltStack master 3007.0.

On FreeBSD, upgraded to 3007.0, it complained it needs Tornado. I Installed Tornado 6.3.3, the master started, but then any command results in this:

root@freebsd:/usr/local/etc/salt # salt '*' test.ping
[WARNING ] TCP Publish Client encountered an exception while connecting to /var/run/salt/master/master_event_pub.ipc: StreamClosedError('Stream is closed'), will reconnect in 1 seconds -   File "/usr/local/bin/salt", line 33, in <module>
    sys.exit(load_entry_point('salt==3007.0', 'console_scripts', 'salt')())

  File "/usr/local/lib/python3.9/site-packages/salt/scripts.py", line 532, in salt_main
    client.run()

  File "/usr/local/lib/python3.9/site-packages/salt/cli/salt.py", line 192, in run
    for full_ret in cmd_func(**kwargs):

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 816, in cmd_cli
    self.pub_data = self.run_job(

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 388, in run_job
    pub_data = self.pub(

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1905, in pub
    if listen and not self.event.connect_pub(timeout=timeout):

  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 323, in connect_pub
    self.subscriber = salt.utils.asynchronous.SyncWrapper(

  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 77, in __init__
    self.obj = cls(*args, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 210, in ipc_publish_client
    return publish_client(opts, io_loop, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 152, in publish_client
    return salt.transport.tcp.PublishClient(

  File "/usr/local/lib/python3.9/site-packages/salt/transport/tcp.py", line 219, in __init__
    super().__init__(opts, io_loop, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 398, in __init__
    super().__init__()

[ERROR   ] An un-handled exception was caught by Salt's global exception handler:
TypeError: argument must be an int, or have a fileno() method.
Traceback (most recent call last):
  File "/usr/local/bin/salt", line 33, in <module>
    sys.exit(load_entry_point('salt==3007.0', 'console_scripts', 'salt')())
  File "/usr/local/lib/python3.9/site-packages/salt/scripts.py", line 532, in salt_main
    client.run()
  File "/usr/local/lib/python3.9/site-packages/salt/cli/salt.py", line 192, in run
    for full_ret in cmd_func(**kwargs):
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 831, in cmd_cli
    for fn_ret in self.get_cli_event_returns(
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1654, in get_cli_event_returns
    for ret in self.get_iter_returns(
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1188, in get_iter_returns
    for raw in ret_iter:
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1105, in get_returns_no_block
    raw = self.event.get_event(
  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 651, in get_event
    ret = self._get_event(wait, tag, match_func, no_block)
  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 553, in _get_event
    raw = self.subscriber.recv(timeout=wait)
  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 139, in wrap
    raise exc_info[1].with_traceback(exc_info[2])
  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 147, in _target
    result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
  File "/usr/local/lib/python3.9/site-packages/tornado/ioloop.py", line 527, in run_sync
    return future_cell[0].result()
  File "/usr/local/lib/python3.9/site-packages/salt/transport/tcp.py", line 374, in recv
    events, _, _ = select.select([self._stream.socket], [], [], 0)
TypeError: argument must be an int, or have a fileno() method.
Traceback (most recent call last):
  File "/usr/local/bin/salt", line 33, in <module>
    sys.exit(load_entry_point('salt==3007.0', 'console_scripts', 'salt')())
  File "/usr/local/lib/python3.9/site-packages/salt/scripts.py", line 532, in salt_main
    client.run()
  File "/usr/local/lib/python3.9/site-packages/salt/cli/salt.py", line 192, in run
    for full_ret in cmd_func(**kwargs):
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 831, in cmd_cli
    for fn_ret in self.get_cli_event_returns(
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1654, in get_cli_event_returns
    for ret in self.get_iter_returns(
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1188, in get_iter_returns
    for raw in ret_iter:
  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1105, in get_returns_no_block
    raw = self.event.get_event(
  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 651, in get_event
    ret = self._get_event(wait, tag, match_func, no_block)
  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 553, in _get_event
    raw = self.subscriber.recv(timeout=wait)
  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 139, in wrap
    raise exc_info[1].with_traceback(exc_info[2])
  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 147, in _target
    result = io_loop.run_sync(lambda: getattr(self.obj, key)(*args, **kwargs))
  File "/usr/local/lib/python3.9/site-packages/tornado/ioloop.py", line 527, in run_sync
    return future_cell[0].result()
  File "/usr/local/lib/python3.9/site-packages/salt/transport/tcp.py", line 374, in recv
    events, _, _ = select.select([self._stream.socket], [], [], 0)
TypeError: argument must be an int, or have a fileno() method.

Also see this bug report on FreeBSD Bugzilla: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=278149

I also installed Salt on a new machine and it behaves the same way.

Setup

  • [x] VM running on a cloud service, please be explicit and add details

FreeBSD running on AWS. No jails.

Steps to Reproduce the behavior Install py39-salt on a fresh FreeBSD 14, install py39-tornado, start salt-master, then try to run any command.

Expected behavior Run the command, don't error out

Screenshots N/A, trace above

Versions Report I killed the machine, sorry, but it's 3007.0.

bgdnlp avatar Apr 25 '24 11:04 bgdnlp

It's not just the master. The minion breaks too

xzenor avatar Apr 25 '24 15:04 xzenor

I'm running into this issue as well on my system. Interestingly after manually installing tornado I get an error about pyzmq version conflict. I'm running salt-call state.apply:

[ERROR   ] Failed to import module config, this is due most likely to a syntax error:
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/pkg_resources/__init__.py", line 581, in _build_master
    ws.require(__requires__)
  File "/usr/local/lib/python3.9/site-packages/pkg_resources/__init__.py", line 909, in require
    needed = self.resolve(parse_requirements(requirements))
  File "/usr/local/lib/python3.9/site-packages/pkg_resources/__init__.py", line 800, in resolve
    raise VersionConflict(dist, req).with_context(dependent_req)
pkg_resources.ContextualVersionConflict: (pyzmq 25.0.2 (/usr/local/lib/python3.9/site-packages), Requirement.parse('pyzmq>=25.1.2'), {'salt'})

ring-pete avatar Apr 26 '24 00:04 ring-pete

Yep had that too so installed from pip but then a bunch of new errors came up in the end installed salt from pip and that resulted in more errors.

# python3.9 -m pip install pyzmq --upgrade
Requirement already satisfied: pyzmq in /usr/local/lib/python3.9/site-packages (25.0.2)
Collecting pyzmq
  Using cached pyzmq-26.0.2.tar.gz (266 kB)
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
  Preparing metadata (pyproject.toml) ... done
Building wheels for collected packages: pyzmq
  Building wheel for pyzmq (pyproject.toml) ... done
  Created wheel for pyzmq: filename=pyzmq-26.0.2-cp39-cp39-freebsd_14_0_release_p6_amd64.whl size=290525 sha256=fdbfc51e83c86d75144f2df02df045ae426ae76928beb93f73e8f77167e3e430
  Stored in directory: /root/.cache/pip/wheels/91/a1/50/7deb937b13e6f21d9463ef937e18d2b801310c2e8170ae2efd
Successfully built pyzmq
Installing collected packages: pyzmq
  Attempting uninstall: pyzmq
    Found existing installation: pyzmq 25.0.2
    Uninstalling pyzmq-25.0.2:
      Successfully uninstalled pyzmq-25.0.2
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
salt 3007.0 requires aiohttp>=3.9.0, which is not installed.
salt 3007.0 requires cherrypy>=18.6.1, which is not installed.
salt 3007.0 requires setproctitle>=1.2.3, which is not installed.
salt 3007.0 requires pyopenssl>=24.0.0, but you have pyopenssl 23.2.0 which is incompatible.
Successfully installed pyzmq-26.0.2
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

After running:
# python3.9 -m pip install salt --upgrade
Requirement already satisfied: salt in /usr/local/lib/python3.9/site-packages (3007.0)
Ignoring rpm-vercmp: markers 'sys_platform == "linux"' don't match your environment
Ignoring pywin32: markers 'sys_platform == "win32"' don't match your environment
Ignoring wmi: markers 'sys_platform == "win32"' don't match your environment
Ignoring pythonnet: markers 'sys_platform == "win32"' don't match your environment
Ignoring pymssql: markers 'sys_platform == "win32"' don't match your environment
Ignoring pymysql: markers 'sys_platform == "win32"' don't match your environment
Ignoring lxml: markers 'sys_platform == "win32"' don't match your environment
Ignoring xmltodict: markers 'sys_platform == "win32"' don't match your environment
Requirement already satisfied: Jinja2 in /usr/local/lib/python3.9/site-packages (from salt) (3.1.3)
Requirement already satisfied: jmespath in /usr/local/lib/python3.9/site-packages (from salt) (1.0.1)
Requirement already satisfied: msgpack>=1.0.0 in /usr/local/lib/python3.9/site-packages (from salt) (1.0.8)
Requirement already satisfied: PyYAML in /usr/local/lib/python3.9/site-packages (from salt) (6.0.1)
Requirement already satisfied: MarkupSafe in /usr/local/lib/python3.9/site-packages (from salt) (2.1.5)
Requirement already satisfied: requests>=2.25.1 in /usr/local/lib/python3.9/site-packages (from salt) (2.31.0)
Requirement already satisfied: distro>=1.0.1 in /usr/local/lib/python3.9/site-packages (from salt) (1.9.0)
Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.9/site-packages (from salt) (5.9.8)
Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.9/site-packages (from salt) (23.2)
Requirement already satisfied: looseversion in /usr/local/lib/python3.9/site-packages (from salt) (1.3.0)
Requirement already satisfied: tornado>=6.3.3 in /usr/local/lib/python3.9/site-packages (from salt) (6.4)
Collecting aiohttp>=3.9.0 (from salt)
  Downloading aiohttp-3.9.5.tar.gz (7.5 MB)
     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.5/7.5 MB 36.7 MB/s eta 0:00:00
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
  Installing backend dependencies ... done
  Preparing metadata (pyproject.toml) ... done
Collecting setproctitle>=1.2.3 (from salt)
  Downloading setproctitle-1.3.3.tar.gz (27 kB)
  Installing build dependencies ... done
  Getting requirements to build wheel ... done
  Preparing metadata (pyproject.toml) ... done
Requirement already satisfied: timelib>=0.2.5 in /usr/local/lib/python3.9/site-packages (from salt) (0.3.0)
Collecting pyopenssl>=24.0.0 (from salt)
  Downloading pyOpenSSL-24.1.0-py3-none-any.whl.metadata (12 kB)
Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/site-packages (from salt) (2.9.0)
Requirement already satisfied: python-gnupg>=0.4.7 in /usr/local/lib/python3.9/site-packages (from salt) (0.5.1)
Collecting cherrypy>=18.6.1 (from salt)
  Downloading CherryPy-18.9.0-py3-none-any.whl.metadata (8.8 kB)
Requirement already satisfied: importlib-metadata>=3.3.0 in /usr/local/lib/python3.9/site-packages (from salt) (7.1.0)
Requirement already satisfied: cryptography>=42.0.0 in /usr/local/lib/python3.9/site-packages (from salt) (42.0.5)
Requirement already satisfied: pyzmq>=25.1.2 in /usr/local/lib/python3.9/site-packages (from salt) (26.0.2)
Requirement already satisfied: pycryptodomex>=3.9.8 in /usr/local/lib/python3.9/site-packages (from salt) (3.20.0)
Collecting aiosignal>=1.1.2 (from aiohttp>=3.9.0->salt)
  Downloading aiosignal-1.3.1-py3-none-any.whl.metadata (4.0 kB)
Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/site-packages (from aiohttp>=3.9.0->salt) (23.2.0)
Collecting frozenlist>=1.1.1 (from aiohttp>=3.9.0->salt)
  Downloading frozenlist-1.4.1-py3-none-any.whl.metadata (12 kB)
Collecting multidict<7.0,>=4.5 (from aiohttp>=3.9.0->salt)
  Downloading multidict-6.0.5-py3-none-any.whl.metadata (4.2 kB)
Collecting yarl<2.0,>=1.0 (from aiohttp>=3.9.0->salt)
  Downloading yarl-1.9.4-py3-none-any.whl.metadata (31 kB)
Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.9/site-packages (from aiohttp>=3.9.0->salt) (4.0.3)
Collecting cheroot>=8.2.1 (from cherrypy>=18.6.1->salt)
  Downloading cheroot-10.0.1-py3-none-any.whl.metadata (7.1 kB)
Collecting portend>=2.1.1 (from cherrypy>=18.6.1->salt)
  Downloading portend-3.2.0-py3-none-any.whl.metadata (3.6 kB)
Collecting more-itertools (from cherrypy>=18.6.1->salt)
  Downloading more_itertools-10.2.0-py3-none-any.whl.metadata (34 kB)
Collecting zc.lockfile (from cherrypy>=18.6.1->salt)
  Downloading zc.lockfile-3.0.post1-py3-none-any.whl.metadata (6.2 kB)
Collecting jaraco.collections (from cherrypy>=18.6.1->salt)
  Downloading jaraco.collections-5.0.1-py3-none-any.whl.metadata (3.8 kB)
Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.9/site-packages (from cryptography>=42.0.0->salt) (1.16.0)
Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/site-packages (from importlib-metadata>=3.3.0->salt) (3.18.1)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/site-packages (from python-dateutil>=2.8.1->salt) (1.16.0)
Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/lib/python3.9/site-packages (from requests>=2.25.1->salt) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/site-packages (from requests>=2.25.1->salt) (3.7)
Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.9/site-packages (from requests>=2.25.1->salt) (1.26.18)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/site-packages (from requests>=2.25.1->salt) (2024.2.2)
Requirement already satisfied: pycparser in /usr/local/lib/python3.9/site-packages (from cffi>=1.12->cryptography>=42.0.0->salt) (2.22)
Collecting jaraco.functools (from cheroot>=8.2.1->cherrypy>=18.6.1->salt)
  Downloading jaraco.functools-4.0.1-py3-none-any.whl.metadata (2.9 kB)
Collecting tempora>=1.8 (from portend>=2.1.1->cherrypy>=18.6.1->salt)
  Downloading tempora-5.5.1-py3-none-any.whl.metadata (3.0 kB)
Collecting jaraco.text (from jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading jaraco.text-3.12.0-py3-none-any.whl.metadata (4.0 kB)
Requirement already satisfied: setuptools in /usr/local/lib/python3.9/site-packages (from zc.lockfile->cherrypy>=18.6.1->salt) (63.1.0)
Requirement already satisfied: pytz in /usr/local/lib/python3.9/site-packages (from tempora>=1.8->portend>=2.1.1->cherrypy>=18.6.1->salt) (2024.1)
Collecting jaraco.context>=4.1 (from jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading jaraco.context-5.3.0-py3-none-any.whl.metadata (4.0 kB)
Collecting autocommand (from jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading autocommand-2.2.2-py3-none-any.whl.metadata (15 kB)
Collecting inflect (from jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading inflect-7.2.1-py3-none-any.whl.metadata (21 kB)
Collecting backports.tarfile (from jaraco.context>=4.1->jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading backports.tarfile-1.1.1-py3-none-any.whl.metadata (2.0 kB)
Collecting typeguard>=4.0.1 (from inflect->jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt)
  Downloading typeguard-4.2.1-py3-none-any.whl.metadata (3.7 kB)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/site-packages (from inflect->jaraco.text->jaraco.collections->cherrypy>=18.6.1->salt) (4.11.0)
Downloading CherryPy-18.9.0-py3-none-any.whl (348 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 348.8/348.8 kB 1.2 MB/s eta 0:00:00
Downloading pyOpenSSL-24.1.0-py3-none-any.whl (56 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 56.9/56.9 kB 1.7 MB/s eta 0:00:00
Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)
Downloading cheroot-10.0.1-py3-none-any.whl (104 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 104.8/104.8 kB 4.7 MB/s eta 0:00:00
Downloading frozenlist-1.4.1-py3-none-any.whl (11 kB)
Downloading more_itertools-10.2.0-py3-none-any.whl (57 kB)
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 57.0/57.0 kB 6.5 MB/s eta 0:00:00
Downloading multidict-6.0.5-py3-none-any.whl (9.7 kB)
Downloading portend-3.2.0-py3-none-any.whl (5.3 kB)
Downloading yarl-1.9.4-py3-none-any.whl (31 kB)
Downloading jaraco.collections-5.0.1-py3-none-any.whl (10 kB)
Downloading zc.lockfile-3.0.post1-py3-none-any.whl (9.8 kB)
Downloading tempora-5.5.1-py3-none-any.whl (13 kB)
Downloading jaraco.functools-4.0.1-py3-none-any.whl (9.8 kB)
Downloading jaraco.text-3.12.0-py3-none-any.whl (11 kB)
Downloading jaraco.context-5.3.0-py3-none-any.whl (6.5 kB)
Downloading autocommand-2.2.2-py3-none-any.whl (19 kB)
Downloading inflect-7.2.1-py3-none-any.whl (34 kB)
Downloading typeguard-4.2.1-py3-none-any.whl (34 kB)
Downloading backports.tarfile-1.1.1-py3-none-any.whl (29 kB)
Building wheels for collected packages: aiohttp, setproctitle
  Building wheel for aiohttp (pyproject.toml) ... done
  Created wheel for aiohttp: filename=aiohttp-3.9.5-cp39-cp39-freebsd_14_0_release_p6_amd64.whl size=406505 sha256=cd0ba8eeb2ff08cef9ab4336f903195a768675eb6daeaa2ea2c0d72d0ad050f0
  Stored in directory: /root/.cache/pip/wheels/56/5e/43/18555983d002ba95a19bf758a33c3251fa81e39847e3b46f23
  Building wheel for setproctitle (pyproject.toml) ... done
  Created wheel for setproctitle: filename=setproctitle-1.3.3-cp39-cp39-freebsd_14_0_release_p6_amd64.whl size=13442 sha256=69cbd69488ff1977414161867feac7de22785821982ff0c14ff4b5f897f8e42b
  Stored in directory: /root/.cache/pip/wheels/6a/ee/ec/0bd85bda0ff784b07950d664e4b009e23a6400f1d1f9474a25
Successfully built aiohttp setproctitle
Installing collected packages: zc.lockfile, setproctitle, multidict, more-itertools, frozenlist, backports.tarfile, autocommand, yarl, typeguard, jaraco.functools, jaraco.context, aiosignal, tempora, pyopenssl, inflect, cheroot, aiohttp, portend, jaraco.text, jaraco.collections, cherrypy
  Attempting uninstall: pyopenssl
    Found existing installation: pyOpenSSL 23.2.0
    Uninstalling pyOpenSSL-23.2.0:
      Successfully uninstalled pyOpenSSL-23.2.0
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
errbot 6.2.0 requires pygments-markdown-lexer>=0.1.0.dev39, which is not installed.
Successfully installed aiohttp-3.9.5 aiosignal-1.3.1 autocommand-2.2.2 backports.tarfile-1.1.1 cheroot-10.0.1 cherrypy-18.9.0 frozenlist-1.4.1 inflect-7.2.1 jaraco.collections-5.0.1 jaraco.context-5.3.0 jaraco.functools-4.0.1 jaraco.text-3.12.0 more-itertools-10.2.0 multidict-6.0.5 portend-3.2.0 pyopenssl-24.1.0 setproctitle-1.3.3 tempora-5.5.1 typeguard-4.2.1 yarl-1.9.4 zc.lockfile-3.0.post1
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv

# salt '*' test.ping
[WARNING ] TCP Publish Client encountered an exception while connecting to /var/run/salt/master/master_event_pub.ipc: StreamClosedError('Stream is closed'), will reconnect in 1 seconds -   File "/usr/local/bin/salt", line 33, in <module>
    sys.exit(load_entry_point('salt==3007.0', 'console_scripts', 'salt')())

  File "/usr/local/lib/python3.9/site-packages/salt/scripts.py", line 532, in salt_main
    client.run()

  File "/usr/local/lib/python3.9/site-packages/salt/cli/salt.py", line 192, in run
    for full_ret in cmd_func(**kwargs):

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 816, in cmd_cli
    self.pub_data = self.run_job(

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 388, in run_job
    pub_data = self.pub(

  File "/usr/local/lib/python3.9/site-packages/salt/client/__init__.py", line 1905, in pub
    if listen and not self.event.connect_pub(timeout=timeout):

  File "/usr/local/lib/python3.9/site-packages/salt/utils/event.py", line 323, in connect_pub
    self.subscriber = salt.utils.asynchronous.SyncWrapper(

  File "/usr/local/lib/python3.9/site-packages/salt/utils/asynchronous.py", line 77, in __init__
    self.obj = cls(*args, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 210, in ipc_publish_client
    return publish_client(opts, io_loop, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 152, in publish_client
    return salt.transport.tcp.PublishClient(

  File "/usr/local/lib/python3.9/site-packages/salt/transport/tcp.py", line 219, in __init__
    super().__init__(opts, io_loop, **kwargs)

  File "/usr/local/lib/python3.9/site-packages/salt/transport/base.py", line 398, in __init__
    super().__init__()

[ERROR   ] Request client send timedout
Salt request timed out. The master is not responding. You may need to run your command with `--async` in order to bypass the congested event bus. With `--async`, the CLI tool will print the job id (jid) and exit immediately without listening for responses. You can then use `salt-run jobs.lookup_jid` to look up the results of the job in the job cache later.

Ofloo avatar Apr 26 '24 07:04 Ofloo

For me the minion was fixed by installing py39-tornado-6.4, at least I don't see any issues there.

After rolling back to previous verison in freebsd I do get this error on the master:

[ERROR   ] Module/package collision: '/usr/local/lib/python3.9/site-packages/salt/utils/vault.py' and '/usr/local/lib/python3.9/site-packages/salt/utils/vault'
[ERROR   ] Module/package collision: '/usr/local/lib/python3.9/site-packages/salt/utils/__pycache__/vault.cpython-39.pyc' and '/usr/local/lib/python3.9/site-packages/salt/utils/vault'
[ERROR   ] Module/package collision: '/usr/local/lib/python3.9/site-packages/salt/utils/vault.py' and '/usr/local/lib/python3.9/site-packages/salt/utils/vault'
[ERROR   ] Module/package collision: '/usr/local/lib/python3.9/site-packages/salt/utils/__pycache__/vault.cpython-39.pyc' and '/usr/local/lib/python3.9/site-packages/salt/utils/vault'

Ofloo avatar Apr 26 '24 07:04 Ofloo

same here, 3007 is totally broken under FreeBSD.. was this even tested?

silenius avatar Apr 26 '24 14:04 silenius

same here, 3007 is totally broken under FreeBSD.. was this even tested?

In the freebsd PR there is a patch that reverts us back to 3006, I can confirm that works on my systems: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=278149

My goal is to test 3007 via pip once we get changes reverted in our ports tree, I am also surprised that this didn't get caught until after the changes were published.

ring-pete avatar Apr 26 '24 16:04 ring-pete

same here, 3007 is totally broken under FreeBSD.. was this even tested?

No, we do not test freebsd in our CI/CD and have not for some time now. That said we'll try to get a patch into a bugfix release.

dwoz avatar Jul 30 '24 00:07 dwoz

No, we do not test freebsd in our CI/CD and have not for some time now. That said we'll try to get a patch into a bugfix release.

Is there any particular reason that you don't test on FreeBSD? Lack of experience, time, CI/CD pipeline limitations, other?

mcgrathd avatar Jul 30 '24 15:07 mcgrathd

same here, 3007 is totally broken under FreeBSD.. was this even tested?

No, we do not test freebsd in our CI/CD and have not for some time now. That said we'll try to get a patch into a bugfix release.

it would be wonderful if you could add support for FreeBSD with something like https://github.com/vmactions/freebsd-vm

silenius avatar Aug 01 '24 12:08 silenius

I'm not sure if that FreeBSD VM action uses a 3rd-party cloud service that costs money or not...but I have a huge pile of small 1U servers sitting around that were running FreeBSD for years as routers. They were recently decommissioned, and I'd be happy to spin up whatever versions are necessary to allow build testing if it would be useful.

darkpixel avatar Aug 01 '24 21:08 darkpixel

I'm able to reliable reproduce the issue with this VagrantFile:

# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
  # Using the Generic FreeBSD box for libvirt
  config.vm.box = "generic/freebsd13"

  # Provider-specific configuration for libvirt
  config.vm.provider :libvirt do |libvirt|
    libvirt.memory = 2048  # Set memory
    libvirt.cpus = 2       # Set number of CPUs
  end

  # Provisioning commands to set up FreeBSD environment
  config.vm.provision "shell", inline: <<-SHELL
    # Update packages and install necessary software
    pkg update
    pkg install -y python39 git cmake

    # Install rustup and the Rust toolchain
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
    source $HOME/.cargo/env
    rustup update
    rustup default stable

    git clone https://github.com/saltstack/salt.git /home/vagrant/salt
    echo env >> /home/vagrant/salt/.git/info/exclude
    chown -R vagrant:vagrant salt

  SHELL
end

And this bash script:

#!/bin/bash
# Navigate to Salt directory
cd /home/vagrant/salt


# Initialize virtual environment
python3.9 -m venv env

source env/bin/activate

# Install the current code
pip install .

# Run salt-master in the foreground
RESULT=0
MASTER_PID=""

sudo ./env/bin/salt-master -l debug 2>&1 | while read -r line; do
    echo "$line"  # Print the output (optional)
    
    if [[ -z "$MASTER_PID" ]]; then
        MASTER_PID=$(pgrep -f 'salt-master')
    fi
    
    if [[ "$line" == *"encountered an exception"* ]]; then
        echo "Exception detected! Forcing salt-master to stop..."
        sudo kill -9 $MASTER_PID
        RESULT=1
        break
    elif [[ "$line" == *"Loaded master key"* ]]; then
        echo "Master started successfully. Forcing salt-master to stop..."
        sudo kill -9 $MASTER_PID
        RESULT=0
        break
    fi
done

# Clean up
deactivate
rm -rf env

# Exit with the result of the test command
exit $RESULT

I'm running a git bisect to see when the issue was introduced to find a fix.

Akm0d avatar Aug 15 '24 16:08 Akm0d

Confirmed that 3006.9 is the last good tag and 3007.0 has the issue

Akm0d avatar Aug 15 '24 19:08 Akm0d

This is the commit that introduced the bug: fb4ce8a741386d1ad4218cfa20648ec38801bdfa

Akm0d avatar Aug 16 '24 21:08 Akm0d