salt icon indicating copy to clipboard operation
salt copied to clipboard
verified publisher icon saltstack

[BUG] "Exception caught while initializing git_pillar remote" after clearing cache on salt masterless configuration

Open modamod opened this issue 5 years ago • 3 comments

Description While testing migration from file_roots to gitfs_roots on vagrant with masterless configuration, I wanted to clear salt cache on the machine by running salt-call --local saltutil.clear_cache. The clear_cache command returned True as shown below:

[root@localhost ~]# salt-call saltutil.clear_cache
local:
    True

After this any salt command would print out this error

Error log 
[root@localhost ~]# salt-call cp.envs
[CRITICAL] Exception caught while initializing git_pillar remote 'branch ssh://<host>/repo.git': Repository not found at /var/cache/salt/minion/git_pillar/0848b69074623f49b4598bfbffca38d6ceb1c4e16307e363db8864f6d0722b6a
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 466, in __init__
    self.new = self.init_remote()
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1769, in init_remote
    self.repo = pygit2.Repository(self.cachedir)
  File "/usr/local/lib64/python3.7/site-packages/pygit2/repository.py", line 1455, in __init__
    path_backend = init_file_backend(path, flags)
_pygit2.GitError: Repository not found at /var/cache/salt/minion/git_pillar/0848b69074623f49b4598bfbffca38d6ceb1c4e16307e363db8864f6d0722b6a
[ERROR   ] Exception caught loading ext_pillar 'git':
  File "/usr/lib/python3.7/site-packages/salt/pillar/__init__.py", line 1145, in ext_pillar
    ext = self._external_pillar_data(pillar, val, key)
  File "/usr/lib/python3.7/site-packages/salt/pillar/__init__.py", line 1075, in _external_pillar_data
    ext = self.ext_pillars[key](self.minion_id, pillar, *val)
  File "/usr/lib/python3.7/site-packages/salt/pillar/git_pillar.py", line 443, in ext_pillar
    global_only=GLOBAL_ONLY,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2297, in __init__
    global_only,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2360, in init_remotes
    self.role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1513, in __init__
    role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 474, in __init__
    failhard(self.role)
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 216, in failhard
    raise FileserverConfigError("Failed to load {}".format(role))

[CRITICAL] Pillar render error: Failed to load ext_pillar git: Failed to load git_pillar
[CRITICAL] Exception caught while initializing gitfs remote 'ssh://<host>/repo.git': Repository not found at /var/cache/salt/minion/gitfs/a1b6302834466d58fa1b3e5ae73f2317b510724e289b612fa1a8c2e60e42d793
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 466, in __init__
    self.new = self.init_remote()
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1769, in init_remote
    self.repo = pygit2.Repository(self.cachedir)
  File "/usr/local/lib64/python3.7/site-packages/pygit2/repository.py", line 1455, in __init__
    path_backend = init_file_backend(path, flags)
_pygit2.GitError: Repository not found at /var/cache/salt/minion/gitfs/a1b6302834466d58fa1b3e5ae73f2317b510724e289b612fa1a8c2e60e42d793
[ERROR   ] An un-handled exception was caught by salt's global exception handler:
FileserverConfigError: Failed to load gitfs
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 466, in __init__
    self.new = self.init_remote()
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1769, in init_remote
    self.repo = pygit2.Repository(self.cachedir)
  File "/usr/local/lib64/python3.7/site-packages/pygit2/repository.py", line 1455, in __init__
    path_backend = init_file_backend(path, flags)
_pygit2.GitError: Repository not found at /var/cache/salt/minion/gitfs/a1b6302834466d58fa1b3e5ae73f2317b510724e289b612fa1a8c2e60e42d793

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/bin/salt-call", line 11, in <module>
    load_entry_point('salt==3002.2', 'console_scripts', 'salt-call')()
  File "/usr/lib/python3.7/site-packages/salt/scripts.py", line 449, in salt_call
    client.run()
  File "/usr/lib/python3.7/site-packages/salt/cli/call.py", line 58, in run
    caller.run()
  File "/usr/lib/python3.7/site-packages/salt/cli/caller.py", line 112, in run
    ret = self.call()
  File "/usr/lib/python3.7/site-packages/salt/cli/caller.py", line 220, in call
    self.opts, data, func, args, kwargs
  File "/usr/lib/python3.7/site-packages/salt/executors/direct_call.py", line 12, in execute
    return func(*args, **kwargs)
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 293, in envs
    return _client().envs()
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 166, in _client
    _mk_client()
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 159, in _mk_client
    ] = salt.fileclient.get_file_client(__opts__)
  File "/usr/lib/python3.7/site-packages/salt/fileclient.py", line 61, in get_file_client
    )(opts)
  File "/usr/lib/python3.7/site-packages/salt/fileclient.py", line 1468, in __init__
    self.channel = salt.fileserver.FSChan(opts)
  File "/usr/lib/python3.7/site-packages/salt/fileserver/__init__.py", line 868, in __init__
    self.fs.init()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/__init__.py", line 542, in init
    self.servers[fstr]()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/gitfs.py", line 173, in init
    _gitfs()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/gitfs.py", line 93, in _gitfs
    init_remotes=init_remotes,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2874, in __new__
    init_remotes=init_remotes,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2297, in __init__
    global_only,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2360, in init_remotes
    self.role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1513, in __init__
    role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 474, in __init__
    failhard(self.role)
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 216, in failhard
    raise FileserverConfigError("Failed to load {}".format(role))
salt.exceptions.FileserverConfigError: Failed to load gitfs
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 466, in __init__
    self.new = self.init_remote()
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1769, in init_remote
    self.repo = pygit2.Repository(self.cachedir)
  File "/usr/local/lib64/python3.7/site-packages/pygit2/repository.py", line 1455, in __init__
    path_backend = init_file_backend(path, flags)
_pygit2.GitError: Repository not found at /var/cache/salt/minion/gitfs/a1b6302834466d58fa1b3e5ae73f2317b510724e289b612fa1a8c2e60e42d793

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/bin/salt-call", line 11, in <module>
    load_entry_point('salt==3002.2', 'console_scripts', 'salt-call')()
  File "/usr/lib/python3.7/site-packages/salt/scripts.py", line 449, in salt_call
    client.run()
  File "/usr/lib/python3.7/site-packages/salt/cli/call.py", line 58, in run
    caller.run()
  File "/usr/lib/python3.7/site-packages/salt/cli/caller.py", line 112, in run
    ret = self.call()
  File "/usr/lib/python3.7/site-packages/salt/cli/caller.py", line 220, in call
    self.opts, data, func, args, kwargs
  File "/usr/lib/python3.7/site-packages/salt/executors/direct_call.py", line 12, in execute
    return func(*args, **kwargs)
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 293, in envs
    return _client().envs()
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 166, in _client
    _mk_client()
  File "/usr/lib/python3.7/site-packages/salt/modules/cp.py", line 159, in _mk_client
    ] = salt.fileclient.get_file_client(__opts__)
  File "/usr/lib/python3.7/site-packages/salt/fileclient.py", line 61, in get_file_client
    )(opts)
  File "/usr/lib/python3.7/site-packages/salt/fileclient.py", line 1468, in __init__
    self.channel = salt.fileserver.FSChan(opts)
  File "/usr/lib/python3.7/site-packages/salt/fileserver/__init__.py", line 868, in __init__
    self.fs.init()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/__init__.py", line 542, in init
    self.servers[fstr]()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/gitfs.py", line 173, in init
    _gitfs()
  File "/usr/lib/python3.7/site-packages/salt/fileserver/gitfs.py", line 93, in _gitfs
    init_remotes=init_remotes,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2874, in __new__
    init_remotes=init_remotes,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2297, in __init__
    global_only,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 2360, in init_remotes
    self.role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 1513, in __init__
    role,
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 474, in __init__
    failhard(self.role)
  File "/usr/lib/python3.7/site-packages/salt/utils/gitfs.py", line 216, in failhard
    raise FileserverConfigError("Failed to load {}".format(role))
salt.exceptions.FileserverConfigError: Failed to load gitfs

Setup

salt masterless configration 
file_client: local
fileserver_backend:
  - gitfs

gitfs_provider: pygit2

gitfs_saltenv_whitelist:
  - develop
  - base

gitfs_remotes:
  - ssh://<host>/<repo>.git:
    - root: salt
    - base: branch
    - ssl_verify: False
    - update_interval: 60
    - pubkey: /root/.ssh/id_ed25519.pub
    - privkey: /root/.ssh/id_ed25519

ext_pillar:
  - git:
    - branch ssh://<host>/<repo>.git:
      # Pillar SLS files will be read from the 'pillar' subdirectory in
      # this repository
      - root: salt/pillar
      - env: base
      - pubkey: /root/.ssh/id_ed25519.pub
      - privkey: /root/.ssh/id_ed25519

Steps to Reproduce the behavior

salt-call --local saltutil.clear_cache
<any salt-call command>
salt-call --local pillars.ls

Expected behavior I expected salt to sync data from the git repository and that salt-call commands to work without issues.

Versions Report

salt --versions-report 
Salt Version:
          Salt: 3002.2

Dependency Versions:
          cffi: 1.14.4
      cherrypy: Not Installed
      dateutil: Not Installed
     docker-py: Not Installed
         gitdb: Not Installed
     gitpython: Not Installed
        Jinja2: 2.10
       libgit2: 1.1.0
      M2Crypto: Not Installed
          Mako: Not Installed
       msgpack: 0.5.6
  msgpack-pure: Not Installed
  mysql-python: Not Installed
     pycparser: 2.20
      pycrypto: Not Installed
  pycryptodome: 3.6.1
        pygit2: 1.4.0
        Python: 3.7.9 (default, Aug 27 2020, 21:59:41)
  python-gnupg: Not Installed
        PyYAML: 4.2
         PyZMQ: 17.0.0
         smmap: Not Installed
       timelib: Not Installed
       Tornado: 4.5.3
           ZMQ: 4.2.3

System Versions:
          dist: amzn 2
        locale: UTF-8
       machine: x86_64
       release: 4.14.186-146.268.amzn2.x86_64
        system: Linux
       version: Amazon Linux 2

Additional context Add any other context about the problem here.

modamod avatar Nov 27 '20 15:11 modamod

Hi there! Welcome to the Salt Community! Thank you for making your first contribution. We have a lengthy process for issues and PRs. Someone from the Core Team will follow up as soon as possible. In the meantime, here’s some information that may help as you continue your Salt journey. Please be sure to review our Code of Conduct. Also, check out some of our community resources including:

There are lots of ways to get involved in our community. Every month, there are around a dozen opportunities to meet with other contributors and the Salt Core team and collaborate in real time. The best way to keep track is by subscribing to the Salt Community Events Calendar. If you have additional questions, email us at [email protected]. We’re glad you’ve joined our community and look forward to doing awesome things with you!

welcome[bot] avatar Nov 27 '20 15:11 welcome[bot]

It looks like we fail to reload data into the cache if the data is missing. Catching the exception and reload that data that we need should do the trick. If the reload fails then we throw the same exception.

cmcmarrow avatar Apr 06 '21 18:04 cmcmarrow

Hi, i'm running into this in a CI pipeline context aswell

$ salt-call --local saltutil.refresh_pillar wait=True
[CRITICAL] Exception caught while initializing git_pillar remote 'master [email protected]:infra/versions.git': Repository not found at /var/cache/salt/minion/git_pillar/c871dea6905364058230d4cf80ed15d8d854f624d596285ba2e62c0f4382283a
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 466, in __init__
    self.new = self.init_remote()
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 1769, in init_remote
    self.repo = pygit2.Repository(self.cachedir)
  File "/usr/lib/python3/dist-packages/pygit2/repository.py", line 1284, in __init__
    path_backend = init_file_backend(path)
_pygit2.GitError: Repository not found at /var/cache/salt/minion/git_pillar/c871dea6905364058230d4cf80ed15d8d854f624d596285ba2e62c0f4382283a
[ERROR   ] Exception caught loading ext_pillar 'git':
  File "/usr/lib/python3/dist-packages/salt/pillar/__init__.py", line 1151, in ext_pillar
    ext = self._external_pillar_data(pillar, val, key)
  File "/usr/lib/python3/dist-packages/salt/pillar/__init__.py", line 1081, in _external_pillar_data
    ext = self.ext_pillars[key](self.minion_id, pillar, *val)
  File "/usr/lib/python3/dist-packages/salt/loader.py", line 1241, in __call__
    return self.loader.run(run_func, *args, **kwargs)
  File "/usr/lib/python3/dist-packages/salt/loader.py", line 2274, in run
    return self._last_context.run(self._run_as, _func_or_method, *args, **kwargs)
  File "/usr/lib/python3/dist-packages/salt/loader.py", line 2289, in _run_as
    return _func_or_method(*args, **kwargs)
  File "/usr/lib/python3/dist-packages/salt/pillar/git_pillar.py", line 438, in ext_pillar
    git_pillar = salt.utils.gitfs.GitPillar(
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 2293, in __init__
    self.init_remotes(
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 2353, in init_remotes
    repo_obj = self.git_providers[self.provider](
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 1506, in __init__
    super().__init__(
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 474, in __init__
    failhard(self.role)
  File "/usr/lib/python3/dist-packages/salt/utils/gitfs.py", line 216, in failhard
    raise FileserverConfigError("Failed to load {}".format(role))

I'm not sure why it's working before clearing the cache.

Deleting the whole local directory

rm -Rf /var/cache/salt/

seems to fix the problem but that's not really scalable

calve avatar Nov 25 '21 12:11 calve