KeyError when generating a list with ls

[kahn10]

Other stig processes may be running in background scripts. I have not been able to get the error to occur with just a regular "stig ls"

conn@instance-1:~$ stig --version stig version 0.11.2a0

conn@instance-1:~$ pipx --version 0.15.4.0

conn@instance-1:~$ /bin/stig ls "stopped&%dn<100|error~space&%dn<100"
Exception in callback _CommandBase.__init__.<locals>.<lambda>(<Task finishe...Error('name')>) at /home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py:154
handle: <Handle _CommandBase.__init__.<locals>.<lambda>(<Task finishe...Error('name')>) at /home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py:154>
Traceback (most recent call last):
  File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 154, in <lambda>
    self._task.add_done_callback(lambda task: self._catch_exceptions(task.result))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/base/torrent.py", line 172, in run
    await self.make_torrent_list(tfilter, sort, columns)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cli/torrent.py", line 104, in make_torrent_list
    torrents = sort.apply(response.torrents)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 114, in apply
    items = sorter(items, inplace=inplace, item_getter=item_getter)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 35, in __call__
    items = sorted(items, key=key_getter, reverse=reverse)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 30, in key_getter
    return keyfunc(item_getter(item))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/torrent.py", line 28, in <lambda>
    'name':              _SortSpec(lambda t: t['name'].casefold(),
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/aiotransmission/torrent.py", line 521, in __getitem__
    value = self._raw[fields[0]]
KeyError: 'name'
Traceback (most recent call last):
  File "/bin/stig", line 8, in <module>
    sys.exit(run())
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/__init__.py", line 25, in run
    main.run()
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/main.py", line 95, in run
    if not run_commands():
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/main.py", line 80, in run_commands
    success = cmdmgr.run_sync(clicmds)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdmanager.py", line 207, in run_sync
    process.wait_sync()
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 195, in wait_sync
    self._catch_exceptions(loop.run_until_complete, self._task)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
    return future.result()
  File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 154, in <lambda>
    self._task.add_done_callback(lambda task: self._catch_exceptions(task.result))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/base/torrent.py", line 172, in run
    await self.make_torrent_list(tfilter, sort, columns)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cli/torrent.py", line 104, in make_torrent_list
    torrents = sort.apply(response.torrents)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 114, in apply
    items = sorter(items, inplace=inplace, item_getter=item_getter)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 35, in __call__
    items = sorted(items, key=key_getter, reverse=reverse)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 30, in key_getter
    return keyfunc(item_getter(item))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/torrent.py", line 28, in <lambda>
    'name':              _SortSpec(lambda t: t['name'].casefold(),
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/aiotransmission/torrent.py", line 521, in __getitem__
    value = self._raw[fields[0]]
KeyError: 'name'
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x7f422bc83730>
Unclosed connector
connections: ['[(<aiohttp.client_proto.ResponseHandler object at 0x7f422b364280>, 6603.940585649)]']
connector: <aiohttp.connector.TCPConnector object at 0x7f422bc83430>

[rndusr]

Does this always happen with that command or did it just happen once?

How often does it happen?

[kahn10]

It happens nearly every time I run that specific command, I haven't seen it happen with any other one. It's weird because it works sometimes, and it used to run just fine in a previous version.

[rndusr]

I've seen this type of bug before but not in a long time. It's a race condition which makes it very hard to debug, especially because I can't reproduce it.

But if you can reproduce it, you could help a lot by hunting down the specific commit that introduced it with git bisect. Are you up for that?

[kahn10]

Sorry, I didn't see this reply. I thought I was getting email updates, but apparently not. I just tested it again and can't reproduce the problem now. I am getting the impression that the problem may have been rooted in a previous stig execution being interrupted mid-run. This happens sometimes when I'm debugging scripts and need to halt them. If you think this is something worth investigating I can try to recreate the issue and do troubleshooting steps.

[rndusr]

I don't think a previous stig process has any way to mess with a currently running stig process. It's more likely that the circumstances that forced this bug to manifest have changed. Maybe you had the perfect number of torrents with the perfect stats while your system was under a perfect load.

stig makes a list of all needed values for each torrent (download rate, size, ratio, etc). It does this by asking every filter, sorter and all the other bits that deal with torrents' values for the keys they need and combines them.

This bug happens when a filter (or something else) tries to use a torrent's value that wasn't requested, for example, when you open a new torrent list with a filter that uses a previously unneeded value.

I'm trying to prevent this by cancelling all ongoing requests and request handlers every time the list of needed values changes and make new requests.

At least that's what I think is going on. When I'm looking at the code, the bug shouldn't occur, so maybe it's something completely different.

If you can reproduce it reliably, I'd be extremely interested. But I'm not sure this is important to fix. I haven't encountered this bug for a long time.

[kahn10]

I think I had a lot more torrents running before, if it gets back up to that level I'll see if it starts happening again.

[kahn10]

I'm getting this error consistently again.

#/bin/stig ls "stopped&%dn<100|error~space&%dn<100"
Exception in callback _CommandBase.__init__.<locals>.<lambda>(<Task finishe...Error('name')>) at /home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py:154
handle: <Handle _CommandBase.__init__.<locals>.<lambda>(<Task finishe...Error('name')>) at /home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py:154>
Traceback (most recent call last):
  File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 154, in <lambda>
    self._task.add_done_callback(lambda task: self._catch_exceptions(task.result))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/base/torrent.py", line 172, in run
    await self.make_torrent_list(tfilter, sort, columns)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cli/torrent.py", line 104, in make_torrent_list
    torrents = sort.apply(response.torrents)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 114, in apply
    items = sorter(items, inplace=inplace, item_getter=item_getter)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 35, in __call__
    items = sorted(items, key=key_getter, reverse=reverse)
File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 30, in key_getter
    return keyfunc(item_getter(item))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/torrent.py", line 28, in <lambda>
    'name':              _SortSpec(lambda t: t['name'].casefold(),
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/aiotransmission/torrent.py", line 521, in __getitem__
    value = self._raw[fields[0]]
KeyError: 'name'
Traceback (most recent call last):
  File "/bin/stig", line 8, in <module>
    sys.exit(run())
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/__init__.py", line 25, in run
    main.run()
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/main.py", line 95, in run
    if not run_commands():
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/main.py", line 80, in run_commands
    success = cmdmgr.run_sync(clicmds)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdmanager.py", line 207, in run_sync
    process.wait_sync()
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 195, in wait_sync
    self._catch_exceptions(loop.run_until_complete, self._task)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/usr/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
    return future.result()
  File "/usr/lib/python3.8/asyncio/events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 154, in <lambda>
    self._task.add_done_callback(lambda task: self._catch_exceptions(task.result))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 164, in _catch_exceptions
    self._finish(exception=e)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 178, in _finish
    raise exception
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cmdbase.py", line 162, in _catch_exceptions
    callabee(*args, **kwargs)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/base/torrent.py", line 172, in run
    await self.make_torrent_list(tfilter, sort, columns)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/commands/cli/torrent.py", line 104, in make_torrent_list
    torrents = sort.apply(response.torrents)
File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 114, in apply
    items = sorter(items, inplace=inplace, item_getter=item_getter)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 35, in __call__
    items = sorted(items, key=key_getter, reverse=reverse)
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/base.py", line 30, in key_getter
    return keyfunc(item_getter(item))
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/sorters/torrent.py", line 28, in <lambda>
    'name':              _SortSpec(lambda t: t['name'].casefold(),
  File "/home/conn/.local/pipx/venvs/stig/lib/python3.8/site-packages/stig/client/aiotransmission/torrent.py", line 521, in __getitem__
    value = self._raw[fields[0]]
KeyError: 'name'
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x7fd6164aea60>
Unclosed connector
connections: ['[(<aiohttp.client_proto.ResponseHandler object at 0x7fd615ba0700>, 91470.478965505)]']
connector: <aiohttp.connector.TCPConnector object at 0x7fd6164aeb50>

[rndusr]

Can you find an earlier stig commit that fails to reproduce this issue?

Otherwise I can't really do anything about if I can't reproduce it myself.

If you know Python, I'd be more than happy to help you with debugging.