internetarchive
internetarchive copied to clipboard
Unhandled exception when doing mass rename
Using Version 2.2.0 under Linux.
command executed (repeatedly):
ia move a/b.dat a/c.dat -H x-archive-keep-old-version:0
I'm batch renaming a few 100 files. They're pretty big and the directory is large, close to 1TB. Renaming takes several hours to finish. Anyways after renaming the first few dozen files, this exception pops up:
Traceback (most recent call last):
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 859, in urlopen
**response_kw
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 859, in urlopen
**response_kw
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 836, in urlopen
retries = retries.increment(method, url, response=response, _pool=self)
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/util/retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='s3.us.archive.org', port=443): Max retries exceeded with url: /xbox-rc4-2/28014.rc4 (Caused by ResponseError('too many 503 error
responses'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 476, in execute
exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 401, in _wrap_coverage
return runner(*args)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 432, in _wrap_profiling
return runner(*args)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 532, in _execute
return self.execute_entry(self._pex_info.entry_point)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 668, in execute_entry
return self.execute_pkg_resources(entry_point)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 700, in execute_pkg_resources
return runner()
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/cli/ia.py", line 170, in main
sys.exit(ia_module.main(argv, session))
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/cli/ia_move.py", line 78, in main
dr = src_file.delete(headers=args['--header'], cascade_delete=True)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/files.py", line 379, in delete
resp = self.item.session.send(prepared_request)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 570, in send
raise e
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 567, in send
reraise_modify(e, e.request.url, prepend=False)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/utils.py", line 326, in reraise_modify
traceback)
File "/home/user/.pex/installed_wheels/035d7c208925c1832def39b592f3477ca36397bf/six-1.16.0-py2.py3-none-any.whl/six.py", line 719, in reraise
raise value
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 564, in send
r = super(ArchiveSession, self).send(request, **kwargs)
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/adapters.py", line 507, in send
raise RetryError(e, request=request)
requests.exceptions.RetryError: (MaxRetryError("HTTPSConnectionPool(host='s3.us.archive.org', port=443): Max retries exceeded with url: /xbox-rc4-2/28014.rc4 (Caused by ResponseError('too many 503 error responses'))"), 'https://s3.us.archive.org/xbox-rc4-2/28014.rc4')
Traceback (most recent call last):
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 859, in urlopen
**response_kw
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 859, in urlopen
**response_kw
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/connectionpool.py", line 836, in urlopen
retries = retries.increment(method, url, response=response, _pool=self)
File "/home/user/.pex/installed_wheels/0468d103d09253d2e6af05983feaa1ab4d39394c/urllib3-1.26.7-py2.py3-none-any.whl/urllib3/util/retry.py", line 574, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='s3.us.archive.org', port=443): Max retries exceeded with url: /xbox-rc4-2/28014.rc4 (Caused by ResponseError('too many 503 error
responses'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 476, in execute
exit_value = self._wrap_coverage(self._wrap_profiling, self._execute)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 401, in _wrap_coverage
return runner(*args)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 432, in _wrap_profiling
return runner(*args)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 532, in _execute
return self.execute_entry(self._pex_info.entry_point)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 668, in execute_entry
return self.execute_pkg_resources(entry_point)
File "/home/user/.pex/unzipped_pexes/09343006bbf9a2afe7a097523a079dc949ab30ec/.bootstrap/pex/pex.py", line 700, in execute_pkg_resources
return runner()
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/cli/ia.py", line 170, in main
sys.exit(ia_module.main(argv, session))
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/cli/ia_move.py", line 78, in main
dr = src_file.delete(headers=args['--header'], cascade_delete=True)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/files.py", line 379, in delete
resp = self.item.session.send(prepared_request)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 570, in send
raise e
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 567, in send
reraise_modify(e, e.request.url, prepend=False)
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/utils.py", line 326, in reraise_modify
traceback)
File "/home/user/.pex/installed_wheels/035d7c208925c1832def39b592f3477ca36397bf/six-1.16.0-py2.py3-none-any.whl/six.py", line 719, in reraise
raise value
File "/home/user/.pex/installed_wheels/b6c52b88d4c3e71ad36a5896944510ea5495e5ea/internetarchive-2.2.0-py2.py3-none-any.whl/internetarchive/session.py", line 564, in send
r = super(ArchiveSession, self).send(request, **kwargs)
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/user/.pex/installed_wheels/fb70e1b8449a6408d983fb8c56f60f1e827890d8/requests-2.26.0-py2.py3-none-any.whl/requests/adapters.py", line 507, in send
raise RetryError(e, request=request)
requests.exceptions.RetryError: (MaxRetryError("HTTPSConnectionPool(host='s3.us.archive.org', port=443): Max retries exceeded with url: /xbox-rc4-2/28014.rc4 (Caused by ResponseError('too many 503 error responses'))"), 'https://s3.us.archive.org/xbox-rc4-2/28014.rc4')
After that when trying to rename another file I get these (handled) error messages:
error: failed to move "a/b.dat" to "a/c.dat" - Please reduce your request rate. - bucket_tasks_queued exceeds rationed amount