b2sdk UploadTokenUsedConcurrently

[mjurbanski-reef]

To utilize b2sdk under asyncio, I'm running it under threadpool executor, but strangely enough, error occurred once per server and not multiple times on a single server (I have ~80 of them running this code, and they upload a file every other minute). Error occurred on around 2023-11-16T04:59:14.000Z on 3 servers. Posting that info, since all my servers reported the same token in the exception, which seems to indicate b2 API server bug.

UploadTokenUsedConcurrently: More than one concurrent upload using auth token 4_005fREDACTED_01b061e5_0b1c7c_uplg_-OREDACTEDgux4J-REDACTED4hORgY=
  File "concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "myapp/worker/tasks_b2.py", line 179, in process_datasource_dir
    return _process_datasource_dir(datasource_dir, claimed_datasource_ids, datasource_id=datasource_id)
  File "myapp/worker/tasks_b2.py", line 219, in _process_datasource_dir
    chunk_data = _upload_merged_chunks(
  File "myapp/worker/tasks_b2.py", line 242, in _upload_merged_chunks
    upload_url = process_upload(BUCKET_dataS, data_file_merged, datasource_id, data_bucket_path)
  File "myapp/worker/tasks_b2.py", line 96, in process_upload
    upload_result = bucket.upload_local_file(
  File "logfury/_logfury/trace_call.py", line 86, in wrapper
    return function(*wrapee_args, **wrapee_kwargs)
  File "b2sdk/version_utils.py", line 116, in wrapper
    return func(*args, **kwargs)
  File "b2sdk/v2/bucket.py", line 83, in upload_local_file
    return super().upload_local_file(
  File "logfury/_logfury/trace_call.py", line 86, in wrapper
    return function(*wrapee_args, **wrapee_kwargs)
  File "b2sdk/bucket.py", line 592, in upload_local_file
    return self.concatenate(
  File "logfury/_logfury/trace_call.py", line 86, in wrapper
    return function(*wrapee_args, **wrapee_kwargs)
  File "b2sdk/bucket.py", line 1018, in concatenate
    return self.create_file(
  File "logfury/_logfury/trace_call.py", line 86, in wrapper
    return function(*wrapee_args, **wrapee_kwargs)
  File "b2sdk/bucket.py", line 842, in create_file
    return self._create_file(
  File "b2sdk/bucket.py", line 953, in _create_file
    return emerger_method(
  File "logfury/_logfury/trace_call.py", line 86, in wrapper
    return function(*wrapee_args, **wrapee_kwargs)
  File "b2sdk/transfer/emerge/emerger.py", line 177, in emerge
    return self._emerge(
  File "b2sdk/transfer/emerge/emerger.py", line 116, in _emerge
    return self.emerge_executor.execute_emerge_plan(
  File "b2sdk/transfer/emerge/executor.py", line 85, in execute_emerge_plan
    return execution.execute_plan(emerge_plan)
  File "b2sdk/transfer/emerge/executor.py", line 241, in execute_plan
    part_sha1_array = [f.result()['contentSha1'] for f in part_futures]
  File "b2sdk/transfer/emerge/executor.py", line 241, in <listcomp>
    part_sha1_array = [f.result()['contentSha1'] for f in part_futures]
  File "concurrent/futures/_base.py", line 456, in result
    return self.__get_result()
  File "concurrent/futures/_base.py", line 401, in __get_result
    raise self._exception
  File "concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "b2sdk/transfer/outbound/upload_manager.py", line 170, in _upload_part
    response = self.services.session.upload_part(
  File "b2sdk/session.py", line 386, in upload_part
    return self._wrap_token(
  File "b2sdk/session.py", line 462, in _wrap_token
    return self._reauthorization_loop(partial_callback)
  File "b2sdk/session.py", line 477, in _reauthorization_loop
    return callback()
  File "b2sdk/session.py", line 552, in _upload_part
    response = f(upload_url, upload_auth_token, *args, **kwargs)
  File "b2sdk/raw_api.py", line 998, in upload_part
    return self.b2_http.post_content_return_json(upload_url, headers, data_stream)
  File "b2sdk/b2http.py", line 262, in post_content_return_json
    response = self._translate_and_retry(do_post, try_count, post_params)
  File "b2sdk/b2http.py", line 507, in _translate_and_retry
    return cls._translate_errors(fcn, post_params)
  File "b2sdk/b2http.py", line 431, in _translate_errors