marie-ai
marie-ai copied to clipboard
marieai
certificate verify failed: unable to get local issuer certificate
Describe the bug During startup we are getting following exception.
INFO marie@33 Kwargs : {'metas': {'name': '', 'description': '', 'workspace': '', 'py_modules': ['marie.executor.text']}, 'requests': {}, 'dynamic_batching':
{}, 'runtime_args': {'workspace': None, 'shard_id': 0, 'shards': 1, 'replicas': 3, 'name': 'extract_t/rep-2', 'provider': [<ProviderType.NONE: 0>],
'metrics_registry': <prometheus_client.registry.CollectorRegistry object at 0x7fd73086ce50>, 'tracer_provider': None, 'meter_provider': None}}
ERROR extract_t/rep-1@32 SSLError(MaxRetryError("HTTPSConnectionPool(host='dl.fbaipublicfiles.com', port=443): Max retries exceeded with url:
/fairseq/gpt2_bpe/vocab.bpe (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get
local issuer certificate (_ssl.c:1007)')))")) during 'WorkerRuntime' initialization
add "--quiet-error" to suppress the exception details
Traceback (most recent call last):
File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 715, in urlopen
httplib_response = self._make_request(
File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 404, in _make_request
self._validate_conn(conn)
File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 1058, in _validate_conn
conn.connect()
File "/opt/venv/lib/python3.10/site-packages/urllib3/connection.py", line 419, in connect
self.sock = ssl_wrap_socket(
File "/opt/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 449, in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(
File "/opt/venv/lib/python3.10/site-packages/urllib3/util/ssl_.py", line 493, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "/usr/lib/python3.10/ssl.py", line 513, in wrap_socket
return self.sslsocket_class._create(
File "/usr/lib/python3.10/ssl.py", line 1071, in _create
self.do_handshake()
File "/usr/lib/python3.10/ssl.py", line 1342, in do_handshake
self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1007)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/venv/lib/python3.10/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/opt/venv/lib/python3.10/site-packages/urllib3/connectionpool.py", line 799, in urlopen
retries = retries.increment(
File "/opt/venv/lib/python3.10/site-packages/urllib3/util/retry.py", line 592, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='dl.fbaipublicfiles.com', port=443): Max retries exceeded with url:
/fairseq/gpt2_bpe/vocab.bpe (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get
local issuer certificate (_ssl.c:1007)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/venv/lib/python3.10/site-packages/marie/serve/executors/run.py", line 143, in run
runtime = AsyncNewLoopRuntime(
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/asyncio.py", line 93, in __init__
self._loop.run_until_complete(self.async_setup())
File "/usr/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
return future.result()
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/asyncio.py", line 310, in async_setup
self.server = self._get_server()
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/asyncio.py", line 215, in _get_server
return GRPCServer(
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/servers/grpc.py", line 34, in __init__
super().__init__(**kwargs)
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/servers/__init__.py", line 63, in __init__
] = (req_handler or self._get_request_handler())
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/servers/__init__.py", line 88, in _get_request_handler
return self.req_handler_cls(
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/worker/request_handling.py", line 140, in __init__
self._load_executor(
File "/opt/venv/lib/python3.10/site-packages/marie/serve/runtimes/worker/request_handling.py", line 377, in _load_executor
self._executor: BaseExecutor = BaseExecutor.load_config(
File "/opt/venv/lib/python3.10/site-packages/marie/jaml/__init__.py", line 792, in load_config
obj = JAML.load(tag_yml, substitute=False, runtime_args=runtime_args)
File "/opt/venv/lib/python3.10/site-packages/marie/jaml/__init__.py", line 174, in load
r = yaml.load(stream, Loader=get_jina_loader_with_runtime(runtime_args))
File "/opt/venv/lib/python3.10/site-packages/yaml/__init__.py", line 81, in load
return loader.get_single_data()
File "/opt/venv/lib/python3.10/site-packages/yaml/constructor.py", line 51, in get_single_data
return self.construct_document(node)
File "/opt/venv/lib/python3.10/site-packages/yaml/constructor.py", line 55, in construct_document
data = self.construct_object(node)
File "/opt/venv/lib/python3.10/site-packages/yaml/constructor.py", line 100, in construct_object
data = constructor(self, node)
File "/opt/venv/lib/python3.10/site-packages/marie/jaml/__init__.py", line 582, in _from_yaml
return get_parser(cls, version=data.get('version', None)).parse(
File "/opt/venv/lib/python3.10/site-packages/marie/jaml/parsers/executor/legacy.py", line 46, in parse
obj = cls(
File "/opt/venv/lib/python3.10/site-packages/marie/serve/executors/decorators.py", line 58, in arg_wrapper
f = func(self, *args, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/marie/serve/helper.py", line 74, in arg_wrapper
f = func(self, *args, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/marie/executor/text/text_extraction_executor.py", line 62, in __init__
self.pipeline = ExtractPipeline(pipeline_config=pipeline, cuda=use_cuda)
File "/opt/venv/lib/python3.10/site-packages/marie/pipe/extract_pipeline.py", line 99, in __init__
self.ocr_engines = get_known_ocr_engines(device=device)
File "/opt/venv/lib/python3.10/site-packages/marie/pipe/components.py", line 110, in get_known_ocr_engines
ocr_engines["default"] = DefaultOcrEngine(cuda=use_cuda)
File "/opt/venv/lib/python3.10/site-packages/marie/ocr/default_ocr_engine.py", line 45, in __init__
self.ocr_processor = TrOcrProcessor(
File "/opt/venv/lib/python3.10/site-packages/marie/document/trocr_ocr_processor.py", line 250, in __init__
) = init(model_path, beam, device)
File "/opt/venv/lib/python3.10/site-packages/marie/document/trocr_ocr_processor.py", line 61, in init
model, cfg, inference_task = fairseq.checkpoint_utils.load_model_ensemble_and_task(
File "/opt/venv/lib/python3.10/site-packages/fairseq/checkpoint_utils.py", line 502, in load_model_ensemble_and_task
model = task.build_model(cfg.model, from_checkpoint=True)
File "/opt/venv/lib/python3.10/site-packages/fairseq/tasks/fairseq_task.py", line 691, in build_model
model = models.build_model(args, self, from_checkpoint)
File "/opt/venv/lib/python3.10/site-packages/fairseq/models/__init__.py", line 106, in build_model
return model.build_model(cfg, task)
File "/opt/venv/lib/python3.10/site-packages/marie/models/unilm/trocr/trocr_models.py", line 169, in build_model
roberta = torch.hub.load('pytorch/fairseq:main', 'roberta.large')
File "/opt/venv/lib/python3.10/site-packages/torch/hub.py", line 566, in load
model = _load_local(repo_or_dir, model, *args, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/torch/hub.py", line 595, in _load_local
model = entry(*args, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/fairseq/models/roberta/model.py", line 380, in from_pretrained
return RobertaHubInterface(x["args"], x["task"], x["models"][0])
File "/opt/venv/lib/python3.10/site-packages/fairseq/models/roberta/hub_interface.py", line 26, in __init__
self.bpe = encoders.build_bpe(cfg.bpe)
File "/opt/venv/lib/python3.10/site-packages/fairseq/registry.py", line 65, in build_x
return builder(cfg, *extra_args, **extra_kwargs)
File "/opt/venv/lib/python3.10/site-packages/fairseq/data/encoders/gpt2_bpe.py", line 33, in __init__
vocab_bpe = file_utils.cached_path(cfg.gpt2_vocab_bpe)
File "/opt/venv/lib/python3.10/site-packages/fairseq/file_utils.py", line 174, in cached_path
return get_from_cache(url_or_filename, cache_dir)
File "/opt/venv/lib/python3.10/site-packages/fairseq/file_utils.py", line 299, in get_from_cache
response = request_wrap_timeout(
File "/opt/venv/lib/python3.10/site-packages/fairseq/file_utils.py", line 251, in request_wrap_timeout
return func(timeout=timeout)
File "/opt/venv/lib/python3.10/site-packages/requests/api.py", line 100, in head
return request("head", url, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/opt/venv/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/opt/venv/lib/python3.10/site-packages/requests/adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='dl.fbaipublicfiles.com', port=443): Max retries exceeded with url: /fairseq/gpt2_bpe/vocab.bpe
(Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate
(_ssl.c:1007)')))
One solution here is to try to cache the file directly and don't attempt to download it.
host='dl.fbaipublicfiles.com' uses ESET-SSL-Filter-CA. However, if it's not included in "/opt/venv/lib/python3.10/site-packages/certifi/cacert.pem" it would cause SSLError. Possible solution: append the content of ESET-SSL-Filter-CA.pem to "opt/venv/lib/python3.10/site-packages/certifi/cacert.pem"
ref: about:certificate?cert=MIID9DCCAtygAwIBAgIQaWa9I7zQtK01rEhHEueegTANBgkqhkiG9w0BAQsFADBIMRswGQYDVQQDDBJFU0VUIFNTTCBGaWx0ZXIgQ0ExHDAaBgNVBAoME0VTRVQsIHNwb2wuIHMgci4gby4xCzAJBgNVBAYTAlNLMB4XDTIzMDMwNjAwMDAwMFoXDTI0MDMwNTIzNTk1OVowcDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExEzARBgNVBAcTCk1lbmxvIFBhcmsxFzAVBgNVBAoTDkZhY2Vib29rLCBJbmMuMR4wHAYDVQQDDBUqLmZiYWlwdWJsaWNmaWxlcy5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCs4Ap4RKHy2SU%2FZ7gwFfKfqc1DlFAs2ntYXmCRLMaVBLn2Y8ZwAsgsLTTkpkb8UYnW%2FO79tOpjR9GxoCC9d5VoUPnXJ%2B%2FpzWH%2BNXA2cjOq4VSQ8TsDx8I8WWqyATY405S%2BL7ghOz7SPoRP3srkpcuCQQzPwjjlZ4KLMqktMbF3MdFjaoQo39g4Y3goBrkYBVpJjM1DPtciuQc4kZVaYveo7oczLa%2FsN29lCU2VLSOJLhZ7UHwcpq7%2FV3ZpmHYctruhyIdAgvT2qBiSrlBox7vR5AMzHEgojlGrKfCYAZ1Wq6C33SnMteaSZzWHSmtWyjjjUhYenypxxuZQpWlffblRAgMBAAGjgbEwga4wCwYDVR0PBAQDAgWgMB0GA1UdDgQWBBSo98jaWaS03U5zdsAYys5bj%2FZsEjAfBgNVHSMEGDAWgBR3QZ5210hxi9rZtJ5OlNlw4eP2cjA1BgNVHREELjAsghUqLmZiYWlwdWJsaWNmaWxlcy5jb22CE2ZiYWlwdWJsaWNmaWxlcy5jb20wHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAkGA1UdEwQCMAAwDQYJKoZIhvcNAQELBQADggEBAHPiJuUa4AHjeWtAxcr9Ry7iDmmOl6fjf9kg6T84lp1ticIAYpNiAL0KsgfOczKJM7Nbx756ycldGLHl1IkIYov7XwEJ3x8UACV%2Fr1xHgeC%2BRKdD42RwWmW6%2Bpvchx9tZixmlaMeWhSGVyAi5%2FJ9oNV7jkDCdhQLP8WWR6mZT3znExlRXJQ24qZL4DjfmWVwePL2D2hcxYIEujaiEPhRSYy6RM1ByE%2Fw6BY9RyiKdYgWidqsBSYXgyDtvaPZNUH%2FmCHDS%2BnhcnujH8h6E6Raj0bJOYB8k5jtN15uo%2B4pWTENUEXC9OwGBDYCyHbPoXCzQh%2FAPVutAw8VWCpfDFBSWeo%3D&cert=MIIDgDCCAmigAwIBAgIQGm3Mlv1c%2BvR098q3bghNcTANBgkqhkiG9w0BAQsFADBIMRswGQYDVQQDDBJFU0VUIFNTTCBGaWx0ZXIgQ0ExHDAaBgNVBAoME0VTRVQsIHNwb2wuIHMgci4gby4xCzAJBgNVBAYTAlNLMB4XDTIzMTAzMDE1NTUxN1oXDTMzMTAyNzE1NTUxN1owSDEbMBkGA1UEAwwSRVNFVCBTU0wgRmlsdGVyIENBMRwwGgYDVQQKDBNFU0VULCBzcG9sLiBzIHIuIG8uMQswCQYDVQQGEwJTSzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJjdOVI2CUSx4ABMhf9Cg4BiqkJzCD%2FMQdhwjvLCrAytOPOvDE2ig8je%2FX3Fn0m2d3QM3rdEZAPHGsJ%2FWgYYa221U7P%2BH%2By5ESj36fF1QwiHsAc6jnFQtHzr1oBaNsBMx9omMGk%2Bxh1OOyfsXSJZue9EPBC1VpnOoV3fV%2BnyLotLk1Aja96SaIqlPwAhX6dFcNMfFjfDM5kvO65ZD4XGT7JPaGB8bZKdMng0p1e38dVbCS2yrGFIuWn82RvMFwNAqAmSGj%2FWAoPKlQvRkoqeFeerR97NboJoS4xcUK4Z5BURueB2xtZS7yzMsAUCY%2FBEHvqAW6vgzC28oqBLFND64ksCAwEAAaNmMGQwDgYDVR0PAQH%2FBAQDAgIEMBIGA1UdEwEB%2FwQIMAYBAf8CAQAwHQYDVR0OBBYEFHdBnnbXSHGL2tm0nk6U2XDh4%2FZyMB8GA1UdIwQYMBaAFHdBnnbXSHGL2tm0nk6U2XDh4%2FZyMA0GCSqGSIb3DQEBCwUAA4IBAQA3s8xsYqE%2BjzIuiz52JcTatGdURjEEnYM6kF4s4rVomc6KGTgBVyk8pTnJHoLaFTneWxKWeLAf4y%2BrPJxLR4u%2Fj9ULkmc9kdRjVpWKd21HQ0zDd5fazT495Q78VWmr2kS8lsOSyvG2EiBMb3YSNo7GG7cViq1p%2FtjxZLxExGy%2BXGGvbmg%2FWdwXUmdwKcvkVT%2BAmeZFyam02DRfLLhrDrC6gwzSZb8ar3vRT%2Fyq0OxgX%2BfJbD3PGYgshU1uiSQK80rYWA2yWm%2BwHiL6N4r4%2F97QZ9Hhj%2Fqhfi881bK4dVGQClqHPTXIkXAyXrbh1Idz7ZrmFXyJRY87CzJCphdcE6l9#
