SSLCertVerificationError when downloading data on Windows 10

[xin-huang]

Hi

When I tried to download the example data with %load https://tskit.dev/tutorials/examples/download.py

I got this SSLCertVerificationError on Windows

SSLCertVerificationError                  Traceback (most recent call last)
c:\users\xin\miniconda3\lib\urllib\request.py in do_open(self, http_class, req, **http_conn_args)
   1349             try:
-> 1350                 h.request(req.get_method(), req.selector, req.data, headers,
   1351                           encode_chunked=req.has_header('Transfer-encoding'))

c:\users\xin\miniconda3\lib\http\client.py in request(self, method, url, body, headers, encode_chunked)
   1239         """Send a complete request to the server."""
-> 1240         self._send_request(method, url, body, headers, encode_chunked)
   1241 

c:\users\xin\miniconda3\lib\http\client.py in _send_request(self, method, url, body, headers, encode_chunked)
   1285             body = _encode(body, 'body')
-> 1286         self.endheaders(body, encode_chunked=encode_chunked)
   1287 

c:\users\xin\miniconda3\lib\http\client.py in endheaders(self, message_body, encode_chunked)
   1234             raise CannotSendHeader()
-> 1235         self._send_output(message_body, encode_chunked=encode_chunked)
   1236 

c:\users\xin\miniconda3\lib\http\client.py in _send_output(self, message_body, encode_chunked)
   1005         del self._buffer[:]
-> 1006         self.send(msg)
   1007 

c:\users\xin\miniconda3\lib\http\client.py in send(self, data)
    945             if self.auto_open:
--> 946                 self.connect()
    947             else:

c:\users\xin\miniconda3\lib\http\client.py in connect(self)
   1408 
-> 1409             self.sock = self._context.wrap_socket(self.sock,
   1410                                                   server_hostname=server_hostname)

c:\users\xin\miniconda3\lib\ssl.py in wrap_socket(self, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, session)
    499         # ctx._wrap_socket()
--> 500         return self.sslsocket_class._create(
    501             sock=sock,

c:\users\xin\miniconda3\lib\ssl.py in _create(cls, sock, server_side, do_handshake_on_connect, suppress_ragged_eofs, server_hostname, context, session)
   1039                         raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets")
-> 1040                     self.do_handshake()
   1041             except (OSError, ValueError):

c:\users\xin\miniconda3\lib\ssl.py in do_handshake(self, block)
   1308                 self.settimeout(None)
-> 1309             self._sslobj.do_handshake()
   1310         finally:

SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1108)

During handling of the above exception, another exception occurred:

URLError                                  Traceback (most recent call last)
~\AppData\Local\Temp/ipykernel_376/3167381492.py in <module>
----> 1 get_ipython().run_line_magic('load', 'https://tskit.dev/tutorials/examples/download.py')

c:\users\xin\miniconda3\lib\site-packages\IPython\core\interactiveshell.py in run_line_magic(self, magic_name, line, _stack_depth)
   2346                 kwargs['local_ns'] = self.get_local_scope(stack_depth)
   2347             with self.builtin_trap:
-> 2348                 result = fn(*args, **kwargs)
   2349             return result
   2350 

<decorator-gen-40> in load(self, arg_s)

c:\users\xin\miniconda3\lib\site-packages\IPython\core\magic.py in <lambda>(f, *a, **k)
    185     # but it's overkill for just that one bit of state.
    186     def magic_deco(arg):
--> 187         call = lambda f, *a, **k: f(*a, **k)
    188 
    189         if callable(arg):

c:\users\xin\miniconda3\lib\site-packages\IPython\core\magics\code.py in load(self, arg_s)
    351         search_ns = 'n' in opts
    352 
--> 353         contents = self.shell.find_user_code(args, search_ns=search_ns)
    354 
    355         if 's' in opts:

c:\users\xin\miniconda3\lib\site-packages\IPython\core\interactiveshell.py in find_user_code(self, target, raw, py_only, skip_encoding_cookie, search_ns)
   3725         try:
   3726             if target.startswith(('http://', 'https://')):
-> 3727                 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
   3728         except UnicodeDecodeError:
   3729             if not py_only :

c:\users\xin\miniconda3\lib\site-packages\IPython\utils\openpy.py in read_py_url(url, errors, skip_encoding_cookie)
     99     # Deferred import for faster start
    100     from urllib.request import urlopen
--> 101     response = urlopen(url)
    102     buffer = io.BytesIO(response.read())
    103     return source_to_unicode(buffer, errors, skip_encoding_cookie)

c:\users\xin\miniconda3\lib\urllib\request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

c:\users\xin\miniconda3\lib\urllib\request.py in open(self, fullurl, data, timeout)
    523 
    524         sys.audit('urllib.Request', req.full_url, req.data, req.headers, req.get_method())
--> 525         response = self._open(req, data)
    526 
    527         # post-process response

c:\users\xin\miniconda3\lib\urllib\request.py in _open(self, req, data)
    540 
    541         protocol = req.type
--> 542         result = self._call_chain(self.handle_open, protocol, protocol +
    543                                   '_open', req)
    544         if result:

c:\users\xin\miniconda3\lib\urllib\request.py in _call_chain(self, chain, kind, meth_name, *args)
    500         for handler in handlers:
    501             func = getattr(handler, meth_name)
--> 502             result = func(*args)
    503             if result is not None:
    504                 return result

c:\users\xin\miniconda3\lib\urllib\request.py in https_open(self, req)
   1391 
   1392         def https_open(self, req):
-> 1393             return self.do_open(http.client.HTTPSConnection, req,
   1394                 context=self._context, check_hostname=self._check_hostname)
   1395 

c:\users\xin\miniconda3\lib\urllib\request.py in do_open(self, http_class, req, **http_conn_args)
   1351                           encode_chunked=req.has_header('Transfer-encoding'))
   1352             except OSError as err: # timeout error
-> 1353                 raise URLError(err)
   1354             r = h.getresponse()
   1355         except:

URLError: <urlopen error [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1108)>

But I can download the data successfully on Ubuntu 20.04

I found a solution according to https://stackoverflow.com/questions/35569042/ssl-certificate-verify-failed-with-python3

and fixed it by adding

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

into https://github.com/tskit-dev/tutorials/blob/main/data/download.py

[benjeffery]

Thanks for the report @xin-huang! It sounds like it isn't happy with our "Let's encrypt" certificate. Glad you found the workaround.

[jeromekelleher]

I think this could be a general issue @benjeffery - we've hit the same thing over here: https://github.com/pystatgen/sgkit/issues/733

[benjeffery]

Eeesh, nasty. Not much we can do but wait for fixes.

[leonard-leo]

@xin-huang Your fix worked!!!!!!

For some context in case others face the same issue: I was trying to import whitebox from the WhiteboxTools python package. I was getting this error:

Unexpected error: <class 'urllib.error.URLError'>

This post on S/O seems to be relevant for this issue, but didn't fix it in my case: https://stackoverflow.com/questions/27835619/urllib-and-ssl-certificate-verify-failed-error

[benjeffery]

Closing, open again if this is still an issue we should try to put in a workaround for.