[BUG] `407: Proxy Authentication Required` for `genai.upload_file` API while proxy has been set to ENV var

[kuri-leo]

Description of the bug:

As user/pass have been set in ENV var as follows:

os.environ["HTTP_PROXY"] = "http://XXX:[email protected]:28888"
os.environ["HTTPS_PROXY"] = "http://XXX:[email protected]:28888"

The uploading API will raise an error from the following code, generated from AI studio:

def upload_to_gemini(path, mime_type=None):
    """Uploads the given file to Gemini.
  
    See https://ai.google.dev/gemini-api/docs/prompting_with_media
    """
    file = genai.upload_file(path, mime_type=mime_type)
    print(f"Uploaded file '{file.display_name}' as: {file.uri}")
    return file

The full error log as follows:

GeneralProxyError                         Traceback (most recent call last)
Cell In[4], line 16
      1 model = genai.GenerativeModel(
      2     model_name="gemini-1.5-pro",
      3     generation_config=generation_config,
   (...)
     10     # See https://ai.google.dev/gemini-api/docs/safety-settings
     11     )
     13 # TODO Make these files available on the local file system
     14 # You may need to update the file paths
     15 files = [
---> 16     upload_to_gemini("XXX.png",
     17                      mime_type="image/png"),
     18     ]
     20 chat_session = model.start_chat(
     21     history=[
     22         {
   (...)
     28         ]
     29     )
     31 response = chat_session.send_message("Please describe the given image in detail.")

Cell In[3], line 12, in upload_to_gemini(path, mime_type)
      7 def upload_to_gemini(path, mime_type=None):
      8     """Uploads the given file to Gemini.
      9   
     10     See https://ai.google.dev/gemini-api/docs/prompting_with_media
     11     """
---> 12     file = genai.upload_file(path, mime_type=mime_type)
     13     print(f"Uploaded file '{file.display_name}' as: {file.uri}")
     14     return file

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/google/generativeai/files.py:69, in upload_file(path, mime_type, name, display_name, resumable)
     66 if display_name is None:
     67     display_name = path.name
---> 69 response = client.create_file(
     70     path=path, mime_type=mime_type, name=name, display_name=display_name, resumable=resumable
     71 )
     72 return file_types.File(response)

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/google/generativeai/client.py:82, in FileServiceClient.create_file(self, path, mime_type, name, display_name, resumable)
     72 def create_file(
     73     self,
     74     path: str | pathlib.Path | os.PathLike,
   (...)
     79     resumable: bool = True,
     80 ) -> protos.File:
     81     if self._discovery_api is None:
---> 82         self._setup_discovery_api()
     84     file = {}
     85     if name is not None:

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/google/generativeai/client.py:65, in FileServiceClient._setup_discovery_api(self)
     56     raise ValueError(
     57         "Invalid operation: Uploading to the File API requires an API key. Please provide a valid API key."
     58     )
     60 request = googleapiclient.http.HttpRequest(
     61     http=httplib2.Http(),
     62     postproc=lambda resp, content: (resp, content),
     63     uri=f"{GENAI_API_DISCOVERY_URL}?version=v1beta&key={api_key}",
     64 )
---> 65 response, content = request.execute()
     67 discovery_doc = content.decode("utf-8")
     68 self._discovery_api = googleapiclient.discovery.build_from_document(
     69     discovery_doc, developerKey=api_key
     70 )

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/googleapiclient/_helpers.py:130, in positional.<locals>.positional_decorator.<locals>.positional_wrapper(*args, **kwargs)
    128     elif positional_parameters_enforcement == POSITIONAL_WARNING:
    129         logger.warning(message)
--> 130 return wrapped(*args, **kwargs)

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/googleapiclient/http.py:923, in HttpRequest.execute(self, http, num_retries)
    920     self.headers["content-length"] = str(len(self.body))
    922 # Handle retries for server-side errors.
--> 923 resp, content = _retry_request(
    924     http,
    925     num_retries,
    926     "request",
    927     self._sleep,
    928     self._rand,
    929     str(self.uri),
    930     method=str(self.method),
    931     body=self.body,
    932     headers=self.headers,
    933 )
    935 for callback in self.response_callbacks:
    936     callback(resp)

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/googleapiclient/http.py:191, in _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs)
    189 try:
    190     exception = None
--> 191     resp, content = http.request(uri, method, *args, **kwargs)
    192 # Retry on SSL errors and socket timeout errors.
    193 except _ssl_SSLError as ssl_error:

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/httplib2/__init__.py:1724, in Http.request(self, uri, method, body, headers, redirections, connection_type)
   1722             content = b""
   1723         else:
-> 1724             (response, content) = self._request(
   1725                 conn, authority, uri, request_uri, method, body, headers, redirections, cachekey,
   1726             )
   1727 except Exception as e:
   1728     is_timeout = isinstance(e, socket.timeout)

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/httplib2/__init__.py:1444, in Http._request(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey)
   1441 if auth:
   1442     auth.request(method, request_uri, headers, body)
-> 1444 (response, content) = self._conn_request(conn, request_uri, method, body, headers)
   1446 if auth:
   1447     if auth.response(response, body):

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/httplib2/__init__.py:1366, in Http._conn_request(self, conn, request_uri, method, body, headers)
   1364 try:
   1365     if conn.sock is None:
-> 1366         conn.connect()
   1367     conn.request(method, request_uri, body, headers)
   1368 except socket.timeout:

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/httplib2/__init__.py:1202, in HTTPSConnectionWithTimeout.connect(self)
   1200     break
   1201 if not self.sock:
-> 1202     raise socket_err

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/httplib2/__init__.py:1156, in HTTPSConnectionWithTimeout.connect(self)
   1154 if has_timeout(self.timeout):
   1155     sock.settimeout(self.timeout)
-> 1156 sock.connect((self.host, self.port))
   1158 self.sock = self._context.wrap_socket(sock, server_hostname=self.host)
   1160 # Python 3.3 compatibility: emulate the check_hostname behavior

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/socks.py:47, in set_self_blocking.<locals>.wrapper(*args, **kwargs)
     45     if _is_blocking == 0:
     46         self.setblocking(True)
---> 47     return function(*args, **kwargs)
     48 except Exception as e:
     49     raise

File ~/.miniconda/envs/jupyter/lib/python3.10/site-packages/socks.py:814, in socksocket.connect(self, dest_pair, catch_errors)
    811 if not catch_errors:
    812     # Wrap socket errors
    813     self.close()
--> 814     raise GeneralProxyError("Socket error", error)
    815 else:
    816     raise error

GeneralProxyError: Socket error: 407: Proxy Authentication Required

Actual vs expected behavior:

Go through an http proxy with authentication correctly, but, it fails.

Any other information you'd like to share?

Maybe related issues in upstream:

• https://github.com/httplib2/httplib2/issues/53
• https://github.com/httplib2/httplib2/issues/154
• https://github.com/httplib2/httplib2/pull/217
• https://github.com/httplib2/httplib2/commit/fa9a3bbfb5667968531613b24529a364f4d16b29Maybe

While everything works fine with these packages alone:

from urllib.parse import urlparse
url = urlparse("http://USER:PASSWORD@HOST:PORT/PATH")
print(url)
print(url.username)
print(url.password)

It returns:

ParseResult(scheme='http', netloc='USER:PASSWORD@HOST:PORT', path='/PATH', params='', query='', fragment='')
USER
PASSWORD

And:

import httplib2

pi = httplib2.proxy_info_from_url("http://USER:PASSWORD@HOST:80/PATH")
print(pi)
print(pi.proxy_host)
print(pi.proxy_port)
print(pi.proxy_user)
print(pi.proxy_pass)

it returns

<ProxyInfo type=3 host:port=host:80 rdns=True user=USER headers=None>
host
80
USER
PASSWORD