PyAirbyte icon indicating copy to clipboard operation
PyAirbyte copied to clipboard

Bug: Cloud API interfaces do not gracefully handle "too many requests" (`Status 429`)

Open aaronsteers opened this issue 2 months ago • 0 comments

This is appearing in CI checks:

______________________ test_create_and_delete_destination ______________________

workspace_id = '19d7a891-8e0e-40ac-8a8c-5faf8d11e47c'
airbyte_cloud_api_root = 'https://api.airbyte.com/v1'
motherduck_api_key = <SecretString: ****>
airbyte_cloud_client_id = <SecretString: ****>
airbyte_cloud_client_secret = <SecretString: ****>

    def test_create_and_delete_destination(
        workspace_id: str,
        airbyte_cloud_api_root: str,
        motherduck_api_key: str,
        airbyte_cloud_client_id: SecretString,
        airbyte_cloud_client_secret: SecretString,
    ) -> None:
        new_resource_name = (
            "deleteme-destination-faker" + text_util.generate_random_suffix()
        )
        destination_config = DestinationDuckdb(
            destination_path="temp_db",
            motherduck_api_key=motherduck_api_key,
        )
    
        destination = api_util.create_destination(
            name=new_resource_name,
            api_root=airbyte_cloud_api_root,
            workspace_id=workspace_id,
            config=destination_config,
            client_id=airbyte_cloud_client_id,
            client_secret=airbyte_cloud_client_secret,
        )
        assert destination.name == new_resource_name
        assert destination.destination_type == "duckdb"
        assert destination.destination_id
    
>       api_util.delete_destination(
            destination_id=destination.destination_id,
            api_root=airbyte_cloud_api_root,
            workspace_id=workspace_id,
            client_id=airbyte_cloud_client_id,
            client_secret=airbyte_cloud_client_secret,
        )

tests/integration_tests/cloud/test_cloud_api_util.py:151: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
airbyte/_util/api_util.py:672: in delete_destination
    response = airbyte_instance.destinations.delete_destination(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <airbyte_api.destinations.Destinations object at 0x7f73e1309780>
request = DeleteDestinationRequest(destination_id='249bc314-b66d-484f-b766-5b5608854263')

    def delete_destination(self, request: api.DeleteDestinationRequest) -> api.DeleteDestinationResponse:
        r"""Delete a Destination"""
        hook_ctx = HookContext(operation_id='deleteDestination', oauth2_scopes=[], security_source=self.sdk_configuration.security)
        base_url = utils.template_url(*self.sdk_configuration.get_server_details())
    
        url = utils.generate_url(base_url, '/destinations/***destinationId***', request)
    
        if callable(self.sdk_configuration.security):
            headers, query_params = utils.get_security(self.sdk_configuration.security())
        else:
            headers, query_params = utils.get_security(self.sdk_configuration.security)
    
        headers['Accept'] = '*/*'
        headers['user-agent'] = self.sdk_configuration.user_agent
        client = self.sdk_configuration.client
    
        try:
            req = client.prepare_request(requests_http.Request('DELETE', url, params=query_params, headers=headers))
            req = self.sdk_configuration.get_hooks().before_request(BeforeRequestContext(hook_ctx), req)
            http_res = client.send(req)
        except Exception as e:
            _, e = self.sdk_configuration.get_hooks().after_error(AfterErrorContext(hook_ctx), None, e)
            if e is not None:
                raise e
    
        if utils.match_status_codes(['403','404','4XX','5XX'], http_res.status_code):
            result, e = self.sdk_configuration.get_hooks().after_error(AfterErrorContext(hook_ctx), http_res, None)
            if e is not None:
                raise e
            if result is not None:
                http_res = result
        else:
            http_res = self.sdk_configuration.get_hooks().after_success(AfterSuccessContext(hook_ctx), http_res)
    
    
    
        res = api.DeleteDestinationResponse(status_code=http_res.status_code, content_type=http_res.headers.get('Content-Type') or '', raw_response=http_res)
    
        if http_res.status_code == 204:
            pass
        elif http_res.status_code == 403 or http_res.status_code == 404 or http_res.status_code >= 400 and http_res.status_code < 500 or http_res.status_code >= 500 and http_res.status_code < 600:
>           raise errors.SDKError('API error occurred', http_res.status_code, http_res.text, http_res)
E           airbyte_api.errors.sdkerror.SDKError: API error occurred: Status 429
E           <!doctype html><meta charset="utf-8"><meta name=viewport content="width=device-width, initial-scale=1"><title>429</title>429 Too Many Requests

aaronsteers avatar Sep 19 '25 20:09 aaronsteers