PyAirbyte icon indicating copy to clipboard operation
PyAirbyte copied to clipboard

Published 20 hours ago • verified publisher icon airbytehq

Failure to run `spec` on `source-airtable` ("Error: Validation against json schema defined in declarative_component_schema.yaml schema failed")

Open aaronsteers opened this issue 6 months ago • 2 comments

Reported in slack:

  • https://airbytehq-team.slack.com/archives/C06FZ238P8W/p1747412543018439

The failure occurs when invoking source-airtable:

Error: Validation against json schema defined in declarative_component_schema.yaml schema failed.

Easy to repro:

uv tool install airbyte-source-airtable
source-airtable spec

Error:

Show/Hide

$ source-airtable spec {"type":"TRACE","trace":{"type":"ERROR","emitted_at":1747413222276,"error":{"message":"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: Validation against json schema defined in declarative_component_schema.yaml schema failed","stack_trace":"Traceback (most recent call last):\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py\", line 394, in _validate_source\n validate(self._source_config, self._declarative_component_schema)\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/jsonschema/validators.py\", line 1121, in validate\n raise error\njsonschema.exceptions.ValidationError: {'type': 'HttpComponentsResolver', 'retriever': {'type': 'SimpleRetriever', 'requester': {'type': 'HttpRequester', 'url_base': 'https://api.airtable.com/v0/', 'authenticator': {'type': 'SelectiveAuthenticator', 'authenticator_selection_path': ['credentials', 'auth_method'], 'authenticators': {'oauth2.0': {'type': 'OAuthAuthenticator', 'client_id': \"{{ config['credentials']['client_id'] }}\", 'client_secret': \"{{ config['credentials']['client_secret'] }}\", 'refresh_token': \"{{ config['credentials']['refresh_token'] }}\", 'grant_type': 'refresh_token', 'refresh_request_headers': {'Authorization': \"Basic {{ [config['credentials']['client_id'], config['credentials']['client_secret']] | join(':') | base64encode }}\", 'Content-Type': 'application/x-www-form-urlencoded'}, 'refresh_token_updater': {'refresh_token_error_status_codes': [400, 401], 'refresh_token_error_key': 'error', 'refresh_token_error_values': ['invalid_grant', 'invalid_client'], 'refresh_token_name': 'refresh_token', 'access_token_config_path': ['credentials', 'access_token'], 'token_expiry_date_config_path': ['credentials', 'token_expiry_date'], 'refresh_token_config_path': ['credentials', 'refresh_token']}, 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token'}, 'api_key': {'type': 'BearerAuthenticator', 'api_token': \"{{ config['credentials']['api_key'] }}\"}}}, 'error_handler': {'type': 'DefaultErrorHandler', 'max_retries': 10, 'backoff_strategies': [{'type': 'ConstantBackoffStrategy', 'backoff_time_in_seconds': 30}], 'response_filters': [{'predicate': \"{{ response.status_code == 403 and response.get('error', {}).get('type') == 'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' }}\", 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': \"{{ 'Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' if config.get('credentials', {}). get('auth_method', '') == 'api_key' else 'Access Token does not have required permissions, please reauthenticate.' }}\", 'type': 'HttpResponseFilter'}, {'http_codes': [403, 422], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Permission denied or entity is unprocessable.', 'type': 'HttpResponseFilter'}, {'http_codes': [401], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Invalid credentials were provided. Please re-authenticate to restore access to Airtable.', 'type': 'HttpResponseFilter'}]}, 'path': 'meta/bases/{{ stream_partition.base_id }}/tables', 'http_method': 'GET'}, 'record_selector': {'type': 'RecordSelector', 'extractor': {'type': 'DpathExtractor', 'field_path': ['tables']}, 'pagination_strategy': {'type': 'CursorPagination', 'cursor_value': '{{ response.get(\"offset\", {}) }}', 'stop_condition': '{{ not response.get(\"offset\", {}) }}'}}, 'partition_router': {'type': 'SubstreamPartitionRouter', 'parent_stream_configs': [{'type': 'ParentStreamConfig', 'parent_key': 'id', 'partition_field': 'base_id', 'extra_fields': [['name']], 'stream': {'type': 'DeclarativeStream', 'name': 'bases', 'retriever': {'type': 'SimpleRetriever', 'requester': {'type': 'HttpRequester', 'url_base': 'https://api.airtable.com/v0/', 'authenticator': {'type': 'SelectiveAuthenticator', 'authenticator_selection_path': ['credentials', 'auth_method'], 'authenticators': {'oauth2.0': {'type': 'OAuthAuthenticator', 'client_id': \"{{ config['credentials']['client_id'] }}\", 'client_secret': \"{{ config['credentials']['client_secret'] }}\", 'refresh_token': \"{{ config['credentials']['refresh_token'] }}\", 'grant_type': 'refresh_token', 'refresh_request_headers': {'Authorization': \"Basic {{ [config['credentials']['client_id'], config['credentials']['client_secret']] | join(':') | base64encode }}\", 'Content-Type': 'application/x-www-form-urlencoded'}, 'refresh_token_updater': {'refresh_token_error_status_codes': [400, 401], 'refresh_token_error_key': 'error', 'refresh_token_error_values': ['invalid_grant', 'invalid_client'], 'refresh_token_name': 'refresh_token', 'access_token_config_path': ['credentials', 'access_token'], 'token_expiry_date_config_path': ['credentials', 'token_expiry_date'], 'refresh_token_config_path': ['credentials', 'refresh_token']}, 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token'}, 'api_key': {'type': 'BearerAuthenticator', 'api_token': \"{{ config['credentials']['api_key'] }}\"}}}, 'error_handler': {'type': 'DefaultErrorHandler', 'max_retries': 10, 'backoff_strategies': [{'type': 'ConstantBackoffStrategy', 'backoff_time_in_seconds': 30}], 'response_filters': [{'predicate': \"{{ response.status_code == 403 and response.get('error', {}).get('type') == 'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' }}\", 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': \"{{ 'Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' if config.get('credentials', {}). get('auth_method', '') == 'api_key' else 'Access Token does not have required permissions, please reauthenticate.' }}\", 'type': 'HttpResponseFilter'}, {'http_codes': [403, 422], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Permission denied or entity is unprocessable.', 'type': 'HttpResponseFilter'}, {'http_codes': [401], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Invalid credentials were provided. Please re-authenticate to restore access to Airtable.', 'type': 'HttpResponseFilter'}]}, 'path': 'meta/bases', 'http_method': 'GET'}, 'record_selector': {'type': 'RecordSelector', 'extractor': {'type': 'DpathExtractor', 'field_path': ['bases']}, 'record_filter': {'type': 'RecordFilter', 'condition': '{{ record.get(\"permissionLevel\") }}'}}, 'paginator': {'type': 'DefaultPaginator', 'page_token_option': {'type': 'RequestOption', 'inject_into': 'request_parameter', 'field_name': 'offset'}, 'pagination_strategy': {'type': 'CursorPagination', 'cursor_value': '{{ response.get(\"offset\", {}) }}', 'stop_condition': '{{ not response.get(\"offset\", {}) }}'}}}, 'schema_loader': {'type': 'InlineSchemaLoader', 'schema': {'type': 'object', '$schema': 'http://json-schema.org/schema#', 'additionalProperties': True, 'properties': {'id': {'type': ['string', 'null']}, 'name': {'type': ['string', 'null']}, 'permissionLevel': {'type': ['string', 'null']}}}}}}]}}, 'components_mapping': [{'type': 'ComponentMappingDefinition', 'field_path': ['name'], 'value': \"{{ stream_slice.extra_fields.name.replace(' ', '_').lower().strip() }}/{{ components_values.name.replace(' ', '_').lower().strip() }}/{{ components_values.id }}\"}, {'type': 'ComponentMappingDefinition', 'field_path': ['retriever', 'requester', 'path'], 'value': '{{ stream_slice.base_id }}/{{ components_values.id }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'requester', 'path'], 'value': '{{ stream_slice.base_id }}/{{ components_values.id }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['transformations', 0, 'fields', 0, 'value'], 'value': '{{ components_values.name }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'requester', 'path'], 'value': 'meta/bases/{{ stream_slice.base_id }}/tables'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'record_selector', 'record_filter', '$parameters', 'table_id'], 'value': '{{ components_values.id }}'}]} is not valid under any of the given schemas\n\nFailed validating 'anyOf' in schema['properties']['dynamic_streams']['items']['properties']['components_resolver']:\n {'anyOf': [{'$ref': '#/definitions/HttpComponentsResolver'},\n {'$ref': '#/definitions/ConfigComponentsResolver'}],\n 'description': 'Component resolve and populates stream templates with '\n 'components values.',\n 'title': 'Components Resolver'}\n\nOn instance['dynamic_streams'][0]['components_resolver']:\n {'components_mapping': [{'field_path': ['name'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ '\n \"stream_slice.extra_fields.name.replace(' \"\n \"', '_').lower().strip() }}/{{ \"\n \"components_values.name.replace(' ', \"\n \"'_').lower().strip() }}/{{ \"\n 'components_values.id }}'},\n {'field_path': ['retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ stream_slice.base_id }}/{{ '\n 'components_values.id }}'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ stream_slice.base_id }}/{{ '\n 'components_values.id }}'},\n {'field_path': ['transformations',\n 0,\n 'fields',\n 0,\n 'value'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ components_values.name }}'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': 'meta/bases/{{ stream_slice.base_id '\n '}}/tables'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'record_selector',\n 'record_filter',\n '$parameters',\n 'table_id'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ components_values.id }}'}],\n 'retriever': {'partition_router': {'parent_stream_configs': [{'extra_fields': [['name']],\n 'parent_key': 'id',\n 'partition_field': 'base_id',\n 'stream': {'name': 'bases',\n 'retriever': {'paginator': {'page_token_option': {'field_name': 'offset',\n 'inject_into': 'request_parameter',\n 'type': 'RequestOption'},\n 'pagination_strategy': {'cursor_value': '{{ '\n 'response.get(\"offset\", '\n '{}) '\n '}}',\n 'stop_condition': '{{ '\n 'not '\n 'response.get(\"offset\", '\n '{}) '\n '}}',\n 'type': 'CursorPagination'},\n 'type': 'DefaultPaginator'},\n 'record_selector': {'extractor': {'field_path': ['bases'],\n 'type': 'DpathExtractor'},\n 'record_filter': {'condition': '{{ '\n 'record.get(\"permissionLevel\") '\n '}}',\n 'type': 'RecordFilter'},\n 'type': 'RecordSelector'},\n 'requester': {'authenticator': {'authenticator_selection_path': ['credentials',\n 'auth_method'],\n 'authenticators': {'api_key': {'api_token': '{{ '\n \"config['credentials']['api_key'] \"\n '}}',\n 'type': 'BearerAuthenticator'},\n 'oauth2.0': {'client_id': '{{ '\n \"config['credentials']['client_id'] \"\n '}}',\n 'client_secret': '{{ '\n \"config['credentials']['client_secret'] \"\n '}}',\n 'grant_type': 'refresh_token',\n 'refresh_request_headers': {'Authorization': 'Basic '\n '{{ '\n \"[config['credentials']['client_id'], \"\n \"config['credentials']['client_secret']] \"\n '| '\n \"join(':') \"\n '| '\n 'base64encode '\n '}}',\n 'Content-Type': 'application/x-www-form-urlencoded'},\n 'refresh_token': '{{ '\n \"config['credentials']['refresh_token'] \"\n '}}',\n 'refresh_token_updater': {'access_token_config_path': ['credentials',\n 'access_token'],\n 'refresh_token_config_path': ['credentials',\n 'refresh_token'],\n 'refresh_token_error_key': 'error',\n 'refresh_token_error_status_codes': [400,\n 401],\n 'refresh_token_error_values': ['invalid_grant',\n 'invalid_client'],\n 'refresh_token_name': 'refresh_token',\n 'token_expiry_date_config_path': ['credentials',\n 'token_expiry_date']},\n 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token',\n 'type': 'OAuthAuthenticator'}},\n 'type': 'SelectiveAuthenticator'},\n 'error_handler': {'backoff_strategies': [{'backoff_time_in_seconds': 30,\n 'type': 'ConstantBackoffStrategy'}],\n 'max_retries': 10,\n 'response_filters': [{'action': 'FAIL',\n 'error_message': '{{ '\n \"'Personal \"\n 'Access '\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n 'add '\n 'all '\n 'required '\n 'permissions '\n 'to '\n 'existed '\n 'one '\n 'or '\n 'create '\n 'new '\n 'PAT, '\n 'see '\n 'docs '\n 'for '\n 'more '\n 'info: '\n \"https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' \"\n 'if '\n \"config.get('credentials', \"\n '{}). '\n \"get('auth_method', \"\n \"'') \"\n '== '\n \"'api_key' \"\n 'else '\n \"'Access \"\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n \"reauthenticate.' \"\n '}}',\n 'failure_type': 'config_error',\n 'predicate': '{{ '\n 'response.status_code '\n '== '\n '403 '\n 'and '\n \"response.get('error', \"\n \"{}).get('type') \"\n '== '\n \"'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' \"\n '}}',\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Permission '\n 'denied '\n 'or '\n 'entity '\n 'is '\n 'unprocessable.',\n 'failure_type': 'config_error',\n 'http_codes': [403,\n 422],\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Invalid '\n 'credentials '\n 'were '\n 'provided. '\n 'Please '\n 're-authenticate '\n 'to '\n 'restore '\n 'access '\n 'to '\n 'Airtable.',\n 'failure_type': 'config_error',\n 'http_codes': [401],\n 'type': 'HttpResponseFilter'}],\n 'type': 'DefaultErrorHandler'},\n 'http_method': 'GET',\n 'path': 'meta/bases',\n 'type': 'HttpRequester',\n 'url_base': 'https://api.airtable.com/v0/'},\n 'type': 'SimpleRetriever'},\n 'schema_loader': {'schema': {'$schema': 'http://json-schema.org/schema#',\n 'additionalProperties': True,\n 'properties': {'id': {'type': ['string',\n 'null']},\n 'name': {'type': ['string',\n 'null']},\n 'permissionLevel': {'type': ['string',\n 'null']}},\n 'type': 'object'},\n 'type': 'InlineSchemaLoader'},\n 'type': 'DeclarativeStream'},\n 'type': 'ParentStreamConfig'}],\n 'type': 'SubstreamPartitionRouter'},\n 'record_selector': {'extractor': {'field_path': ['tables'],\n 'type': 'DpathExtractor'},\n 'pagination_strategy': {'cursor_value': '{{ '\n 'response.get(\"offset\", '\n '{}) '\n '}}',\n 'stop_condition': '{{ '\n 'not '\n 'response.get(\"offset\", '\n '{}) '\n '}}',\n 'type': 'CursorPagination'},\n 'type': 'RecordSelector'},\n 'requester': {'authenticator': {'authenticator_selection_path': ['credentials',\n 'auth_method'],\n 'authenticators': {'api_key': {'api_token': '{{ '\n \"config['credentials']['api_key'] \"\n '}}',\n 'type': 'BearerAuthenticator'},\n 'oauth2.0': {'client_id': '{{ '\n \"config['credentials']['client_id'] \"\n '}}',\n 'client_secret': '{{ '\n \"config['credentials']['client_secret'] \"\n '}}',\n 'grant_type': 'refresh_token',\n 'refresh_request_headers': {'Authorization': 'Basic '\n '{{ '\n \"[config['credentials']['client_id'], \"\n \"config['credentials']['client_secret']] \"\n '| '\n \"join(':') \"\n '| '\n 'base64encode '\n '}}',\n 'Content-Type': 'application/x-www-form-urlencoded'},\n 'refresh_token': '{{ '\n \"config['credentials']['refresh_token'] \"\n '}}',\n 'refresh_token_updater': {'access_token_config_path': ['credentials',\n 'access_token'],\n 'refresh_token_config_path': ['credentials',\n 'refresh_token'],\n 'refresh_token_error_key': 'error',\n 'refresh_token_error_status_codes': [400,\n 401],\n 'refresh_token_error_values': ['invalid_grant',\n 'invalid_client'],\n 'refresh_token_name': 'refresh_token',\n 'token_expiry_date_config_path': ['credentials',\n 'token_expiry_date']},\n 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token',\n 'type': 'OAuthAuthenticator'}},\n 'type': 'SelectiveAuthenticator'},\n 'error_handler': {'backoff_strategies': [{'backoff_time_in_seconds': 30,\n 'type': 'ConstantBackoffStrategy'}],\n 'max_retries': 10,\n 'response_filters': [{'action': 'FAIL',\n 'error_message': '{{ '\n \"'Personal \"\n 'Access '\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n 'add '\n 'all '\n 'required '\n 'permissions '\n 'to '\n 'existed '\n 'one '\n 'or '\n 'create '\n 'new '\n 'PAT, '\n 'see '\n 'docs '\n 'for '\n 'more '\n 'info: '\n \"https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' \"\n 'if '\n \"config.get('credentials', \"\n '{}). '\n \"get('auth_method', \"\n \"'') \"\n '== '\n \"'api_key' \"\n 'else '\n \"'Access \"\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n \"reauthenticate.' \"\n '}}',\n 'failure_type': 'config_error',\n 'predicate': '{{ '\n 'response.status_code '\n '== '\n '403 '\n 'and '\n \"response.get('error', \"\n \"{}).get('type') \"\n '== '\n \"'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' \"\n '}}',\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Permission '\n 'denied '\n 'or '\n 'entity '\n 'is '\n 'unprocessable.',\n 'failure_type': 'config_error',\n 'http_codes': [403,\n 422],\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Invalid '\n 'credentials '\n 'were '\n 'provided. '\n 'Please '\n 're-authenticate '\n 'to '\n 'restore '\n 'access '\n 'to '\n 'Airtable.',\n 'failure_type': 'config_error',\n 'http_codes': [401],\n 'type': 'HttpResponseFilter'}],\n 'type': 'DefaultErrorHandler'},\n 'http_method': 'GET',\n 'path': 'meta/bases/{{ '\n 'stream_partition.base_id '\n '}}/tables',\n 'type': 'HttpRequester',\n 'url_base': 'https://api.airtable.com/v0/'},\n 'type': 'SimpleRetriever'},\n 'type': 'HttpComponentsResolver'}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/source_airtable/run.py\", line 25, in _get_source\n return SourceAirtable(\n ^^^^^^^^^^^^^^^\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/source_airtable/source.py\", line 13, in __init__\n super().__init__(catalog=catalog, config=config, state=state, **{\"path_to_yaml\": \"manifest.yaml\"})\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/airbyte_cdk/sources/declarative/yaml_declarative_source.py\", line 34, in __init__\n super().__init__(\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/airbyte_cdk/sources/declarative/concurrent_declarative_source.py\", line 93, in __init__\n super().__init__(\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py\", line 138, in __init__\n self._validate_source()\n File \"/Users/ajsteers/.local/share/uv/tools/airbyte-source-airtable/lib/python3.12/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py\", line 396, in _validate_source\n raise ValidationError(\njsonschema.exceptions.ValidationError: Validation against json schema defined in declarative_component_schema.yaml schema failed\n"}}}


</details>

aaronsteers avatar May 16 '25 16:05 aaronsteers

I've tested this workaround successfully:

Workaround tested successfully:
uv tool install airbyte-source-airtable --with="airbyte-cdk==6.32.0"
source-airtable spec

Steps to debug:

  1. Confirm (as in the original PR description) that the issue exists outside of PyAirbyte.
  2. Find the version of the CDK pinned in poetry.lock:

https://github.com/airbytehq/airbyte/blob/02ae7c24927be9962c220604867af74ada9df445/airbyte-integrations/connectors/source-airtable/poetry.lock#L4-L6

  1. Retry using uv install --with arg (as noted above in this comment).

To adapt this workaround for PyAirbyte, you can use:

source = airtable = ab.get_source(
    "source-airtable",
    # Pin to the CDK version from the connector's `poetry.lock` file:
    pip_url="source-airtable airbyte-cdk==6.32.0"
)

aaronsteers avatar May 16 '25 16:05 aaronsteers

Hey team, I have tried that solution and no luck.

import airbyte as ab  

source = airtable =  ab.get_source(
    "source-airtable",
    pip_url="source-airtable airbyte-cdk==6.32.0"
)
credentials = {
    "credentials": {
        "auth_method": "api_key",        
        "api_key":     "pat"
    }
}
source.set_config(config=credentials)

Logs

`

2025-05-16 11:03:38 - ERROR - Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: Validation against json schema defined in declarative_component_schema.yaml schema failed 2025-05-16 11:03:38 - INFO - {"type":"TRACE","trace":{"type":"ERROR","emitted_at":1747415018612,"error":{"message":"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: Validation against json schema defined in declarative_component_schema.yaml schema failed","stack_trace":"Traceback (most recent call last):\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py", line 394, in validate_source\n validate(self.source_config, self.declarative_component_schema)\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/jsonschema/validators.py", line 1121, in validate\n raise error\njsonschema.exceptions.ValidationError: {'type': 'HttpComponentsResolver', 'retriever': {'type': 'SimpleRetriever', 'requester': {'type': 'HttpRequester', 'url_base': 'https://api.airtable.com/v0/', 'authenticator': {'type': 'SelectiveAuthenticator', 'authenticator_selection_path': ['credentials', 'auth_method'], 'authenticators': {'oauth2.0': {'type': 'OAuthAuthenticator', 'client_id': "{{ config['credentials']['client_id'] }}", 'client_secret': "{{ config['credentials']['client_secret'] }}", 'refresh_token': "{{ config['credentials']['refresh_token'] }}", 'grant_type': 'refresh_token', 'refresh_request_headers': {'Authorization': "Basic {{ [config['credentials']['client_id'], config['credentials']['client_secret']] | join(':') | base64encode }}", 'Content-Type': 'application/x-www-form-urlencoded'}, 'refresh_token_updater': {'refresh_token_error_status_codes': [400, 401], 'refresh_token_error_key': 'error', 'refresh_token_error_values': ['invalid_grant', 'invalid_client'], 'refresh_token_name': 'refresh_token', 'access_token_config_path': ['credentials', 'access_token'], 'token_expiry_date_config_path': ['credentials', 'token_expiry_date'], 'refresh_token_config_path': ['credentials', 'refresh_token']}, 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token'}, 'api_key': {'type': 'BearerAuthenticator', 'api_token': "{{ config['credentials']['api_key'] }}"}}}, 'error_handler': {'type': 'DefaultErrorHandler', 'max_retries': 10, 'backoff_strategies': [{'type': 'ConstantBackoffStrategy', 'backoff_time_in_seconds': 30}], 'response_filters': [{'predicate': "{{ response.status_code == 403 and response.get('error', {}).get('type') == 'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' }}", 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': "{{ 'Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' if config.get('credentials', {}). get('auth_method', '') == 'api_key' else 'Access Token does not have required permissions, please reauthenticate.' }}", 'type': 'HttpResponseFilter'}, {'http_codes': [403, 422], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Permission denied or entity is unprocessable.', 'type': 'HttpResponseFilter'}, {'http_codes': [401], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Invalid credentials were provided. Please re-authenticate to restore access to Airtable.', 'type': 'HttpResponseFilter'}]}, 'path': 'meta/bases/{{ stream_partition.base_id }}/tables', 'http_method': 'GET'}, 'record_selector': {'type': 'RecordSelector', 'extractor': {'type': 'DpathExtractor', 'field_path': ['tables']}, 'pagination_strategy': {'type': 'CursorPagination', 'cursor_value': '{{ response.get("offset", {}) }}', 'stop_condition': '{{ not response.get("offset", {}) }}'}}, 'partition_router': {'type': 'SubstreamPartitionRouter', 'parent_stream_configs': [{'type': 'ParentStreamConfig', 'parent_key': 'id', 'partition_field': 'base_id', 'extra_fields': [['name']], 'stream': {'type': 'DeclarativeStream', 'name': 'bases', 'retriever': {'type': 'SimpleRetriever', 'requester': {'type': 'HttpRequester', 'url_base': 'https://api.airtable.com/v0/', 'authenticator': {'type': 'SelectiveAuthenticator', 'authenticator_selection_path': ['credentials', 'auth_method'], 'authenticators': {'oauth2.0': {'type': 'OAuthAuthenticator', 'client_id': "{{ config['credentials']['client_id'] }}", 'client_secret': "{{ config['credentials']['client_secret'] }}", 'refresh_token': "{{ config['credentials']['refresh_token'] }}", 'grant_type': 'refresh_token', 'refresh_request_headers': {'Authorization': "Basic {{ [config['credentials']['client_id'], config['credentials']['client_secret']] | join(':') | base64encode }}", 'Content-Type': 'application/x-www-form-urlencoded'}, 'refresh_token_updater': {'refresh_token_error_status_codes': [400, 401], 'refresh_token_error_key': 'error', 'refresh_token_error_values': ['invalid_grant', 'invalid_client'], 'refresh_token_name': 'refresh_token', 'access_token_config_path': ['credentials', 'access_token'], 'token_expiry_date_config_path': ['credentials', 'token_expiry_date'], 'refresh_token_config_path': ['credentials', 'refresh_token']}, 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token'}, 'api_key': {'type': 'BearerAuthenticator', 'api_token': "{{ config['credentials']['api_key'] }}"}}}, 'error_handler': {'type': 'DefaultErrorHandler', 'max_retries': 10, 'backoff_strategies': [{'type': 'ConstantBackoffStrategy', 'backoff_time_in_seconds': 30}], 'response_filters': [{'predicate': "{{ response.status_code == 403 and response.get('error', {}).get('type') == 'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' }}", 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': "{{ 'Personal Access Token does not have required permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' if config.get('credentials', {}). get('auth_method', '') == 'api_key' else 'Access Token does not have required permissions, please reauthenticate.' }}", 'type': 'HttpResponseFilter'}, {'http_codes': [403, 422], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Permission denied or entity is unprocessable.', 'type': 'HttpResponseFilter'}, {'http_codes': [401], 'action': 'FAIL', 'failure_type': 'config_error', 'error_message': 'Invalid credentials were provided. Please re-authenticate to restore access to Airtable.', 'type': 'HttpResponseFilter'}]}, 'path': 'meta/bases', 'http_method': 'GET'}, 'record_selector': {'type': 'RecordSelector', 'extractor': {'type': 'DpathExtractor', 'field_path': ['bases']}, 'record_filter': {'type': 'RecordFilter', 'condition': '{{ record.get("permissionLevel") }}'}}, 'paginator': {'type': 'DefaultPaginator', 'page_token_option': {'type': 'RequestOption', 'inject_into': 'request_parameter', 'field_name': 'offset'}, 'pagination_strategy': {'type': 'CursorPagination', 'cursor_value': '{{ response.get("offset", {}) }}', 'stop_condition': '{{ not response.get("offset", {}) }}'}}}, 'schema_loader': {'type': 'InlineSchemaLoader', 'schema': {'type': 'object', '$schema': 'http://json-schema.org/schema#', 'additionalProperties': True, 'properties': {'id': {'type': ['string', 'null']}, 'name': {'type': ['string', 'null']}, 'permissionLevel': {'type': ['string', 'null']}}}}}}]}}, 'components_mapping': [{'type': 'ComponentMappingDefinition', 'field_path': ['name'], 'value': "{{ stream_slice.extra_fields.name.replace(' ', '').lower().strip() }}/{{ components_values.name.replace(' ', '').lower().strip() }}/{{ components_values.id }}"}, {'type': 'ComponentMappingDefinition', 'field_path': ['retriever', 'requester', 'path'], 'value': '{{ stream_slice.base_id }}/{{ components_values.id }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'requester', 'path'], 'value': '{{ stream_slice.base_id }}/{{ components_values.id }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['transformations', 0, 'fields', 0, 'value'], 'value': '{{ components_values.name }}'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'requester', 'path'], 'value': 'meta/bases/{{ stream_slice.base_id }}/tables'}, {'type': 'ComponentMappingDefinition', 'field_path': ['schema_loader', 'retriever', 'record_selector', 'record_filter', '$parameters', 'table_id'], 'value': '{{ components_values.id }}'}]} is not valid under any of the given schemas\n\nFailed validating 'anyOf' in schema['properties']['dynamic_streams']['items']['properties']['components_resolver']:\n {'anyOf': [{'$ref': '#/definitions/HttpComponentsResolver'},\n {'$ref': '#/definitions/ConfigComponentsResolver'}],\n 'description': 'Component resolve and populates stream templates with '\n 'components values.',\n 'title': 'Components Resolver'}\n\nOn instance['dynamic_streams'][0]['components_resolver']:\n {'components_mapping': [{'field_path': ['name'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ '\n "stream_slice.extra_fields.name.replace(' "\n "', '').lower().strip() }}/{{ "\n "components_values.name.replace(' ', "\n "'_').lower().strip() }}/{{ "\n 'components_values.id }}'},\n {'field_path': ['retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ stream_slice.base_id }}/{{ '\n 'components_values.id }}'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ stream_slice.base_id }}/{{ '\n 'components_values.id }}'},\n {'field_path': ['transformations',\n 0,\n 'fields',\n 0,\n 'value'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ components_values.name }}'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'requester',\n 'path'],\n 'type': 'ComponentMappingDefinition',\n 'value': 'meta/bases/{{ stream_slice.base_id '\n '}}/tables'},\n {'field_path': ['schema_loader',\n 'retriever',\n 'record_selector',\n 'record_filter',\n '$parameters',\n 'table_id'],\n 'type': 'ComponentMappingDefinition',\n 'value': '{{ components_values.id }}'}],\n 'retriever': {'partition_router': {'parent_stream_configs': [{'extra_fields': [['name']],\n 'parent_key': 'id',\n 'partition_field': 'base_id',\n 'stream': {'name': 'bases',\n 'retriever': {'paginator': {'page_token_option': {'field_name': 'offset',\n 'inject_into': 'request_parameter',\n 'type': 'RequestOption'},\n 'pagination_strategy': {'cursor_value': '{{ '\n 'response.get("offset", '\n '{}) '\n '}}',\n 'stop_condition': '{{ '\n 'not '\n 'response.get("offset", '\n '{}) '\n '}}',\n 'type': 'CursorPagination'},\n 'type': 'DefaultPaginator'},\n 'record_selector': {'extractor': {'field_path': ['bases'],\n 'type': 'DpathExtractor'},\n 'record_filter': {'condition': '{{ '\n 'record.get("permissionLevel") '\n '}}',\n 'type': 'RecordFilter'},\n 'type': 'RecordSelector'},\n 'requester': {'authenticator': {'authenticator_selection_path': ['credentials',\n 'auth_method'],\n 'authenticators': {'api_key': {'api_token': '{{ '\n "config['credentials']['api_key'] "\n '}}',\n 'type': 'BearerAuthenticator'},\n 'oauth2.0': {'client_id': '{{ '\n "config['credentials']['client_id'] "\n '}}',\n 'client_secret': '{{ '\n "config['credentials']['client_secret'] "\n '}}',\n 'grant_type': 'refresh_token',\n 'refresh_request_headers': {'Authorization': 'Basic '\n '{{ '\n "[config['credentials']['client_id'], "\n "config['credentials']['client_secret']] "\n '| '\n "join(':') "\n '| '\n 'base64encode '\n '}}',\n 'Content-Type': 'application/x-www-form-urlencoded'},\n 'refresh_token': '{{ '\n "config['credentials']['refresh_token'] "\n '}}',\n 'refresh_token_updater': {'access_token_config_path': ['credentials',\n 'access_token'],\n 'refresh_token_config_path': ['credentials',\n 'refresh_token'],\n 'refresh_token_error_key': 'error',\n 'refresh_token_error_status_codes': [400,\n 401],\n 'refresh_token_error_values': ['invalid_grant',\n 'invalid_client'],\n 'refresh_token_name': 'refresh_token',\n 'token_expiry_date_config_path': ['credentials',\n 'token_expiry_date']},\n 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token',\n 'type': 'OAuthAuthenticator'}},\n 'type': 'SelectiveAuthenticator'},\n 'error_handler': {'backoff_strategies': [{'backoff_time_in_seconds': 30,\n 'type': 'ConstantBackoffStrategy'}],\n 'max_retries': 10,\n 'response_filters': [{'action': 'FAIL',\n 'error_message': '{{ '\n "'Personal "\n 'Access '\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n 'add '\n 'all '\n 'required '\n 'permissions '\n 'to '\n 'existed '\n 'one '\n 'or '\n 'create '\n 'new '\n 'PAT, '\n 'see '\n 'docs '\n 'for '\n 'more '\n 'info: '\n "https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' "\n 'if '\n "config.get('credentials', "\n '{}). '\n "get('auth_method', "\n "'') "\n '== '\n "'api_key' "\n 'else '\n "'Access "\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n "reauthenticate.' "\n '}}',\n 'failure_type': 'config_error',\n 'predicate': '{{ '\n 'response.status_code '\n '== '\n '403 '\n 'and '\n "response.get('error', "\n "{}).get('type') "\n '== '\n "'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' "\n '}}',\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Permission '\n 'denied '\n 'or '\n 'entity '\n 'is '\n 'unprocessable.',\n 'failure_type': 'config_error',\n 'http_codes': [403,\n 422],\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Invalid '\n 'credentials '\n 'were '\n 'provided. '\n 'Please '\n 're-authenticate '\n 'to '\n 'restore '\n 'access '\n 'to '\n 'Airtable.',\n 'failure_type': 'config_error',\n 'http_codes': [401],\n 'type': 'HttpResponseFilter'}],\n 'type': 'DefaultErrorHandler'},\n 'http_method': 'GET',\n 'path': 'meta/bases',\n 'type': 'HttpRequester',\n 'url_base': 'https://api.airtable.com/v0/'},\n 'type': 'SimpleRetriever'},\n 'schema_loader': {'schema': {'$schema': 'http://json-schema.org/schema#',\n 'additionalProperties': True,\n 'properties': {'id': {'type': ['string',\n 'null']},\n 'name': {'type': ['string',\n 'null']},\n 'permissionLevel': {'type': ['string',\n 'null']}},\n 'type': 'object'},\n 'type': 'InlineSchemaLoader'},\n 'type': 'DeclarativeStream'},\n 'type': 'ParentStreamConfig'}],\n 'type': 'SubstreamPartitionRouter'},\n 'record_selector': {'extractor': {'field_path': ['tables'],\n 'type': 'DpathExtractor'},\n 'pagination_strategy': {'cursor_value': '{{ '\n 'response.get("offset", '\n '{}) '\n '}}',\n 'stop_condition': '{{ '\n 'not '\n 'response.get("offset", '\n '{}) '\n '}}',\n 'type': 'CursorPagination'},\n 'type': 'RecordSelector'},\n 'requester': {'authenticator': {'authenticator_selection_path': ['credentials',\n 'auth_method'],\n 'authenticators': {'api_key': {'api_token': '{{ '\n "config['credentials']['api_key'] "\n '}}',\n 'type': 'BearerAuthenticator'},\n 'oauth2.0': {'client_id': '{{ '\n "config['credentials']['client_id'] "\n '}}',\n 'client_secret': '{{ '\n "config['credentials']['client_secret'] "\n '}}',\n 'grant_type': 'refresh_token',\n 'refresh_request_headers': {'Authorization': 'Basic '\n '{{ '\n "[config['credentials']['client_id'], "\n "config['credentials']['client_secret']] "\n '| '\n "join(':') "\n '| '\n 'base64encode '\n '}}',\n 'Content-Type': 'application/x-www-form-urlencoded'},\n 'refresh_token': '{{ '\n "config['credentials']['refresh_token'] "\n '}}',\n 'refresh_token_updater': {'access_token_config_path': ['credentials',\n 'access_token'],\n 'refresh_token_config_path': ['credentials',\n 'refresh_token'],\n 'refresh_token_error_key': 'error',\n 'refresh_token_error_status_codes': [400,\n 401],\n 'refresh_token_error_values': ['invalid_grant',\n 'invalid_client'],\n 'refresh_token_name': 'refresh_token',\n 'token_expiry_date_config_path': ['credentials',\n 'token_expiry_date']},\n 'token_refresh_endpoint': 'https://airtable.com/oauth2/v1/token',\n 'type': 'OAuthAuthenticator'}},\n 'type': 'SelectiveAuthenticator'},\n 'error_handler': {'backoff_strategies': [{'backoff_time_in_seconds': 30,\n 'type': 'ConstantBackoffStrategy'}],\n 'max_retries': 10,\n 'response_filters': [{'action': 'FAIL',\n 'error_message': '{{ '\n "'Personal "\n 'Access '\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n 'add '\n 'all '\n 'required '\n 'permissions '\n 'to '\n 'existed '\n 'one '\n 'or '\n 'create '\n 'new '\n 'PAT, '\n 'see '\n 'docs '\n 'for '\n 'more '\n 'info: '\n "https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable' "\n 'if '\n "config.get('credentials', "\n '{}). '\n "get('auth_method', "\n "'') "\n '== '\n "'api_key' "\n 'else '\n "'Access "\n 'Token '\n 'does '\n 'not '\n 'have '\n 'required '\n 'permissions, '\n 'please '\n "reauthenticate.' "\n '}}',\n 'failure_type': 'config_error',\n 'predicate': '{{ '\n 'response.status_code '\n '== '\n '403 '\n 'and '\n "response.get('error', "\n "{}).get('type') "\n '== '\n "'INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND' "\n '}}',\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Permission '\n 'denied '\n 'or '\n 'entity '\n 'is '\n 'unprocessable.',\n 'failure_type': 'config_error',\n 'http_codes': [403,\n 422],\n 'type': 'HttpResponseFilter'},\n {'action': 'FAIL',\n 'error_message': 'Invalid '\n 'credentials '\n 'were '\n 'provided. '\n 'Please '\n 're-authenticate '\n 'to '\n 'restore '\n 'access '\n 'to '\n 'Airtable.',\n 'failure_type': 'config_error',\n 'http_codes': [401],\n 'type': 'HttpResponseFilter'}],\n 'type': 'DefaultErrorHandler'},\n 'http_method': 'GET',\n 'path': 'meta/bases/{{ '\n 'stream_partition.base_id '\n '}}/tables',\n 'type': 'HttpRequester',\n 'url_base': 'https://api.airtable.com/v0/'},\n 'type': 'SimpleRetriever'},\n 'type': 'HttpComponentsResolver'}\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/source_airtable/run.py", line 25, in _get_source\n return SourceAirtable(\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/source_airtable/source.py", line 13, in init\n super().init(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"})\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/airbyte_cdk/sources/declarative/yaml_declarative_source.py", line 34, in init\n super().init(\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/airbyte_cdk/sources/declarative/concurrent_declarative_source.py", line 93, in init\n super().init(\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py", line 138, in init\n self._validate_source()\n File "/Users/mauricio.perez/Documents/git/connectors/.venv-source-airtable/lib/python3.10/site-packages/airbyte_cdk/sources/declarative/manifest_declarative_source.py", line 396, in _validate_source\n raise ValidationError(\njsonschema.exceptions.ValidationError: Validation against json schema defined in declarative_component_schema.yaml schema failed\n"}}}

`

MauPerez avatar May 16 '25 17:05 MauPerez