dbt-fal
dbt-fal copied to clipboard
Cannot instantiate and use more than one FalDbt objects at a time
Describe the bug When a FalDbt object is instantiated, it touches some data information on dbt's side, like the profile.yml that was read. We need to encapsulate this per instance.
Your environment
- OS: macOS 11.6.6
- Paste the output of the following command:
fal 0.1.0
Core:
- installed: 1.1.0
- latest: 1.1.0 - Up to date!
Plugins:
- bigquery: 1.1.0 - Up to date!
- postgres: 1.1.0 - Up to date!
~/Projects/fal/fal/integration_tests/projects/008_pure_python_models main* ≡ 12s
fal-eFX98vrn-py3.10 ❯
- Adapter being used: postgres and bigquery
How to reproduce
Setup the following in a file in integration_tests/projects
dir of this repo
import os
os.environ['temp_dir'] = '.'
os.environ['DBT_DATABASE'] = ''
os.environ['DBT_SCHEMA'] = ''
os.environ['GCLOUD_PROJECT'] = ''
os.environ['BQ_DATASET'] = ''
os.environ['KEYFILE_DIR'] = ''
pgfal = FalDbt(project_dir="008_pure_python_models", profiles_dir="../profiles/postgres", profile_target="dev")
# works:
pgfal.ref('model_a')
bqfal = FalDbt(project_dir="008_pure_python_models", profiles_dir="../profiles/bigquery", profile_target="integration_tests")
bqfal.ref("model_a")
# fails:
pgfal.ref('model_a')
pgfal = FalDbt(project_dir="008_pure_python_models", profiles_dir="../profiles/postgres", profile_target="dev")
# works:
pgfal.ref('model_a')
Expected behavior
to be able to ref
from either of the instances.
Actual behavior
Can only ref
(and others) from the latest built instance.
Additional context Error example:
---------------------------------------------------------------------------
DbtProfileError Traceback (most recent call last)
/var/folders/ht/prv18mgd1xx6bgvccwp_5v4m0000gn/T/ipykernel_37576/3791264918.py in <cell line: 1>()
----> 1 pgfal.ref('model_a')
.../fal/src/faldbt/project.py in ref(self, target_1, target_2)
422 target_model = self._model(target_model_name, target_package_name)
423
--> 424 result = lib.fetch_target(
425 self.project_dir,
426 self.profiles_dir,
.../fal/src/faldbt/lib.py in fetch_target(project_dir, profiles_dir, target, profile_target)
166 project_dir: str, profiles_dir: str, target: CompileResultNode, profile_target=None
167 ) -> RemoteRunResult:
--> 168 relation = _get_target_relation(
169 target, project_dir, profiles_dir, profile_target=profile_target
170 )
.../fal/src/faldbt/lib.py in _get_target_relation(target, project_dir, profiles_dir, profile_target)
124 profile_target: str = None,
125 ) -> Optional[BaseRelation]:
--> 126 adapter = _get_adapter(project_dir, profiles_dir, profile_target)
127 config = parse.get_dbt_config(
128 project_dir, profiles_dir, profile_target=profile_target
.../fal/src/faldbt/lib.py in _get_adapter(project_dir, profiles_dir, profile_target, config)
76 ) -> SQLAdapter:
77 if config is None:
---> 78 config = parse.get_dbt_config(
79 project_dir, profiles_dir, profile_target=profile_target
80 )
.../fal/src/faldbt/parse.py in get_dbt_config(project_dir, profiles_dir, threads, profile_target)
61 target=profile_target,
62 )
---> 63 return RuntimeConfig.from_args(args)
64
65
.../lib/python3.10/site-packages/dbt/config/runtime.py in from_args(cls, args)
224 :raises ValidationException: If the cli variables are invalid.
225 """
--> 226 project, profile = cls.collect_parts(args)
227
228 return cls.from_parts(
.../lib/python3.10/site-packages/dbt/config/runtime.py in collect_parts(cls, args)
200 profile_renderer = ProfileRenderer(cli_vars)
201 profile_name = partial.render_profile_name(profile_renderer)
--> 202 profile = cls._get_rendered_profile(args, profile_renderer, profile_name)
203 # Save env_vars encountered in rendering for partial parsing
204 profile.profile_env_vars = profile_renderer.ctx_obj.env_vars
.../lib/python3.10/site-packages/dbt/config/runtime.py in _get_rendered_profile(cls, args, profile_renderer, profile_name)
185 ) -> Profile:
186
--> 187 return Profile.render_from_args(args, profile_renderer, profile_name)
188
189 @classmethod
.../lib/python3.10/site-packages/dbt/config/profile.py in render_from_args(cls, args, renderer, project_profile_name)
426 raw_profiles = read_profile(flags.PROFILES_DIR)
427 profile_name = cls.pick_profile_name(getattr(args, "profile", None), project_profile_name)
--> 428 return cls.from_raw_profiles(
429 raw_profiles=raw_profiles,
430 profile_name=profile_name,
.../lib/python3.10/site-packages/dbt/config/profile.py in from_raw_profiles(cls, raw_profiles, profile_name, renderer, target_override, threads_override)
392 user_config = raw_profiles.get("config")
393
--> 394 return cls.from_raw_profile_info(
395 raw_profile=raw_profile,
396 profile_name=profile_name,
.../lib/python3.10/site-packages/dbt/config/profile.py in from_raw_profile_info(cls, raw_profile, profile_name, renderer, user_config, target_override, threads_override)
336 user_config = raw_profile.get("config")
337 # TODO: should it be, and the values coerced to bool?
--> 338 target_name, profile_data = cls.render_profile(
339 raw_profile, profile_name, target_override, renderer
340 )
.../lib/python3.10/site-packages/dbt/config/profile.py in render_profile(cls, raw_profile, profile_name, target_override, renderer)
296 fire_event(MissingProfileTarget(profile_name=profile_name, target_name=target_name))
297
--> 298 raw_profile_data = cls._get_profile_data(raw_profile, profile_name, target_name)
299
300 try:
.../lib/python3.10/site-packages/dbt/config/profile.py in _get_profile_data(profile, profile_name, target_name)
221 )
222 )
--> 223 raise DbtProfileError(msg, result_type="invalid_target")
224 profile_data = outputs[target_name]
225
DbtProfileError: Runtime Error
The profile 'fal_test' does not have a target named 'dev'. The valid target names for this profile are:
- integration_tests
Is there an existing bug report for this? The only related one is #111